Development

This commit is contained in:
Matthew Grotke 2026-05-25 01:04:47 -04:00
parent a4652866c3
commit 27eaea3d73
19 changed files with 602 additions and 427 deletions

View file

@ -20,19 +20,18 @@ These scripts do not run continuously in the background. They install and facili
## Capabilities
The suite is organized into three independent but complementary scripts, each managing one layer of the stack:
The suite is organized into independent but complementary scripts, each managing one layer of the stack:
### Core: DHCP, DNS, Blocklists, Firewall, RADIUS, mDNS, and WireGuard VPN (`core.py`)
### Core: DHCP, DNS, Firewall, RADIUS, mDNS, and WireGuard VPN (`core.py`)
- Configures VLAN sub-interfaces via `systemd-networkd`
- Assigns static or dynamic DHCP reservations by MAC address and hostname
- Defines dynamic IP pools per VLAN
- Manages per-VLAN gateway, DNS, and NTP settings derived from `server_identities`
- Runs one `dnsmasq` instance per VLAN, each bound exclusively to its gateway IP, giving true per-VLAN DNS filtering
- Downloads and merges blocklists from upstream providers you choose (e.g. OISD, Hagezi)
- Applies per-VLAN content filtering - VLANs with different blocklist sets each get their own merged blocklist
- Applies per-VLAN content filtering - VLANs with different blocklist sets each get their own merged blocklist (blocklists are downloaded and merged by `dns-blocklists.py`)
- Supports local hostname overrides (split DNS for DDNS hostnames)
- Installs a daily `systemd` timer to refresh blocklists
- Installs a daily `systemd` timer that runs `dns-blocklists.py` to refresh blocklists
- Tracks lifetime DNS metrics (queries forwarded, cache hits, authoritative, TCP peaks, pool usage)
- Builds `nftables` tables atomically - safe to re-apply without service disruption
- Handles port forwarding (DNAT/SNAT) for externally accessible services
@ -50,6 +49,13 @@ The suite is organized into three independent but complementary scripts, each ma
- Supports any number of WireGuard VPN interfaces (`is_vpn: true` VLANs); generates the server keypair on first apply, writes the server conf to `/etc/wireguard/`, and brings the interface up with `wg-quick`; subsequent applies sync peer changes live without restarting the interface
- Supports per-peer split-tunnel (VPN subnet only) or full-tunnel (all traffic) routing; peer data is stored directly in `core.json`
### Optional: DNS Blocklists (`dns-blocklists.py`)
- Downloads blocklists from upstream providers you choose (e.g. OISD, Hagezi)
- Merges them per unique VLAN combination into conf files loaded by `dnsmasq`
- Runs `core.py --apply` after a successful download to reload all instances
- Invoked by the daily `systemd` timer installed by `core.py --apply`
### Optional: DDNS (`ddns.py`)
- Detects the current public IP by rotating through multiple IP-check services

View file

@ -33,7 +33,7 @@ def add_vlan():
radius_default = 'radius_default' in request.form
mdns_reflection = 'mdns_reflection' in request.form
use_blocklists = sanitize.filterlist(request.form.getlist('use_blocklists'),
{b.get('name') for b in load_core().get('blocklists', [])})
{b.get('name') for b in load_core().get('dns_blocking', {}).get('blocklists', [])})
if not name:
flash('Name is required.', 'error')
@ -104,7 +104,7 @@ def edit_vlan():
radius_default = 'radius_default' in request.form
mdns_reflection = 'mdns_reflection' in request.form
use_blocklists = sanitize.filterlist(request.form.getlist('use_blocklists'),
{b.get('name') for b in load_core().get('blocklists', [])})
{b.get('name') for b in load_core().get('dns_blocking', {}).get('blocklists', [])})
# subnet_mask is only present when the column is visible (not all edit paths send it).
# Validate if submitted; fall back to the stored value otherwise.

View file

@ -111,9 +111,9 @@ def ddns_tableaccounts_rowdelete():
return redirect(VIEW)
@bp.route('/action/ddns_cardcheckinterval_save', methods=['POST'])
@bp.route('/action/ddns_cardipcheckinterval_save', methods=['POST'])
@require_level('administrator')
def ddns_cardcheckinterval_save():
def ddns_cardipcheckinterval_save():
raw = request.form.get('timer_interval', '').strip()
try:
mins = int(raw)
@ -157,9 +157,9 @@ def ddns_cardipcheckservices_save():
return redirect(VIEW)
@bp.route('/action/ddns_cardddnslog_save', methods=['POST'])
@bp.route('/action/ddns_cardlogging_save', methods=['POST'])
@require_level('administrator')
def ddns_cardddnslog_save():
def ddns_cardlogging_save():
log_max_kb = validate.int_range(request.form.get('log_max_kb', '').strip(), 64, None)
if log_max_kb is None:
flash('Max Log Size must be a number >= 64.', 'error')
@ -178,9 +178,9 @@ def ddns_cardddnslog_save():
return redirect(VIEW)
@bp.route('/action/ddns_cardddnslog_clear', methods=['POST'])
@bp.route('/action/ddns_cardlogging_clear', methods=['POST'])
@require_level('administrator')
def ddns_cardddnslog_clear():
def ddns_cardlogging_clear():
try:
open(LOG_FILE, 'w').close()
flash('DDNS log cleared.', 'success')
@ -189,9 +189,9 @@ def ddns_cardddnslog_clear():
return redirect(VIEW)
@bp.route('/action/ddns_cardddnslog_download', methods=['GET'])
@bp.route('/action/ddns_cardlogging_download', methods=['GET'])
@require_level('administrator')
def ddns_cardddnslog_download():
def ddns_cardlogging_download():
if not os.path.isfile(LOG_FILE):
abort(404)
return send_file(LOG_FILE, as_attachment=True, download_name='ddns.log', mimetype='text/plain')

View file

@ -1,12 +1,13 @@
import re
from flask import Blueprint, request, redirect, flash
from auth import require_level
from config_utils import load_core, save_core, verify_core_hash, queued_msg
import sanitize
import validation as validate
bp = Blueprint('action_dnsblocklists', __name__)
bp = Blueprint('action_dnsblocking', __name__)
VIEW = '/view/view_dns_blocklists'
VIEW = '/view/view_dns_blocking'
_VALID_FORMATS_STR = ', '.join(sorted(validate.VALID_BLOCKLIST_FORMATS))
@ -50,9 +51,9 @@ def _parse_fields():
return {'name': name, 'description': description, 'format': fmt, 'url': url}, None
@bp.route('/action/dnsblocklists_tableblocklists_rowdelete', methods=['POST'])
@bp.route('/action/dnsblocking_tableblocklists_rowdelete', methods=['POST'])
@require_level('administrator')
def dnsblocklists_tableblocklists_rowdelete():
def dnsblocking_tableblocklists_rowdelete():
idx = _row_index()
if idx is None:
flash('Invalid request.', 'error')
@ -62,7 +63,7 @@ def dnsblocklists_tableblocklists_rowdelete():
return redirect(VIEW)
core = load_core()
items = core.get('blocklists', [])
items = core.get('dns_blocking', {}).get('blocklists', [])
if idx < 0 or idx >= len(items):
flash('Entry not found.', 'error')
return redirect(VIEW)
@ -79,9 +80,9 @@ def dnsblocklists_tableblocklists_rowdelete():
return redirect(VIEW)
@bp.route('/action/dnsblocklists_tableblocklists_rowedit', methods=['POST'])
@bp.route('/action/dnsblocking_tableblocklists_rowedit', methods=['POST'])
@require_level('administrator')
def dnsblocklists_tableblocklists_rowedit():
def dnsblocking_tableblocklists_rowedit():
idx = _row_index()
if idx is None:
flash('Invalid request.', 'error')
@ -95,7 +96,7 @@ def dnsblocklists_tableblocklists_rowedit():
return redirect(VIEW)
core = load_core()
items = core.get('blocklists', [])
items = core.get('dns_blocking', {}).get('blocklists', [])
if idx < 0 or idx >= len(items):
flash('Entry not found.', 'error')
return redirect(VIEW)
@ -117,9 +118,9 @@ def dnsblocklists_tableblocklists_rowedit():
return redirect(VIEW)
@bp.route('/action/dnsblocklists_cardaddblocklist_add', methods=['POST'])
@bp.route('/action/dnsblocking_cardaddblocklist_add', methods=['POST'])
@require_level('administrator')
def dnsblocklists_cardaddblocklist_add():
def dnsblocking_cardaddblocklist_add():
fields, err = _parse_fields()
if err:
return redirect(VIEW)
@ -128,7 +129,7 @@ def dnsblocklists_cardaddblocklist_add():
return redirect(VIEW)
core = load_core()
blocklists = core.setdefault('blocklists', [])
blocklists = core.setdefault('dns_blocking', {}).setdefault('blocklists', [])
if any(b.get('name', '').lower() == fields['name'].lower() for b in blocklists):
flash('The configuration has not been saved because a blocklist with that name already exists.', 'error')
@ -152,9 +153,9 @@ def dnsblocklists_cardaddblocklist_add():
return redirect(VIEW)
@bp.route('/action/dnsblocklists_cardblocklistrefresh_save', methods=['POST'])
@bp.route('/action/dnsblocking_cardblocklistrefresh_save', methods=['POST'])
@require_level('administrator')
def dnsblocklists_cardblocklistrefresh_save():
def dnsblocking_cardblocklistrefresh_save():
daily_execute_time = validate.time_24h(sanitize.time_24h(request.form.get('daily_execute_time_24hr_local', '')))
if not daily_execute_time:
@ -166,15 +167,48 @@ def dnsblocklists_cardblocklistrefresh_save():
return redirect(VIEW)
core = load_core()
core.setdefault('general', {})['daily_execute_time_24hr_local'] = daily_execute_time
core.setdefault('dns_blocking', {}).setdefault('general', {})['daily_execute_time_24hr_local'] = daily_execute_time
save_core(core)
flash(queued_msg('core apply'), 'success')
return redirect(VIEW)
@bp.route('/action/dnsblocklists_cardblocklistrefresh_refresh', methods=['POST'])
@bp.route('/action/dnsblocking_cardblocklistrefresh_refreshnow', methods=['POST'])
@require_level('administrator')
def dnsblocklists_cardblocklistrefresh_refresh():
def dnsblocking_cardblocklistrefresh_refreshnow():
flash(queued_msg('core update-blocklists', action_label='Blocklist refresh queued'), 'success')
return redirect(VIEW)
@bp.route('/action/dnsblocking_cardlogging_save', methods=['POST'])
@require_level('administrator')
def dnsblocking_cardlogging_save():
log_max_kb_raw = request.form.get('log_max_kb', '').strip()
log_errors_only = 'log_errors_only' in request.form
dnsmasq_log_queries = 'dnsmasq_log_queries' in request.form
log_max_kb = validate.int_range(log_max_kb_raw, 64, None)
if log_max_kb is None:
flash('Max Log Size must be a number >= 64.', 'error')
return redirect(VIEW)
if not verify_core_hash(request.form.get('config_hash', '')):
flash('Configuration was modified by another session. Please refresh and try again.', 'error')
return redirect(VIEW)
core = load_core()
core.setdefault('dns_blocking', {}).setdefault('general', {}).update({
'log_max_kb': log_max_kb,
'log_errors_only': log_errors_only,
})
core.setdefault('network_interfaces', {})['dnsmasq_log_queries'] = dnsmasq_log_queries
errors = validate.validate_config(core)
if errors:
for msg in errors:
flash(msg, 'error')
return redirect(VIEW)
save_core(core)
flash(queued_msg('core apply'), 'success')
return redirect(VIEW)

View file

@ -10,39 +10,6 @@ bp = Blueprint('action_general', __name__)
_VIEW = '/view/view_general'
@bp.route('/action/general_cardlogging_save', methods=['POST'])
@require_level('administrator')
def general_cardlogging_save():
log_max_kb_raw = request.form.get('log_max_kb', '').strip()
log_errors_only = 'log_errors_only' in request.form
dnsmasq_log_queries = 'dnsmasq_log_queries' in request.form
log_max_kb = validate.int_range(log_max_kb_raw, 64, None)
if log_max_kb is None:
flash('Max Log Size must be a number >= 64.', 'error')
return redirect(_VIEW)
if not verify_core_hash(request.form.get('config_hash', '')):
flash('Configuration was modified by another session. Please refresh and try again.', 'error')
return redirect(_VIEW)
core = load_core()
core.setdefault('general', {}).update({
'log_max_kb': log_max_kb,
'log_errors_only': log_errors_only,
'dnsmasq_log_queries': dnsmasq_log_queries,
})
errors = validate.validate_config(core)
if errors:
for msg in errors:
flash(msg, 'error')
return redirect(_VIEW)
save_core(core)
flash(queued_msg('core apply'), 'success')
return redirect(_VIEW)
@bp.route('/action/general_cardpendingchanges_save', methods=['POST'])
@require_level('administrator')
def general_cardpendingchanges_save():
@ -51,7 +18,7 @@ def general_cardpendingchanges_save():
return redirect(_VIEW)
core = load_core()
core.setdefault('general', {})['apply_on_save'] = 'apply_on_save' in request.form
core.setdefault('network_interfaces', {})['apply_on_save'] = 'apply_on_save' in request.form
save_core(core)
flash(queued_msg('core apply'), 'success')

View file

@ -55,7 +55,7 @@ def networkinterfaces_cardnetworkinterface_save():
return redirect(_VIEW)
core = load_core()
gen = core.setdefault('general', {})
gen = core.setdefault('network_interfaces', {})
gen['wan_interface'] = wan
gen['lan_interface'] = lan
errors = validate.validate_config(core)

View file

@ -5,12 +5,12 @@ from config_utils import (
_seconds_until_next_run, _entry_ts_from_queue,
)
bp = Blueprint('api_apply_status', __name__)
bp = Blueprint('api_apply_health', __name__)
@bp.route('/api/apply-status')
@bp.route('/api/apply-health')
@require_level('viewer')
def apply_status():
def apply_health():
entry_uuid = request.args.get('uuid', '')
if not entry_uuid:
return jsonify({'status': 'unknown'})

View file

@ -11,7 +11,7 @@ DASHBOARD_DONE = f'{CONFIGS_DIR}/.dashboard-done'
DASHBOARD_LAST_RUN = f'{CONFIGS_DIR}/.dashboard-last-run'
DASHBOARD_LOCK = f'{CONFIGS_DIR}/.dashboard-lock'
DASHBOARD_PENDING = f'{CONFIGS_DIR}/.dashboard-pending'
STATUS_FILE = f'{CONFIGS_DIR}/.status'
HEALTH_FILE = f'{CONFIGS_DIR}/.health'
PRODUCT_NAME = os.environ.get('PRODUCT_NAME', 'routlin')
DASHB_TIMER_NAME = f'{PRODUCT_NAME}-dashboard-queue'
DDNS_TIMER_NAME = f'{PRODUCT_NAME}-ddns-update'
@ -111,7 +111,7 @@ def _trim_if_needed():
def _apply_on_save():
try:
return load_core().get('general', {}).get('apply_on_save', True)
return load_core().get('network_interfaces', {}).get('apply_on_save', True)
except Exception:
return True

View file

@ -8,7 +8,7 @@ from action_apply_mdns import bp as action_apply_mdns_bp
from action_apply_vpn import bp as action_apply_vpn_bp
from action_apply_banned_ips import bp as action_apply_banned_ips_bp
from action_apply_host_overrides import bp as action_apply_host_overrides_bp
from action_dnsblocklists import bp as action_dnsblocklists_bp
from action_dnsblocking import bp as action_dnsblocking_bp
from action_apply_vlans import bp as action_apply_vlans_bp
from action_apply_inter_vlan import bp as action_apply_inter_vlan_bp
from action_apply_port_forwarding import bp as action_apply_port_forwarding_bp
@ -22,7 +22,7 @@ from action_delete_account import bp as action_delete_account_bp
from action_save_preferences import bp as action_save_preferences_bp
from action_change_password import bp as action_change_password_bp
from action_ddns import bp as action_ddns_bp
from api_apply_status import bp as api_apply_status_bp
from api_apply_health import bp as api_apply_health_bp
app = Flask(__name__)
app.secret_key = os.environ.get('SECRET_KEY', os.urandom(24))
@ -34,7 +34,7 @@ app.register_blueprint(action_apply_mdns_bp)
app.register_blueprint(action_apply_vpn_bp)
app.register_blueprint(action_apply_banned_ips_bp)
app.register_blueprint(action_apply_host_overrides_bp)
app.register_blueprint(action_dnsblocklists_bp)
app.register_blueprint(action_dnsblocking_bp)
app.register_blueprint(action_apply_vlans_bp)
app.register_blueprint(action_apply_inter_vlan_bp)
app.register_blueprint(action_apply_port_forwarding_bp)
@ -48,7 +48,7 @@ app.register_blueprint(action_delete_account_bp)
app.register_blueprint(action_save_preferences_bp)
app.register_blueprint(action_change_password_bp)
app.register_blueprint(action_ddns_bp)
app.register_blueprint(api_apply_status_bp)
app.register_blueprint(api_apply_health_bp)
def _seed_initial_account():
email = os.environ.get('INITIAL_MANAGER_EMAIL', '').strip().lower()

View file

@ -159,7 +159,7 @@ def _resolve_iface(vlan, core):
))
idx = next((i for i, v in enumerate(wg_sorted) if v is vlan), 0)
return f'wg{idx}'
lan = core.get('general', {}).get('lan_interface', 'eth0')
lan = core.get('network_interfaces', {}).get('lan_interface', 'eth0')
vid = validate.derive_vlan_id(vlan.get('subnet', ''), vlan.get('subnet_mask', 24)) or 1
return lan if vid == 1 else f'{lan}.{vid}'
@ -258,7 +258,7 @@ def _config_datasource(name):
vlans = core.get('vlans', [])
if name == 'interfaces':
gen = core.get('general', {})
gen = core.get('network_interfaces', {})
wan = gen.get('wan_interface', '')
lan = gen.get('lan_interface', '')
return [
@ -274,7 +274,7 @@ def _config_datasource(name):
if name == 'blocklists':
rows = []
for bl in core.get('blocklists', []):
for bl in core.get('dns_blocking', {}).get('blocklists', []):
row = dict(bl)
bl_path = f'{CONFIGS_DIR}/blocklists/{bl.get("save_as", "")}'
try:
@ -288,7 +288,7 @@ def _config_datasource(name):
return rows
if name == 'vlans':
bl_desc = {b['name']: b.get('description', b['name']) for b in core.get('blocklists', []) if 'name' in b}
bl_desc = {b['name']: b.get('description', b['name']) for b in core.get('dns_blocking', {}).get('blocklists', []) if 'name' in b}
rows = []
for v in sorted(vlans, key=lambda x: validate.derive_vlan_id(x.get('subnet', ''), x.get('subnet_mask', 24)) or 0):
row = {k: v.get(k) for k in ('name', 'subnet', 'subnet_mask', 'radius_default', 'mdns_reflection', 'is_vpn')}
@ -421,7 +421,7 @@ def _bl_last_update():
def _blocklist_stats_html(core):
bl_dir = f'{CONFIGS_DIR}/blocklists'
rows = ''
for bl in core.get('blocklists', []):
for bl in core.get('dns_blocking', {}).get('blocklists', []):
name = e(bl.get('name', ''))
save_as = bl.get('save_as', '')
bl_path = f'{bl_dir}/{save_as}' if save_as else ''
@ -557,18 +557,19 @@ def _vpn_info():
def collect_tokens():
tokens = {}
core = _load_core()
gen = core.get('general', {})
net = core.get('network_interfaces', {})
dns_blk_gen = core.get('dns_blocking', {}).get('general', {})
dns = core.get('upstream_dns', {})
vlans = core.get('vlans', [])
tokens['GENERAL_WAN_INTERFACE'] = str(gen.get('wan_interface', '-'))
tokens['GENERAL_LAN_INTERFACE'] = str(gen.get('lan_interface', '-'))
tokens['GENERAL_WAN_STATUS'] = _iface_status(gen.get('wan_interface', ''))
tokens['GENERAL_LAN_STATUS'] = _iface_status(gen.get('lan_interface', ''))
tokens['GENERAL_LOG_MAX_KB'] = str(gen.get('log_max_kb', '-'))
tokens['GENERAL_WAN_INTERFACE'] = str(net.get('wan_interface', '-'))
tokens['GENERAL_LAN_INTERFACE'] = str(net.get('lan_interface', '-'))
tokens['GENERAL_WAN_STATUS'] = _iface_status(net.get('wan_interface', ''))
tokens['GENERAL_LAN_STATUS'] = _iface_status(net.get('lan_interface', ''))
tokens['GENERAL_LOG_MAX_KB'] = str(dns_blk_gen.get('log_max_kb', '-'))
sys_ifaces = _get_system_interfaces()
# Always include currently-configured values so dropdowns are never blank.
for configured in [gen.get('wan_interface', ''), gen.get('lan_interface', '')]:
for configured in [net.get('wan_interface', ''), net.get('lan_interface', '')]:
if configured and configured not in sys_ifaces:
sys_ifaces.append(configured)
sys_ifaces.sort()
@ -586,10 +587,10 @@ def collect_tokens():
)
tokens['NETWORK_INTERFACE_STATS_SPEED_PAD'] = str(max(max_speed_len, len('Speed')))
tokens['GENERAL_LOG_ERRORS_ONLY'] = 'true' if gen.get('log_errors_only') else 'false'
tokens['GENERAL_DNSMASQ_LOG_QUERIES'] = 'true' if gen.get('dnsmasq_log_queries') else 'false'
tokens['GENERAL_DAILY_EXECUTE_TIME'] = str(gen.get('daily_execute_time_24hr_local', '-'))
tokens['GENERAL_APPLY_ON_SAVE'] = 'true' if gen.get('apply_on_save', True) else 'false'
tokens['GENERAL_LOG_ERRORS_ONLY'] = 'true' if dns_blk_gen.get('log_errors_only') else 'false'
tokens['GENERAL_DNSMASQ_LOG_QUERIES'] = 'true' if net.get('dnsmasq_log_queries') else 'false'
tokens['GENERAL_DAILY_EXECUTE_TIME'] = str(dns_blk_gen.get('daily_execute_time_24hr_local', '-'))
tokens['GENERAL_APPLY_ON_SAVE'] = 'true' if net.get('apply_on_save', True) else 'false'
pending_items = get_dashboard_pending()
if pending_items:
@ -645,7 +646,7 @@ def collect_tokens():
tokens['EXISTING_VLAN_NAMES_JSON'] = json.dumps([v.get('name', '') for v in vlans])
tokens['EXISTING_VLAN_INTERFACES_JSON'] = json.dumps([_resolve_iface(v, core) for v in vlans])
tokens['STAT_BANNED_IP_COUNT'] = str(sum(1 for b in core.get('banned_ips', []) if b.get('enabled', True)))
tokens['STAT_BLOCKLIST_COUNT'] = str(len(core.get('blocklists', [])))
tokens['STAT_BLOCKLIST_COUNT'] = str(len(core.get('dns_blocking', {}).get('blocklists', [])))
tokens['BLOCKLIST_STATS_HTML'] = _blocklist_stats_html(core)
ddns = _load_ddns()
@ -745,7 +746,7 @@ def collect_tokens():
tokens['BLOCKLIST_NAME_OPTIONS'] = json.dumps([
{'value': bl.get('name', ''), 'label': bl.get('description', bl.get('name', ''))}
for bl in core.get('blocklists', [])
for bl in core.get('dns_blocking', {}).get('blocklists', [])
])
tokens['ACCOUNT_LEVEL_OPTIONS'] = json.dumps([
@ -1542,7 +1543,7 @@ def render_layout(view_id, content_html, tokens):
problem_bars = ''
try:
import json as _j
st = _j.load(open(f'{CONFIGS_DIR}/.status'))
st = _j.load(open(f'{CONFIGS_DIR}/.health'))
grouped = {'error': [], 'warning': []}
for section in ('configurations', 'logs'):
for item in st.get(section, []):
@ -2449,7 +2450,7 @@ function startApplyPoller(uuid, bar, mine) {
}
function doPoll() {
fetch('/api/apply-status?uuid=' + encodeURIComponent(uuid))
fetch('/api/apply-health?uuid=' + encodeURIComponent(uuid))
.then(function(r) { return r.json(); })
.then(onStatus)
.catch(function() { pollTimer = setTimeout(doPoll, 3000); });

View file

@ -14,7 +14,7 @@
{ "type": "nav_item", "label": "General", "map_to": "view_general", "client_requirement": "client_is_administrator+" },
{ "type": "nav_item", "label": "Network Interfaces", "map_to": "view_network_interfaces", "client_requirement": "client_is_administrator+" },
{ "type": "nav_item", "label": "Upstream DNS", "map_to": "view_upstream_dns", "client_requirement": "client_is_administrator+" },
{ "type": "nav_item", "label": "DNS Blocklists", "map_to": "view_dns_blocklists", "client_requirement": "client_is_administrator+" },
{ "type": "nav_item", "label": "DNS Blocking", "map_to": "view_dns_blocking", "client_requirement": "client_is_administrator+" },
{ "type": "nav_item", "label": "VLANs", "map_to": "view_vlans", "client_requirement": "client_is_administrator+" },
{ "type": "nav_item", "label": "Inter-VLAN Exceptions", "map_to": "view_inter_vlan", "client_requirement": "client_is_administrator+" },
{ "type": "nav_item", "label": "Port Forwarding", "map_to": "view_port_forwarding", "client_requirement": "client_is_administrator+" },

View file

@ -317,10 +317,10 @@
},
{
"type": "stat_card",
"label": "Check Interval",
"label": "IP Check Interval",
"value": "%DDNS_TIMER_INTERVAL%",
"sub": "%STAT_PUBLIC_IP_LAST_CHECKED%",
"edit_action": "/action/ddns_cardcheckinterval_save",
"edit_action": "/action/ddns_cardipcheckinterval_save",
"edit_field": "timer_interval",
"edit_input_type": "number",
"edit_min": "1",
@ -509,7 +509,7 @@
},
{
"type": "card",
"label": "DDNS Log",
"label": "Logging",
"client_requirement": "client_is_administrator+",
"items": [
{
@ -527,12 +527,12 @@
"items": [
{
"type": "button_ghost",
"action": "/action/ddns_cardddnslog_download",
"action": "/action/ddns_cardlogging_download",
"text": "Download Log"
},
{
"type": "button_danger",
"action": "/action/ddns_cardddnslog_clear",
"action": "/action/ddns_cardlogging_clear",
"method": "post",
"text": "Clear Log"
}
@ -543,7 +543,7 @@
},
{
"type": "form",
"action": "/action/ddns_cardddnslog_save",
"action": "/action/ddns_cardlogging_save",
"method": "post",
"items": [
{
@ -568,7 +568,7 @@
"items": [
{
"type": "button_primary",
"action": "/action/ddns_cardddnslog_save",
"action": "/action/ddns_cardlogging_save",
"method": "post",
"text": "Save"
},
@ -601,60 +601,6 @@
}
]
},
{
"type": "card",
"label": "Logging",
"client_requirement": "client_is_administrator+",
"items": [
{
"type": "form",
"action": "/action/general_cardlogging_save",
"method": "post",
"items": [
{
"type": "field",
"label": "Max Log Size (KB)",
"name": "log_max_kb",
"input_type": "number",
"value": "%GENERAL_LOG_MAX_KB%",
"min": 64,
"hint": "Log is cleared and restarted when it exceeds this size."
},
{
"type": "field",
"label": "Errors Only",
"name": "log_errors_only",
"input_type": "checkbox",
"value": "%GENERAL_LOG_ERRORS_ONLY%",
"hint": "Only write error-level messages to the log."
},
{
"type": "field",
"label": "Log DNS Queries",
"name": "dnsmasq_log_queries",
"input_type": "checkbox",
"value": "%GENERAL_DNSMASQ_LOG_QUERIES%",
"hint": "Log every DNS query. High volume \u2014 enable for debugging only."
},
{
"type": "button_row",
"items": [
{
"type": "button_primary",
"action": "/action/general_cardlogging_save",
"method": "post",
"text": "Save"
},
{
"type": "button_cancel",
"text": "Cancel"
}
]
}
]
}
]
},
{
"type": "card",
"label": "Pending Changes",
@ -858,7 +804,10 @@
}
]
},
{ "type": "raw_html", "html": "<br /><br /><br /><br /><br />" }
{
"type": "raw_html",
"html": "<br /><br /><br /><br /><br />"
}
]
},
{
@ -1191,7 +1140,7 @@
]
},
{
"id": "view_dns_blocklists",
"id": "view_dns_blocking",
"client_requirement": "client_is_viewer+",
"items": [
{
@ -1199,7 +1148,7 @@
"items": [
{
"type": "h1",
"text": "DNS Blocklists"
"text": "DNS Blocking"
},
{
"type": "p",
@ -1207,6 +1156,60 @@
}
]
},
{
"type": "card",
"label": "Logging",
"client_requirement": "client_is_administrator+",
"items": [
{
"type": "form",
"action": "/action/dnsblocking_cardlogging_save",
"method": "post",
"items": [
{
"type": "field",
"label": "Max Log Size (KB)",
"name": "log_max_kb",
"input_type": "number",
"value": "%GENERAL_LOG_MAX_KB%",
"min": 64,
"hint": "Log is cleared and restarted when it exceeds this size."
},
{
"type": "field",
"label": "Errors Only",
"name": "log_errors_only",
"input_type": "checkbox",
"value": "%GENERAL_LOG_ERRORS_ONLY%",
"hint": "Only write error-level messages to the log."
},
{
"type": "field",
"label": "Log DNS Queries",
"name": "dnsmasq_log_queries",
"input_type": "checkbox",
"value": "%GENERAL_DNSMASQ_LOG_QUERIES%",
"hint": "Log every DNS query. High volume \u2014 enable for debugging only."
},
{
"type": "button_row",
"items": [
{
"type": "button_primary",
"action": "/action/dnsblocking_cardlogging_save",
"method": "post",
"text": "Save"
},
{
"type": "button_cancel",
"text": "Cancel"
}
]
}
]
}
]
},
{
"type": "table",
"datasource": "config:blocklists",
@ -1234,7 +1237,7 @@
"row_actions": [
{
"client_requirement": "client_is_administrator+",
"action": "/action/dnsblocklists_tableblocklists_rowedit",
"action": "/action/dnsblocking_tableblocklists_rowedit",
"method": "inline_edit",
"text": "Edit",
"class": "btn-ghost btn-sm",
@ -1262,7 +1265,7 @@
},
{
"client_requirement": "client_is_administrator+",
"action": "/action/dnsblocklists_tableblocklists_rowdelete",
"action": "/action/dnsblocking_tableblocklists_rowdelete",
"method": "post",
"text": "Delete",
"class": "btn-danger btn-sm"
@ -1277,7 +1280,7 @@
"items": [
{
"type": "form",
"action": "/action/dnsblocklists_cardaddblocklist_add",
"action": "/action/dnsblocking_cardaddblocklist_add",
"method": "post",
"items": [
{
@ -1315,7 +1318,7 @@
"items": [
{
"type": "button_primary",
"action": "/action/dnsblocklists_cardaddblocklist_add",
"action": "/action/dnsblocking_cardaddblocklist_add",
"method": "post",
"text": "Add Blocklist"
},
@ -1347,7 +1350,7 @@
"items": [
{
"type": "button_secondary",
"action": "/action/dnsblocklists_cardblocklistrefresh_refresh",
"action": "/action/dnsblocking_cardblocklistrefresh_refreshnow",
"method": "post",
"text": "Refresh All Now"
}
@ -1359,7 +1362,7 @@
},
{
"type": "form",
"action": "/action/dnsblocklists_cardblocklistrefresh_save",
"action": "/action/dnsblocking_cardblocklistrefresh_save",
"method": "post",
"items": [
{
@ -1377,7 +1380,7 @@
"items": [
{
"type": "button_primary",
"action": "/action/dnsblocklists_cardblocklistrefresh_save",
"action": "/action/dnsblocking_cardblocklistrefresh_save",
"method": "post",
"text": "Save"
},

View file

@ -24,7 +24,7 @@ All configuration lives in two JSON files. Edit these to match your network befo
| `.dashboard-last-run` | Epoch timestamp of the last timer execution. |
| `.dashboard-lock` | PID lock file preventing concurrent timer runs. |
| `.dashboard-pending` | Changes held back when Apply on Save is disabled; flushed to `.dashboard-queue` when Apply Now is clicked. |
| `.status` | JSON health check results written by `core.py --apply`, `core.py --status`, and the `routlin-status-check` timer (every 5 minutes). Read by the dashboard to display problem alerts. |
| `.health` | JSON health check results written by `core.py --apply`, `core.py --status`, and the `routlin-health-check` timer (every 5 minutes). Read by the dashboard to display problem alerts. |
| `.dns-metrics` | Cumulative lifetime DNS metrics across all VLAN instances. Created and updated each time `--view-metrics` is run. |
| `.ddns-last-ip-*` | Cached public IP per DDNS provider. Managed by `ddns.py`. |
| `.ddns-last-service` | Tracks IP-check service rotation. Managed by `ddns.py`. |
@ -35,14 +35,14 @@ All configuration lives in two JSON files. Edit these to match your network befo
### 1. Edit Core Configuration (`core.json`)
Edit the top-level `general` block:
Edit the top-level `network_interfaces` block:
- Set `wan_interface` to the name of your WAN-facing NIC (e.g. `eno2`). Run `ip link` to find it.
Edit the top-level blocks:
- Set `upstream_dns.upstream_servers` to your preferred DNS resolvers (e.g. `1.1.1.1`, `8.8.8.8`)
- Add blocklist sources under `blocklists` with a name, URL, and format for each (e.g. OISD, Hagezi)
- Add blocklist sources under `dns_blocking.blocklists` with a name, URL, and format for each (e.g. OISD, Hagezi)
- Add entries to `host_overrides` for any local hostnames that should resolve to a specific IP (e.g. a DDNS hostname pointing to an internal server)
- Add entries to `port_forwarding` for any services that should be reachable from the internet (specify protocol, external port, destination IP, and destination port)
- Add entries to `banned_ips` to block traffic from specific IPs or networks (see below)
@ -179,7 +179,7 @@ Configure mDNS reflection with the top-level `mdns_reflection` block in `core.js
```bash
sudo python3 install.py # Install required packages; optionally set up dashboard and HTTPS
sudo python3 core.py --apply # Apply VLANs, DHCP, DNS, firewall, RADIUS, mDNS, timers
sudo python3 core.py --update-blocklists # Download and apply blocklists
sudo python3 dns-blocklists.py # Download and apply blocklists
```
Optional (if DDNS is desired):
@ -224,19 +224,26 @@ Commands that modify system state require `sudo`. Read-only commands do not.
```
sudo python3 core.py --apply # Apply full config: networkd, dnsmasq, nftables, RADIUS, mDNS, timers, boot service; runs health checks at end
sudo python3 core.py --apply --dry-run # Preview --apply actions without making changes
sudo python3 core.py --update-blocklists # Download and merge blocklists, then --apply
sudo python3 core.py --disable # Revert to network client (interactive wizard)
sudo python3 core.py --disable --dry-run # Preview --disable wizard without making changes
sudo python3 core.py --reset-leases # Stop dnsmasq, delete all lease files, restart (forces devices to re-acquire)
sudo python3 core.py --reset-leases VLAN # Reset leases for a specific VLAN only (e.g. trusted, iot, guest)
python3 core.py --status # Service status, config checks, and log alerts for all managed components; writes .status
python3 core.py --status # Service status, config checks, and log alerts for all managed components; writes .health
python3 core.py --view-configs # Active per-VLAN dnsmasq config files
python3 core.py --view-leases # Active DHCP leases across all VLANs with VLAN, type, and description
python3 core.py --view-rules # Active nftables ruleset
python3 core.py --view-metrics # Lifetime DNS metrics across all VLAN instances
```
### dns-blocklists.py
```
sudo python3 dns-blocklists.py
```
Downloads every blocklist referenced by at least one VLAN, merges them into per-combination conf files, then calls `core.py --apply` to reload dnsmasq instances. Run this after initial deployment and any time you add or change blocklist sources. The daily `systemd` timer installed by `core.py --apply` runs this automatically.
### create_vpn_peer.py
Does not require `sudo`. Requires `wireguard-tools` (`wg` must be on PATH) and a prior `core.py --apply` to generate the server keypair.

View file

@ -1,11 +1,8 @@
{
"general": {
"network_interfaces": {
"wan_interface": "eno2",
"lan_interface": "enp6s0",
"log_max_kb": 1024,
"log_errors_only": false,
"dnsmasq_log_queries": false,
"daily_execute_time_24hr_local": "02:30"
"dnsmasq_log_queries": false
},
"upstream_dns": {
"strict_order": false,
@ -72,29 +69,6 @@
"ip": "192.168.1.20"
}
],
"blocklists": [
{
"name": "oisd-big",
"description": "OISD Big (ads, phishing, malware, telemetry)",
"save_as": "oisd-big.conf",
"url": "https://big.oisd.nl/dnsmasq2",
"format": "dnsmasq"
},
{
"name": "hagezi-light",
"description": "Hagezi Light (ads, tracking, metrics, badware)",
"save_as": "hagezi-light.conf",
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/dnsmasq/light.txt",
"format": "dnsmasq"
},
{
"name": "hagezi-pro-plus",
"description": "Hagezi Pro Plus (ads, tracking, porn, gambling)",
"save_as": "hagezi-pro-plus.conf",
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/dnsmasq/pro.plus.txt",
"format": "dnsmasq"
}
],
"inter_vlan_exceptions": [
{
"description": "IoT TV -> Plex",
@ -724,22 +698,100 @@
}
],
"ip_check_services": [
{"type": "http", "url": "https://api.ipify.org"},
{"type": "http", "url": "https://ifconfig.me/ip"},
{"type": "http", "url": "https://icanhazip.com"},
{"type": "http", "url": "https://api4.my-ip.io/ip"},
{"type": "http", "url": "https://ipv4.icanhazip.com"},
{"type": "http", "url": "https://checkip.amazonaws.com"},
{"type": "http", "url": "https://1.1.1.1/cdn-cgi/trace"},
{"type": "http", "url": "https://ipinfo.io/ip"},
{"type": "http", "url": "https://ipecho.net/plain"},
{"type": "http", "url": "https://ident.me"},
{"type": "http", "url": "https://myip.dnsomatic.com"},
{"type": "http", "url": "https://wtfismyip.com/text"},
{"type": "dig", "url": "@1.1.1.1 ch txt whoami.cloudflare"},
{"type": "dig", "url": "whoami.akamai.net @ns1-1.akamaitech.net"},
{"type": "dig", "url": "-4 TXT o-o.myaddr.l.google.com @ns1.google.com"},
{"type": "dig", "url": "-4 @ns3.cloudflare.com whoami.cloudflare.com txt"}
{
"type": "http",
"url": "https://api.ipify.org"
},
{
"type": "http",
"url": "https://ifconfig.me/ip"
},
{
"type": "http",
"url": "https://icanhazip.com"
},
{
"type": "http",
"url": "https://api4.my-ip.io/ip"
},
{
"type": "http",
"url": "https://ipv4.icanhazip.com"
},
{
"type": "http",
"url": "https://checkip.amazonaws.com"
},
{
"type": "http",
"url": "https://1.1.1.1/cdn-cgi/trace"
},
{
"type": "http",
"url": "https://ipinfo.io/ip"
},
{
"type": "http",
"url": "https://ipecho.net/plain"
},
{
"type": "http",
"url": "https://ident.me"
},
{
"type": "http",
"url": "https://myip.dnsomatic.com"
},
{
"type": "http",
"url": "https://wtfismyip.com/text"
},
{
"type": "dig",
"url": "@1.1.1.1 ch txt whoami.cloudflare"
},
{
"type": "dig",
"url": "whoami.akamai.net @ns1-1.akamaitech.net"
},
{
"type": "dig",
"url": "-4 TXT o-o.myaddr.l.google.com @ns1.google.com"
},
{
"type": "dig",
"url": "-4 @ns3.cloudflare.com whoami.cloudflare.com txt"
}
]
},
"dns_blocking": {
"general": {
"log_max_kb": 1024,
"log_errors_only": false,
"daily_execute_time_24hr_local": "02:30"
},
"blocklists": [
{
"name": "oisd-big",
"description": "OISD Big (ads, phishing, malware, telemetry)",
"save_as": "oisd-big.conf",
"url": "https://big.oisd.nl/dnsmasq2",
"format": "dnsmasq"
},
{
"name": "hagezi-light",
"description": "Hagezi Light (ads, tracking, metrics, badware)",
"save_as": "hagezi-light.conf",
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/dnsmasq/light.txt",
"format": "dnsmasq"
},
{
"name": "hagezi-pro-plus",
"description": "Hagezi Pro Plus (ads, tracking, porn, gambling)",
"save_as": "hagezi-pro-plus.conf",
"url": "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/dnsmasq/pro.plus.txt",
"format": "dnsmasq"
}
]
}
}

View file

@ -74,7 +74,6 @@ Validation:
Usage:
sudo python3 core.py --apply Apply config fast: restart running services only
sudo python3 core.py --update-blocklists Refresh blocklists and apply (used by timer)
sudo python3 core.py --status Show service and timer status
sudo python3 core.py --view-configs Show active per-VLAN dnsmasq config files
sudo python3 core.py --view-leases Show active DHCP leases
@ -94,8 +93,6 @@ import re
import subprocess
import sys
import time
import urllib.request
import urllib.error
import argparse
from datetime import datetime
from pathlib import Path
@ -128,11 +125,11 @@ DASHB_TIMER_FILE = SYSTEMD_DIR / f"{DASHB_TIMER_NAME}.timer"
DASHB_TIMER_SVC_FILE = SYSTEMD_DIR / f"{DASHB_TIMER_NAME}.service"
DASHB_TIMER_INTERVAL_SEC = 60
DASHB_QUEUE_FILE = SCRIPT_DIR / ".dashboard-queue"
STATUS_TIMER_NAME = f"{PRODUCT_NAME}-status-check"
STATUS_TIMER_FILE = SYSTEMD_DIR / f"{STATUS_TIMER_NAME}.timer"
STATUS_TIMER_SVC_FILE = SYSTEMD_DIR / f"{STATUS_TIMER_NAME}.service"
STATUS_TIMER_INTERVAL_SEC = 300
STATUS_FILE = SCRIPT_DIR / ".status"
HEALTH_TIMER_NAME = f"{PRODUCT_NAME}-health-check"
HEALTH_TIMER_FILE = SYSTEMD_DIR / f"{HEALTH_TIMER_NAME}.timer"
HEALTH_TIMER_SVC_FILE = SYSTEMD_DIR / f"{HEALTH_TIMER_NAME}.service"
HEALTH_TIMER_INTERVAL_SEC = 300
HEALTH_FILE = SCRIPT_DIR / ".health"
DASHB_DONE_FILE = SCRIPT_DIR / ".dashboard-done"
DASHB_LAST_RUN_FILE = SCRIPT_DIR / ".dashboard-last-run"
DASHB_LOCK_FILE = SCRIPT_DIR / ".dashboard-lock"
@ -463,132 +460,6 @@ def blocklists_available(data):
combos.add(combo_hash(names))
return any(merged_path(h).exists() for h in combos)
def parse_dnsmasq_format(content):
domains = set()
for ln in content.splitlines():
ln = ln.strip()
if not ln or ln.startswith("#"):
continue
if ln.startswith("local=/"):
domain = ln.removeprefix("local=/").rstrip("/")
if domain:
domains.add(domain)
elif ln.startswith("address=/"):
parts = ln.removeprefix("address=/").split("/")
if parts:
domains.add(parts[0])
return domains
def parse_hosts_format(content):
domains = set()
for ln in content.splitlines():
ln = ln.strip()
if not ln or ln.startswith("#"):
continue
parts = ln.split()
if len(parts) >= 2:
domains.add(parts[1])
return domains
def parse_blocklist(content, fmt):
if fmt == "dnsmasq":
return parse_dnsmasq_format(content)
return parse_hosts_format(content)
def build_merged_conf(domains, bl_names):
"""Build a merged dnsmasq conf blocking all domains and their subdomains."""
lines = [
"# Generated by core.py -- do not edit manually.",
f"# Blocklist combination: {', '.join(sorted(bl_names))}",
f"# Merged: {len(domains):,} unique domains.",
"#",
"# Blocks domain and all subdomains via local=/domain/ syntax.",
"",
]
for domain in sorted(domains):
lines.append(f"local=/{domain}/")
return "\n".join(lines)
def download_all_blocklists(data):
"""
Download every blocklist referenced by at least one VLAN.
Returns dict: name -> (content_str, entry) or (None, entry) on failure.
"""
bl_library = {bl["name"]: bl for bl in data.get("blocklists", [])}
needed = set()
for vlan in data["vlans"]:
needed.update(vlan.get("use_blocklists", []))
results = {}
for name in needed:
entry = bl_library[name]
url = entry["url"]
try:
req = urllib.request.Request(url, headers={"User-Agent": "dns-dhcp.py/1.0"})
with urllib.request.urlopen(req, timeout=30) as r:
content = r.read().decode("utf-8", errors="ignore")
log.info(f"Downloaded: {entry['description']} ({len(content):,} bytes)")
results[name] = (content, entry)
except Exception as e:
log.error(f"Failed to download '{entry['description']}' from {url}: {e}")
results[name] = (None, entry)
return results
def update_blocklists(data):
"""
Download all referenced blocklists, build per-combo merged files,
and clean up stale merged files. Returns active hashes set.
"""
BLOCKLIST_DIR.mkdir(exist_ok=True)
log.info("Downloading blocklists...")
downloaded = download_all_blocklists(data)
# Parse domains per blocklist name; save raw files
domains_by_name = {}
for name, (content, entry) in downloaded.items():
if content is None:
log.error(f"Blocklist '{name}' failed to download -- it will be skipped.")
domains_by_name[name] = set()
else:
(BLOCKLIST_DIR / entry["save_as"]).write_text(content)
domains = parse_blocklist(content, entry.get("format", "dnsmasq"))
log.info(f"Parsed {len(domains):,} domains from '{name}'")
domains_by_name[name] = domains
# Build one merged file per unique combo
active_hashes = set()
combos = {}
for vlan in data["vlans"]:
names = frozenset(vlan.get("use_blocklists", []))
if names:
h = combo_hash(names)
combos[h] = names
for h, names in combos.items():
combo_domains = set()
for name in names:
combo_domains.update(domains_by_name.get(name, set()))
merged = build_merged_conf(combo_domains, names)
merged_path(h).write_text(merged)
active_hashes.add(h)
log.info(
f"Merged [{h}] ({', '.join(sorted(names))}): "
f"{len(combo_domains):,} unique domains."
)
# Remove stale merged files (hashes no longer in active combos)
for f in BLOCKLIST_DIR.glob("merged-*.conf"):
h = f.stem.removeprefix("merged-")
if h not in active_hashes:
f.unlink()
log.info(f"Removed stale merged file: {f.name}")
# Return True if all blocklists downloaded successfully
any_failed = any(content is None for content, _ in downloaded.values())
return not any_failed
# ===================================================================
# Build per-VLAN dnsmasq config
# ===================================================================
@ -608,7 +479,7 @@ def _wan_has_ipv6(iface):
def build_vlan_dnsmasq_conf(vlan, data, iface):
"""Generate the complete dnsmasq config for one VLAN instance."""
dns_cfg = data.get("upstream_dns", {})
general = data.get("general", {})
general = data.get("network_interfaces", {})
overrides = [o for o in data.get("host_overrides", []) if o.get("enabled") is True]
name = vlan["name"]
d = vlan.get("dhcp_information", {})
@ -714,7 +585,7 @@ def build_vlan_dnsmasq_conf(vlan, data, iface):
line("no-resolv")
if dns_cfg.get("strict_order"):
line("strict-order")
wan = data["general"]["wan_interface"]
wan = data["network_interfaces"]["wan_interface"]
wan_has_ipv6 = _wan_has_ipv6(wan)
for srv in dns_cfg.get("upstream_servers", []):
if ":" in srv and not wan_has_ipv6:
@ -737,7 +608,7 @@ def build_vlan_dnsmasq_conf(vlan, data, iface):
line(f"conf-file={bl_file}")
line()
elif bl_names:
line("# Blocklist not yet downloaded -- run --update-blocklists to fetch")
line("# Blocklist not yet downloaded -- run: sudo python3 dns-blocklists.py")
line()
return "\n".join(L)
@ -1201,10 +1072,9 @@ def parse_time_to_calendar(time_str):
return f"*-*-* {hh.zfill(2)}:{mm.zfill(2)}:00"
def install_timer(data):
general = data.get("general", {})
general = data.get("dns_blocking", {}).get("general", {})
execute_time = general.get("daily_execute_time_24hr_local", "02:30")
on_calendar = parse_time_to_calendar(execute_time)
script_path = Path(__file__).resolve()
timer_content = "\n".join([
"# Generated by core.py -- do not edit manually.",
@ -1221,17 +1091,18 @@ def install_timer(data):
"",
])
blocklist_script = SCRIPT_DIR / "dns-blocklists.py"
service_content = "\n".join([
"# Generated by core.py -- do not edit manually.",
"",
"[Unit]",
"Description=core.py daily blocklist refresh",
"Description=Daily blocklist refresh",
"After=network-online.target",
"Wants=network-online.target",
"",
"[Service]",
"Type=oneshot",
f"ExecStart=/usr/bin/python3 {script_path} --update-blocklists",
f"ExecStart=/usr/bin/python3 {blocklist_script}",
"",
])
@ -1534,7 +1405,7 @@ def banned_ip_sets(data):
# ===================================================================
def build_nft_config(data, dry_run=False):
wan = data["general"]["wan_interface"]
wan = data["network_interfaces"]["wan_interface"]
# Exclude WG VLANs whose interface is not up -- nft rejects rules that
# reference non-existent interfaces, which would leave no firewall at all.
vlans = [v for v in data["vlans"]
@ -2232,8 +2103,8 @@ def disable_avahi():
def show_status(data):
import status as _status
_status.print_table(_status.run_and_write(data))
import health as _health
_health.print_table(_health.run_and_write(data))
def show_configs(data):
for vlan in data["vlans"]:
@ -2550,9 +2421,9 @@ def show_metrics(data):
def stop_instances(data):
"""Remove timers and stop all per-VLAN instances (config files preserved)."""
_remove_timers(
names=[BLIST_TIMER_NAME, DASHB_TIMER_NAME, STATUS_TIMER_NAME, DDNS_TIMER_NAME],
timer_files=[BLIST_TIMER_FILE, DASHB_TIMER_FILE, STATUS_TIMER_FILE, DDNS_TIMER_FILE],
svc_files=[BLIST_TIMER_SVC_FILE, DASHB_TIMER_SVC_FILE, STATUS_TIMER_SVC_FILE, DDNS_TIMER_SVC_FILE],
names=[BLIST_TIMER_NAME, DASHB_TIMER_NAME, HEALTH_TIMER_NAME, DDNS_TIMER_NAME],
timer_files=[BLIST_TIMER_FILE, DASHB_TIMER_FILE, HEALTH_TIMER_FILE, DDNS_TIMER_FILE],
svc_files=[BLIST_TIMER_SVC_FILE, DASHB_TIMER_SVC_FILE, HEALTH_TIMER_SVC_FILE, DDNS_TIMER_SVC_FILE],
daemon_reload=True,
)
print()
@ -2746,7 +2617,7 @@ def _dry_run_conflicting_services(data):
def _dry_run_blocklists(data):
print("Blocklists (dry-run) ================================================")
for entry in data.get("blocklists", []):
for entry in data.get("dns_blocking", {}).get("blocklists", []):
print(f" Would download: {entry['description']}")
print(f" URL: {entry['url']}")
seen = {}
@ -2763,7 +2634,7 @@ def _dry_run_blocklists(data):
def _dry_run_timer(data):
print("Timer (dry-run) =====================================================")
general = data.get("general", {})
general = data.get("dns_blocking", {}).get("general", {})
execute_time = general.get("daily_execute_time_24hr_local", "02:30")
for path, label in [(BLIST_TIMER_FILE, "timer unit"), (BLIST_TIMER_SVC_FILE, "service unit")]:
action = "update" if path.exists() else "create and enable"
@ -3133,7 +3004,7 @@ def cmd_apply(data, dry_run=False):
print("dnsmasq instances ===================================================")
if not blocklists_available(data):
print(" NOTE: No merged blocklist files found -- blocklist rules will be absent.")
print(" Run --update-blocklists to download and merge blocklists.")
print(" Run: sudo python3 dns-blocklists.py")
apply_dnsmasq_instances(data, start_if_needed=True)
print()
@ -3147,12 +3018,12 @@ def cmd_apply(data, dry_run=False):
print("Interval timers =====================================================")
# build parallel lists; dashboard timer only installed when queue file exists
t_names = [STATUS_TIMER_NAME]
t_files = [STATUS_TIMER_FILE]
s_files = [STATUS_TIMER_SVC_FILE]
t_names = [HEALTH_TIMER_NAME]
t_files = [HEALTH_TIMER_FILE]
s_files = [HEALTH_TIMER_SVC_FILE]
t_descs = ["Router status health check"]
t_execs = [f"/usr/bin/python3 {SCRIPT_DIR / 'status.py'}"]
t_intervals = [STATUS_TIMER_INTERVAL_SEC]
t_execs = [f"/usr/bin/python3 {SCRIPT_DIR / 'health.py'}"]
t_intervals = [HEALTH_TIMER_INTERVAL_SEC]
if DASHB_QUEUE_FILE.exists():
t_names += [DASHB_TIMER_NAME]
t_files += [DASHB_TIMER_FILE]
@ -3204,24 +3075,8 @@ def cmd_apply(data, dry_run=False):
print("Done.")
import status as _status
_status.print_table(_status.run_and_write(data))
def cmd_update_blocklists(data):
"""--update-blocklists: download and merge blocklists. On success, call
cmd_apply to reload dnsmasq instances with the new blocklists.
"""
check_root()
print("Updating blocklists =================================================")
success = update_blocklists(data)
print()
if success:
print("Applying updated configs ============================================")
cmd_apply(data)
else:
print("WARNING: Blocklist update had errors -- skipping --apply.")
print(" Existing merged files (if any) are unchanged.")
import health as _health
_health.print_table(_health.run_and_write(data))
def main():
@ -3231,7 +3086,6 @@ def main():
epilog=(
"examples:\n"
" sudo python3 core.py --apply Apply full config (idempotent, safe to re-run)\n"
" sudo python3 core.py --update-blocklists Refresh blocklists and apply\n"
" sudo python3 core.py --status Show service and timer status\n"
" sudo python3 core.py --view-configs Show active per-VLAN dnsmasq config files\n"
" sudo python3 core.py --view-leases Show active DHCP leases\n"
@ -3245,9 +3099,8 @@ def main():
" sudo python3 core.py --disable --dry-run\n"
)
)
parser.add_argument("--apply", action="store_true", help="Apply full config: services, networkd, dnsmasq, nftables, timer, boot service")
parser.add_argument("--update-blocklists", action="store_true", help="Refresh blocklists and apply (used by timer)")
parser.add_argument("--dry-run", action="store_true", help="Preview all actions without making changes (combine with --apply or --disable)")
parser.add_argument("--apply", action="store_true", help="Apply full config: services, networkd, dnsmasq, nftables, timer, boot service")
parser.add_argument("--dry-run", action="store_true", help="Preview all actions without making changes (combine with --apply or --disable)")
parser.add_argument("--status", action="store_true", help="Show service and timer status")
parser.add_argument("--view-configs", action="store_true", help="Show active per-VLAN dnsmasq config files")
parser.add_argument("--view-leases", action="store_true", help="Show active DHCP leases")
@ -3260,9 +3113,7 @@ def main():
args = parser.parse_args()
update_blocklists_flag = getattr(args, "update_blocklists", False)
if not any([args.apply, update_blocklists_flag,
if not any([args.apply,
args.dry_run, args.status, args.view_configs, args.view_leases,
args.view_rules, args.disable, args.view_metrics,
args.reset_leases]):
@ -3281,7 +3132,7 @@ def main():
print(f" - {e}", file=sys.stderr)
sys.exit(1)
general = data.get("general", {})
general = data.get("dns_blocking", {}).get("general", {})
setup_logging(
general.get("log_max_kb", 1024),
general.get("log_errors_only", False)
@ -3318,10 +3169,6 @@ def main():
cmd_disable(data, dry_run=args.dry_run)
return
if update_blocklists_flag:
cmd_update_blocklists(data)
return
if args.apply:
cmd_apply(data, dry_run=args.dry_run)
return

258
routlin/dns-blocklists.py Normal file
View file

@ -0,0 +1,258 @@
#!/usr/bin/env python3
"""
dns-blocklists.py -- Download and merge DNS blocklists defined in core.json.
Reads the blocklists library from core.json, downloads every blocklist referenced
by at least one VLAN, merges them into per-combo conf files (one per unique
combination of blocklist names), then sends SIGHUP to each running dnsmasq
instance so it reloads its config without restarting.
Usage:
sudo python3 dns-blocklists.py
"""
import hashlib
import json
import logging
import os
import subprocess
import sys
import urllib.request
import urllib.error
from pathlib import Path
PRODUCT_NAME = "routlin"
SCRIPT_DIR = Path(__file__).parent
CONFIG_FILE = SCRIPT_DIR / "core.json"
BLOCKLIST_DIR = SCRIPT_DIR / "blocklists"
LOG_FILE = SCRIPT_DIR / "dns-blocklists.log"
log = None
def _chown_to_script_dir_owner(path):
try:
stat = SCRIPT_DIR.stat()
os.chown(path, stat.st_uid, stat.st_gid)
except Exception:
pass
def setup_logging(max_kb, errors_only):
global log
try:
if LOG_FILE.exists() and LOG_FILE.stat().st_size > max_kb * 1024:
LOG_FILE.write_text("")
if not LOG_FILE.exists():
LOG_FILE.touch()
_chown_to_script_dir_owner(LOG_FILE)
file_handler = logging.FileHandler(LOG_FILE)
except PermissionError:
print(f"WARNING: Cannot write to {LOG_FILE} (permission denied). "
f"Run with sudo or fix ownership: sudo chown $USER {LOG_FILE}")
file_handler = None
level = logging.ERROR if errors_only else logging.INFO
handlers = [logging.StreamHandler(sys.stdout)]
if file_handler:
handlers.insert(0, file_handler)
logging.basicConfig(
level=level,
format="%(asctime)s %(levelname)-8s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
handlers=handlers,
)
log = logging.getLogger("dns-blocklists")
def die(msg):
print(f"ERROR: {msg}", file=sys.stderr)
sys.exit(1)
def check_root():
if os.geteuid() != 0:
die("This script must be run as root (sudo).")
def load_config():
if not CONFIG_FILE.exists():
die(f"Config file not found: {CONFIG_FILE}")
with open(CONFIG_FILE) as f:
data = json.load(f)
if not data.get("vlans"):
die("No vlans defined in core.json.")
return data
def combo_hash(names):
key = ",".join(sorted(names))
return hashlib.sha256(key.encode()).hexdigest()[:8]
def merged_path(h):
return BLOCKLIST_DIR / f"merged-{h}.conf"
def parse_dnsmasq_format(content):
domains = set()
for ln in content.splitlines():
ln = ln.strip()
if not ln or ln.startswith("#"):
continue
if ln.startswith("local=/"):
domain = ln.removeprefix("local=/").rstrip("/")
if domain:
domains.add(domain)
elif ln.startswith("address=/"):
parts = ln.removeprefix("address=/").split("/")
if parts:
domains.add(parts[0])
return domains
def parse_hosts_format(content):
domains = set()
for ln in content.splitlines():
ln = ln.strip()
if not ln or ln.startswith("#"):
continue
parts = ln.split()
if len(parts) >= 2:
domains.add(parts[1])
return domains
def parse_blocklist(content, fmt):
if fmt == "dnsmasq":
return parse_dnsmasq_format(content)
return parse_hosts_format(content)
def build_merged_conf(domains, bl_names):
lines = [
"# Generated by dns-blocklists.py -- do not edit manually.",
f"# Blocklist combination: {', '.join(sorted(bl_names))}",
f"# Merged: {len(domains):,} unique domains.",
"#",
"# Blocks domain and all subdomains via local=/domain/ syntax.",
"",
]
for domain in sorted(domains):
lines.append(f"local=/{domain}/")
return "\n".join(lines)
def download_all_blocklists(data):
bl_library = {bl["name"]: bl for bl in data.get("dns_blocking", {}).get("blocklists", [])}
needed = set()
for vlan in data["vlans"]:
needed.update(vlan.get("use_blocklists", []))
results = {}
for name in needed:
entry = bl_library[name]
url = entry["url"]
try:
req = urllib.request.Request(url, headers={"User-Agent": "dns-blocklists.py/1.0"})
with urllib.request.urlopen(req, timeout=30) as r:
content = r.read().decode("utf-8", errors="ignore")
log.info(f"Downloaded: {entry['description']} ({len(content):,} bytes)")
results[name] = (content, entry)
except Exception as e:
log.error(f"Failed to download '{entry['description']}' from {url}: {e}")
results[name] = (None, entry)
return results
def update_blocklists(data):
BLOCKLIST_DIR.mkdir(exist_ok=True)
log.info("Downloading blocklists...")
downloaded = download_all_blocklists(data)
domains_by_name = {}
for name, (content, entry) in downloaded.items():
if content is None:
log.error(f"Blocklist '{name}' failed to download -- it will be skipped.")
domains_by_name[name] = set()
else:
(BLOCKLIST_DIR / entry["save_as"]).write_text(content)
domains = parse_blocklist(content, entry.get("format", "dnsmasq"))
log.info(f"Parsed {len(domains):,} domains from '{name}'")
domains_by_name[name] = domains
active_hashes = set()
combos = {}
for vlan in data["vlans"]:
names = frozenset(vlan.get("use_blocklists", []))
if names:
h = combo_hash(names)
combos[h] = names
for h, names in combos.items():
combo_domains = set()
for name in names:
combo_domains.update(domains_by_name.get(name, set()))
merged = build_merged_conf(combo_domains, names)
merged_path(h).write_text(merged)
active_hashes.add(h)
log.info(
f"Merged [{h}] ({', '.join(sorted(names))}): "
f"{len(combo_domains):,} unique domains."
)
for f in BLOCKLIST_DIR.glob("merged-*.conf"):
h = f.stem.removeprefix("merged-")
if h not in active_hashes:
f.unlink()
log.info(f"Removed stale merged file: {f.name}")
any_failed = any(content is None for content, _ in downloaded.values())
return not any_failed
def reload_dnsmasq_instances():
"""Send SIGHUP to every active dnsmasq-routlin-* instance so it reloads
its conf-file inclusions without restarting. No DNS or DHCP interruption."""
result = subprocess.run(
["systemctl", "list-units", "--state=active", "--no-legend", "--plain",
f"dnsmasq-{PRODUCT_NAME}-*.service"],
capture_output=True, text=True,
)
units = [line.split()[0] for line in result.stdout.splitlines() if line.strip()]
if not units:
print(" No active dnsmasq instances found.")
return
for unit in units:
r = subprocess.run(["systemctl", "kill", "--signal=SIGHUP", unit],
capture_output=True, text=True)
if r.returncode == 0:
print(f" Reloaded: {unit}")
else:
print(f" WARNING: Failed to reload {unit}: {r.stderr.strip()}")
def main():
check_root()
data = load_config()
general = data.get("dns_blocking", {}).get("general", {})
setup_logging(
general.get("log_max_kb", 1024),
general.get("log_errors_only", False),
)
print("Updating blocklists =================================================")
success = update_blocklists(data)
print()
if success:
print("Reloading dnsmasq instances =========================================")
reload_dnsmasq_instances()
else:
print("WARNING: Blocklist update had errors -- skipping reload.")
print(" Existing merged files (if any) are unchanged.")
if __name__ == "__main__":
main()

View file

@ -1,11 +1,11 @@
"""
status.py -- System health checks for Routlin.
health.py -- System health checks for Routlin.
Reads core.json, checks services, configuration files, and logs, then writes
.status JSON. Imported by core.py; also runnable standalone.
.health JSON. Imported by core.py; also runnable standalone.
Public API:
run_and_write(data) -> dict run all checks, write .status, return dict
run_and_write(data) -> dict run all checks, write .health, return dict
print_table(status: dict) render the CLI service table from status dict
"""
import hashlib
@ -28,7 +28,7 @@ from validation import derive_interface, derive_vlan_id, is_wg
PRODUCT_NAME = "routlin"
SCRIPT_DIR = Path(__file__).parent
STATUS_FILE = SCRIPT_DIR / ".status"
HEALTH_FILE = SCRIPT_DIR / ".health"
CONFIG_FILE = SCRIPT_DIR / "core.json"
BLOCKLIST_DIR = SCRIPT_DIR / "blocklists"
DNSMASQ_CONF_DIR = Path(f"/etc/dnsmasq-{PRODUCT_NAME}")
@ -44,7 +44,7 @@ RADIUS_CLIENTS_CONF = Path("/etc/freeradius/3.0/clients.conf")
RADIUS_USERS_FILE = Path("/etc/freeradius/3.0/users")
BLIST_TIMER_NAME = f"{PRODUCT_NAME}-dns-blocklist-update"
DASHB_TIMER_NAME = f"{PRODUCT_NAME}-dashboard-queue"
STATUS_TIMER_NAME = f"{PRODUCT_NAME}-status-check"
HEALTH_TIMER_NAME = f"{PRODUCT_NAME}-health-check"
DDNS_TIMER_NAME = f"{PRODUCT_NAME}-ddns-update"
DASHB_QUEUE_FILE = SCRIPT_DIR / ".dashboard-queue"
NAT_SERVICE_NAME = f"{PRODUCT_NAME}-nat"
@ -166,8 +166,8 @@ def check_services(data):
"expected_enabled": "enabled",
"severity": "error"})
units.append({"id": f"{STATUS_TIMER_NAME}.timer",
"name": f"{STATUS_TIMER_NAME}.timer",
units.append({"id": f"{HEALTH_TIMER_NAME}.timer",
"name": f"{HEALTH_TIMER_NAME}.timer",
"expected_active": "active", "expected_enabled": "enabled",
"severity": "warning"})
@ -542,7 +542,7 @@ def check_configurations(data):
pass
# --- Blocklist file freshness ---
blocklists = data.get("blocklists", [])
blocklists = data.get("dns_blocking", {}).get("blocklists", [])
if blocklists:
combos = {}
for vlan in vlans:
@ -557,13 +557,13 @@ def check_configurations(data):
results.append(_problem(
f"blocklist_{h}", f"blocklist ({label})", "warning",
f"Merged blocklist file for '{label}' does not exist.",
"Run `sudo python3 core.py --update-blocklists` to download blocklists."))
"Run `sudo python3 dns-blocklists.py` to download blocklists."))
elif now - path.stat().st_mtime > BLOCKLIST_STALE_SECS:
age_h = int((now - path.stat().st_mtime) / 3600)
results.append(_problem(
f"blocklist_{h}", f"blocklist ({label})", "warning",
f"Merged blocklist for '{label}' is {age_h}h old (threshold 36h).",
"Run `sudo python3 core.py --update-blocklists` to refresh."))
"Run `sudo python3 dns-blocklists.py` to refresh."))
else:
results.append(_ok(f"blocklist_{h}", f"blocklist ({label})"))
@ -712,7 +712,7 @@ def _next_blocklist_update():
# ===================================================================
def run_and_write(data):
"""Run all checks, write .status atomically, return the status dict."""
"""Run all checks, write .health atomically, return the status dict."""
status = {
"checked_at": datetime.now().strftime("%Y-%m-%dT%H:%M:%S"),
"services": check_services(data),
@ -720,9 +720,9 @@ def run_and_write(data):
"logs": check_logs(data),
"next_blocklist_update": _next_blocklist_update(),
}
tmp = STATUS_FILE.with_suffix(".tmp")
tmp = HEALTH_FILE.with_suffix(".tmp")
tmp.write_text(json.dumps(status, indent=2))
tmp.replace(STATUS_FILE)
tmp.replace(HEALTH_FILE)
return status

View file

@ -30,7 +30,7 @@ DASHB_DONE_FILE = SCRIPT_DIR / ".dashboard-done"
DASHB_LAST_RUN_FILE = SCRIPT_DIR / ".dashboard-last-run"
DASHB_LOCK_FILE = SCRIPT_DIR / ".dashboard-lock"
DASHB_PENDING_FILE = SCRIPT_DIR / ".dashboard-pending"
STATUS_FILE = SCRIPT_DIR / ".status"
HEALTH_FILE = SCRIPT_DIR / ".health"
# ===================================================================
@ -331,7 +331,7 @@ def setup_docker_compose(reuse_config=False):
def create_dotfiles():
for f in (DASHB_QUEUE_FILE, DASHB_DONE_FILE, DASHB_LAST_RUN_FILE, DASHB_LOCK_FILE, DASHB_PENDING_FILE, STATUS_FILE):
for f in (DASHB_QUEUE_FILE, DASHB_DONE_FILE, DASHB_LAST_RUN_FILE, DASHB_LOCK_FILE, DASHB_PENDING_FILE, HEALTH_FILE):
if not f.exists():
f.touch()
# chown to the routlin dir owner so the timer can write

View file

@ -283,7 +283,7 @@ def derive_vlan_id(subnet, prefix):
def derive_interface(vlan, data):
"""Derive the interface name for a VLAN without mutating data."""
lan = data.get('general', {}).get('lan_interface', 'eth0')
lan = data.get('network_interfaces', {}).get('lan_interface', 'eth0')
if is_wg(vlan):
wg_vlans = [v for v in data.get('vlans', []) if is_wg(v)]
wg_sorted = sorted(
@ -312,7 +312,7 @@ def validate_config(data):
seen_listen_ports = {}
# Pre-compute per-VLAN vlan_ids and interface names without mutating data
_lan = data.get("general", {}).get("lan_interface", "eth0")
_lan = data.get("network_interfaces", {}).get("lan_interface", "eth0")
_all_vlans = data.get("vlans", [])
_derived_ids = [
derive_vlan_id(_v.get("subnet", ""), _v.get("subnet_mask", 24))
@ -336,13 +336,13 @@ def validate_config(data):
errors.append("upstream_dns.upstream_servers is missing or empty.")
# -- WAN / LAN interfaces --------------------------------------------------
gen = data.get("general", {})
gen = data.get("network_interfaces", {})
wan = gen.get("wan_interface", "")
lan = gen.get("lan_interface", "")
if not wan:
errors.append("general.wan_interface is missing or empty.")
errors.append("network_interfaces.wan_interface is missing or empty.")
if not lan:
errors.append("general.lan_interface is missing or empty.")
errors.append("network_interfaces.lan_interface is missing or empty.")
if wan and lan:
available_interfaces = set()
try:
@ -351,17 +351,17 @@ def validate_config(data):
pass
if available_interfaces:
if wan not in available_interfaces:
errors.append(f"general.wan_interface: '{wan}' does not exist on this system.")
errors.append(f"network_interfaces.wan_interface: '{wan}' does not exist on this system.")
if lan not in available_interfaces:
errors.append(f"general.lan_interface: '{lan}' does not exist on this system.")
errors.append(f"network_interfaces.lan_interface: '{lan}' does not exist on this system.")
if wan == lan:
errors.append(f"general.wan_interface and general.lan_interface must be different (both set to '{wan}').")
errors.append(f"network_interfaces.wan_interface and network_interfaces.lan_interface must be different (both set to '{wan}').")
# -- Blocklist library -----------------------------------------------------
blocklists_by_name = {}
for idx, bl in enumerate(data.get("blocklists", [])):
for idx, bl in enumerate(data.get("dns_blocking", {}).get("blocklists", [])):
name = bl.get("name", "")
label = f"blocklists[{idx}] '{name}'"
label = f"dns_blocking.blocklists[{idx}] '{name}'"
for field in ("name", "description", "save_as", "url", "format"):
if not bl.get(field):
errors.append(f"{label}: missing or empty field '{field}'.")