diff --git a/docker/routlin-dash/app/config_utils.py b/docker/routlin-dash/app/config_utils.py index 7ba1d45..a0df9f4 100644 --- a/docker/routlin-dash/app/config_utils.py +++ b/docker/routlin-dash/app/config_utils.py @@ -619,10 +619,10 @@ def run_apply(): pass -def run_update_blocklists(): +def run_merge_blocklists(): try: subprocess.run( - ['python3', f'{CONFIGS_DIR}/core.py', '--update-blocklists'], + ['python3', f'{CONFIGS_DIR}/core.py', '--merge-blocklists'], capture_output=True, timeout=120 ) except Exception: diff --git a/docker/routlin-dash/app/pages/dnsblocking/action.py b/docker/routlin-dash/app/pages/dnsblocking/action.py index 92dc63b..141435e 100644 --- a/docker/routlin-dash/app/pages/dnsblocking/action.py +++ b/docker/routlin-dash/app/pages/dnsblocking/action.py @@ -7,7 +7,7 @@ import config_utils import sanitize import mod_validation as validate -DNS_LOG_FILE = Path(config_utils.CONFIGS_DIR) / 'dns-blocklists.log' +DNS_LOG_FILE = Path(config_utils.CONFIGS_DIR) / 'blocklists.log' _PAGE = Path(__file__).parent.name @@ -102,7 +102,7 @@ def blocklists_delete(): flash(msg, 'error') return redirect(f'/{_PAGE}') changes = config_utils.diff_fields(before, None) - flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', name, changes, 'core apply', queue=False), 'success') + flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', name, changes, 'merge blocklists'), 'success') return redirect(f'/{_PAGE}') @@ -178,7 +178,7 @@ def blocklists_edit(): flash(msg, 'error') return redirect(f'/{_PAGE}') changes = config_utils.diff_fields(before, items[idx]) - flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', fields['name'], changes, 'core apply', queue=False), 'success') + flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', fields['name'], changes, 'merge blocklists'), 'success') return redirect(f'/{_PAGE}') @@ -239,7 +239,7 @@ def addblocklist_add(): flash(msg, 'error') return redirect(f'/{_PAGE}') changes = config_utils.diff_fields(None, entry) - flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', fields['name'], changes, 'core apply', queue=False), 'success') + flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', fields['name'], changes, 'merge blocklists'), 'success') return redirect(f'/{_PAGE}') @@ -267,7 +267,9 @@ def blocklistrefresh_save(): @bp.route('/action/dnsblocking/blocklistrefresh_refresh', methods=['POST']) @auth.require_level('administrator') def blocklistrefresh_refresh(): - flash(config_utils.queued_msg('core update-blocklists', action_label='Blocklist refresh queued'), 'success') + config_utils.queue_command('download blocklists') + config_utils.queue_command('merge blocklists') + flash('Blocklist download and merge queued.', 'success') return redirect(f'/{_PAGE}') @@ -319,4 +321,4 @@ def logging_download(): if not DNS_LOG_FILE.is_file(): flash('Log file not found.', 'error') return redirect(f'/{_PAGE}') - return send_file(DNS_LOG_FILE, as_attachment=True, download_name='dns-blocklists.log', mimetype='text/plain') + return send_file(DNS_LOG_FILE, as_attachment=True, download_name='blocklists.log', mimetype='text/plain') diff --git a/docker/routlin-dash/app/pages/dnsblocking/view.py b/docker/routlin-dash/app/pages/dnsblocking/view.py index ac5985f..ce150b0 100644 --- a/docker/routlin-dash/app/pages/dnsblocking/view.py +++ b/docker/routlin-dash/app/pages/dnsblocking/view.py @@ -4,7 +4,7 @@ from datetime import datetime, timezone import config_utils import factory -DNS_LOG_FILE = f'{config_utils.CONFIGS_DIR}/dns-blocklists.log' +DNS_LOG_FILE = f'{config_utils.CONFIGS_DIR}/blocklists.log' DNS_LOG_MAX = 50 BL_TYPE_LABELS = {'community': 'Community', 'local': 'Local'} diff --git a/routlin/core.py b/routlin/core.py index 3deaac3..d812416 100644 --- a/routlin/core.py +++ b/routlin/core.py @@ -370,20 +370,13 @@ def _dry_run_conflicting_services(data): def _dry_run_blocklists(data): print("Blocklists (dry-run) ================================================") - for entry in data.get("dns_blocking", {}).get("blocklists", []): - print(f" Would download: {entry['description']}") - print(f" URL: {entry['url']}") - seen = {} - for vlan in data["vlans"]: + for vlan in data.get("vlans", []): names = vlan.get("use_blocklists", []) if names: - h = dnsmasq.combo_hash(names) - if h not in seen: - seen[h] = sorted(names) - path = dnsmasq.merged_path(h) - action = "update" if path.exists() else "create" - print(f" Would {action} merged blocklist: {path}") - print(f" Sources: {', '.join(sorted(names))}") + f = dnsmasq.vlan_hosts_file(vlan) + action = "update" if f.exists() else "create" + print(f" Would {action}: {f}") + print(f" Sources: {', '.join(sorted(names))}") def _dry_run_timer(data): print("Timer (dry-run) =====================================================") @@ -751,7 +744,7 @@ def cmd_apply(data, dry_run=False): print("dnsmasq instances ===================================================") if not dnsmasq.blocklists_available(data): print(" NOTE: No merged blocklist files found -- blocklist rules will be absent.") - print(" Run: sudo python3 dns-blocklists.py") + print(" Run: sudo python3 dl_blocklists.py") dnsmasq.apply_dnsmasq_instances(data, start_if_needed=True) print() @@ -856,7 +849,8 @@ def main(): " sudo python3 core.py --disable --dry-run\n" ) ) - parser.add_argument("--apply", action="store_true", help="Apply full config: services, networkd, dnsmasq, nftables, timer, boot service") + parser.add_argument("--apply", action="store_true", help="Apply full config: services, networkd, dnsmasq, nftables, timer, boot service") + parser.add_argument("--merge-blocklists", action="store_true", help="Merge downloaded blocklists and reload dnsmasq via SIGHUP (no restart)") parser.add_argument("--dry-run", action="store_true", help="Preview all actions without making changes (combine with --apply or --disable)") parser.add_argument("--status", action="store_true", help="Show service and timer status") parser.add_argument("--view-configs", action="store_true", help="Show active per-VLAN dnsmasq config files") @@ -870,7 +864,7 @@ def main(): args = parser.parse_args() - if not any([args.apply, + if not any([args.apply, args.merge_blocklists, args.dry_run, args.status, args.view_configs, args.view_leases, args.view_rules, args.disable, args.view_metrics, args.reset_leases]): @@ -924,6 +918,22 @@ def main(): cmd_disable(data, dry_run=args.dry_run) return + if args.merge_blocklists: + if not shared.is_root(): + die("This script must be run as root (sudo).") + general = data.get("dns_blocking", {}).get("general", {}) + dnsmasq.setup_blocklist_logging(general) + print("Merging blocklists ==================================================") + success = dnsmasq.update_blocklist_hosts(data) + print() + if success: + print("Reloading dnsmasq instances =========================================") + dnsmasq.sighup_all_instances() + else: + print("WARNING: Some blocklists failed -- reloading anyway with available data.") + dnsmasq.sighup_all_instances() + return + if args.apply: cmd_apply(data, dry_run=args.dry_run) return diff --git a/routlin/dl_blocklists.py b/routlin/dl_blocklists.py new file mode 100644 index 0000000..5ef4d88 --- /dev/null +++ b/routlin/dl_blocklists.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Download community blocklists defined in config.json to disk. + +Saves each community blocklist to blocklists/. +Local blocklists are managed by the dashboard and are not downloaded here. + +Run this on a schedule (e.g. daily) to refresh remote lists, then run + sudo python3 core.py --merge-blocklists +to merge, update SQLite, and reload dnsmasq without restarting it. + +Usage: + sudo python3 dl_blocklists.py +""" + +import json +import os +import sys +import urllib.request +from pathlib import Path + +SCRIPT_DIR = Path(__file__).parent +CONFIG_FILE = SCRIPT_DIR / "config.json" +BLOCKLIST_DIR = SCRIPT_DIR / "blocklists" + + +def die(msg): + print(f"ERROR: {msg}", file=sys.stderr) + sys.exit(1) + + +def check_root(): + if os.geteuid() != 0: + die("This script must be run as root (sudo).") + + +def load_config(): + if not CONFIG_FILE.exists(): + die(f"Config file not found: {CONFIG_FILE}") + with open(CONFIG_FILE) as f: + return json.load(f) + + +def download_blocklists(data): + any_fail = False + BLOCKLIST_DIR.mkdir(exist_ok=True) + for bl in data.get("dns_blocking", {}).get("blocklists", []): + if bl.get("bl_type") == "local": + continue + url = bl.get("url", "") + save_as = bl.get("save_as", "") + name = bl.get("name", "?") + if not url or not save_as: + print(f" Skipped '{name}': missing url or save_as") + continue + try: + req = urllib.request.Request(url, headers={"User-Agent": "routlin/1.0"}) + with urllib.request.urlopen(req, timeout=30) as r: + content = r.read() + (BLOCKLIST_DIR / save_as).write_bytes(content) + print(f" Downloaded: {name} ({len(content):,} bytes)") + except Exception as e: + print(f" ERROR: Failed to download '{name}': {e}") + any_fail = True + return not any_fail + + +def main(): + check_root() + data = load_config() + print("Downloading blocklists ==============================================") + success = download_blocklists(data) + if not success: + print("WARNING: One or more downloads failed.") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/routlin/dns-blocklists.py b/routlin/dns-blocklists.py deleted file mode 100644 index 5c42545..0000000 --- a/routlin/dns-blocklists.py +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/env python3 -""" -dns-blocklists.py -- Download and merge DNS blocklists defined in config.json. - -Reads the blocklists library from config.json, downloads every blocklist referenced -by at least one VLAN, and upserts normalized domains into a SQLite database -(blocklists/domains.db). Downloads are skipped when the content hash is unchanged. -Merged per-combo conf files are only rewritten when a constituent blocklist changed. -Sends SIGHUP to each running dnsmasq instance so it reloads without restarting. - -Usage: - sudo python3 dns-blocklists.py -""" - -import hashlib -import json -import logging -import os -import sqlite3 -import subprocess -import sys -import time -import urllib.request -import urllib.error -from pathlib import Path - -PRODUCT_NAME = "routlin" -SCRIPT_DIR = Path(__file__).parent -CONFIG_FILE = SCRIPT_DIR / "config.json" -BLOCKLIST_DIR = SCRIPT_DIR / "blocklists" -DB_FILE = BLOCKLIST_DIR / "domains.db" -LOG_FILE = SCRIPT_DIR / "dns-blocklists.log" - -log = None - - -def _chown_to_script_dir_owner(path): - try: - stat = SCRIPT_DIR.stat() - os.chown(path, stat.st_uid, stat.st_gid) - except Exception: - pass - - -def setup_logging(max_kb, errors_only): - global log - try: - if LOG_FILE.exists() and LOG_FILE.stat().st_size > max_kb * 1024: - LOG_FILE.write_text("") - if not LOG_FILE.exists(): - LOG_FILE.touch() - _chown_to_script_dir_owner(LOG_FILE) - file_handler = logging.FileHandler(LOG_FILE) - except PermissionError: - print(f"WARNING: Cannot write to {LOG_FILE} (permission denied). " - f"Run with sudo or fix ownership: sudo chown $USER {LOG_FILE}") - file_handler = None - level = logging.ERROR if errors_only else logging.INFO - handlers = [logging.StreamHandler(sys.stdout)] - if file_handler: - handlers.insert(0, file_handler) - logging.basicConfig( - level=level, - format="%(asctime)s %(levelname)-8s %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", - handlers=handlers, - ) - log = logging.getLogger("dns-blocklists") - - -def die(msg): - print(f"ERROR: {msg}", file=sys.stderr) - sys.exit(1) - - -def check_root(): - if os.geteuid() != 0: - die("This script must be run as root (sudo).") - - -def load_config(): - if not CONFIG_FILE.exists(): - die(f"Config file not found: {CONFIG_FILE}") - with open(CONFIG_FILE) as f: - data = json.load(f) - if not data.get("vlans"): - die("No vlans defined in config.json.") - return data - - -def combo_hash(names): - key = ",".join(sorted(names)) - return hashlib.sha256(key.encode()).hexdigest()[:8] - - -def merged_path(h): - return BLOCKLIST_DIR / f"merged-{h}.conf" - - -# Parse / detect ====================================================== - -def parse_dnsmasq_format(content): - domains = set() - for ln in content.splitlines(): - ln = ln.strip() - if not ln or ln.startswith("#"): - continue - if ln.startswith("local=/"): - domain = ln.removeprefix("local=/").rstrip("/") - if domain: - domains.add(domain) - elif ln.startswith("address=/"): - parts = ln.removeprefix("address=/").split("/") - if parts: - domains.add(parts[0]) - return domains - - -def parse_hosts_format(content): - domains = set() - for ln in content.splitlines(): - ln = ln.strip() - if not ln or ln.startswith("#"): - continue - parts = ln.split() - if len(parts) >= 2: - domains.add(parts[1]) - return domains - - -def parse_local_format(content): - domains = set() - for ln in content.splitlines(): - ln = ln.strip() - if ln and not ln.startswith("#"): - domains.add(ln) - return domains - - -def detect_format(content): - for ln in content.splitlines(): - ln = ln.strip() - if not ln or ln.startswith("#"): - continue - if ln.startswith("local=/") or ln.startswith("address=/"): - return "dnsmasq" - if ln[0].isdigit(): - return "hosts" - return "dnsmasq" - - -def parse_blocklist(content, is_local=False): - if is_local: - return parse_local_format(content) - fmt = detect_format(content) - if fmt == "dnsmasq": - return parse_dnsmasq_format(content) - return parse_hosts_format(content) - - -def content_hash(content): - return hashlib.sha256(content.encode()).hexdigest() - - -# SQLite ============================================================== - -def open_db(): - db = sqlite3.connect(DB_FILE) - db.execute("PRAGMA journal_mode=WAL") - db.execute("PRAGMA foreign_keys=ON") - db.executescript(""" - CREATE TABLE IF NOT EXISTS blocklists ( - id INTEGER PRIMARY KEY, - name TEXT UNIQUE NOT NULL, - content_hash TEXT, - fetched_at INTEGER, - domain_count INTEGER - ); - CREATE TABLE IF NOT EXISTS domains ( - domain TEXT NOT NULL, - blocklist_id INTEGER NOT NULL REFERENCES blocklists(id) ON DELETE CASCADE, - PRIMARY KEY (domain, blocklist_id) - ); - CREATE INDEX IF NOT EXISTS idx_domains_domain ON domains(domain); - """) - db.commit() - return db - - -def get_stored_hash(db, name): - row = db.execute("SELECT content_hash FROM blocklists WHERE name = ?", (name,)).fetchone() - return row[0] if row else None - - -def upsert_blocklist(db, name, domains, raw_hash): - now = int(time.time()) - db.execute(""" - INSERT INTO blocklists (name, content_hash, fetched_at, domain_count) - VALUES (?, ?, ?, ?) - ON CONFLICT(name) DO UPDATE SET - content_hash = excluded.content_hash, - fetched_at = excluded.fetched_at, - domain_count = excluded.domain_count - """, (name, raw_hash, now, len(domains))) - bl_id = db.execute("SELECT id FROM blocklists WHERE name = ?", (name,)).fetchone()[0] - db.execute("DELETE FROM domains WHERE blocklist_id = ?", (bl_id,)) - db.executemany("INSERT INTO domains (domain, blocklist_id) VALUES (?, ?)", - ((d, bl_id) for d in domains)) - db.commit() - - -def query_merged_domains(db, names): - placeholders = ",".join("?" * len(names)) - rows = db.execute(f""" - SELECT DISTINCT d.domain - FROM domains d - JOIN blocklists b ON d.blocklist_id = b.id - WHERE b.name IN ({placeholders}) - ORDER BY d.domain - """, list(names)).fetchall() - return [r[0] for r in rows] - - -# Conf file output ==================================================== - -def build_merged_conf(domains, bl_names): - lines = [ - "# Generated by dns-blocklists.py -- do not edit manually.", - f"# Blocklist combination: {', '.join(sorted(bl_names))}", - f"# Merged: {len(domains):,} unique domains.", - "#", - "# Blocks domain and all subdomains via local=/domain/ syntax.", - "", - ] - for domain in domains: - lines.append(f"local=/{domain}/") - return "\n".join(lines) - - -# Fetch =============================================================== - -def fetch_community(entry): - url = entry["url"] - req = urllib.request.Request(url, headers={"User-Agent": "dns-blocklists.py/1.0"}) - with urllib.request.urlopen(req, timeout=30) as r: - return r.read().decode("utf-8", errors="ignore") - - -def read_local(entry): - save_as = entry.get("save_as", "") - path = BLOCKLIST_DIR / save_as if save_as else None - if not path: - return "" - return path.read_text() - - -# Main update ========================================================= - -def update_blocklists(data): - BLOCKLIST_DIR.mkdir(exist_ok=True) - _chown_to_script_dir_owner(BLOCKLIST_DIR) - - db = open_db() - - bl_library = {bl["name"]: bl for bl in data.get("dns_blocking", {}).get("blocklists", [])} - needed = set() - for vlan in data["vlans"]: - needed.update(vlan.get("use_blocklists", [])) - - changed = set() - any_fail = False - - for name in needed: - entry = bl_library[name] - is_local = entry.get("bl_type") == "local" - - try: - raw = read_local(entry) if is_local else fetch_community(entry) - except Exception as e: - log.error(f"Failed to fetch '{name}': {e}") - any_fail = True - continue - - h = content_hash(raw) - if h == get_stored_hash(db, name): - log.info(f"Unchanged: '{name}' -- skipping") - continue - - domains = parse_blocklist(raw, is_local=is_local) - upsert_blocklist(db, name, domains, h) - log.info(f"Updated '{name}': {len(domains):,} domains") - changed.add(name) - - active_hashes = set() - combos = {} - for vlan in data["vlans"]: - names = frozenset(vlan.get("use_blocklists", [])) - if names: - h = combo_hash(names) - combos[h] = names - - for h, names in combos.items(): - active_hashes.add(h) - if not changed.intersection(names) and merged_path(h).exists(): - log.info(f"Combo [{h}] unchanged -- skipping rewrite") - continue - domains = query_merged_domains(db, names) - merged_path(h).write_text(build_merged_conf(domains, names)) - log.info(f"Merged [{h}] ({', '.join(sorted(names))}): {len(domains):,} unique domains") - - for f in BLOCKLIST_DIR.glob("merged-*.conf"): - h = f.stem.removeprefix("merged-") - if h not in active_hashes: - f.unlink() - log.info(f"Removed stale merged file: {f.name}") - - db.close() - return not any_fail - - -def reload_dnsmasq_instances(): - """Send SIGHUP to every active dnsmasq-routlin-* instance so it reloads - its conf-file inclusions without restarting. No DNS or DHCP interruption.""" - result = subprocess.run( - ["systemctl", "list-units", "--state=active", "--no-legend", "--plain", - f"dnsmasq-{PRODUCT_NAME}-*.service"], - capture_output=True, text=True, - ) - units = [line.split()[0] for line in result.stdout.splitlines() if line.strip()] - if not units: - print(" No active dnsmasq instances found.") - return - for unit in units: - r = subprocess.run(["systemctl", "kill", "--signal=SIGHUP", unit], - capture_output=True, text=True) - if r.returncode == 0: - print(f" Reloaded: {unit}") - else: - print(f" WARNING: Failed to reload {unit}: {r.stderr.strip()}") - - -def main(): - check_root() - data = load_config() - general = data.get("dns_blocking", {}).get("general", {}) - setup_logging( - general.get("log_max_kb", 1024), - general.get("log_errors_only", False), - ) - - print("Updating blocklists =================================================") - success = update_blocklists(data) - print() - - if success: - print("Reloading dnsmasq instances =========================================") - reload_dnsmasq_instances() - else: - print("WARNING: Blocklist update had errors -- skipping reload.") - print(" Existing merged files (if any) are unchanged.") - - -if __name__ == "__main__": - main() diff --git a/routlin/health.py b/routlin/health.py index c6f0ae3..8597c44 100644 --- a/routlin/health.py +++ b/routlin/health.py @@ -534,13 +534,13 @@ def check_configurations(data): results.append(problem( f"blocklist_{h}", f"blocklist ({label})", "warning", f"Merged blocklist file for '{label}' does not exist.", - "Run `sudo python3 dns-blocklists.py` to download blocklists.")) + "Run `sudo python3 dl_blocklists.py` to download blocklists.")) elif now - path.stat().st_mtime > BLOCKLIST_STALE_SECS: age_h = int((now - path.stat().st_mtime) / 3600) results.append(problem( f"blocklist_{h}", f"blocklist ({label})", "warning", f"Merged blocklist for '{label}' is {age_h}h old (threshold 36h).", - "Run `sudo python3 dns-blocklists.py` to refresh.")) + "Run `sudo python3 dl_blocklists.py` to refresh.")) else: results.append(ok(f"blocklist_{h}", f"blocklist ({label})")) diff --git a/routlin/mod_dnsmasq.py b/routlin/mod_dnsmasq.py index d9a5333..80bf677 100644 --- a/routlin/mod_dnsmasq.py +++ b/routlin/mod_dnsmasq.py @@ -6,9 +6,11 @@ generation, applying/reloading instances, system service conflict resolution (systemd-resolved, dnsmasq, chrony, ufw), and DHCP lease display. """ -import hashlib import json +import logging +import sqlite3 import subprocess +import time from datetime import datetime from pathlib import Path @@ -17,29 +19,303 @@ import mod_wireguard as wireguard import mod_validation as validation BLOCKLIST_DIR = shared.SCRIPT_DIR / "blocklists" +DB_FILE = BLOCKLIST_DIR / "domains.db" +LOG_FILE = shared.SCRIPT_DIR / "blocklists.log" RESOLV_CONF = Path("/etc/resolv.conf") +_log = logging.getLogger("blocklists") + # =================================================================== # Blocklist management # =================================================================== -def combo_hash(names): - """Return a stable 8-char hex hash for a list/set of blocklist names.""" - key = ",".join(sorted(names)) - return hashlib.sha256(key.encode()).hexdigest()[:8] +def vlan_hosts_file(vlan): + """Stable per-VLAN hosts file path (always the same regardless of blocklist combo).""" + return BLOCKLIST_DIR / f"for-{vlan['name']}.hosts" -def merged_path(h): - return BLOCKLIST_DIR / f"merged-{h}.conf" def blocklists_available(data): - """Return True if at least one merged blocklist file exists on disk.""" - combos = set() + """Return True if at least one per-VLAN hosts file is non-empty.""" for vlan in data.get("vlans", []): - names = vlan.get("use_blocklists", []) - if names: - combos.add(combo_hash(names)) - return any(merged_path(h).exists() for h in combos) + if vlan.get("use_blocklists"): + f = vlan_hosts_file(vlan) + if f.exists() and f.stat().st_size > 0: + return True + return False + + +# =================================================================== +# Blocklist parse / detect +# =================================================================== + +def _parse_dnsmasq_format(content): + domains = set() + for ln in content.splitlines(): + ln = ln.strip() + if not ln or ln.startswith("#"): + continue + if ln.startswith("local=/"): + domain = ln.removeprefix("local=/").rstrip("/") + if domain: + domains.add(domain) + elif ln.startswith("address=/"): + parts = ln.removeprefix("address=/").split("/") + if parts: + domains.add(parts[0]) + return domains + + +def _parse_hosts_format(content): + domains = set() + for ln in content.splitlines(): + ln = ln.strip() + if not ln or ln.startswith("#"): + continue + parts = ln.split() + if len(parts) >= 2: + domains.add(parts[1]) + return domains + + +def _parse_local_format(content): + domains = set() + for ln in content.splitlines(): + ln = ln.strip() + if ln and not ln.startswith("#"): + domains.add(ln) + return domains + + +def _detect_format(content): + for ln in content.splitlines(): + ln = ln.strip() + if not ln or ln.startswith("#"): + continue + if ln.startswith("local=/") or ln.startswith("address=/"): + return "dnsmasq" + if ln[0].isdigit(): + return "hosts" + return "dnsmasq" + + +def _parse_blocklist(content, is_local=False): + if is_local: + return _parse_local_format(content) + fmt = _detect_format(content) + if fmt == "dnsmasq": + return _parse_dnsmasq_format(content) + return _parse_hosts_format(content) + + +# =================================================================== +# Blocklist SQLite +# =================================================================== + +def _open_db(): + db = sqlite3.connect(DB_FILE) + db.execute("PRAGMA journal_mode=WAL") + db.execute("PRAGMA foreign_keys=ON") + db.executescript(""" + CREATE TABLE IF NOT EXISTS blocklists ( + id INTEGER PRIMARY KEY, + name TEXT UNIQUE NOT NULL, + mtime REAL, + fetched_at INTEGER, + domain_count INTEGER + ); + CREATE TABLE IF NOT EXISTS domains ( + domain TEXT NOT NULL, + blocklist_id INTEGER NOT NULL REFERENCES blocklists(id) ON DELETE CASCADE, + PRIMARY KEY (domain, blocklist_id) + ); + CREATE INDEX IF NOT EXISTS idx_domains_domain ON domains(domain); + """) + db.commit() + return db + + +def _get_stored_mtime(db, name): + row = db.execute("SELECT mtime FROM blocklists WHERE name = ?", (name,)).fetchone() + return row[0] if row else None + + +def _upsert_blocklist(db, name, domains, mtime): + now = int(time.time()) + db.execute(""" + INSERT INTO blocklists (name, mtime, fetched_at, domain_count) + VALUES (?, ?, ?, ?) + ON CONFLICT(name) DO UPDATE SET + mtime = excluded.mtime, + fetched_at = excluded.fetched_at, + domain_count = excluded.domain_count + """, (name, mtime, now, len(domains))) + bl_id = db.execute("SELECT id FROM blocklists WHERE name = ?", (name,)).fetchone()[0] + db.execute("DELETE FROM domains WHERE blocklist_id = ?", (bl_id,)) + db.executemany("INSERT INTO domains (domain, blocklist_id) VALUES (?, ?)", + ((d, bl_id) for d in domains)) + db.commit() + + +def _query_merged_domains(db, names): + placeholders = ",".join("?" * len(names)) + rows = db.execute(f""" + SELECT DISTINCT d.domain + FROM domains d + JOIN blocklists b ON d.blocklist_id = b.id + WHERE b.name IN ({placeholders}) + ORDER BY d.domain + """, list(names)).fetchall() + return [r[0] for r in rows] + + +# =================================================================== +# Blocklist merge and SIGHUP +# =================================================================== + +def _build_merged_hosts(domains, bl_names): + lines = [ + "# Generated by core.py -- do not edit manually.", + f"# Blocklists: {', '.join(sorted(bl_names))}", + f"# Domains: {len(domains):,}", + "", + ] + for domain in domains: + lines.append(f"0.0.0.0 {domain}") + return "\n".join(lines) + "\n" + + +def setup_blocklist_logging(general): + """Configure file + stdout logging for blocklists logger.""" + max_kb = general.get("log_max_kb", 1024) + errors_only = general.get("log_errors_only", False) + try: + if LOG_FILE.exists() and LOG_FILE.stat().st_size > max_kb * 1024: + LOG_FILE.write_text("") + if not LOG_FILE.exists(): + LOG_FILE.touch() + file_handler = logging.FileHandler(LOG_FILE) + except PermissionError: + print(f"WARNING: Cannot write to {LOG_FILE} -- run with sudo.") + file_handler = None + level = logging.ERROR if errors_only else logging.INFO + handlers = [logging.StreamHandler()] + if file_handler: + handlers.insert(0, file_handler) + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)-8s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=handlers, + force=True, + ) + + +def update_blocklist_hosts(data): + """Parse downloaded/local blocklist files, upsert into SQLite, and write + per-VLAN hosts files (0.0.0.0 format). Always writes every VLAN's file + (empty if no blocklists assigned) so addn-hosts= in the dnsmasq conf is + always valid and SIGHUP can update it without a restart. + + Returns True on full success, False if any fetch/parse failed. + """ + BLOCKLIST_DIR.mkdir(exist_ok=True) + + db = _open_db() + bl_library = {bl["name"]: bl for bl in data.get("dns_blocking", {}).get("blocklists", [])} + + needed = set() + for vlan in data.get("vlans", []): + needed.update(vlan.get("use_blocklists", [])) + + changed = set() + any_fail = False + + for name in needed: + if name not in bl_library: + _log.warning(f"Blocklist '{name}' referenced by a VLAN but not defined -- skipping") + continue + entry = bl_library[name] + is_local = entry.get("bl_type") == "local" + save_as = entry.get("save_as", "") + + try: + path = BLOCKLIST_DIR / save_as if save_as else None + if not path or not path.exists(): + _log.warning(f"'{name}': file not found ({path}) -- skipping") + any_fail = True + continue + current_mtime = path.stat().st_mtime + except Exception as e: + _log.error(f"Failed to stat '{name}': {e}") + any_fail = True + continue + + if current_mtime == _get_stored_mtime(db, name): + _log.info(f"Unchanged: '{name}' -- skipping") + continue + + try: + raw = path.read_text("utf-8", errors="ignore") + except Exception as e: + _log.error(f"Failed to read '{name}': {e}") + any_fail = True + continue + + domains = _parse_blocklist(raw, is_local=is_local) + _upsert_blocklist(db, name, domains, current_mtime) + _log.info(f"Updated '{name}': {len(domains):,} domains") + changed.add(name) + + active_vlan_names = set() + for vlan in data.get("vlans", []): + vlan_name = vlan["name"] + active_vlan_names.add(vlan_name) + bl_names = [n for n in vlan.get("use_blocklists", []) if n in bl_library] + + hosts_file = vlan_hosts_file(vlan) + if not bl_names: + if not hosts_file.exists(): + hosts_file.write_text("") + continue + + if not changed.intersection(bl_names) and hosts_file.exists(): + _log.info(f"VLAN '{vlan_name}' blocklists unchanged -- skipping rewrite") + continue + + domains = _query_merged_domains(db, bl_names) + hosts_file.write_text(_build_merged_hosts(domains, bl_names)) + _log.info(f"VLAN '{vlan_name}': wrote {len(domains):,} domains from [{', '.join(sorted(bl_names))}]") + + for f in BLOCKLIST_DIR.glob("for-*.hosts"): + vlan_name = f.stem.removeprefix("for-") + if vlan_name not in active_vlan_names: + f.unlink() + _log.info(f"Removed stale hosts file: {f.name}") + + db.close() + return not any_fail + + +def sighup_all_instances(): + """Send SIGHUP to every active dnsmasq-routlin-* instance to reload addn-hosts + files without restarting. No DNS or DHCP interruption.""" + result = subprocess.run( + ["systemctl", "list-units", "--state=active", "--no-legend", "--plain", + f"dnsmasq-{shared.PRODUCT_NAME}-*.service"], + capture_output=True, text=True, + ) + units = [line.split()[0] for line in result.stdout.splitlines() if line.strip()] + if not units: + print(" No active dnsmasq instances found.") + return + for unit in units: + r = subprocess.run(["systemctl", "kill", "--signal=SIGHUP", unit], + capture_output=True, text=True) + if r.returncode == 0: + print(f" Reloaded: {unit}") + else: + print(f" WARNING: Failed to SIGHUP {unit}: {r.stderr.strip()}") # =================================================================== @@ -95,12 +371,7 @@ def build_vlan_dnsmasq_conf(vlan, data, iface): opts = shared.resolve_vlan_options(vlan) gateway = opts["gateway"] - bl_names = vlan.get("use_blocklists", []) - bl_file = None - if bl_names: - p = merged_path(combo_hash(bl_names)) - if p.exists(): - bl_file = p + hosts_file = vlan_hosts_file(vlan) L = [ "# Generated by core.py -- do not edit manually.", @@ -216,14 +487,12 @@ def build_vlan_dnsmasq_conf(vlan, data, iface): for o in overrides: L += [f"# {o['description']}", f"address=/{o['host']}/{o['ip']}", ""] - if bl_file: + if hosts_file.exists(): L += [ "# -- Blocklist ------------------------------------------------------", - f"conf-file={bl_file}", + f"addn-hosts={hosts_file}", "", ] - elif bl_names: - L += ["# Blocklist not yet downloaded -- run: sudo python3 dns-blocklists.py", ""] return "\n".join(L) @@ -419,6 +688,9 @@ def apply_dnsmasq_instances(data, dry_run=False, start_if_needed=True): if not dry_run: shared.DNSMASQ_CONF_DIR.mkdir(exist_ok=True) + print("Updating blocklist hosts files ======================================") + update_blocklist_hosts(data) + print() disable_system_dnsmasq(data) print() diff --git a/routlin/mod_timers.py b/routlin/mod_timers.py index d7305bd..482c7c2 100644 --- a/routlin/mod_timers.py +++ b/routlin/mod_timers.py @@ -69,7 +69,7 @@ def install_timer(data): "", ]) - blocklist_script = shared.SCRIPT_DIR / "dns-blocklists.py" + blocklist_script = shared.SCRIPT_DIR / "dl_blocklists.py" service_content = "\n".join([ "# Generated by core.py -- do not edit manually.", "",