Development

docker/routlin-dash/app/config_utils.py

@@ -619,10 +619,10 @@ def run_apply():
         pass
 
 
-def run_update_blocklists():
+def run_merge_blocklists():
     try:
         subprocess.run(
-            ['python3', f'{CONFIGS_DIR}/core.py', '--update-blocklists'],
+            ['python3', f'{CONFIGS_DIR}/core.py', '--merge-blocklists'],
             capture_output=True, timeout=120
         )
     except Exception:

docker/routlin-dash/app/pages/dnsblocking/action.py

@@ -7,7 +7,7 @@ import config_utils
 import sanitize
 import mod_validation as validate
 
-DNS_LOG_FILE = Path(config_utils.CONFIGS_DIR) / 'dns-blocklists.log'
+DNS_LOG_FILE = Path(config_utils.CONFIGS_DIR) / 'blocklists.log'
 
 _PAGE = Path(__file__).parent.name
 
@@ -102,7 +102,7 @@ def blocklists_delete():
             flash(msg, 'error')
         return redirect(f'/{_PAGE}')
     changes = config_utils.diff_fields(before, None)
-    flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', name, changes, 'core apply', queue=False), 'success')
+    flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', name, changes, 'merge blocklists'), 'success')
     return redirect(f'/{_PAGE}')
 
 
@@ -178,7 +178,7 @@ def blocklists_edit():
             flash(msg, 'error')
         return redirect(f'/{_PAGE}')
     changes = config_utils.diff_fields(before, items[idx])
-    flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', fields['name'], changes, 'core apply', queue=False), 'success')
+    flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', fields['name'], changes, 'merge blocklists'), 'success')
     return redirect(f'/{_PAGE}')
 
 
@@ -239,7 +239,7 @@ def addblocklist_add():
             flash(msg, 'error')
         return redirect(f'/{_PAGE}')
     changes = config_utils.diff_fields(None, entry)
-    flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', fields['name'], changes, 'core apply', queue=False), 'success')
+    flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', fields['name'], changes, 'merge blocklists'), 'success')
     return redirect(f'/{_PAGE}')
 
 
@@ -267,7 +267,9 @@ def blocklistrefresh_save():
 @bp.route('/action/dnsblocking/blocklistrefresh_refresh', methods=['POST'])
 @auth.require_level('administrator')
 def blocklistrefresh_refresh():
-    flash(config_utils.queued_msg('core update-blocklists', action_label='Blocklist refresh queued'), 'success')
+    config_utils.queue_command('download blocklists')
+    config_utils.queue_command('merge blocklists')
+    flash('Blocklist download and merge queued.', 'success')
     return redirect(f'/{_PAGE}')
 
 
@@ -319,4 +321,4 @@ def logging_download():
     if not DNS_LOG_FILE.is_file():
         flash('Log file not found.', 'error')
         return redirect(f'/{_PAGE}')
-    return send_file(DNS_LOG_FILE, as_attachment=True, download_name='dns-blocklists.log', mimetype='text/plain')
+    return send_file(DNS_LOG_FILE, as_attachment=True, download_name='blocklists.log', mimetype='text/plain')

docker/routlin-dash/app/pages/dnsblocking/view.py

@@ -4,7 +4,7 @@ from datetime import datetime, timezone
 import config_utils
 import factory
 
-DNS_LOG_FILE = f'{config_utils.CONFIGS_DIR}/dns-blocklists.log'
+DNS_LOG_FILE = f'{config_utils.CONFIGS_DIR}/blocklists.log'
 DNS_LOG_MAX  = 50
 
 BL_TYPE_LABELS = {'community': 'Community', 'local': 'Local'}

routlin/core.py

@@ -370,19 +370,12 @@ def _dry_run_conflicting_services(data):
 
 def _dry_run_blocklists(data):
     print("Blocklists (dry-run) ================================================")
-    for entry in data.get("dns_blocking", {}).get("blocklists", []):
+    for vlan in data.get("vlans", []):
-        print(f"  Would download: {entry['description']}")
-        print(f"    URL: {entry['url']}")
-    seen = {}
-    for vlan in data["vlans"]:
         names = vlan.get("use_blocklists", [])
         if names:
-            h = dnsmasq.combo_hash(names)
+            f      = dnsmasq.vlan_hosts_file(vlan)
-            if h not in seen:
+            action = "update" if f.exists() else "create"
-                seen[h] = sorted(names)
+            print(f"  Would {action}: {f}")
-                path = dnsmasq.merged_path(h)
-                action = "update" if path.exists() else "create"
-                print(f"  Would {action} merged blocklist: {path}")
             print(f"    Sources: {', '.join(sorted(names))}")
 
 def _dry_run_timer(data):
@@ -751,7 +744,7 @@ def cmd_apply(data, dry_run=False):
     print("dnsmasq instances ===================================================")
     if not dnsmasq.blocklists_available(data):
         print("  NOTE: No merged blocklist files found -- blocklist rules will be absent.")
-        print("        Run: sudo python3 dns-blocklists.py")
+        print("        Run: sudo python3 dl_blocklists.py")
     dnsmasq.apply_dnsmasq_instances(data, start_if_needed=True)
     print()
 
@@ -857,6 +850,7 @@ def main():
         )
     )
     parser.add_argument("--apply",              action="store_true", help="Apply full config: services, networkd, dnsmasq, nftables, timer, boot service")
+    parser.add_argument("--merge-blocklists",  action="store_true", help="Merge downloaded blocklists and reload dnsmasq via SIGHUP (no restart)")
     parser.add_argument("--dry-run", action="store_true", help="Preview all actions without making changes (combine with --apply or --disable)")
     parser.add_argument("--status",            action="store_true", help="Show service and timer status")
     parser.add_argument("--view-configs",      action="store_true", help="Show active per-VLAN dnsmasq config files")
@@ -870,7 +864,7 @@ def main():
 
     args = parser.parse_args()
 
-    if not any([args.apply,
+    if not any([args.apply, args.merge_blocklists,
                 args.dry_run, args.status, args.view_configs, args.view_leases,
                 args.view_rules, args.disable, args.view_metrics,
                 args.reset_leases]):
@@ -924,6 +918,22 @@ def main():
         cmd_disable(data, dry_run=args.dry_run)
         return
 
+    if args.merge_blocklists:
+        if not shared.is_root():
+            die("This script must be run as root (sudo).")
+        general = data.get("dns_blocking", {}).get("general", {})
+        dnsmasq.setup_blocklist_logging(general)
+        print("Merging blocklists ==================================================")
+        success = dnsmasq.update_blocklist_hosts(data)
+        print()
+        if success:
+            print("Reloading dnsmasq instances =========================================")
+            dnsmasq.sighup_all_instances()
+        else:
+            print("WARNING: Some blocklists failed -- reloading anyway with available data.")
+            dnsmasq.sighup_all_instances()
+        return
+
     if args.apply:
         cmd_apply(data, dry_run=args.dry_run)
         return

routlin/dl_blocklists.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""Download community blocklists defined in config.json to disk.
+
+Saves each community blocklist to blocklists/<save_as>.
+Local blocklists are managed by the dashboard and are not downloaded here.
+
+Run this on a schedule (e.g. daily) to refresh remote lists, then run
+  sudo python3 core.py --merge-blocklists
+to merge, update SQLite, and reload dnsmasq without restarting it.
+
+Usage:
+  sudo python3 dl_blocklists.py
+"""
+
+import json
+import os
+import sys
+import urllib.request
+from pathlib import Path
+
+SCRIPT_DIR    = Path(__file__).parent
+CONFIG_FILE   = SCRIPT_DIR / "config.json"
+BLOCKLIST_DIR = SCRIPT_DIR / "blocklists"
+
+
+def die(msg):
+    print(f"ERROR: {msg}", file=sys.stderr)
+    sys.exit(1)
+
+
+def check_root():
+    if os.geteuid() != 0:
+        die("This script must be run as root (sudo).")
+
+
+def load_config():
+    if not CONFIG_FILE.exists():
+        die(f"Config file not found: {CONFIG_FILE}")
+    with open(CONFIG_FILE) as f:
+        return json.load(f)
+
+
+def download_blocklists(data):
+    any_fail = False
+    BLOCKLIST_DIR.mkdir(exist_ok=True)
+    for bl in data.get("dns_blocking", {}).get("blocklists", []):
+        if bl.get("bl_type") == "local":
+            continue
+        url     = bl.get("url", "")
+        save_as = bl.get("save_as", "")
+        name    = bl.get("name", "?")
+        if not url or not save_as:
+            print(f"  Skipped '{name}': missing url or save_as")
+            continue
+        try:
+            req = urllib.request.Request(url, headers={"User-Agent": "routlin/1.0"})
+            with urllib.request.urlopen(req, timeout=30) as r:
+                content = r.read()
+            (BLOCKLIST_DIR / save_as).write_bytes(content)
+            print(f"  Downloaded: {name} ({len(content):,} bytes)")
+        except Exception as e:
+            print(f"  ERROR: Failed to download '{name}': {e}")
+            any_fail = True
+    return not any_fail
+
+
+def main():
+    check_root()
+    data = load_config()
+    print("Downloading blocklists ==============================================")
+    success = download_blocklists(data)
+    if not success:
+        print("WARNING: One or more downloads failed.")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

routlin/dns-blocklists.py

@@ -1,364 +0,0 @@
-#!/usr/bin/env python3
-"""
-dns-blocklists.py -- Download and merge DNS blocklists defined in config.json.
-
-Reads the blocklists library from config.json, downloads every blocklist referenced
-by at least one VLAN, and upserts normalized domains into a SQLite database
-(blocklists/domains.db). Downloads are skipped when the content hash is unchanged.
-Merged per-combo conf files are only rewritten when a constituent blocklist changed.
-Sends SIGHUP to each running dnsmasq instance so it reloads without restarting.
-
-Usage:
-  sudo python3 dns-blocklists.py
-"""
-
-import hashlib
-import json
-import logging
-import os
-import sqlite3
-import subprocess
-import sys
-import time
-import urllib.request
-import urllib.error
-from pathlib import Path
-
-PRODUCT_NAME  = "routlin"
-SCRIPT_DIR    = Path(__file__).parent
-CONFIG_FILE   = SCRIPT_DIR / "config.json"
-BLOCKLIST_DIR = SCRIPT_DIR / "blocklists"
-DB_FILE       = BLOCKLIST_DIR / "domains.db"
-LOG_FILE      = SCRIPT_DIR / "dns-blocklists.log"
-
-log = None
-
-
-def _chown_to_script_dir_owner(path):
-    try:
-        stat = SCRIPT_DIR.stat()
-        os.chown(path, stat.st_uid, stat.st_gid)
-    except Exception:
-        pass
-
-
-def setup_logging(max_kb, errors_only):
-    global log
-    try:
-        if LOG_FILE.exists() and LOG_FILE.stat().st_size > max_kb * 1024:
-            LOG_FILE.write_text("")
-        if not LOG_FILE.exists():
-            LOG_FILE.touch()
-        _chown_to_script_dir_owner(LOG_FILE)
-        file_handler = logging.FileHandler(LOG_FILE)
-    except PermissionError:
-        print(f"WARNING: Cannot write to {LOG_FILE} (permission denied). "
-              f"Run with sudo or fix ownership: sudo chown $USER {LOG_FILE}")
-        file_handler = None
-    level = logging.ERROR if errors_only else logging.INFO
-    handlers = [logging.StreamHandler(sys.stdout)]
-    if file_handler:
-        handlers.insert(0, file_handler)
-    logging.basicConfig(
-        level=level,
-        format="%(asctime)s  %(levelname)-8s  %(message)s",
-        datefmt="%Y-%m-%d %H:%M:%S",
-        handlers=handlers,
-    )
-    log = logging.getLogger("dns-blocklists")
-
-
-def die(msg):
-    print(f"ERROR: {msg}", file=sys.stderr)
-    sys.exit(1)
-
-
-def check_root():
-    if os.geteuid() != 0:
-        die("This script must be run as root (sudo).")
-
-
-def load_config():
-    if not CONFIG_FILE.exists():
-        die(f"Config file not found: {CONFIG_FILE}")
-    with open(CONFIG_FILE) as f:
-        data = json.load(f)
-    if not data.get("vlans"):
-        die("No vlans defined in config.json.")
-    return data
-
-
-def combo_hash(names):
-    key = ",".join(sorted(names))
-    return hashlib.sha256(key.encode()).hexdigest()[:8]
-
-
-def merged_path(h):
-    return BLOCKLIST_DIR / f"merged-{h}.conf"
-
-
-# Parse / detect ======================================================
-
-def parse_dnsmasq_format(content):
-    domains = set()
-    for ln in content.splitlines():
-        ln = ln.strip()
-        if not ln or ln.startswith("#"):
-            continue
-        if ln.startswith("local=/"):
-            domain = ln.removeprefix("local=/").rstrip("/")
-            if domain:
-                domains.add(domain)
-        elif ln.startswith("address=/"):
-            parts = ln.removeprefix("address=/").split("/")
-            if parts:
-                domains.add(parts[0])
-    return domains
-
-
-def parse_hosts_format(content):
-    domains = set()
-    for ln in content.splitlines():
-        ln = ln.strip()
-        if not ln or ln.startswith("#"):
-            continue
-        parts = ln.split()
-        if len(parts) >= 2:
-            domains.add(parts[1])
-    return domains
-
-
-def parse_local_format(content):
-    domains = set()
-    for ln in content.splitlines():
-        ln = ln.strip()
-        if ln and not ln.startswith("#"):
-            domains.add(ln)
-    return domains
-
-
-def detect_format(content):
-    for ln in content.splitlines():
-        ln = ln.strip()
-        if not ln or ln.startswith("#"):
-            continue
-        if ln.startswith("local=/") or ln.startswith("address=/"):
-            return "dnsmasq"
-        if ln[0].isdigit():
-            return "hosts"
-    return "dnsmasq"
-
-
-def parse_blocklist(content, is_local=False):
-    if is_local:
-        return parse_local_format(content)
-    fmt = detect_format(content)
-    if fmt == "dnsmasq":
-        return parse_dnsmasq_format(content)
-    return parse_hosts_format(content)
-
-
-def content_hash(content):
-    return hashlib.sha256(content.encode()).hexdigest()
-
-
-# SQLite ==============================================================
-
-def open_db():
-    db = sqlite3.connect(DB_FILE)
-    db.execute("PRAGMA journal_mode=WAL")
-    db.execute("PRAGMA foreign_keys=ON")
-    db.executescript("""
-        CREATE TABLE IF NOT EXISTS blocklists (
-            id           INTEGER PRIMARY KEY,
-            name         TEXT UNIQUE NOT NULL,
-            content_hash TEXT,
-            fetched_at   INTEGER,
-            domain_count INTEGER
-        );
-        CREATE TABLE IF NOT EXISTS domains (
-            domain       TEXT    NOT NULL,
-            blocklist_id INTEGER NOT NULL REFERENCES blocklists(id) ON DELETE CASCADE,
-            PRIMARY KEY (domain, blocklist_id)
-        );
-        CREATE INDEX IF NOT EXISTS idx_domains_domain ON domains(domain);
-    """)
-    db.commit()
-    return db
-
-
-def get_stored_hash(db, name):
-    row = db.execute("SELECT content_hash FROM blocklists WHERE name = ?", (name,)).fetchone()
-    return row[0] if row else None
-
-
-def upsert_blocklist(db, name, domains, raw_hash):
-    now = int(time.time())
-    db.execute("""
-        INSERT INTO blocklists (name, content_hash, fetched_at, domain_count)
-        VALUES (?, ?, ?, ?)
-        ON CONFLICT(name) DO UPDATE SET
-            content_hash = excluded.content_hash,
-            fetched_at   = excluded.fetched_at,
-            domain_count = excluded.domain_count
-    """, (name, raw_hash, now, len(domains)))
-    bl_id = db.execute("SELECT id FROM blocklists WHERE name = ?", (name,)).fetchone()[0]
-    db.execute("DELETE FROM domains WHERE blocklist_id = ?", (bl_id,))
-    db.executemany("INSERT INTO domains (domain, blocklist_id) VALUES (?, ?)",
-                   ((d, bl_id) for d in domains))
-    db.commit()
-
-
-def query_merged_domains(db, names):
-    placeholders = ",".join("?" * len(names))
-    rows = db.execute(f"""
-        SELECT DISTINCT d.domain
-        FROM domains d
-        JOIN blocklists b ON d.blocklist_id = b.id
-        WHERE b.name IN ({placeholders})
-        ORDER BY d.domain
-    """, list(names)).fetchall()
-    return [r[0] for r in rows]
-
-
-# Conf file output ====================================================
-
-def build_merged_conf(domains, bl_names):
-    lines = [
-        "# Generated by dns-blocklists.py -- do not edit manually.",
-        f"# Blocklist combination: {', '.join(sorted(bl_names))}",
-        f"# Merged: {len(domains):,} unique domains.",
-        "#",
-        "# Blocks domain and all subdomains via local=/domain/ syntax.",
-        "",
-    ]
-    for domain in domains:
-        lines.append(f"local=/{domain}/")
-    return "\n".join(lines)
-
-
-# Fetch ===============================================================
-
-def fetch_community(entry):
-    url = entry["url"]
-    req = urllib.request.Request(url, headers={"User-Agent": "dns-blocklists.py/1.0"})
-    with urllib.request.urlopen(req, timeout=30) as r:
-        return r.read().decode("utf-8", errors="ignore")
-
-
-def read_local(entry):
-    save_as = entry.get("save_as", "")
-    path = BLOCKLIST_DIR / save_as if save_as else None
-    if not path:
-        return ""
-    return path.read_text()
-
-
-# Main update =========================================================
-
-def update_blocklists(data):
-    BLOCKLIST_DIR.mkdir(exist_ok=True)
-    _chown_to_script_dir_owner(BLOCKLIST_DIR)
-
-    db = open_db()
-
-    bl_library = {bl["name"]: bl for bl in data.get("dns_blocking", {}).get("blocklists", [])}
-    needed     = set()
-    for vlan in data["vlans"]:
-        needed.update(vlan.get("use_blocklists", []))
-
-    changed  = set()
-    any_fail = False
-
-    for name in needed:
-        entry    = bl_library[name]
-        is_local = entry.get("bl_type") == "local"
-
-        try:
-            raw = read_local(entry) if is_local else fetch_community(entry)
-        except Exception as e:
-            log.error(f"Failed to fetch '{name}': {e}")
-            any_fail = True
-            continue
-
-        h = content_hash(raw)
-        if h == get_stored_hash(db, name):
-            log.info(f"Unchanged: '{name}' -- skipping")
-            continue
-
-        domains = parse_blocklist(raw, is_local=is_local)
-        upsert_blocklist(db, name, domains, h)
-        log.info(f"Updated '{name}': {len(domains):,} domains")
-        changed.add(name)
-
-    active_hashes = set()
-    combos        = {}
-    for vlan in data["vlans"]:
-        names = frozenset(vlan.get("use_blocklists", []))
-        if names:
-            h = combo_hash(names)
-            combos[h] = names
-
-    for h, names in combos.items():
-        active_hashes.add(h)
-        if not changed.intersection(names) and merged_path(h).exists():
-            log.info(f"Combo [{h}] unchanged -- skipping rewrite")
-            continue
-        domains = query_merged_domains(db, names)
-        merged_path(h).write_text(build_merged_conf(domains, names))
-        log.info(f"Merged [{h}] ({', '.join(sorted(names))}): {len(domains):,} unique domains")
-
-    for f in BLOCKLIST_DIR.glob("merged-*.conf"):
-        h = f.stem.removeprefix("merged-")
-        if h not in active_hashes:
-            f.unlink()
-            log.info(f"Removed stale merged file: {f.name}")
-
-    db.close()
-    return not any_fail
-
-
-def reload_dnsmasq_instances():
-    """Send SIGHUP to every active dnsmasq-routlin-* instance so it reloads
-    its conf-file inclusions without restarting. No DNS or DHCP interruption."""
-    result = subprocess.run(
-        ["systemctl", "list-units", "--state=active", "--no-legend", "--plain",
-         f"dnsmasq-{PRODUCT_NAME}-*.service"],
-        capture_output=True, text=True,
-    )
-    units = [line.split()[0] for line in result.stdout.splitlines() if line.strip()]
-    if not units:
-        print("  No active dnsmasq instances found.")
-        return
-    for unit in units:
-        r = subprocess.run(["systemctl", "kill", "--signal=SIGHUP", unit],
-                           capture_output=True, text=True)
-        if r.returncode == 0:
-            print(f"  Reloaded: {unit}")
-        else:
-            print(f"  WARNING: Failed to reload {unit}: {r.stderr.strip()}")
-
-
-def main():
-    check_root()
-    data    = load_config()
-    general = data.get("dns_blocking", {}).get("general", {})
-    setup_logging(
-        general.get("log_max_kb", 1024),
-        general.get("log_errors_only", False),
-    )
-
-    print("Updating blocklists =================================================")
-    success = update_blocklists(data)
-    print()
-
-    if success:
-        print("Reloading dnsmasq instances =========================================")
-        reload_dnsmasq_instances()
-    else:
-        print("WARNING: Blocklist update had errors -- skipping reload.")
-        print("         Existing merged files (if any) are unchanged.")
-
-
-if __name__ == "__main__":
-    main()

routlin/health.py

@@ -534,13 +534,13 @@ def check_configurations(data):
                 results.append(problem(
                     f"blocklist_{h}", f"blocklist ({label})", "warning",
                     f"Merged blocklist file for '{label}' does not exist.",
-                    "Run `sudo python3 dns-blocklists.py` to download blocklists."))
+                    "Run `sudo python3 dl_blocklists.py` to download blocklists."))
             elif now - path.stat().st_mtime > BLOCKLIST_STALE_SECS:
                 age_h = int((now - path.stat().st_mtime) / 3600)
                 results.append(problem(
                     f"blocklist_{h}", f"blocklist ({label})", "warning",
                     f"Merged blocklist for '{label}' is {age_h}h old (threshold 36h).",
-                    "Run `sudo python3 dns-blocklists.py` to refresh."))
+                    "Run `sudo python3 dl_blocklists.py` to refresh."))
             else:
                 results.append(ok(f"blocklist_{h}", f"blocklist ({label})"))
 

routlin/mod_dnsmasq.py

@@ -6,9 +6,11 @@ generation, applying/reloading instances, system service conflict resolution
 (systemd-resolved, dnsmasq, chrony, ufw), and DHCP lease display.
 """
 
-import hashlib
 import json
+import logging
+import sqlite3
 import subprocess
+import time
 from datetime import datetime
 from pathlib import Path
 
@@ -17,29 +19,303 @@ import mod_wireguard as wireguard
 import mod_validation as validation
 
 BLOCKLIST_DIR = shared.SCRIPT_DIR / "blocklists"
+DB_FILE       = BLOCKLIST_DIR / "domains.db"
+LOG_FILE      = shared.SCRIPT_DIR / "blocklists.log"
 RESOLV_CONF   = Path("/etc/resolv.conf")
 
+_log = logging.getLogger("blocklists")
+
 
 # ===================================================================
 # Blocklist management
 # ===================================================================
 
-def combo_hash(names):
+def vlan_hosts_file(vlan):
-    """Return a stable 8-char hex hash for a list/set of blocklist names."""
+    """Stable per-VLAN hosts file path (always the same regardless of blocklist combo)."""
-    key = ",".join(sorted(names))
+    return BLOCKLIST_DIR / f"for-{vlan['name']}.hosts"
-    return hashlib.sha256(key.encode()).hexdigest()[:8]
 
-def merged_path(h):
-    return BLOCKLIST_DIR / f"merged-{h}.conf"
 
 def blocklists_available(data):
-    """Return True if at least one merged blocklist file exists on disk."""
+    """Return True if at least one per-VLAN hosts file is non-empty."""
-    combos = set()
     for vlan in data.get("vlans", []):
-        names = vlan.get("use_blocklists", [])
+        if vlan.get("use_blocklists"):
-        if names:
+            f = vlan_hosts_file(vlan)
-            combos.add(combo_hash(names))
+            if f.exists() and f.stat().st_size > 0:
-    return any(merged_path(h).exists() for h in combos)
+                return True
+    return False
+
+
+# ===================================================================
+# Blocklist parse / detect
+# ===================================================================
+
+def _parse_dnsmasq_format(content):
+    domains = set()
+    for ln in content.splitlines():
+        ln = ln.strip()
+        if not ln or ln.startswith("#"):
+            continue
+        if ln.startswith("local=/"):
+            domain = ln.removeprefix("local=/").rstrip("/")
+            if domain:
+                domains.add(domain)
+        elif ln.startswith("address=/"):
+            parts = ln.removeprefix("address=/").split("/")
+            if parts:
+                domains.add(parts[0])
+    return domains
+
+
+def _parse_hosts_format(content):
+    domains = set()
+    for ln in content.splitlines():
+        ln = ln.strip()
+        if not ln or ln.startswith("#"):
+            continue
+        parts = ln.split()
+        if len(parts) >= 2:
+            domains.add(parts[1])
+    return domains
+
+
+def _parse_local_format(content):
+    domains = set()
+    for ln in content.splitlines():
+        ln = ln.strip()
+        if ln and not ln.startswith("#"):
+            domains.add(ln)
+    return domains
+
+
+def _detect_format(content):
+    for ln in content.splitlines():
+        ln = ln.strip()
+        if not ln or ln.startswith("#"):
+            continue
+        if ln.startswith("local=/") or ln.startswith("address=/"):
+            return "dnsmasq"
+        if ln[0].isdigit():
+            return "hosts"
+    return "dnsmasq"
+
+
+def _parse_blocklist(content, is_local=False):
+    if is_local:
+        return _parse_local_format(content)
+    fmt = _detect_format(content)
+    if fmt == "dnsmasq":
+        return _parse_dnsmasq_format(content)
+    return _parse_hosts_format(content)
+
+
+# ===================================================================
+# Blocklist SQLite
+# ===================================================================
+
+def _open_db():
+    db = sqlite3.connect(DB_FILE)
+    db.execute("PRAGMA journal_mode=WAL")
+    db.execute("PRAGMA foreign_keys=ON")
+    db.executescript("""
+        CREATE TABLE IF NOT EXISTS blocklists (
+            id           INTEGER PRIMARY KEY,
+            name         TEXT UNIQUE NOT NULL,
+            mtime        REAL,
+            fetched_at   INTEGER,
+            domain_count INTEGER
+        );
+        CREATE TABLE IF NOT EXISTS domains (
+            domain       TEXT    NOT NULL,
+            blocklist_id INTEGER NOT NULL REFERENCES blocklists(id) ON DELETE CASCADE,
+            PRIMARY KEY (domain, blocklist_id)
+        );
+        CREATE INDEX IF NOT EXISTS idx_domains_domain ON domains(domain);
+    """)
+    db.commit()
+    return db
+
+
+def _get_stored_mtime(db, name):
+    row = db.execute("SELECT mtime FROM blocklists WHERE name = ?", (name,)).fetchone()
+    return row[0] if row else None
+
+
+def _upsert_blocklist(db, name, domains, mtime):
+    now = int(time.time())
+    db.execute("""
+        INSERT INTO blocklists (name, mtime, fetched_at, domain_count)
+        VALUES (?, ?, ?, ?)
+        ON CONFLICT(name) DO UPDATE SET
+            mtime        = excluded.mtime,
+            fetched_at   = excluded.fetched_at,
+            domain_count = excluded.domain_count
+    """, (name, mtime, now, len(domains)))
+    bl_id = db.execute("SELECT id FROM blocklists WHERE name = ?", (name,)).fetchone()[0]
+    db.execute("DELETE FROM domains WHERE blocklist_id = ?", (bl_id,))
+    db.executemany("INSERT INTO domains (domain, blocklist_id) VALUES (?, ?)",
+                   ((d, bl_id) for d in domains))
+    db.commit()
+
+
+def _query_merged_domains(db, names):
+    placeholders = ",".join("?" * len(names))
+    rows = db.execute(f"""
+        SELECT DISTINCT d.domain
+        FROM domains d
+        JOIN blocklists b ON d.blocklist_id = b.id
+        WHERE b.name IN ({placeholders})
+        ORDER BY d.domain
+    """, list(names)).fetchall()
+    return [r[0] for r in rows]
+
+
+# ===================================================================
+# Blocklist merge and SIGHUP
+# ===================================================================
+
+def _build_merged_hosts(domains, bl_names):
+    lines = [
+        "# Generated by core.py -- do not edit manually.",
+        f"# Blocklists: {', '.join(sorted(bl_names))}",
+        f"# Domains: {len(domains):,}",
+        "",
+    ]
+    for domain in domains:
+        lines.append(f"0.0.0.0 {domain}")
+    return "\n".join(lines) + "\n"
+
+
+def setup_blocklist_logging(general):
+    """Configure file + stdout logging for blocklists logger."""
+    max_kb      = general.get("log_max_kb", 1024)
+    errors_only = general.get("log_errors_only", False)
+    try:
+        if LOG_FILE.exists() and LOG_FILE.stat().st_size > max_kb * 1024:
+            LOG_FILE.write_text("")
+        if not LOG_FILE.exists():
+            LOG_FILE.touch()
+        file_handler = logging.FileHandler(LOG_FILE)
+    except PermissionError:
+        print(f"WARNING: Cannot write to {LOG_FILE} -- run with sudo.")
+        file_handler = None
+    level    = logging.ERROR if errors_only else logging.INFO
+    handlers = [logging.StreamHandler()]
+    if file_handler:
+        handlers.insert(0, file_handler)
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s  %(levelname)-8s  %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        handlers=handlers,
+        force=True,
+    )
+
+
+def update_blocklist_hosts(data):
+    """Parse downloaded/local blocklist files, upsert into SQLite, and write
+    per-VLAN hosts files (0.0.0.0 format). Always writes every VLAN's file
+    (empty if no blocklists assigned) so addn-hosts= in the dnsmasq conf is
+    always valid and SIGHUP can update it without a restart.
+
+    Returns True on full success, False if any fetch/parse failed.
+    """
+    BLOCKLIST_DIR.mkdir(exist_ok=True)
+
+    db         = _open_db()
+    bl_library = {bl["name"]: bl for bl in data.get("dns_blocking", {}).get("blocklists", [])}
+
+    needed = set()
+    for vlan in data.get("vlans", []):
+        needed.update(vlan.get("use_blocklists", []))
+
+    changed  = set()
+    any_fail = False
+
+    for name in needed:
+        if name not in bl_library:
+            _log.warning(f"Blocklist '{name}' referenced by a VLAN but not defined -- skipping")
+            continue
+        entry    = bl_library[name]
+        is_local = entry.get("bl_type") == "local"
+        save_as  = entry.get("save_as", "")
+
+        try:
+            path = BLOCKLIST_DIR / save_as if save_as else None
+            if not path or not path.exists():
+                _log.warning(f"'{name}': file not found ({path}) -- skipping")
+                any_fail = True
+                continue
+            current_mtime = path.stat().st_mtime
+        except Exception as e:
+            _log.error(f"Failed to stat '{name}': {e}")
+            any_fail = True
+            continue
+
+        if current_mtime == _get_stored_mtime(db, name):
+            _log.info(f"Unchanged: '{name}' -- skipping")
+            continue
+
+        try:
+            raw = path.read_text("utf-8", errors="ignore")
+        except Exception as e:
+            _log.error(f"Failed to read '{name}': {e}")
+            any_fail = True
+            continue
+
+        domains = _parse_blocklist(raw, is_local=is_local)
+        _upsert_blocklist(db, name, domains, current_mtime)
+        _log.info(f"Updated '{name}': {len(domains):,} domains")
+        changed.add(name)
+
+    active_vlan_names = set()
+    for vlan in data.get("vlans", []):
+        vlan_name = vlan["name"]
+        active_vlan_names.add(vlan_name)
+        bl_names = [n for n in vlan.get("use_blocklists", []) if n in bl_library]
+
+        hosts_file = vlan_hosts_file(vlan)
+        if not bl_names:
+            if not hosts_file.exists():
+                hosts_file.write_text("")
+            continue
+
+        if not changed.intersection(bl_names) and hosts_file.exists():
+            _log.info(f"VLAN '{vlan_name}' blocklists unchanged -- skipping rewrite")
+            continue
+
+        domains = _query_merged_domains(db, bl_names)
+        hosts_file.write_text(_build_merged_hosts(domains, bl_names))
+        _log.info(f"VLAN '{vlan_name}': wrote {len(domains):,} domains from [{', '.join(sorted(bl_names))}]")
+
+    for f in BLOCKLIST_DIR.glob("for-*.hosts"):
+        vlan_name = f.stem.removeprefix("for-")
+        if vlan_name not in active_vlan_names:
+            f.unlink()
+            _log.info(f"Removed stale hosts file: {f.name}")
+
+    db.close()
+    return not any_fail
+
+
+def sighup_all_instances():
+    """Send SIGHUP to every active dnsmasq-routlin-* instance to reload addn-hosts
+    files without restarting. No DNS or DHCP interruption."""
+    result = subprocess.run(
+        ["systemctl", "list-units", "--state=active", "--no-legend", "--plain",
+         f"dnsmasq-{shared.PRODUCT_NAME}-*.service"],
+        capture_output=True, text=True,
+    )
+    units = [line.split()[0] for line in result.stdout.splitlines() if line.strip()]
+    if not units:
+        print("  No active dnsmasq instances found.")
+        return
+    for unit in units:
+        r = subprocess.run(["systemctl", "kill", "--signal=SIGHUP", unit],
+                           capture_output=True, text=True)
+        if r.returncode == 0:
+            print(f"  Reloaded: {unit}")
+        else:
+            print(f"  WARNING: Failed to SIGHUP {unit}: {r.stderr.strip()}")
 
 
 # ===================================================================
@@ -95,12 +371,7 @@ def build_vlan_dnsmasq_conf(vlan, data, iface):
     opts      = shared.resolve_vlan_options(vlan)
     gateway   = opts["gateway"]
 
-    bl_names = vlan.get("use_blocklists", [])
+    hosts_file = vlan_hosts_file(vlan)
-    bl_file  = None
-    if bl_names:
-        p = merged_path(combo_hash(bl_names))
-        if p.exists():
-            bl_file = p
 
     L = [
         "# Generated by core.py -- do not edit manually.",
@@ -216,14 +487,12 @@ def build_vlan_dnsmasq_conf(vlan, data, iface):
         for o in overrides:
             L += [f"# {o['description']}", f"address=/{o['host']}/{o['ip']}", ""]
 
-    if bl_file:
+    if hosts_file.exists():
         L += [
             "# -- Blocklist ------------------------------------------------------",
-            f"conf-file={bl_file}",
+            f"addn-hosts={hosts_file}",
             "",
         ]
-    elif bl_names:
-        L += ["# Blocklist not yet downloaded -- run: sudo python3 dns-blocklists.py", ""]
 
     return "\n".join(L)
 
@@ -419,6 +688,9 @@ def apply_dnsmasq_instances(data, dry_run=False, start_if_needed=True):
 
     if not dry_run:
         shared.DNSMASQ_CONF_DIR.mkdir(exist_ok=True)
+        print("Updating blocklist hosts files ======================================")
+        update_blocklist_hosts(data)
+        print()
         disable_system_dnsmasq(data)
         print()
 

routlin/mod_timers.py

@@ -69,7 +69,7 @@ def install_timer(data):
         "",
     ])
 
-    blocklist_script = shared.SCRIPT_DIR / "dns-blocklists.py"
+    blocklist_script = shared.SCRIPT_DIR / "dl_blocklists.py"
     service_content = "\n".join([
         "# Generated by core.py -- do not edit manually.",
         "",