Development
This commit is contained in:
parent
49dd4a2cf8
commit
e33133df1e
9 changed files with 412 additions and 414 deletions
|
|
@ -619,10 +619,10 @@ def run_apply():
|
|||
pass
|
||||
|
||||
|
||||
def run_update_blocklists():
|
||||
def run_merge_blocklists():
|
||||
try:
|
||||
subprocess.run(
|
||||
['python3', f'{CONFIGS_DIR}/core.py', '--update-blocklists'],
|
||||
['python3', f'{CONFIGS_DIR}/core.py', '--merge-blocklists'],
|
||||
capture_output=True, timeout=120
|
||||
)
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import config_utils
|
|||
import sanitize
|
||||
import mod_validation as validate
|
||||
|
||||
DNS_LOG_FILE = Path(config_utils.CONFIGS_DIR) / 'dns-blocklists.log'
|
||||
DNS_LOG_FILE = Path(config_utils.CONFIGS_DIR) / 'blocklists.log'
|
||||
|
||||
_PAGE = Path(__file__).parent.name
|
||||
|
||||
|
|
@ -102,7 +102,7 @@ def blocklists_delete():
|
|||
flash(msg, 'error')
|
||||
return redirect(f'/{_PAGE}')
|
||||
changes = config_utils.diff_fields(before, None)
|
||||
flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', name, changes, 'core apply', queue=False), 'success')
|
||||
flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', name, changes, 'merge blocklists'), 'success')
|
||||
return redirect(f'/{_PAGE}')
|
||||
|
||||
|
||||
|
|
@ -178,7 +178,7 @@ def blocklists_edit():
|
|||
flash(msg, 'error')
|
||||
return redirect(f'/{_PAGE}')
|
||||
changes = config_utils.diff_fields(before, items[idx])
|
||||
flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', fields['name'], changes, 'core apply', queue=False), 'success')
|
||||
flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', fields['name'], changes, 'merge blocklists'), 'success')
|
||||
return redirect(f'/{_PAGE}')
|
||||
|
||||
|
||||
|
|
@ -239,7 +239,7 @@ def addblocklist_add():
|
|||
flash(msg, 'error')
|
||||
return redirect(f'/{_PAGE}')
|
||||
changes = config_utils.diff_fields(None, entry)
|
||||
flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', fields['name'], changes, 'core apply', queue=False), 'success')
|
||||
flash(config_utils.record_group(cfg, 'dns_blocking.blocklists', 'name', fields['name'], changes, 'merge blocklists'), 'success')
|
||||
return redirect(f'/{_PAGE}')
|
||||
|
||||
|
||||
|
|
@ -267,7 +267,9 @@ def blocklistrefresh_save():
|
|||
@bp.route('/action/dnsblocking/blocklistrefresh_refresh', methods=['POST'])
|
||||
@auth.require_level('administrator')
|
||||
def blocklistrefresh_refresh():
|
||||
flash(config_utils.queued_msg('core update-blocklists', action_label='Blocklist refresh queued'), 'success')
|
||||
config_utils.queue_command('download blocklists')
|
||||
config_utils.queue_command('merge blocklists')
|
||||
flash('Blocklist download and merge queued.', 'success')
|
||||
return redirect(f'/{_PAGE}')
|
||||
|
||||
|
||||
|
|
@ -319,4 +321,4 @@ def logging_download():
|
|||
if not DNS_LOG_FILE.is_file():
|
||||
flash('Log file not found.', 'error')
|
||||
return redirect(f'/{_PAGE}')
|
||||
return send_file(DNS_LOG_FILE, as_attachment=True, download_name='dns-blocklists.log', mimetype='text/plain')
|
||||
return send_file(DNS_LOG_FILE, as_attachment=True, download_name='blocklists.log', mimetype='text/plain')
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from datetime import datetime, timezone
|
|||
import config_utils
|
||||
import factory
|
||||
|
||||
DNS_LOG_FILE = f'{config_utils.CONFIGS_DIR}/dns-blocklists.log'
|
||||
DNS_LOG_FILE = f'{config_utils.CONFIGS_DIR}/blocklists.log'
|
||||
DNS_LOG_MAX = 50
|
||||
|
||||
BL_TYPE_LABELS = {'community': 'Community', 'local': 'Local'}
|
||||
|
|
|
|||
|
|
@ -370,19 +370,12 @@ def _dry_run_conflicting_services(data):
|
|||
|
||||
def _dry_run_blocklists(data):
|
||||
print("Blocklists (dry-run) ================================================")
|
||||
for entry in data.get("dns_blocking", {}).get("blocklists", []):
|
||||
print(f" Would download: {entry['description']}")
|
||||
print(f" URL: {entry['url']}")
|
||||
seen = {}
|
||||
for vlan in data["vlans"]:
|
||||
for vlan in data.get("vlans", []):
|
||||
names = vlan.get("use_blocklists", [])
|
||||
if names:
|
||||
h = dnsmasq.combo_hash(names)
|
||||
if h not in seen:
|
||||
seen[h] = sorted(names)
|
||||
path = dnsmasq.merged_path(h)
|
||||
action = "update" if path.exists() else "create"
|
||||
print(f" Would {action} merged blocklist: {path}")
|
||||
f = dnsmasq.vlan_hosts_file(vlan)
|
||||
action = "update" if f.exists() else "create"
|
||||
print(f" Would {action}: {f}")
|
||||
print(f" Sources: {', '.join(sorted(names))}")
|
||||
|
||||
def _dry_run_timer(data):
|
||||
|
|
@ -751,7 +744,7 @@ def cmd_apply(data, dry_run=False):
|
|||
print("dnsmasq instances ===================================================")
|
||||
if not dnsmasq.blocklists_available(data):
|
||||
print(" NOTE: No merged blocklist files found -- blocklist rules will be absent.")
|
||||
print(" Run: sudo python3 dns-blocklists.py")
|
||||
print(" Run: sudo python3 dl_blocklists.py")
|
||||
dnsmasq.apply_dnsmasq_instances(data, start_if_needed=True)
|
||||
print()
|
||||
|
||||
|
|
@ -857,6 +850,7 @@ def main():
|
|||
)
|
||||
)
|
||||
parser.add_argument("--apply", action="store_true", help="Apply full config: services, networkd, dnsmasq, nftables, timer, boot service")
|
||||
parser.add_argument("--merge-blocklists", action="store_true", help="Merge downloaded blocklists and reload dnsmasq via SIGHUP (no restart)")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Preview all actions without making changes (combine with --apply or --disable)")
|
||||
parser.add_argument("--status", action="store_true", help="Show service and timer status")
|
||||
parser.add_argument("--view-configs", action="store_true", help="Show active per-VLAN dnsmasq config files")
|
||||
|
|
@ -870,7 +864,7 @@ def main():
|
|||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not any([args.apply,
|
||||
if not any([args.apply, args.merge_blocklists,
|
||||
args.dry_run, args.status, args.view_configs, args.view_leases,
|
||||
args.view_rules, args.disable, args.view_metrics,
|
||||
args.reset_leases]):
|
||||
|
|
@ -924,6 +918,22 @@ def main():
|
|||
cmd_disable(data, dry_run=args.dry_run)
|
||||
return
|
||||
|
||||
if args.merge_blocklists:
|
||||
if not shared.is_root():
|
||||
die("This script must be run as root (sudo).")
|
||||
general = data.get("dns_blocking", {}).get("general", {})
|
||||
dnsmasq.setup_blocklist_logging(general)
|
||||
print("Merging blocklists ==================================================")
|
||||
success = dnsmasq.update_blocklist_hosts(data)
|
||||
print()
|
||||
if success:
|
||||
print("Reloading dnsmasq instances =========================================")
|
||||
dnsmasq.sighup_all_instances()
|
||||
else:
|
||||
print("WARNING: Some blocklists failed -- reloading anyway with available data.")
|
||||
dnsmasq.sighup_all_instances()
|
||||
return
|
||||
|
||||
if args.apply:
|
||||
cmd_apply(data, dry_run=args.dry_run)
|
||||
return
|
||||
|
|
|
|||
78
routlin/dl_blocklists.py
Normal file
78
routlin/dl_blocklists.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Download community blocklists defined in config.json to disk.
|
||||
|
||||
Saves each community blocklist to blocklists/<save_as>.
|
||||
Local blocklists are managed by the dashboard and are not downloaded here.
|
||||
|
||||
Run this on a schedule (e.g. daily) to refresh remote lists, then run
|
||||
sudo python3 core.py --merge-blocklists
|
||||
to merge, update SQLite, and reload dnsmasq without restarting it.
|
||||
|
||||
Usage:
|
||||
sudo python3 dl_blocklists.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).parent
|
||||
CONFIG_FILE = SCRIPT_DIR / "config.json"
|
||||
BLOCKLIST_DIR = SCRIPT_DIR / "blocklists"
|
||||
|
||||
|
||||
def die(msg):
|
||||
print(f"ERROR: {msg}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def check_root():
|
||||
if os.geteuid() != 0:
|
||||
die("This script must be run as root (sudo).")
|
||||
|
||||
|
||||
def load_config():
|
||||
if not CONFIG_FILE.exists():
|
||||
die(f"Config file not found: {CONFIG_FILE}")
|
||||
with open(CONFIG_FILE) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def download_blocklists(data):
|
||||
any_fail = False
|
||||
BLOCKLIST_DIR.mkdir(exist_ok=True)
|
||||
for bl in data.get("dns_blocking", {}).get("blocklists", []):
|
||||
if bl.get("bl_type") == "local":
|
||||
continue
|
||||
url = bl.get("url", "")
|
||||
save_as = bl.get("save_as", "")
|
||||
name = bl.get("name", "?")
|
||||
if not url or not save_as:
|
||||
print(f" Skipped '{name}': missing url or save_as")
|
||||
continue
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "routlin/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
content = r.read()
|
||||
(BLOCKLIST_DIR / save_as).write_bytes(content)
|
||||
print(f" Downloaded: {name} ({len(content):,} bytes)")
|
||||
except Exception as e:
|
||||
print(f" ERROR: Failed to download '{name}': {e}")
|
||||
any_fail = True
|
||||
return not any_fail
|
||||
|
||||
|
||||
def main():
|
||||
check_root()
|
||||
data = load_config()
|
||||
print("Downloading blocklists ==============================================")
|
||||
success = download_blocklists(data)
|
||||
if not success:
|
||||
print("WARNING: One or more downloads failed.")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,364 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
dns-blocklists.py -- Download and merge DNS blocklists defined in config.json.
|
||||
|
||||
Reads the blocklists library from config.json, downloads every blocklist referenced
|
||||
by at least one VLAN, and upserts normalized domains into a SQLite database
|
||||
(blocklists/domains.db). Downloads are skipped when the content hash is unchanged.
|
||||
Merged per-combo conf files are only rewritten when a constituent blocklist changed.
|
||||
Sends SIGHUP to each running dnsmasq instance so it reloads without restarting.
|
||||
|
||||
Usage:
|
||||
sudo python3 dns-blocklists.py
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
|
||||
PRODUCT_NAME = "routlin"
|
||||
SCRIPT_DIR = Path(__file__).parent
|
||||
CONFIG_FILE = SCRIPT_DIR / "config.json"
|
||||
BLOCKLIST_DIR = SCRIPT_DIR / "blocklists"
|
||||
DB_FILE = BLOCKLIST_DIR / "domains.db"
|
||||
LOG_FILE = SCRIPT_DIR / "dns-blocklists.log"
|
||||
|
||||
log = None
|
||||
|
||||
|
||||
def _chown_to_script_dir_owner(path):
|
||||
try:
|
||||
stat = SCRIPT_DIR.stat()
|
||||
os.chown(path, stat.st_uid, stat.st_gid)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def setup_logging(max_kb, errors_only):
|
||||
global log
|
||||
try:
|
||||
if LOG_FILE.exists() and LOG_FILE.stat().st_size > max_kb * 1024:
|
||||
LOG_FILE.write_text("")
|
||||
if not LOG_FILE.exists():
|
||||
LOG_FILE.touch()
|
||||
_chown_to_script_dir_owner(LOG_FILE)
|
||||
file_handler = logging.FileHandler(LOG_FILE)
|
||||
except PermissionError:
|
||||
print(f"WARNING: Cannot write to {LOG_FILE} (permission denied). "
|
||||
f"Run with sudo or fix ownership: sudo chown $USER {LOG_FILE}")
|
||||
file_handler = None
|
||||
level = logging.ERROR if errors_only else logging.INFO
|
||||
handlers = [logging.StreamHandler(sys.stdout)]
|
||||
if file_handler:
|
||||
handlers.insert(0, file_handler)
|
||||
logging.basicConfig(
|
||||
level=level,
|
||||
format="%(asctime)s %(levelname)-8s %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
handlers=handlers,
|
||||
)
|
||||
log = logging.getLogger("dns-blocklists")
|
||||
|
||||
|
||||
def die(msg):
|
||||
print(f"ERROR: {msg}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def check_root():
|
||||
if os.geteuid() != 0:
|
||||
die("This script must be run as root (sudo).")
|
||||
|
||||
|
||||
def load_config():
|
||||
if not CONFIG_FILE.exists():
|
||||
die(f"Config file not found: {CONFIG_FILE}")
|
||||
with open(CONFIG_FILE) as f:
|
||||
data = json.load(f)
|
||||
if not data.get("vlans"):
|
||||
die("No vlans defined in config.json.")
|
||||
return data
|
||||
|
||||
|
||||
def combo_hash(names):
|
||||
key = ",".join(sorted(names))
|
||||
return hashlib.sha256(key.encode()).hexdigest()[:8]
|
||||
|
||||
|
||||
def merged_path(h):
|
||||
return BLOCKLIST_DIR / f"merged-{h}.conf"
|
||||
|
||||
|
||||
# Parse / detect ======================================================
|
||||
|
||||
def parse_dnsmasq_format(content):
|
||||
domains = set()
|
||||
for ln in content.splitlines():
|
||||
ln = ln.strip()
|
||||
if not ln or ln.startswith("#"):
|
||||
continue
|
||||
if ln.startswith("local=/"):
|
||||
domain = ln.removeprefix("local=/").rstrip("/")
|
||||
if domain:
|
||||
domains.add(domain)
|
||||
elif ln.startswith("address=/"):
|
||||
parts = ln.removeprefix("address=/").split("/")
|
||||
if parts:
|
||||
domains.add(parts[0])
|
||||
return domains
|
||||
|
||||
|
||||
def parse_hosts_format(content):
|
||||
domains = set()
|
||||
for ln in content.splitlines():
|
||||
ln = ln.strip()
|
||||
if not ln or ln.startswith("#"):
|
||||
continue
|
||||
parts = ln.split()
|
||||
if len(parts) >= 2:
|
||||
domains.add(parts[1])
|
||||
return domains
|
||||
|
||||
|
||||
def parse_local_format(content):
|
||||
domains = set()
|
||||
for ln in content.splitlines():
|
||||
ln = ln.strip()
|
||||
if ln and not ln.startswith("#"):
|
||||
domains.add(ln)
|
||||
return domains
|
||||
|
||||
|
||||
def detect_format(content):
|
||||
for ln in content.splitlines():
|
||||
ln = ln.strip()
|
||||
if not ln or ln.startswith("#"):
|
||||
continue
|
||||
if ln.startswith("local=/") or ln.startswith("address=/"):
|
||||
return "dnsmasq"
|
||||
if ln[0].isdigit():
|
||||
return "hosts"
|
||||
return "dnsmasq"
|
||||
|
||||
|
||||
def parse_blocklist(content, is_local=False):
|
||||
if is_local:
|
||||
return parse_local_format(content)
|
||||
fmt = detect_format(content)
|
||||
if fmt == "dnsmasq":
|
||||
return parse_dnsmasq_format(content)
|
||||
return parse_hosts_format(content)
|
||||
|
||||
|
||||
def content_hash(content):
|
||||
return hashlib.sha256(content.encode()).hexdigest()
|
||||
|
||||
|
||||
# SQLite ==============================================================
|
||||
|
||||
def open_db():
|
||||
db = sqlite3.connect(DB_FILE)
|
||||
db.execute("PRAGMA journal_mode=WAL")
|
||||
db.execute("PRAGMA foreign_keys=ON")
|
||||
db.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS blocklists (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT UNIQUE NOT NULL,
|
||||
content_hash TEXT,
|
||||
fetched_at INTEGER,
|
||||
domain_count INTEGER
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS domains (
|
||||
domain TEXT NOT NULL,
|
||||
blocklist_id INTEGER NOT NULL REFERENCES blocklists(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (domain, blocklist_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_domains_domain ON domains(domain);
|
||||
""")
|
||||
db.commit()
|
||||
return db
|
||||
|
||||
|
||||
def get_stored_hash(db, name):
|
||||
row = db.execute("SELECT content_hash FROM blocklists WHERE name = ?", (name,)).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
def upsert_blocklist(db, name, domains, raw_hash):
|
||||
now = int(time.time())
|
||||
db.execute("""
|
||||
INSERT INTO blocklists (name, content_hash, fetched_at, domain_count)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
content_hash = excluded.content_hash,
|
||||
fetched_at = excluded.fetched_at,
|
||||
domain_count = excluded.domain_count
|
||||
""", (name, raw_hash, now, len(domains)))
|
||||
bl_id = db.execute("SELECT id FROM blocklists WHERE name = ?", (name,)).fetchone()[0]
|
||||
db.execute("DELETE FROM domains WHERE blocklist_id = ?", (bl_id,))
|
||||
db.executemany("INSERT INTO domains (domain, blocklist_id) VALUES (?, ?)",
|
||||
((d, bl_id) for d in domains))
|
||||
db.commit()
|
||||
|
||||
|
||||
def query_merged_domains(db, names):
|
||||
placeholders = ",".join("?" * len(names))
|
||||
rows = db.execute(f"""
|
||||
SELECT DISTINCT d.domain
|
||||
FROM domains d
|
||||
JOIN blocklists b ON d.blocklist_id = b.id
|
||||
WHERE b.name IN ({placeholders})
|
||||
ORDER BY d.domain
|
||||
""", list(names)).fetchall()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
# Conf file output ====================================================
|
||||
|
||||
def build_merged_conf(domains, bl_names):
|
||||
lines = [
|
||||
"# Generated by dns-blocklists.py -- do not edit manually.",
|
||||
f"# Blocklist combination: {', '.join(sorted(bl_names))}",
|
||||
f"# Merged: {len(domains):,} unique domains.",
|
||||
"#",
|
||||
"# Blocks domain and all subdomains via local=/domain/ syntax.",
|
||||
"",
|
||||
]
|
||||
for domain in domains:
|
||||
lines.append(f"local=/{domain}/")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# Fetch ===============================================================
|
||||
|
||||
def fetch_community(entry):
|
||||
url = entry["url"]
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "dns-blocklists.py/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.read().decode("utf-8", errors="ignore")
|
||||
|
||||
|
||||
def read_local(entry):
|
||||
save_as = entry.get("save_as", "")
|
||||
path = BLOCKLIST_DIR / save_as if save_as else None
|
||||
if not path:
|
||||
return ""
|
||||
return path.read_text()
|
||||
|
||||
|
||||
# Main update =========================================================
|
||||
|
||||
def update_blocklists(data):
|
||||
BLOCKLIST_DIR.mkdir(exist_ok=True)
|
||||
_chown_to_script_dir_owner(BLOCKLIST_DIR)
|
||||
|
||||
db = open_db()
|
||||
|
||||
bl_library = {bl["name"]: bl for bl in data.get("dns_blocking", {}).get("blocklists", [])}
|
||||
needed = set()
|
||||
for vlan in data["vlans"]:
|
||||
needed.update(vlan.get("use_blocklists", []))
|
||||
|
||||
changed = set()
|
||||
any_fail = False
|
||||
|
||||
for name in needed:
|
||||
entry = bl_library[name]
|
||||
is_local = entry.get("bl_type") == "local"
|
||||
|
||||
try:
|
||||
raw = read_local(entry) if is_local else fetch_community(entry)
|
||||
except Exception as e:
|
||||
log.error(f"Failed to fetch '{name}': {e}")
|
||||
any_fail = True
|
||||
continue
|
||||
|
||||
h = content_hash(raw)
|
||||
if h == get_stored_hash(db, name):
|
||||
log.info(f"Unchanged: '{name}' -- skipping")
|
||||
continue
|
||||
|
||||
domains = parse_blocklist(raw, is_local=is_local)
|
||||
upsert_blocklist(db, name, domains, h)
|
||||
log.info(f"Updated '{name}': {len(domains):,} domains")
|
||||
changed.add(name)
|
||||
|
||||
active_hashes = set()
|
||||
combos = {}
|
||||
for vlan in data["vlans"]:
|
||||
names = frozenset(vlan.get("use_blocklists", []))
|
||||
if names:
|
||||
h = combo_hash(names)
|
||||
combos[h] = names
|
||||
|
||||
for h, names in combos.items():
|
||||
active_hashes.add(h)
|
||||
if not changed.intersection(names) and merged_path(h).exists():
|
||||
log.info(f"Combo [{h}] unchanged -- skipping rewrite")
|
||||
continue
|
||||
domains = query_merged_domains(db, names)
|
||||
merged_path(h).write_text(build_merged_conf(domains, names))
|
||||
log.info(f"Merged [{h}] ({', '.join(sorted(names))}): {len(domains):,} unique domains")
|
||||
|
||||
for f in BLOCKLIST_DIR.glob("merged-*.conf"):
|
||||
h = f.stem.removeprefix("merged-")
|
||||
if h not in active_hashes:
|
||||
f.unlink()
|
||||
log.info(f"Removed stale merged file: {f.name}")
|
||||
|
||||
db.close()
|
||||
return not any_fail
|
||||
|
||||
|
||||
def reload_dnsmasq_instances():
|
||||
"""Send SIGHUP to every active dnsmasq-routlin-* instance so it reloads
|
||||
its conf-file inclusions without restarting. No DNS or DHCP interruption."""
|
||||
result = subprocess.run(
|
||||
["systemctl", "list-units", "--state=active", "--no-legend", "--plain",
|
||||
f"dnsmasq-{PRODUCT_NAME}-*.service"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
units = [line.split()[0] for line in result.stdout.splitlines() if line.strip()]
|
||||
if not units:
|
||||
print(" No active dnsmasq instances found.")
|
||||
return
|
||||
for unit in units:
|
||||
r = subprocess.run(["systemctl", "kill", "--signal=SIGHUP", unit],
|
||||
capture_output=True, text=True)
|
||||
if r.returncode == 0:
|
||||
print(f" Reloaded: {unit}")
|
||||
else:
|
||||
print(f" WARNING: Failed to reload {unit}: {r.stderr.strip()}")
|
||||
|
||||
|
||||
def main():
|
||||
check_root()
|
||||
data = load_config()
|
||||
general = data.get("dns_blocking", {}).get("general", {})
|
||||
setup_logging(
|
||||
general.get("log_max_kb", 1024),
|
||||
general.get("log_errors_only", False),
|
||||
)
|
||||
|
||||
print("Updating blocklists =================================================")
|
||||
success = update_blocklists(data)
|
||||
print()
|
||||
|
||||
if success:
|
||||
print("Reloading dnsmasq instances =========================================")
|
||||
reload_dnsmasq_instances()
|
||||
else:
|
||||
print("WARNING: Blocklist update had errors -- skipping reload.")
|
||||
print(" Existing merged files (if any) are unchanged.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -534,13 +534,13 @@ def check_configurations(data):
|
|||
results.append(problem(
|
||||
f"blocklist_{h}", f"blocklist ({label})", "warning",
|
||||
f"Merged blocklist file for '{label}' does not exist.",
|
||||
"Run `sudo python3 dns-blocklists.py` to download blocklists."))
|
||||
"Run `sudo python3 dl_blocklists.py` to download blocklists."))
|
||||
elif now - path.stat().st_mtime > BLOCKLIST_STALE_SECS:
|
||||
age_h = int((now - path.stat().st_mtime) / 3600)
|
||||
results.append(problem(
|
||||
f"blocklist_{h}", f"blocklist ({label})", "warning",
|
||||
f"Merged blocklist for '{label}' is {age_h}h old (threshold 36h).",
|
||||
"Run `sudo python3 dns-blocklists.py` to refresh."))
|
||||
"Run `sudo python3 dl_blocklists.py` to refresh."))
|
||||
else:
|
||||
results.append(ok(f"blocklist_{h}", f"blocklist ({label})"))
|
||||
|
||||
|
|
|
|||
|
|
@ -6,9 +6,11 @@ generation, applying/reloading instances, system service conflict resolution
|
|||
(systemd-resolved, dnsmasq, chrony, ufw), and DHCP lease display.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -17,29 +19,303 @@ import mod_wireguard as wireguard
|
|||
import mod_validation as validation
|
||||
|
||||
BLOCKLIST_DIR = shared.SCRIPT_DIR / "blocklists"
|
||||
DB_FILE = BLOCKLIST_DIR / "domains.db"
|
||||
LOG_FILE = shared.SCRIPT_DIR / "blocklists.log"
|
||||
RESOLV_CONF = Path("/etc/resolv.conf")
|
||||
|
||||
_log = logging.getLogger("blocklists")
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Blocklist management
|
||||
# ===================================================================
|
||||
|
||||
def combo_hash(names):
|
||||
"""Return a stable 8-char hex hash for a list/set of blocklist names."""
|
||||
key = ",".join(sorted(names))
|
||||
return hashlib.sha256(key.encode()).hexdigest()[:8]
|
||||
def vlan_hosts_file(vlan):
|
||||
"""Stable per-VLAN hosts file path (always the same regardless of blocklist combo)."""
|
||||
return BLOCKLIST_DIR / f"for-{vlan['name']}.hosts"
|
||||
|
||||
def merged_path(h):
|
||||
return BLOCKLIST_DIR / f"merged-{h}.conf"
|
||||
|
||||
def blocklists_available(data):
|
||||
"""Return True if at least one merged blocklist file exists on disk."""
|
||||
combos = set()
|
||||
"""Return True if at least one per-VLAN hosts file is non-empty."""
|
||||
for vlan in data.get("vlans", []):
|
||||
names = vlan.get("use_blocklists", [])
|
||||
if names:
|
||||
combos.add(combo_hash(names))
|
||||
return any(merged_path(h).exists() for h in combos)
|
||||
if vlan.get("use_blocklists"):
|
||||
f = vlan_hosts_file(vlan)
|
||||
if f.exists() and f.stat().st_size > 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Blocklist parse / detect
|
||||
# ===================================================================
|
||||
|
||||
def _parse_dnsmasq_format(content):
|
||||
domains = set()
|
||||
for ln in content.splitlines():
|
||||
ln = ln.strip()
|
||||
if not ln or ln.startswith("#"):
|
||||
continue
|
||||
if ln.startswith("local=/"):
|
||||
domain = ln.removeprefix("local=/").rstrip("/")
|
||||
if domain:
|
||||
domains.add(domain)
|
||||
elif ln.startswith("address=/"):
|
||||
parts = ln.removeprefix("address=/").split("/")
|
||||
if parts:
|
||||
domains.add(parts[0])
|
||||
return domains
|
||||
|
||||
|
||||
def _parse_hosts_format(content):
|
||||
domains = set()
|
||||
for ln in content.splitlines():
|
||||
ln = ln.strip()
|
||||
if not ln or ln.startswith("#"):
|
||||
continue
|
||||
parts = ln.split()
|
||||
if len(parts) >= 2:
|
||||
domains.add(parts[1])
|
||||
return domains
|
||||
|
||||
|
||||
def _parse_local_format(content):
|
||||
domains = set()
|
||||
for ln in content.splitlines():
|
||||
ln = ln.strip()
|
||||
if ln and not ln.startswith("#"):
|
||||
domains.add(ln)
|
||||
return domains
|
||||
|
||||
|
||||
def _detect_format(content):
|
||||
for ln in content.splitlines():
|
||||
ln = ln.strip()
|
||||
if not ln or ln.startswith("#"):
|
||||
continue
|
||||
if ln.startswith("local=/") or ln.startswith("address=/"):
|
||||
return "dnsmasq"
|
||||
if ln[0].isdigit():
|
||||
return "hosts"
|
||||
return "dnsmasq"
|
||||
|
||||
|
||||
def _parse_blocklist(content, is_local=False):
|
||||
if is_local:
|
||||
return _parse_local_format(content)
|
||||
fmt = _detect_format(content)
|
||||
if fmt == "dnsmasq":
|
||||
return _parse_dnsmasq_format(content)
|
||||
return _parse_hosts_format(content)
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Blocklist SQLite
|
||||
# ===================================================================
|
||||
|
||||
def _open_db():
|
||||
db = sqlite3.connect(DB_FILE)
|
||||
db.execute("PRAGMA journal_mode=WAL")
|
||||
db.execute("PRAGMA foreign_keys=ON")
|
||||
db.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS blocklists (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT UNIQUE NOT NULL,
|
||||
mtime REAL,
|
||||
fetched_at INTEGER,
|
||||
domain_count INTEGER
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS domains (
|
||||
domain TEXT NOT NULL,
|
||||
blocklist_id INTEGER NOT NULL REFERENCES blocklists(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (domain, blocklist_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_domains_domain ON domains(domain);
|
||||
""")
|
||||
db.commit()
|
||||
return db
|
||||
|
||||
|
||||
def _get_stored_mtime(db, name):
|
||||
row = db.execute("SELECT mtime FROM blocklists WHERE name = ?", (name,)).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
def _upsert_blocklist(db, name, domains, mtime):
|
||||
now = int(time.time())
|
||||
db.execute("""
|
||||
INSERT INTO blocklists (name, mtime, fetched_at, domain_count)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
mtime = excluded.mtime,
|
||||
fetched_at = excluded.fetched_at,
|
||||
domain_count = excluded.domain_count
|
||||
""", (name, mtime, now, len(domains)))
|
||||
bl_id = db.execute("SELECT id FROM blocklists WHERE name = ?", (name,)).fetchone()[0]
|
||||
db.execute("DELETE FROM domains WHERE blocklist_id = ?", (bl_id,))
|
||||
db.executemany("INSERT INTO domains (domain, blocklist_id) VALUES (?, ?)",
|
||||
((d, bl_id) for d in domains))
|
||||
db.commit()
|
||||
|
||||
|
||||
def _query_merged_domains(db, names):
|
||||
placeholders = ",".join("?" * len(names))
|
||||
rows = db.execute(f"""
|
||||
SELECT DISTINCT d.domain
|
||||
FROM domains d
|
||||
JOIN blocklists b ON d.blocklist_id = b.id
|
||||
WHERE b.name IN ({placeholders})
|
||||
ORDER BY d.domain
|
||||
""", list(names)).fetchall()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Blocklist merge and SIGHUP
|
||||
# ===================================================================
|
||||
|
||||
def _build_merged_hosts(domains, bl_names):
|
||||
lines = [
|
||||
"# Generated by core.py -- do not edit manually.",
|
||||
f"# Blocklists: {', '.join(sorted(bl_names))}",
|
||||
f"# Domains: {len(domains):,}",
|
||||
"",
|
||||
]
|
||||
for domain in domains:
|
||||
lines.append(f"0.0.0.0 {domain}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def setup_blocklist_logging(general):
|
||||
"""Configure file + stdout logging for blocklists logger."""
|
||||
max_kb = general.get("log_max_kb", 1024)
|
||||
errors_only = general.get("log_errors_only", False)
|
||||
try:
|
||||
if LOG_FILE.exists() and LOG_FILE.stat().st_size > max_kb * 1024:
|
||||
LOG_FILE.write_text("")
|
||||
if not LOG_FILE.exists():
|
||||
LOG_FILE.touch()
|
||||
file_handler = logging.FileHandler(LOG_FILE)
|
||||
except PermissionError:
|
||||
print(f"WARNING: Cannot write to {LOG_FILE} -- run with sudo.")
|
||||
file_handler = None
|
||||
level = logging.ERROR if errors_only else logging.INFO
|
||||
handlers = [logging.StreamHandler()]
|
||||
if file_handler:
|
||||
handlers.insert(0, file_handler)
|
||||
logging.basicConfig(
|
||||
level=level,
|
||||
format="%(asctime)s %(levelname)-8s %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
handlers=handlers,
|
||||
force=True,
|
||||
)
|
||||
|
||||
|
||||
def update_blocklist_hosts(data):
|
||||
"""Parse downloaded/local blocklist files, upsert into SQLite, and write
|
||||
per-VLAN hosts files (0.0.0.0 format). Always writes every VLAN's file
|
||||
(empty if no blocklists assigned) so addn-hosts= in the dnsmasq conf is
|
||||
always valid and SIGHUP can update it without a restart.
|
||||
|
||||
Returns True on full success, False if any fetch/parse failed.
|
||||
"""
|
||||
BLOCKLIST_DIR.mkdir(exist_ok=True)
|
||||
|
||||
db = _open_db()
|
||||
bl_library = {bl["name"]: bl for bl in data.get("dns_blocking", {}).get("blocklists", [])}
|
||||
|
||||
needed = set()
|
||||
for vlan in data.get("vlans", []):
|
||||
needed.update(vlan.get("use_blocklists", []))
|
||||
|
||||
changed = set()
|
||||
any_fail = False
|
||||
|
||||
for name in needed:
|
||||
if name not in bl_library:
|
||||
_log.warning(f"Blocklist '{name}' referenced by a VLAN but not defined -- skipping")
|
||||
continue
|
||||
entry = bl_library[name]
|
||||
is_local = entry.get("bl_type") == "local"
|
||||
save_as = entry.get("save_as", "")
|
||||
|
||||
try:
|
||||
path = BLOCKLIST_DIR / save_as if save_as else None
|
||||
if not path or not path.exists():
|
||||
_log.warning(f"'{name}': file not found ({path}) -- skipping")
|
||||
any_fail = True
|
||||
continue
|
||||
current_mtime = path.stat().st_mtime
|
||||
except Exception as e:
|
||||
_log.error(f"Failed to stat '{name}': {e}")
|
||||
any_fail = True
|
||||
continue
|
||||
|
||||
if current_mtime == _get_stored_mtime(db, name):
|
||||
_log.info(f"Unchanged: '{name}' -- skipping")
|
||||
continue
|
||||
|
||||
try:
|
||||
raw = path.read_text("utf-8", errors="ignore")
|
||||
except Exception as e:
|
||||
_log.error(f"Failed to read '{name}': {e}")
|
||||
any_fail = True
|
||||
continue
|
||||
|
||||
domains = _parse_blocklist(raw, is_local=is_local)
|
||||
_upsert_blocklist(db, name, domains, current_mtime)
|
||||
_log.info(f"Updated '{name}': {len(domains):,} domains")
|
||||
changed.add(name)
|
||||
|
||||
active_vlan_names = set()
|
||||
for vlan in data.get("vlans", []):
|
||||
vlan_name = vlan["name"]
|
||||
active_vlan_names.add(vlan_name)
|
||||
bl_names = [n for n in vlan.get("use_blocklists", []) if n in bl_library]
|
||||
|
||||
hosts_file = vlan_hosts_file(vlan)
|
||||
if not bl_names:
|
||||
if not hosts_file.exists():
|
||||
hosts_file.write_text("")
|
||||
continue
|
||||
|
||||
if not changed.intersection(bl_names) and hosts_file.exists():
|
||||
_log.info(f"VLAN '{vlan_name}' blocklists unchanged -- skipping rewrite")
|
||||
continue
|
||||
|
||||
domains = _query_merged_domains(db, bl_names)
|
||||
hosts_file.write_text(_build_merged_hosts(domains, bl_names))
|
||||
_log.info(f"VLAN '{vlan_name}': wrote {len(domains):,} domains from [{', '.join(sorted(bl_names))}]")
|
||||
|
||||
for f in BLOCKLIST_DIR.glob("for-*.hosts"):
|
||||
vlan_name = f.stem.removeprefix("for-")
|
||||
if vlan_name not in active_vlan_names:
|
||||
f.unlink()
|
||||
_log.info(f"Removed stale hosts file: {f.name}")
|
||||
|
||||
db.close()
|
||||
return not any_fail
|
||||
|
||||
|
||||
def sighup_all_instances():
|
||||
"""Send SIGHUP to every active dnsmasq-routlin-* instance to reload addn-hosts
|
||||
files without restarting. No DNS or DHCP interruption."""
|
||||
result = subprocess.run(
|
||||
["systemctl", "list-units", "--state=active", "--no-legend", "--plain",
|
||||
f"dnsmasq-{shared.PRODUCT_NAME}-*.service"],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
units = [line.split()[0] for line in result.stdout.splitlines() if line.strip()]
|
||||
if not units:
|
||||
print(" No active dnsmasq instances found.")
|
||||
return
|
||||
for unit in units:
|
||||
r = subprocess.run(["systemctl", "kill", "--signal=SIGHUP", unit],
|
||||
capture_output=True, text=True)
|
||||
if r.returncode == 0:
|
||||
print(f" Reloaded: {unit}")
|
||||
else:
|
||||
print(f" WARNING: Failed to SIGHUP {unit}: {r.stderr.strip()}")
|
||||
|
||||
|
||||
# ===================================================================
|
||||
|
|
@ -95,12 +371,7 @@ def build_vlan_dnsmasq_conf(vlan, data, iface):
|
|||
opts = shared.resolve_vlan_options(vlan)
|
||||
gateway = opts["gateway"]
|
||||
|
||||
bl_names = vlan.get("use_blocklists", [])
|
||||
bl_file = None
|
||||
if bl_names:
|
||||
p = merged_path(combo_hash(bl_names))
|
||||
if p.exists():
|
||||
bl_file = p
|
||||
hosts_file = vlan_hosts_file(vlan)
|
||||
|
||||
L = [
|
||||
"# Generated by core.py -- do not edit manually.",
|
||||
|
|
@ -216,14 +487,12 @@ def build_vlan_dnsmasq_conf(vlan, data, iface):
|
|||
for o in overrides:
|
||||
L += [f"# {o['description']}", f"address=/{o['host']}/{o['ip']}", ""]
|
||||
|
||||
if bl_file:
|
||||
if hosts_file.exists():
|
||||
L += [
|
||||
"# -- Blocklist ------------------------------------------------------",
|
||||
f"conf-file={bl_file}",
|
||||
f"addn-hosts={hosts_file}",
|
||||
"",
|
||||
]
|
||||
elif bl_names:
|
||||
L += ["# Blocklist not yet downloaded -- run: sudo python3 dns-blocklists.py", ""]
|
||||
|
||||
return "\n".join(L)
|
||||
|
||||
|
|
@ -419,6 +688,9 @@ def apply_dnsmasq_instances(data, dry_run=False, start_if_needed=True):
|
|||
|
||||
if not dry_run:
|
||||
shared.DNSMASQ_CONF_DIR.mkdir(exist_ok=True)
|
||||
print("Updating blocklist hosts files ======================================")
|
||||
update_blocklist_hosts(data)
|
||||
print()
|
||||
disable_system_dnsmasq(data)
|
||||
print()
|
||||
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ def install_timer(data):
|
|||
"",
|
||||
])
|
||||
|
||||
blocklist_script = shared.SCRIPT_DIR / "dns-blocklists.py"
|
||||
blocklist_script = shared.SCRIPT_DIR / "dl_blocklists.py"
|
||||
service_content = "\n".join([
|
||||
"# Generated by core.py -- do not edit manually.",
|
||||
"",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue