101 lines
2.9 KiB
Python
101 lines
2.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Download community blocklists defined in config.json to disk.
|
|
|
|
Saves each community blocklist to blocklists/<save_as>.
|
|
Local blocklists are managed by the dashboard and are not downloaded here.
|
|
|
|
Run this on a schedule (e.g. daily) to refresh remote lists, then run
|
|
sudo python3 core.py --merge-blocklists
|
|
to merge, update SQLite, and reload dnsmasq without restarting it.
|
|
|
|
Usage:
|
|
sudo python3 dl_blocklists.py
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
SCRIPT_DIR = Path(__file__).parent
|
|
CONFIG_FILE = SCRIPT_DIR / "config.json"
|
|
BLOCKLIST_DIR = SCRIPT_DIR / "blocklists"
|
|
LOG_FILE = BLOCKLIST_DIR / ".log"
|
|
LAST_DL_FILE = BLOCKLIST_DIR / ".last-dl"
|
|
|
|
|
|
def die(msg):
|
|
print(f"ERROR: {msg}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
def check_root():
|
|
if os.geteuid() != 0:
|
|
die("This script must be run as root (sudo).")
|
|
|
|
|
|
def load_config():
|
|
if not CONFIG_FILE.exists():
|
|
die(f"Config file not found: {CONFIG_FILE}")
|
|
with open(CONFIG_FILE) as f:
|
|
return json.load(f)
|
|
|
|
|
|
def setup_logging():
|
|
BLOCKLIST_DIR.mkdir(exist_ok=True)
|
|
try:
|
|
file_handler = logging.FileHandler(LOG_FILE, mode='a')
|
|
except PermissionError:
|
|
print(f"WARNING: Cannot write to {LOG_FILE} -- run with sudo.")
|
|
file_handler = None
|
|
handlers = [logging.StreamHandler()]
|
|
if file_handler:
|
|
handlers.insert(0, file_handler)
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)-8s %(message)s",
|
|
datefmt="%Y-%m-%d %H:%M:%S",
|
|
handlers=handlers,
|
|
force=True,
|
|
)
|
|
|
|
|
|
def download_blocklists(data):
|
|
any_fail = False
|
|
for bl in data.get("dns_blocking", {}).get("blocklists", []):
|
|
if bl.get("bl_type") == "local":
|
|
continue
|
|
url = bl.get("url", "")
|
|
save_as = bl.get("save_as", "")
|
|
name = bl.get("name", "?")
|
|
if not url or not save_as:
|
|
logging.warning("Skipped '%s': missing url or save_as", name)
|
|
continue
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "routlin/1.0"})
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
content = r.read()
|
|
(BLOCKLIST_DIR / save_as).write_bytes(content)
|
|
logging.info("Downloaded: %s (%s bytes)", name, f"{len(content):,}")
|
|
except Exception as e:
|
|
logging.error("Failed to download '%s': %s", name, e)
|
|
any_fail = True
|
|
return not any_fail
|
|
|
|
|
|
def main():
|
|
check_root()
|
|
setup_logging()
|
|
data = load_config()
|
|
LAST_DL_FILE.write_text(str(int(__import__('time').time())))
|
|
logging.info("Downloading blocklists ==========================================")
|
|
success = download_blocklists(data)
|
|
if not success:
|
|
logging.warning("One or more downloads failed.")
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|