#!/usr/bin/env python3
"""宁国市人民政府 - 建设项目环评文件审批 (ningguo.gov.cn)"""
import os, sys, re, time, sqlite3
from datetime import datetime, timedelta
from urllib.parse import urljoin
import argparse
import requests

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
BASE_URL = "https://www.ningguo.gov.cn"
LIST_URL = BASE_URL + "/Jczwgk/showList/0/111001001/page_{n}.html"
SITE_NAME = "宁国市建设项目环评文件审批"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
parser = argparse.ArgumentParser()
parser.add_argument("--pages", type=int, default=0)
_args = parser.parse_args()
MAX_PAGES = _args.pages

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def log(msg): print(f"[{datetime.now().strftime('%H:%M:%S')}] {msg}", flush=True)

def safe_get(url, retries=3):
    for a in range(retries):
        try:
            r = requests.get(url, headers=HEADERS, timeout=30); r.encoding = 'utf-8'
            if r.status_code == 200: return r.text
            log(f"  HTTP {r.status_code}")
        except Exception as e:
            log(f"  Attempt {a+1}: {e}"); time.sleep(2)
    return None

def parse_list(html):
    items = []
    for part in re.split(r'<li[^>]*>', html):
        if '</li>' not in part: continue
        li = part[:part.index('</li>')]
        dm = re.search(r'<span[^>]*>(\d{4}-\d{2}-\d{2})</span>', li)
        am = re.search(r'<a\s+href="([^"]+)"[^>]*title="([^"]*)"', li)
        if dm and am:
            href = am.group(1)
            if not href.startswith('http'): href = urljoin(BASE_URL, href)
            items.append((am.group(2).strip(), href, dm.group(1)))
    return items

def fetch_detail(url):
    html = safe_get(url)
    if not html: return None, None
    dm = re.search(r'<meta[^>]*PubDate[^>]*content="(\d{4}-\d{2}-\d{2})', html)
    pub_date = dm.group(1) if dm else None
    cm = re.search(r'<div[^>]*id="zoom"[^>]*>(.*?)</div>\s*</div>\s*</div>', html, re.DOTALL)
    if not cm: cm = re.search(r'<div[^>]*id="zoom"[^>]*>(.*?)</div>', html, re.DOTALL)
    return (cm.group(1).strip() if cm else ""), pub_date

def main():
    log(f"Starting: {SITE_NAME}, cutoff={CUTOFF}")
    conn = sqlite3.connect(DB_PATH, timeout=60)
    html = safe_get(LIST_URL.format(n=1))
    total = int(re.search(r'pagecount="(\d+)"', html).group(1)) if html else 1
    log(f"Pages: {total}")
    n, o, e = 0, 0, 0
    if MAX_PAGES > 0:
        total = min(total, MAX_PAGES)
        log(f"Limited to {total} pages")
    for p in range(1, total+1):
        log(f"Page {p}/{total}...")
        html = safe_get(LIST_URL.format(n=p))
        if not html: e += 1; continue
        items = parse_list(html)
        if not items: log("  no items"); continue
        log(f"  {len(items)} items")
        all_old = True
        for t, l, d in items:
            if d < CUTOFF: o += 1; continue
            all_old = False
            c, pd = fetch_detail(l)
            if not c: continue
            conn.execute("INSERT OR IGNORE INTO gov_raw (page_url,title,content,publish_date,site_name,source_url,summary) VALUES (?,?,?,?,?,?,?)",
                         (l, t, c, pd or d, SITE_NAME, l, t))
            if conn.total_changes > 0: n += 1
        if all_old and p > 1: log("  past cutoff, stopping"); break
        conn.commit(); time.sleep(0.5)
    conn.commit(); conn.close()
    log(f"Done: new={n}, old={o}, err={e}")

if __name__ == "__main__":
    main()