#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
金沙县人民政府 - 环境保护 (gzjinsha.gov.cn/xxgk/zxgk/tgrye/)
列表: /xxgk/zxgk/tgrye/ (P1), index_{N}.html (PN, 共12页)
列表项: JS变量 var str_1=URL str_3=日期 str_4=标题 (document.write生成)
详情: TRS_UEDITOR
用法: python3 crawl_gzjinsha_hjbh.py [--pages=N] [--json out.jsonl]
"""
import re, sys, os, json, time, sqlite3, requests
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from urllib.parse import urljoin

SITE_NAME = "金沙县人民政府-环境保护"
BASE_URL = "https://www.gzjinsha.gov.cn"
LIST_PATH = "/xxgk/zxgk/tgrye"
LIST_URL = BASE_URL + LIST_PATH + "/"
DOMAIN = "www.gzjinsha.gov.cn"
GROUP = "贵州"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}

def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)

def clean_title(t):
    t = re.sub(r" | |\u200b|\ufeff", "", t or "")
    t = re.sub(r"\s+", " ", t)
    t = re.sub(r"\.\.\.|…$", "", t).strip()
    return t

def fetch(url, timeout=30):
    for attempt in range(4):
        try:
            r = requests.get(url, headers=HEADERS, timeout=timeout)
            r.encoding = "utf-8"
            return r.text
        except Exception as e:
            if attempt == 3:
                log(f"fetch ERR {url[-50:]}: {e}")
                return ""
            time.sleep(2 * (attempt + 1))
    return ""

def fetch_list(page):
    url = LIST_URL if page == 1 else f"{LIST_URL}index_{page}.html"
    html = fetch(url)
    if not html:
        return []
    items = []
    # JS 变量: var str_1 = "./202512/t20251212_89039416.html"; var str_3 = "2025-12-12"; var str_4 = "标题"
    block_pat = re.compile(
        r'var str_1\s*=\s*"([^"]*)"\s*;.*?'
        r'var str_3\s*=\s*"([^"]*)"\s*;.*?'
        r'var str_4\s*=\s*"([^"]*)"\s*;',
        re.S)
    for m in block_pat.finditer(html):
        href, date, title = m.group(1), m.group(2), m.group(3)
        title = clean_title(title)
        full = href if href.startswith("http") else urljoin(LIST_URL, href)
        if title and ("t20" in href or "20" in href):
            items.append({"title": title, "url": full, "date": date.strip()[:10]})
    return items

def extract_clean_text(content_html, page_url=None):
    if not content_html:
        return ""
    soup = BeautifulSoup(content_html, "html.parser")
    for a_tag in soup.find_all("a", href=True):
        href = a_tag.get("href", "")
        text = a_tag.get_text(strip=True) or "附件"
        full_url = urljoin(page_url or BASE_URL, href)
        a_tag.replace_with(f'<p><a href="{full_url}">{text}</a></p>')
    for tag in soup.find_all(['span', 'b', 'strong', 'font', 'em', 'i', 'u', 's']):
        tag.unwrap()
    parts = []
    for el in soup.find_all(['table', 'p']):
        if el.name == 'table':
            parts.append(str(el))
        elif el.name == 'p' and not el.find_parent('table'):
            t = el.get_text(separator='', strip=True)
            if t:
                parts.append(f'<p>{t}</p>')
    return '\n\n'.join(parts) if parts else content_html.strip()

def parse_detail(url):
    html = fetch(url)
    if not html:
        return {"title": "", "publish_date": "", "content": ""}
    soup = BeautifulSoup(html, "html.parser")
    result = {"title": "", "publish_date": "", "content": ""}
    ttag = soup.find("title")
    if ttag:
        result["title"] = clean_title(ttag.get_text(strip=True))
    m = re.search(r"(\d{4}-\d{2}-\d{2})", html[:8000])
    if m:
        result["publish_date"] = m.group(1)
    content_div = soup.find("div", class_="TRS_UEDITOR")
    if not content_div:
        content_div = soup.find("div", class_="content")
    if content_div:
        result["content"] = extract_clean_text(str(content_div), url)
    return result

def crawl(pages=5, json_out=None):
    log(f"Crawl pages={pages} cutoff={CUTOFF}")
    items_all = []
    seen = set()
    for p in range(1, pages + 1):
        items = fetch_list(p)
        new = 0
        for it in items:
            if it["url"] not in seen:
                seen.add(it["url"])
                items_all.append(it)
                new += 1
        log(f"P{p}: {len(items)} items ({new} new), total {len(items_all)}")
        if not items:
            break
        time.sleep(0.6)

    enriched = []
    for idx, it in enumerate(items_all, 1):
        d = parse_detail(it["url"])
        if (d["publish_date"] or it["date"]) and (d["publish_date"] or it["date"]) < CUTOFF:
            continue
        enriched.append({
            "title": d["title"] or it["title"],
            "page_url": it["url"],
            "publish_date": d["publish_date"] or it["date"],
            "content": d["content"],
            "site_name": SITE_NAME,
            "column": "环境保护",
        })
        if idx % 10 == 0:
            log(f"  {idx}/{len(items_all)}")
        time.sleep(0.3)

    if json_out:
        with open(json_out, "w", encoding="utf-8") as f:
            for item in enriched:
                f.write(json.dumps(item, ensure_ascii=False) + "\n")
        log(f"JSONL: {len(enriched)} items -> {json_out}")
        return

    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=10000")
    c = conn.cursor()
    stored = skipped = 0
    for item in enriched:
        try:
            c.execute(
                "INSERT OR IGNORE INTO gov_raw "
                "(site_name, source_url, page_url, title, publish_date, content, summary, category, script_name)"
                " VALUES (?,?,?,?,?,?,?,?,?)",
                (item["site_name"], item["page_url"], item["page_url"],
                 item["title"], item["publish_date"], item["content"],
                 (item["content"] or "")[:500], item["column"], "crawl_gzjinsha_hjbh.py"))
            if c.rowcount > 0:
                stored += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"DB error: {e}")
            skipped += 1
    conn.commit()
    conn.close()
    log(f"Result: {stored} new, {skipped} skipped")

if __name__ == "__main__":
    args = sys.argv[1:]
    pages = 5
    json_out = None
    if "--pages=" in " ".join(args):
        pages = int(re.search(r"--pages=(\d+)", " ".join(args)).group(1))
    if "--json" in args:
        json_out = args[args.index("--json") + 1]
    crawl(pages=pages, json_out=json_out)