#!/usr/bin/env python3
"""Crawler for 白银区人民政府 - 公示公告 (baiyinqu.gov.cn xwzx/GSGG)
   JPAAS API: /api-gateway/jpaas-publish-server/front/page/build/unit
   列表: tagId=分页 pageId=9139131b15e244b48c146e567381178d, 20条/页, paramJson pageNo/pageSize
   详情: /xwzx/GSGG/art/YYYY/art_xxx.html 静态HTML, 正文 div[class*=content] / .article
   2026-08-21 重建（原脚本残缺 crawl() 未定义）
"""
import requests, re, sqlite3, json, os, sys, time
from urllib.parse import urljoin

DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "白银区人民政府"
SCRIPT_NAME = "crawl_baiyinqu.py"
BASE_URL = "https://www.baiyinqu.gov.cn"
API_URL = BASE_URL + "/api-gateway/jpaas-publish-server/front/page/build/unit"
LIST_URL = "https://www.baiyinqu.gov.cn/xwzx/GSGG/index.html"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
CUTOFF = "2023-08-18"

API_PARAMS = {
    "parseType": "bulidstatic",
    "webId": "1953af59ba95408c9ebfffd5af1eea96",
    "tplSetId": "e7f893abbbd14083b0cdc69bfdf92239",
    "pageType": "column",
    "tagId": "分页",
    "editType": "null",
    "pageId": "9139131b15e244b48c146e567381178d",
}
HEADERS = {"User-Agent": UA, "Referer": LIST_URL}

stats = {"new": 0, "skip": 0, "err": 0}


def fetch_list(page):
    params = dict(API_PARAMS)
    params["paramJson"] = json.dumps({"pageNo": page, "pageSize": 20})
    try:
        r = requests.get(API_URL, params=params, headers=HEADERS, timeout=(5, 20), verify=False)
        d = r.json()
        html = d.get("data", {}).get("html", "") if isinstance(d.get("data"), dict) else ""
        return html
    except Exception as e:
        print(f"  列表API错误(页{page}): {e}", file=sys.stderr)
        return None


def extract_list_items(html):
    items = []
    seen = set()
    # 列表: <li><i>日期</i><a href="/xwzx/GSGG/art/..." title="标题">标题</a></li>
    for m in re.finditer(r'<i>\s*(\d{4}-\d{2}-\d{2})\s*</i>\s*<a[^>]*href="([^"]*art[^"]*\.html)"[^>]*title="([^"]*)"', html):
        date, href, title = m.group(1), m.group(2), m.group(3).strip()
        url = urljoin(BASE_URL, href)
        if url in seen:
            continue
        seen.add(url)
        items.append({"url": url, "title": title, "date": date})
    return items


def extract_detail(html):
    result = {"title": "", "content": "", "date": ""}
    m = re.search(r'<title[^>]*>(.*?)</title>', html, re.DOTALL)
    if m:
        result["title"] = m.group(1).strip()
    # 正文容器: 多模式尝试（JPAAS art 详情常见）
    for pattern in [r'<div[^>]*class="[^"]*content-txt[^"]*"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="[^"]*article[^"]*"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="[^"]*zoom[^"]*"[^>]*>(.*?)</div>',
                    r'<div[^>]*id="zoom"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="[^"]*TRS_Editor[^"]*"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="[^"]*news-content[^"]*"[^>]*>(.*?)</div>']:
        m = re.search(pattern, html, re.DOTALL)
        if m:
            inner = m.group(1)
            inner = re.sub(r'<script[^>]*>.*?</script>', '', inner, flags=re.DOTALL)
            inner = re.sub(r'<style[^>]*>.*?</style>', '', inner, flags=re.DOTALL)
            text_len = len(re.sub(r'<[^>]+>', '', inner).strip())
            if text_len >= 50:
                result["content"] = inner.strip()
                break
    # 日期: 发布时间
    m = re.search(r'(20\d{2}-\d{2}-\d{2})', html)
    if m:
        result["date"] = m.group(1)
    return result


def main():
    max_pages = 3
    for i, a in enumerate(sys.argv):
        if a == "--pages" and i + 1 < len(sys.argv):
            max_pages = int(sys.argv[i + 1])
        elif a.startswith("--pages="):
            max_pages = int(a.split("=", 1)[1])
    print(f"=== 白银区公示公告 ===", flush=True)

    conn = sqlite3.connect(DB, timeout=30)
    cur = conn.cursor()
    t0 = time.time()

    for page in range(1, max_pages + 1):
        html = fetch_list(page)
        if not html or "art" not in html:
            print(f"  页{page} 无内容，停止")
            break
        items = extract_list_items(html)
        if not items:
            print(f"  页{page} 无条目，停止")
            break
        print(f"  页{page}: {len(items)} 条", flush=True)
        for item in items:
            url = item["url"]
            cur.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,))
            if cur.fetchone():
                stats["skip"] += 1
                continue
            try:
                r = requests.get(url, headers=HEADERS, timeout=(5, 20), verify=False)
                if r.status_code != 200:
                    stats["err"] += 1
                    continue
                det = extract_detail(r.text)
            except Exception as e:
                stats["err"] += 1
                continue
            title = det["title"] or item["title"]
            if not title:
                stats["err"] += 1
                continue
            content = det["content"] or f'<p><a href="{url}">{title}</a></p>'
            date = det["date"] or item["date"] or "1900-01-01"
            try:
                cur.execute("""
                    INSERT OR IGNORE INTO gov_raw
                    (site_name, source_url, page_url, title, publish_date, date_rank, summary, status,
                     category, visits, content, tags, industry, attachments, group_name, has_table, script_name)
                    VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
                """, (
                    SITE_NAME, LIST_URL, url, title, date, date,
                    re.sub(r'<[^>]+>', '', content)[:200],
                    "published", "公示公告", 0, content, "", "", "[]",
                    "甘肃", 1 if "<table" in content else 0, SCRIPT_NAME
                ))
                if cur.rowcount > 0:
                    stats["new"] += 1
                else:
                    stats["skip"] += 1
                conn.commit()
            except Exception as e:
                print(f"  DB错误: {e}", file=sys.stderr)
                stats["err"] += 1
        time.sleep(1)

    conn.close()
    print(f"完成: 新增{stats['new']} 跳过{stats['skip']} 错误{stats['err']} 用时{time.time()-t0:.0f}s", flush=True)


if __name__ == "__main__":
    import urllib3
    urllib3.disable_warnings()
    main()