#!/usr/bin/env python3
"""
五原县人民政府 — 通知公告 爬虫
site: www.wuyuan.gov.cn
栏目: /tzgg/ (内蒙古五原县，非江西婺源)
列表: JS API POST /rmt/tmpl/api/site/{siteId}/channelid/{channelId}
      body: {"currpage":N,"pagesize":10,"params":"..."} → {"errcode":0,"data":[{docno,title,inputTime(ms),...}]}
      1069条, 10条/页, 107页
详情: http://www.wuyuan.gov.cn/tzgg/{docno}.html
详情结构: meta ArticleTitle/PubDate, div#content.hl_data=hlText(正文, 标题首p+表格)
"""
import re, sys, os, sqlite3, time, json, urllib.request, ssl

SITE_NAME = "五原县-通知公告"
DOMAIN = "www.wuyuan.gov.cn"
BASE_URL = "http://www.wuyuan.gov.cn"
API_URL = BASE_URL + "/rmt/tmpl/api/site/f4f8152cf0cd47c0a69f16be6bce423f/channelid/14bd7ef54c7f4a6f987f96e9e45d4c97"
API_PARAMS = ""  # 运行时从列表页提取
TOTAL_COUNT = 1069  # 10条/页 → 107页

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
CTX = ssl._create_unverified_context()
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"
}

def http_get(url):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=30, context=CTX)
        return resp.read().decode("utf-8", errors="replace")
    except Exception:
        return None

def http_post_json(url, payload):
    data = json.dumps(payload).encode("utf-8")
    req = urllib.request.Request(url, data=data, headers={
        "User-Agent": HEADERS["User-Agent"],
        "Content-Type": "application/json;charset=utf-8",
    })
    try:
        resp = urllib.request.urlopen(req, timeout=30, context=CTX)
        return json.loads(resp.read().decode("utf-8", errors="replace"))
    except Exception:
        return None

def get_api_params():
    """从列表页提取 params 签名"""
    html = http_get(BASE_URL + "/tzgg/")
    if not html:
        return None
    m = re.search(r'params:\s*"([^"]+)"', html)
    return m.group(1) if m else None

def fetch_list_page(page):
    """调 API 拿一页列表 → [{docno, title, ts}]"""
    payload = {"currpage": page, "pagesize": 10, "params": API_PARAMS}
    j = http_post_json(API_URL, payload)
    if not j or j.get("errcode") != 0:
        return []
    data = j.get("data") or []
    items = []
    for it in data:
        docno = it.get("docno")
        title = (it.get("title") or "").strip()
        ts = it.get("inputTime") or 0
        date = time.strftime("%Y-%m-%d", time.localtime(ts / 1000)) if ts else ""
        if docno:
            items.append({
                "docno": docno,
                "title": title,
                "publish_date": date,
                "url": f"{BASE_URL}/tzgg/{docno}.html",
            })
    return items

def fetch_detail(url):
    """获取详情：标题、正文HTML、日期"""
    html = http_get(url)
    if not html:
        return None

    title = ""
    m_t = re.search(r'<meta name="ArticleTitle" content="([^"]+)"', html)
    if m_t:
        title = m_t.group(1).strip()

    date = ""
    m_d = re.search(r'<meta name="PubDate" content="([^"]+)"', html)
    if m_d:
        date = m_d.group(1).strip()[:10]

    content = ""
    m_c = re.search(r'<div id="content"[^>]*>(.*?)</div>\s*<div class="xglj"', html, re.DOTALL)
    if not m_c:
        m_c = re.search(r'<div id="content"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m_c:
        content = m_c.group(1).strip()
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        content = content.strip()

    return {"title": title or "", "content": content, "date": date}

def init_db():
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT, source_url TEXT, page_url TEXT,
        title TEXT, publish_date TEXT, date_rank INTEGER DEFAULT 0,
        summary TEXT, status TEXT, category TEXT DEFAULT '',
        visits INTEGER DEFAULT 0, content TEXT DEFAULT '', tags TEXT DEFAULT ''
    )''')
    try:
        conn.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
            title, content, site_name,
            content='gov_raw', content_rowid='id', tokenize='unicode61'
        )''')
    except sqlite3.OperationalError:
        pass
    conn.commit()
    conn.close()

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--full", action="store_true", help="全量107页")
    parser.add_argument("--pages", type=int, default=1, help="指定页数")
    args = parser.parse_args()

    global API_PARAMS
    API_PARAMS = get_api_params()
    if not API_PARAMS:
        print("✗ 无法获取 API params")
        return
    print(f"API params 获取成功 ({len(API_PARAMS)} 字符)")

    init_db()
    total_pages = (TOTAL_COUNT + 9) // 10
    pages_to_crawl = total_pages if args.full else args.pages
    mode = "全量" if args.full else f"增量({pages_to_crawl}页)"

    print(f"🔍 {SITE_NAME} — {mode}模式")
    print(f"  总条数: {TOTAL_COUNT}, 本次爬取: {pages_to_crawl}页")
    print()

    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    cur = conn.cursor()

    new_total = 0
    skip_total = 0
    for page in range(1, pages_to_crawl + 1):
        items = fetch_list_page(page)
        if not items:
            print(f"  [p{page:2d}/{pages_to_crawl}] ✗ API无数据")
            continue

        page_new = 0
        page_skip = 0
        for item in items:
            exists = cur.execute(
                "SELECT 1 FROM gov_raw WHERE page_url=? AND site_name=?",
                (item["url"], SITE_NAME)
            ).fetchone()
            if exists:
                page_skip += 1
                skip_total += 1
                continue

            detail = fetch_detail(item["url"])
            if detail:
                content = detail.get("content", "")
                title = detail.get("title") or item["title"]
                date = detail.get("date", item.get("publish_date", ""))
            else:
                content = ""
                title = item["title"]
                date = item.get("publish_date", "")

            if not content:
                page_skip += 1
                skip_total += 1
                continue

            try:
                cur.execute(
                    "INSERT OR IGNORE INTO gov_raw "
                    "(title, content, publish_date, page_url, source_url, site_name, status) "
                    "VALUES (?,?,?,?,?,?,?)",
                    (title, content, date, item["url"], DOMAIN, SITE_NAME, "published")
                )
                if cur.rowcount > 0:
                    row_id = cur.lastrowid
                    try:
                        plain = re.sub(r"<[^>]+>", " ", content)
                        plain = re.sub(r"\s+", " ", plain).strip()[:200] or title
                        cur.execute(
                            "INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES (?,?,?,?)",
                            (row_id, title, SITE_NAME, plain)
                        )
                    except Exception:
                        pass
                    page_new += 1
                    new_total += 1
            except Exception:
                pass

        conn.commit()
        print(f"  [p{page:2d}/{pages_to_crawl}] ✓ {len(items)}条 (新增{page_new} 跳过{page_skip})")
        time.sleep(0.3)

    conn.close()
    print(f"\n📊 完成！新增: {new_total} | 跳过: {skip_total}")

if __name__ == "__main__":
    main()