#!/usr/bin/env python3
"""
crawl_ninghai_hpgs.py - 宁海县生态环境专题子栏目 (JPAAS 平台)
支持 --col 参数: 1229116620=环评批前公示, 1229116621=环评审批结果公示
列表: GET /api-gateway/jpaas-publish-server/front/page/build/unit
  webId=3575&pageId=<col>&parseType=bulidstatic&pageType=column&tagId=栏目列表&tplSetId=Kj4AQALilei1D88cYSuZt&editType=null
  paramJson={"pageNo":N,"pageSize":"15"} 分页
详情: /col/col<col>/art/YYYY/art_xxx.html, meta ArticleTitle/PubDate/ContentSource, 正文 id=zoom
"""

import sys, os, requests, re, json
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE = "http://www.ninghai.gov.cn"
LIST_API = BASE + "/api-gateway/jpaas-publish-server/front/page/build/unit"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0",
}

COL_MAP = {
    "1229116620": "宁海县-环评批前公示",
    "1229116621": "宁海县-环评审批结果公示",
    "1229116613": "宁海县-信息公示",
    "1229116619": "宁海县-环评受理公示",
}


def fetch_page(col, page_num, rows=15):
    q = {
        "webId": "3575",
        "pageId": col,
        "parseType": "bulidstatic",
        "pageType": "column",
        "tagId": "栏目列表",
        "tplSetId": "Kj4AQALilei1D88cYSuZt",
        "editType": "null",
        "paramJson": json.dumps({"pageNo": page_num, "pageSize": str(rows)}, ensure_ascii=False),
    }
    try:
        r = requests.get(LIST_API, params=q, headers=HEADERS, timeout=25)
        r.encoding = "utf-8"
        d = r.json()
        if not d.get("success"):
            print(f"  [API FAIL] {d.get('message')}")
            return [], 0
        html = (d.get("data") or {}).get("html", "") or ""
        m = re.search(r'count="(\d+)"', html)
        total = int(m.group(1)) if m else 0
        items = []
        pat = re.compile(r'<a[^>]+href="(/col/col' + col + r'/art/\d+/art_[a-f0-9]+\.html)"[^>]*>([^<]{10,200})</a>\s*<span>(\d{4}-\d{2}-\d{2})</span>')
        for m2 in pat.finditer(html):
            items.append({"url": BASE + m2.group(1), "title": m2.group(2).strip(), "date": m2.group(3)})
        return items, total
    except Exception as e:
        print(f"  [API ERR] {e}")
        return [], 0


def fetch_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=25)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  [FETCH ERR] {e}")
        return "", "", "", ""
    c = r.text
    title = ""
    m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', c)
    if m:
        title = m.group(1).strip()
    pub_date = ""
    m = re.search(r'<meta\s+name="PubDate"\s+content="([^"]*)"', c)
    if m:
        dm = re.search(r"\d{4}-\d{2}-\d{2}", m.group(1))
        pub_date = dm.group() if dm else ""
    src = ""
    m = re.search(r'<meta\s+name="ContentSource"\s+content="([^"]*)"', c)
    if m:
        src = m.group(1).strip()
    # 正文 id=zoom
    content = ""
    idx = c.find('id="zoom"')
    if idx > 0:
        start = c.find(">", idx) + 1
        depth = 1
        pos = start
        while pos < len(c) and depth > 0:
            if c[pos:pos+4] == "<div":
                depth += 1
                pos += 4
            elif c[pos:pos+6] == "</div>":
                depth -= 1
                pos += 6
            else:
                pos += 1
        content = c[start:pos-6].strip()
        # 附件/链接相对路径转绝对 URL
        content = re.sub(r'href="/', 'href="' + BASE + '/', content)
        content = re.sub(r'src="/', 'src="' + BASE + '/', content)
    return title, pub_date, src, content


def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--col", default="1229116620")
    parser.add_argument("--pages", type=int, default=1)
    parser.add_argument("--all", action="store_true")
    args = parser.parse_args()

    col = args.col
    site_name = COL_MAP.get(col, f"宁海县-{col}")
    print(f"  栏目: {site_name} (col{col})")

    items_all = []
    seen = set()
    first_items, total = fetch_page(col, 1)
    print(f"  total={total} 条")
    if args.all:
        max_pages = (total + 14) // 15
    else:
        max_pages = args.pages
    for pg in range(1, max_pages + 1):
        if pg > 1:
            items, _ = fetch_page(col, pg)
        else:
            items = first_items
        print(f"  第{pg}页: {len(items)} 条")
        for it in items:
            if it["url"] not in seen:
                seen.add(it["url"])
                items_all.append(it)

    records = []
    for idx, it in enumerate(items_all):
        print(f"  [{idx+1}/{len(items_all)}] {it['title'][:40]}...")
        t, d, s, content = fetch_detail(it["url"])
        if not content.strip():
            print("    [SKIP] 正文空")
            continue
        records.append({
            "title": t or it["title"],
            "url": it["url"],
            "pub_date": d or it["date"],
            "site_name": site_name,
            "content": content,
            "summary": "",
        })
    print(f"  共 {len(records)} 条有效")
    if records:
        push_to_searchdb(records, f"ninghai_{col}")
    return len(records)


if __name__ == "__main__":
    cnt = main()
    print(f"Done: {cnt} records")