#!/usr/bin/env python3
"""
crawl_linqing_hpgs.py - 临清市人民政府-环评审批信息 (channel_j_lqsrmzfmhwz_253d)
列表: POST /api/mDocument {method:"mList", params:{channel_name, per_page, page}}
  total=382 条, 详情字段全在 API 响应里 (content 富文本HTML / release 日期 / organization 单位 / router URL)
"""

import sys, os, requests, re, json
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE = "http://www.linqing.gov.cn"
API = BASE + "/api/mDocument"
SITE_NAME = "临清市-环评审批信息"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0",
    "Content-Type": "application/json",
}


def fetch_page(page_num, per_page=10):
    payload = {
        "method": "mList",
        "params": {"channel_name": "j_lqsrmzfmhwz_253d", "per_page": per_page, "page": page_num},
        "url": BASE + "/channel_j_lqsrmzfmhwz_253d/",
    }
    try:
        r = requests.post(API, json=payload, timeout=(5, 20), headers=HEADERS)
        r.encoding = "utf-8"
        return r.json()
    except Exception as e:
        print(f"  [API ERR] {e}")
        return {}


def fix_content(content):
    """相对路径附件/图片转绝对 URL"""
    if not content:
        return ""
    def _abs(m):
        attr, url = m.group(1), m.group(2)
        if url.startswith("/"):
            return f'{attr}="{BASE}{url}"'
        return m.group(0)
    content = re.sub(r'(href|src)="(/[^"]+)"', _abs, content)
    return content


def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--pages", type=int, default=1)
    parser.add_argument("--all", action="store_true")
    args = parser.parse_args()

    # 第一页拿 total
    d = fetch_page(1)
    total = d.get("total", 0)
    print(f"  total={total} 条")
    if args.all:
        max_pages = (total + 9) // 10
    else:
        max_pages = args.pages

    records = []
    seen = set()
    for pg in range(1, max_pages + 1):
        if pg > 1:
            d = fetch_page(pg)
        items = d.get("items") or []
        print(f"  第{pg}页: {len(items)} 条")
        for it in items:
            url = it.get("router") or ""
            if not url or url in seen:
                continue
            seen.add(url)
            content = fix_content(it.get("content") or "")
            records.append({
                "title": (it.get("title") or "").strip(),
                "url": url,
                "pub_date": (it.get("release") or it.get("_createtime") or "")[:10],
                "site_name": SITE_NAME,
                "content": content,
                "summary": "",
                "author": (it.get("organization") or ""),
            })
    print(f"  共 {len(records)} 条")
    valid = [r for r in records if r["content"].strip()]
    if valid:
        push_to_searchdb(valid, "linqing_hpgs")
    return len(valid)


if __name__ == "__main__":
    cnt = main()
    print(f"Done: {cnt} records")