#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
仪征市自然资源和规划局 - 信息公开（江苏省自然资源厅 gtapp 应用）
https://zrzy.jiangsu.gov.cn/gtapp/nrglIndex.action?classID=2c9082b55b652546015b65ec3be2004f

站点特征（2026-08-11 探测）:
- Struts2 action 动态站；列表 POST cpage=N 分页（topage JS form），18条/页，共94页
- 列表标题 <a title='完整标题' href='/gtapp/nrglIndex.action?type=2&messageID={UUID}'>，显示文本截断须取 title
- 日期 <td align="center">YYYY-MM-DD</td>
- 详情 = meta PubDate + table.xxgk-gjy-top 元数据(索引号/文号/发布机构/生成日期) + 标题 + 附件
- 附件 /gtapp/nrgl/GJAttach/{ts}.{ext}（相对路径，urljoin 详情页URL 绝对化）
- 正文 = 标题 + 附件段落（源站详情无独立正文 div，证照类公告以附件为主）

用法:
  python3 crawl_zrzy_yz_xxgk.py --pages=67   # CUTOFF 3年窗口 (2023-08-10 起)
  python3 crawl_zrzy_yz_xxgk.py --pages=1    # 测试
"""
import re
import sys
import json
import html
import urllib3
import requests
from urllib.parse import urljoin

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

from crawler_lib import push_to_searchdb

SITE_NAME = "仪征市自然资源和规划局-信息公开"
BASE_URL = "https://zrzy.jiangsu.gov.cn"
LIST_URL = BASE_URL + "/gtapp/nrglIndex.action?classID=2c9082b55b652546015b65ec3be2004f"
CUTOFF = "2023-08-10"   # 3 年窗口
CATEGORY = "自然资源"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
    "Referer": LIST_URL,
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
}

def clean_title(t):
    t = html.unescape(t or "")
    t = t.replace("\u200b", "").replace("\ufeff", "")
    t = re.sub(r"^[•·\s]+", "", t)
    return t.strip()

def fetch_list_page(page):
    """POST cpage=N 返回 (items, next_page)"""
    r = requests.post(LIST_URL, headers=HEADERS, data={"cpage": str(page)}, timeout=30, verify=False)
    r.raise_for_status()
    r.encoding = r.apparent_encoding or "utf-8"
    raw = r.text
    items = []
    # 每条: <a title='完整标题' href='/gtapp/nrglIndex.action?type=2&messageID={id}' target="_blank"> 截断标题 </a> ... <td align="center">日期</td>
    for m in re.finditer(
        r"<a title='([^']*)' href='([^']*nrglIndex\.action\?type=2&(?:amp;)?messageID=[^']*)'[^>]*>(.*?)</a>\s*</td>\s*<td align=\"center\">(\d{4}-\d{2}-\d{2})</td>",
        raw, re.S):
        title, href, disp, date = m.group(1), m.group(2), m.group(3), m.group(4)
        title = clean_title(title)
        if not title:
            continue
        href = html.unescape(href)
        detail_url = urljoin(LIST_URL, href)
        items.append({"title": title, "url": detail_url, "date": date})
    return items

def fetch_detail(detail_url, list_title):
    """返回 (content_html, attachments)"""
    try:
        r = requests.get(detail_url, headers=HEADERS, timeout=30, verify=False)
        r.raise_for_status()
        r.encoding = r.apparent_encoding or "utf-8"
        raw = r.text
    except Exception:
        return None, []
    # 附件: <a href='/gtapp/nrgl/GJAttach/{ts}.{ext}' target="blank">文件名</a>
    atts = []
    for am in re.finditer(r"<a[^>]*href='([^']*GJAttach/[^']+)'[^>]*>(.*?)</a>", raw, re.S):
        href, name = am.group(1), am.group(2)
        href = html.unescape(href)
        abs_url = urljoin(detail_url, href)
        name = clean_title(name) or abs_url.split("/")[-1]
        atts.append({"name": name, "url": abs_url})
    # 正文: 标题段落 + 附件段落（标题用列表 title，源站详情 td.bt 是元数据表头）
    body_parts = []
    if list_title:
        body_parts.append(f"<p>{list_title}</p>")
    for a in atts:
        body_parts.append(f'<p><a href="{a["url"]}">{a["name"]}</a></p>')
    content = "".join(body_parts)
    return content, atts

def main():
    max_pages = None
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            max_pages = int(a.split("=")[1])
        elif a.isdigit():
            max_pages = int(a)
    all_items = []
    seen = set()
    page = 1
    while True:
        if max_pages and page > max_pages:
            break
        items = fetch_list_page(page)
        if not items:
            print(f"[P{page}] empty, stop")
            break
        new = 0
        for it in items:
            if it["url"] in seen:
                continue
            if it["date"] < CUTOFF:
                print(f"[P{page}] {it['date']} < CUTOFF {CUTOFF}, stop")
                page = 999999
                break
            seen.add(it["url"])
            all_items.append(it)
            new += 1
        print(f"[P{page}] got {len(items)} items, new {new}, total {len(all_items)}")
        if page == 999999:
            break
        page += 1
    print(f"[AutoPg] max_pages={max_pages}, collected {len(all_items)}")

    # 抓详情
    valid = []
    for i, it in enumerate(all_items):
        content, atts = fetch_detail(it["url"], it["title"])
        if content is None:
            content = f"<p>{it['title']}</p>"
        valid.append({
            "site_name": SITE_NAME,
            "title": it["title"],
            "pub_date": it["date"],
            "content": content,
            "source_url": it["url"],
            "url": it["url"],
            "attachments": json.dumps(atts, ensure_ascii=False) if atts else "[]",
        })
        if (i + 1) % 50 == 0:
            print(f"[Detail] {i+1}/{len(all_items)}")
    print(f"[Detail] done {len(valid)}")
    push_to_searchdb(valid, CATEGORY)

if __name__ == "__main__":
    main()