#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
抚州国家高新区 - 重点领域/社会公益事业/环境保护 (col8400) 爬虫
列表: https://gxq.jxfz.gov.cn/col/col8350/index.html?number=I00007I00004I00002
  (col8350 与 col8400 是同一 XXGK 通用模板页, 栏目内容由 URL number 参数决定)
  XXGK 动态模块: 页面 JS loadDynamic POST /module/xxgk/search.jsp
  真实请求: POST search.jsp?infotypeId=I00007I00004I00002
    body: infotypeId=0&jdid=53&divid=div7299&vc_title=&vc_number=&currpage=N&...&infotypeId=I00007I00004I00002&area=
  共 90 条 / 18 条每页 = 5 页 (curl/GET 被 WAF 拒, 完整 POST body 稳定)
  列表: <li><a href='http://gxq.jxfz.gov.cn/art/...' title="标题">标题</a><b>2026-08-06</b></li>
详情: /art/YYYY/M/D/art_8400_NUMBER.html (TRS, http 301 -> https)
  标题: meta ArticleTitle; 日期: meta PubDate; 正文: div#zoom
用法: python3 crawl_gxq_jxfz_hb.py [--pages=N]
"""
import os, sys, re, time, json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE_URL = "https://gxq.jxfz.gov.cn"
SITE_NAME = "抚州高新区-环境保护"
CATEGORY = "环境保护"
LIST_URL = BASE_URL + "/col/col8350/index.html?number=I00007I00004I00002"
SEARCH_URL = BASE_URL + "/module/xxgk/search.jsp"
MAX_PAGES = 5
INFOTYPE = "I00007I00004I00002"

UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
HEADERS = {"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"}

LI_RE = re.compile(
    r"<li[^>]*>\s*<a[^>]+href='([^']+)'[^>]*title=\"([^\"]*)\"[^>]*>(?:.*?)</a>\s*<b>\s*([^<]+?)\s*</b>\s*</li>",
    re.S
)


def parse_args():
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            return int(a.split("=")[1])
        if a.isdigit():
            return int(a)
    return MAX_PAGES


def clean_title(t):
    t = re.sub(r"\s+", " ", t or "")
    t = re.sub(r"^[•·\-—]\s*", "", t)
    return t.strip()


def fetch_list_requests(max_pages):
    """直接 POST search.jsp (复刻 loadDynamic 完整 body), currpage 翻页, 稳定"""
    session = requests.Session()
    session.headers.update(HEADERS)
    session.headers["Referer"] = LIST_URL
    all_items = []
    seen = set()
    for pg in range(1, max_pages + 1):
        data = {
            "infotypeId": "0", "jdid": "53", "divid": "div7299",
            "vc_title": "", "vc_number": "", "currpage": str(pg),
            "vc_filenumber": "", "vc_all": "", "texttype": "", "fbtime": "",
            "infotypeId": INFOTYPE,
            "vc_title": "", "vc_number": "", "area": "",
        }
        try:
            r = session.post(SEARCH_URL, data=data, timeout=30)
            r.encoding = "utf-8"
            html = r.text
        except Exception as e:
            print(f"  Page {pg}: ERR {e}", flush=True)
            break
        n = 0
        for m in LI_RE.finditer(html):
            href, title, date = m.group(1).strip(), clean_title(m.group(2)), m.group(3).strip()
            if not title or len(title) < 4:
                continue
            if not href.startswith("http"):
                href = urljoin(BASE_URL, href)
            if href in seen:
                continue
            seen.add(href)
            dm = re.search(r"(\d{4}-\d{2}-\d{2})", date)
            date = dm.group(1) if dm else date
            all_items.append({"title": title, "url": href, "date": date})
            n += 1
        print(f"  Page {pg}: +{n} 累计{len(all_items)}", flush=True)
        if n == 0:
            break
        time.sleep(0.5)
    return all_items


def fetch(session, url):
    try:
        r = session.get(url, timeout=25)
        if r.status_code != 200:
            return ""
        r.encoding = "utf-8"
        return r.text
    except Exception:
        return ""


def extract_detail(html, url):
    """标题/日期/正文"""
    soup = BeautifulSoup(html, "html.parser")

    title = ""
    mt = soup.find("meta", attrs={"name": "ArticleTitle"})
    if mt and mt.get("content"):
        title = clean_title(mt["content"])
    if not title:
        h1 = soup.find("h1")
        if h1:
            title = clean_title(h1.get_text())

    date = ""
    mp = soup.find("meta", attrs={"name": "PubDate"})
    if mp and mp.get("content"):
        date = mp["content"].strip()[:10]
    if not date:
        m = re.search(r"(\d{4}-\d{2}-\d{2})", html)
        if m:
            date = m.group(1)

    body = ""
    zoom = soup.find("div", id="zoom")
    if zoom:
        parts = []
        for el in zoom.find_all(["p", "table", "img"]):
            if el.name == "p":
                if el.find_parent("table") or el.find("table") or el.find("p"):
                    continue
                if any(
                    re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|ofd|wps)$", a.get("href", ""), re.I)
                    or "download" in a.get("href", "").lower()
                    for a in el.find_all("a", href=True)
                ):
                    continue
                for a in el.find_all("a", href=True):
                    h = a["href"].strip()
                    if not h.startswith("http"):
                        a["href"] = urljoin(url, h)
                txt = el.get_text(" ", strip=True)
                txt = re.sub(r"\s+", " ", txt)
                if txt:
                    parts.append(f"<p>{txt}</p>")
            elif el.name == "table":
                parts.append(str(el))
            elif el.name == "img":
                src = el.get("src", "")
                if src:
                    if not src.startswith("http"):
                        src = urljoin(url, src)
                    parts.append(f'<p><img src="{src}"></p>')
        body = "\n".join(parts)
        if not body:
            txt = zoom.get_text(" ", strip=True)
            if txt:
                body = f"<p>{txt}</p>"

    # 附件
    attachments = []
    if zoom:
        for a in zoom.find_all("a", href=True):
            h = a["href"].strip()
            if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|ofd|wps)$", h, re.I) or "download" in h.lower():
                full = urljoin(url, h)
                name = a.get_text(strip=True) or h.split("/")[-1]
                attachments.append({"name": name, "url": full})

    return title, date, body, attachments


def main():
    max_pages = parse_args()
    print(f"[GXQ-JXFZ-HB] SITE={SITE_NAME} max_pages={max_pages}", flush=True)

    all_items = fetch_list_requests(max_pages)
    print(f"\n列表完成: {len(all_items)} 条, 抓详情...", flush=True)

    session = requests.Session()
    session.headers.update(HEADERS)
    results = []
    for i, it in enumerate(all_items):
        html = fetch(session, it["url"])
        if not html:
            results.append({
                "site_name": SITE_NAME, "source_url": it["url"], "url": it["url"],
                "title": it["title"], "pub_date": it["date"], "content": "",
                "summary": it["title"], "category": CATEGORY, "attachments": "",
            })
            continue
        d_title, d_date, body, atts = extract_detail(html, it["url"])
        title = d_title or it["title"]
        pub_date = d_date or it.get("date", "")
        summary = re.sub(r"<[^>]+>", " ", body) if body else title
        summary = re.sub(r"\s+", " ", summary).strip()[:300]
        results.append({
            "site_name": SITE_NAME, "source_url": it["url"], "url": it["url"],
            "title": title, "pub_date": pub_date, "content": body,
            "summary": summary, "category": CATEGORY,
            "attachments": json.dumps(atts, ensure_ascii=False) if atts else "",
        })
        if (i + 1) % 10 == 0:
            print(f"  detail {i+1}/{len(all_items)}", flush=True)
        time.sleep(0.3)

    print(f"  pushing {len(results)} items...", flush=True)
    push_to_searchdb(results, batch_label=SITE_NAME)
    empty = sum(1 for r in results if not (r.get("content") or "").strip())
    dates = [r["pub_date"] for r in results if r["pub_date"]]
    print(f"  DONE. pushed={len(results)} 空正文={empty}", flush=True)
    print(f"  日期范围: {min(dates) if dates else '-'} ~ {max(dates) if dates else '-'}", flush=True)


if __name__ == "__main__":
    main()