#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
南充经开区-公示公告 爬虫 (TRS dataproxy 分页)
站点: ncsjkq.sczwfw.gov.cn
栏目: /col/col40915/ (公示公告)
列表: dataproxy.jsp?page=N&columnid=40915&webid=242&unitid=52420 (HTML片段)
详情: div.zoom 正文
"""
import re, os, sys, time, json, urllib.parse
import requests, warnings
warnings.filterwarnings('ignore')
from datetime import datetime, timedelta

_MAX_PG = None
for _a in sys.argv[1:]:
    if _a.startswith("--pages="):
        try:
            _MAX_PG = int(_a.split("=", 1)[1])
        except Exception:
            pass
    elif _a.isdigit():
        _MAX_PG = int(_a)
if _MAX_PG is not None:
    print(f'[AutoPg] max_pages={_MAX_PG}')

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "南充经开区-公示公告"
BASE_URL = "https://ncsjkq.sczwfw.gov.cn"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0",
    "Referer": "https://ncsjkq.sczwfw.gov.cn/col/col40915/index.html",
}

# dataproxy 基础参数 (从页面提取)
DP_BASE = "https://ncsjkq.sczwfw.gov.cn/module/jpage/dataproxy.jsp"
DP_WEBNAME = "%25E5%258D%2597%25E5%2585%2585%25E7%25BB%258F%25E6%25B5%258E%25E5%25BC%2580%25E5%258F%2591%25E5%258C%25BA%25E6%2594%25BF%25E5%258A%25A1%25E6%259C%258D%25E5%258A%25A1%25E7%25BD%2591"


def fetch(url):
    for retry in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=20, verify=False, allow_redirects=False)
            if r.status_code == 200:
                r.encoding = "utf-8"
                return r.text
            if r.status_code == 302:
                print(f"  [WARN] {url} -> 302 to {r.headers.get('Location','?')[:60]} retry{retry+1}")
            else:
                print(f"  [WARN] {url} HTTP {r.status_code} retry{retry+1}")
        except Exception as e:
            print(f"  [WARN] {url} err: {e} retry{retry+1}")
        time.sleep(1)
    return None


def fetch_list_page(page):
    # dataproxy.jsp 是装饰性假分页(302 主站)，真实数据在首页 HTML recordset CDATA 全量输出
    if page > 1:
        return []
    url = f"{BASE_URL}/col/col40915/index.html"
    html = fetch(url)
    if not html:
        return []
    return parse_list(html)


def parse_list(html):
    """dataproxy 返回的 li 条目"""
    items = []
    for li in re.findall(r'<li[^>]*>(.*?)</li>', html, re.DOTALL):
        m = re.search(r'href="([^"]+)"[^>]*>(.*?)</a>\s*<span[^>]*>(\d{4}-\d{2}-\d{2})</span>', li, re.DOTALL)
        if not m:
            m = re.search(r'href="([^"]+)"[^>]*title="([^"]*)"[^>]*>(.*?)</a>', li, re.DOTALL)
            if m:
                title = m.group(2).strip()
                dm = re.search(r'(\d{4}-\d{2}-\d{2})', li)
                if not dm:
                    continue
                items.append({"url": m.group(1), "title": title, "date": dm.group(1)})
                continue
            else:
                continue
        title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
        items.append({"url": m.group(1), "title": title, "date": m.group(3)})
    return items


def extract_detail(html):
    """div.zoom 平衡匹配"""
    i = html.find('<div class="zoom">')
    if i < 0:
        i = html.find('class="zoom"')
        if i < 0:
            return ""
    ds = html.rfind('<div', 0, i)
    if ds < 0:
        return ""
    s = html[ds:]
    d = 0
    for j in range(len(s)):
        if s[j:j+4] == "<div" and (j+4 >= len(s) or s[j+4] in " >\n\r\t"):
            d += 1
        elif s[j:j+6] == "</div>":
            d -= 1
            if d == 0:
                gt = s.find(">", 0, j)
                c = s[gt+1:j] if gt > 0 else ""
                c = re.sub(r'<(script|style)[^>]*>.*?</\1>', '', c, flags=re.DOTALL|re.I)
                c = re.sub(r'<p[^>]*>', '<p>', c)
                text = re.sub(r'<[^>]+>', '', c).strip()
                if text:
                    return c.strip()
    return ""


def run(max_pages=200):
    print(f"\n{'='*50}")
    print(f"🚀 {SITE_NAME}")
    print(f"{'='*50}")

    all_items = []
    for pg in range(1, min(max_pages, _MAX_PG or max_pages) + 1):
        its = fetch_list_page(pg)
        if not its:
            print(f"  第{pg}页: 空 -> 结束")
            break
        dates = [it["date"] for it in its if it["date"]]
        print(f"  第{pg}页: {len(its)} 条 ({min(dates) if dates else '?'} ~ {max(dates) if dates else '?'})")
        all_items.extend(its)
        if dates and max(dates) < CUTOFF:
            print(f"  该页已全部早于截断日{CUTOFF}，停止翻页")
            break
        time.sleep(0.3)

    print(f"\n📊 列表共 {len(all_items)} 条")
    kept = [it for it in all_items if not it["date"] or it["date"] >= CUTOFF]
    print(f"  窗口内(>= {CUTOFF}): {len(kept)} 条")

    out = []
    err = 0
    for i, it in enumerate(kept, 1):
        url = it["url"]
        if not url.startswith("http"):
            url = BASE_URL + url
        detail_html = fetch(url)
        if not detail_html:
            err += 1
            continue
        content = extract_detail(detail_html)
        text_len = len(re.sub(r'<[^>]+>', '', content).strip()) if content else 0
        if text_len < 5 and '<img' not in (content or ''):
            print(f"  ⚠ 空正文: {it['title'][:40]}")
            err += 1
            continue
        # 标题: 页面内提取
        title = it["title"]
        tm = re.search(r'<meta name="ArticleTitle" content="([^"]*)"', detail_html)
        if tm:
            title = tm.group(1).strip()
        out.append({
            "site_name": SITE_NAME,
            "title": title,
            "pub_date": it["date"],
            "content": content,
            "source_url": url,
            "url": url,
        })
        if i % 20 == 0:
            print(f"  ...{i}/{len(kept)}")
        time.sleep(0.2)

    print(f"\n📦 待入库: {len(out)}, 失败: {err}")
    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
    from crawler_lib import push_to_searchdb
    push_to_searchdb(out, batch_label=SITE_NAME)


if __name__ == "__main__":
    run(max_pages=200)