#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
crawl_jmdbq_hjbh.py — 荆门市东宝区人民政府「生态环境」3 个精准子栏目
http://www.jmdbq.gov.cn/col/col2535/index.html (聚合页: 环境影响评价27235 + 生态环保宣传27236 + 环保督察27237)
TRS WCM 系:
- 列表: col2535 聚合页仅各子栏目 5 条; 精准子栏目 col27235/27236/27237 单页全量(24/20/12 条)
- 分页 API: POST /module/web/jpage/dataproxy.jsp (form编码, startrecord/endrecord/perpage + col/webid/columnid/unitid/webname/permissiontype)
    返回 XML: <record><![CDATA[<li><a href title>标题</a><span>日期</span></li>]]></record>
- 详情: /art/YYYY/M/D/art_{col}_{id}.html
    标题 div.con-title, 日期 span.date(发布日期：2026-06-29 16:09), 正文 ContentStart→ZJEG_RSS.content.end
- 附件: 正文内 a 标签(如有), 内嵌URL保留
"""
import argparse
import html as html_lib
import os
import re
import sqlite3
import time
import urllib.parse

import requests

BASE = "http://www.jmdbq.gov.cn"
SITE_NAME = "荆门市东宝区生态环境"
SCRIPT_NAME = "crawl_jmdbq_hjbh.py"
GROUP_NAME = "湖北"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
    "Accept-Language": "zh-CN,zh;q=0.9",
}
TIMEOUT = 30

# 精准子栏目: col -> (栏目名)
COLUMNS = [
    (27235, "环境影响评价"),
    (27236, "生态环保宣传"),
    (27237, "环保督察"),
]
# dataproxy 固定参数(从页面 jpage 配置提取)
PROXY_BASE = {
    "col": 1,
    "webid": 38,
    "path": "/",
    "sourceContentType": 1,
    "unitid": "321696",
    "webname": "东宝区人民政府",
    "permissiontype": 0,
}


def clean_title(t):
    """标题清洗: strip &middot;&nbsp;&#32; 实体前缀与空白"""
    if not t:
        return ""
    t = t.strip()
    t = re.sub(r"&middot;|&nbsp;|&#160;|&#32;", "", t, flags=re.I)
    t = t.strip()
    t = re.sub(r"\s+", " ", t)
    return t


def html_to_text(html_seg):
    """HTML -> 文本, 保留 a 链接(内嵌URL)与表格, 去 script/style, \n\n 分段"""
    seg = html_seg
    links = []

    def _link_repl(m):
        href, txt = m.group(1), m.group(2)
        txt = re.sub(r"<[^>]+>", "", txt)
        txt = html_lib.unescape(txt).strip()
        if not txt or href.startswith("#") or href.startswith("javascript"):
            return txt or ""
        abs_url = urllib.parse.urljoin(BASE, href)
        links.append((href, abs_url, txt))
        return f"__LINK__{len(links)}__"

    seg = re.sub(r'<a\s[^>]*href="([^"]+)"[^>]*>(.*?)</a>', _link_repl, seg, flags=re.S | re.I)
    # 去 script/style
    seg = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", seg, flags=re.S | re.I)
    # 段落边界: 块级标签 -> \n\n
    seg = re.sub(r"</(p|div|li|tr|h[1-6]|br)>", "\n\n", seg, flags=re.I)
    seg = re.sub(r"<(p|div|li|tr|h[1-6]|br)[^>]*>", "\n", seg, flags=re.I)
    seg = re.sub(r"<[^>]+>", "", seg)
    seg = html_lib.unescape(seg)
    # 还原链接(内嵌URL)
    for i, (href, abs_url, txt) in enumerate(links, 1):
        seg = seg.replace(f"__LINK__{i}__", f'<a href="{abs_url}" target="_blank">{txt}</a>')
    seg = re.sub(r"\r", "", seg)
    seg = re.sub(r"\n{3,}", "\n\n", seg)
    seg = re.sub(r"[ \t\u00a0]+", " ", seg)
    return seg.strip(), links


def fetch_column(session, columnid, name):
    """通过 dataproxy API 获取子栏目全部条目 -> [(title, art_url, pub_date)]"""
    data = dict(PROXY_BASE)
    data["columnid"] = columnid
    data["startrecord"] = 1
    data["endrecord"] = 10000
    data["perpage"] = 10000
    try:
        r = session.post(
            BASE + "/module/web/jpage/dataproxy.jsp",
            headers={**HEADERS, "Referer": f"{BASE}/col/col{columnid}/index.html",
                     "X-Requested-With": "XMLHttpRequest"},
            data=data, timeout=TIMEOUT)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"[ERR] col{columnid} dataproxy: {e}")
        return []
    xml = r.text
    out = []
    for m in re.finditer(r"<record><!\[CDATA\[(.*?)\]\]></record>", xml, re.S):
        seg = m.group(1)
        lm = re.search(r'href="([^"]+)"[^>]*title="([^"]*)"[^>]*>(.*?)</a><span>([^<]+)</span>', seg, re.S)
        if not lm:
            continue
        href = lm.group(1)
        title = clean_title(lm.group(2) or re.sub(r"<[^>]+>", "", lm.group(3)))
        art_url = urllib.parse.urljoin(BASE, href)
        pub = lm.group(4).strip()
        out.append((title, art_url, pub))
    return out


def parse_detail(html_src, art_url):
    """详情页 -> (title, publish_date, body_html, links)"""
    # 标题
    m = re.search(r'<p class="con-title">\s*(.*?)\s*</p>', html_src, re.S)
    title = clean_title(m.group(1)) if m else ""
    if not title:
        m = re.search(r"<title>(.*?)</title>", html_src, re.S)
        if m:
            title = clean_title(m.group(1).split("-")[0])
    # 日期 发布日期：2026-06-29 16:09
    m = re.search(r"发布日期[：:]\s*(\d{4})-(\d{1,2})-(\d{1,2})", html_src)
    publish = ""
    if m:
        publish = "%s-%02d-%02d" % (m.group(1), int(m.group(2)), int(m.group(3)))
    # 正文: ContentStart -> ZJEG_RSS.content.end
    body_html = ""
    start_m = re.search(r"<meta name=\"ContentStart\">", html_src)
    end_m = re.search(r"ZJEG_RSS\.content\.end", html_src)
    if start_m:
        s = start_m.end()
        e = end_m.start() if end_m else len(html_src)
        body_html = html_src[s:e].strip()
    return title or None, publish or None, body_html, []


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--pages", type=int, default=1, help="抓取页数(每子栏目页数, 该站单页全量)")
    ap.add_argument("--db", default="/mnt/data/search.db", help="SQLite 数据库路径")
    ap.add_argument("--script-name", default=SCRIPT_NAME)
    args = ap.parse_args()

    db_path = args.db
    script_name = args.script_name
    if not db_path.startswith("/mnt/"):
        db_path = os.path.expanduser(db_path)

    conn = sqlite3.connect(db_path, timeout=60)
    cur = conn.cursor()
    cur.execute("PRAGMA busy_timeout=60000")
    new_count = 0
    dup_count = 0
    err_count = 0

    session = requests.Session()

    for columnid, cname in COLUMNS:
        items = fetch_column(session, columnid, cname)
        print(f"[{cname} col{columnid}] API {len(items)} 条")
        for title, art_url, pub_date in items:
            try:
                cur.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=? AND script_name=?", (art_url, script_name))
                if cur.fetchone()[0] > 0:
                    dup_count += 1
                    continue
                dr = session.get(art_url, headers=HEADERS, timeout=TIMEOUT)
                dr.encoding = "utf-8"
                d_title, d_date, content_html, links = parse_detail(dr.text, art_url)
                if not content_html.strip():
                    print(f"  跳过空正文: {title[:30]}")
                    dup_count += 1
                    continue
                if d_title:
                    title = d_title
                if d_date:
                    pub_date = d_date
                content, links_out = html_to_text(content_html)
                att_txt = "\n\n".join(a[2] for a in links_out) if links_out else ""
                summary = content[:500] if content else ""
                cur.execute(
                    "INSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, status, category, visits, content, tags, industry, attachments, group_name, has_table, script_name) "
                    "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
                    (SITE_NAME, art_url, art_url, title, pub_date, 0, summary, "", "", 0, content, "",
                     "other", att_txt, GROUP_NAME, 1 if "<table" in content else 0, script_name),
                )
                # 2026-09-22: 先提交 gov_raw —— 库上触发器已维护 FTS，下面这条手动写入会因
                #   rowid 重复而 IntegrityError；不先 commit 会把 gov_raw 那条一并回滚（静默丢数据）
                conn.commit()
                cur.execute(
                    "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?,?,?,?)",
                    (cur.lastrowid, title, SITE_NAME, summary),
                )
                conn.commit()
                new_count += 1
                print(f"  + {pub_date} {title[:40]}")
            except Exception as e:
                err_count += 1
                print(f"  ERR {title[:30]}: {e}")
        time.sleep(0.3)

    conn.close()
    print(f"完成: 新增 {new_count} / 重复 {dup_count} / 错误 {err_count}")


if __name__ == "__main__":
    main()