#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""连云港经济技术开发区 — 通知公告爬虫
URL: http://www.lda.gov.cn/lygjjjskfq/tzgg/tzgg.html
CMS: TrueCMS (jpage AJAX分页)
列表: AJAX接口 /TrueCMS/messageController/getMessage.do
  jsonpCallback 必需(缺则404), 参数 startrecord/endrecord/perpage=15, columnId固定
  totalRecord=946, 63页
详情: /lygjjjskfq/tzgg/content/{UUID}.html
  标题: meta ArticleTitle / div.pp1>h1
  日期: meta PubDate / div.pp2 "发布时间：2026-07-16 17:15"
  来源: div.pp2 "信息来源：生态环境局"
  正文: div#zoom
"""
import requests, sqlite3, re, sys, time, json
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB = "/root/search.db"
BASE = "http://www.lda.gov.cn"
LIST_DIR = BASE + "/lygjjjskfq/tzgg"
SITE = "连云港经开区-通知公告"
GROUP = "江苏"
SCRIPT = "crawl_lda_tzgg.py"
MAX_PAGES = 63  # 946条 / 15 = 63.07
PER_PAGE = 15
COLUMN_ID = "75b959b9-e957-4642-996b-16ef4602f58f"
AJAX_URL = BASE + "/TrueCMS/messageController/getMessage.do"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
    "Referer": LIST_DIR + "/tzgg.html",
}


def log(msg):
    print(msg, file=sys.stderr, flush=True)


def fetch(url, headers=None, retries=3):
    h = headers or HEADERS
    for attempt in range(retries):
        try:
            r = requests.get(url, headers=h, timeout=30)
            r.encoding = "utf-8"
            return r.text
        except Exception as e:
            log(f"  [WARN] {url} 请求失败({attempt+1}/{retries}): {e}")
            time.sleep(2)
    return ""


def crawl_list(pg):
    """AJAX分页抓列表, 返回 [{title, url, date}]"""
    startrecord = (pg - 1) * PER_PAGE + 1
    endrecord = pg * PER_PAGE
    params = {
        "startrecord": startrecord,
        "endrecord": endrecord,
        "perpage": PER_PAGE,
        "columnId": COLUMN_ID,
        "jsonpCallback": f"callback{int(time.time()*1000)}",
    }
    try:
        r = requests.get(AJAX_URL, headers=HEADERS, params=params, timeout=30)
        r.encoding = "utf-8"
        data = r.json()
    except Exception as e:
        log(f"  [WARN] AJAX {pg}页失败: {e}")
        return []
    result = data.get("result", "")
    records = re.findall(r"<record>\s*<!\[CDATA\[(.*?)\]\]>", result, re.S)
    items = []
    for rec in records:
        # <li><span>[2026-07-21]</span><a href="..." title="...">标题</a></li>
        dm = re.search(r"\[(20\d{2}-\d{2}-\d{2})\]", rec)
        am = re.search(r'href="([^"]*)"\s+title="([^"]*)"', rec)
        if not am:
            continue
        href = am.group(1)
        if not href.startswith("http"):
            href = urljoin(BASE, href)
        title = am.group(2).strip()
        # strip 实体前缀
        title = re.sub(r"^(?:&middot;|·|&nbsp;|\s)+", "", title).strip()
        date = dm.group(1) if dm else ""
        items.append({"title": title, "url": href, "date": date})
    return items


def extract_detail(url):
    """返回 (title, date, content)"""
    html = fetch(url)
    if not html:
        return None, None, None
    soup = BeautifulSoup(html, "html.parser")
    # 标题
    title = ""
    mt = soup.find("meta", attrs={"name": "ArticleTitle"})
    if mt and mt.get("content"):
        title = mt["content"].strip()
    if not title:
        h1 = soup.find("div", class_="pp1")
        if h1 and h1.find("h1"):
            title = h1.find("h1").get_text(strip=True)
    # 日期
    date = ""
    md = soup.find("meta", attrs={"name": "PubDate"})
    if md and md.get("content"):
        m = re.search(r"(20\d{2}-\d{2}-\d{2})", md["content"])
        if m:
            date = m.group(1)
    # 正文: div#zoom
    content = ""
    dc = soup.find("div", id="zoom")
    if dc:
        # 附件: a[href] 指向文件
        attachments = []
        seen_att = set()
        for a in dc.find_all("a", href=True):
            href = a["href"]
            if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|txt)$", href, re.I):
                if not href.startswith("http"):
                    href = urljoin(BASE, href)
                name = a.get_text(strip=True) or "附件"
                if href not in seen_att:
                    seen_att.add(href)
                    attachments.append({"name": name, "url": href})

        parts = []
        for node in dc.find_all(["p", "div", "table"], recursive=True):
            # 跳过表格内部节点(表格已整体保留, 内部 p/div 不拍平, 避免重复)
            if node.name != "table" and node.find_parent("table"):
                continue
            if node.name == "div" and node.get("id") in ("xgwdP", "qrcode"):
                continue
            if node.find_parent("div", id="xgwdP"):
                continue
            if node.find_parent(class_="filelink") or (node.get("class") and "filelink" in node.get("class")):
                continue
            # 跳过含已提取附件的p(避免正文+附件区重复)
            if node.name == "p":
                def _att_url(el):
                    u = ""
                    if el.name == "a":
                        u = el.get("href", "")
                    if u and not u.startswith("http"):
                        u = urljoin(BASE, u)
                    return u
                has_att = any(
                    _att_url(el) in seen_att
                    for el in node.find_all(["a"]) if _att_url(el))
                if has_att:
                    continue
            html_str = str(node).strip()
            if not html_str:
                continue
            # 去嵌套: p 内再套 p 只取最内层文本段落
            if node.name == "p":
                # 外层p包含表格则跳过(表格内部节点独立处理)
                if node.find("table"):
                    continue
                inner_p = node.find("p", recursive=False)
                if inner_p:
                    html_str = str(inner_p).strip()
            if node.name == "div" and node.find_parent("div") and not node.find(["p", "table"]):
                parts.append(html_str)
            elif node.name in ("p", "table"):
                parts.append(html_str)
        content = "\n\n".join(parts).strip()
        # 去重: 相同段落只保留一次 + 过滤空/纯<br>段落(含 span 包裹)
        uniq_parts = []
        for part in content.split("\n\n"):
            part_stripped = part.strip()
            if not part_stripped:
                continue
            # 剥标签后无文本(仅空白/br) → 过滤
            if not re.sub(r"<[^>]+>", "", part_stripped).strip():
                continue
            if part_stripped not in uniq_parts:
                uniq_parts.append(part_stripped)
        content = "\n\n".join(uniq_parts).strip()
        if attachments:
            att_links = "\n".join(f'<p><a href="{att["url"]}">{att["name"]}</a></p>' for att in attachments)
            content = content + "\n\n" + att_links if content else att_links
    return title, date, content


def main():
    import argparse
    ap = argparse.ArgumentParser()
    ap.add_argument("--pages", type=int, default=1)
    args = ap.parse_args()
    pages = max(1, min(args.pages, MAX_PAGES))

    conn = sqlite3.connect(DB, timeout=15)
    conn.execute("PRAGMA busy_timeout=10000")
    c = conn.cursor()
    c.execute("CREATE TABLE IF NOT EXISTS gov_raw (id INTEGER PRIMARY KEY, site_name TEXT, source_url TEXT, page_url TEXT, title TEXT, publish_date TEXT, date_rank INTEGER DEFAULT 0, summary TEXT, status TEXT, category TEXT DEFAULT '', visits INTEGER DEFAULT 0, content TEXT DEFAULT '', tags TEXT DEFAULT '', industry TEXT DEFAULT 'other', attachments TEXT DEFAULT '', group_name TEXT, has_table INTEGER DEFAULT 0, script_name TEXT DEFAULT '')")
    c.execute("CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(title, site_name, summary, tokenize='trigram')")

    total_new = total_dup = total_skip = 0
    seen = set()
    for pg in range(1, pages + 1):
        log(f"--- 第{pg}页 ---")
        items = crawl_list(pg)
        if not items:
            log("列表为空, 停止翻页")
            break
        log(f"找到{len(items)}条")
        for item in items:
            if item["url"] in seen:
                continue
            seen.add(item["url"])
            c.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],))
            if c.fetchone():
                total_dup += 1
                continue
            log(f"  [{item['date']}] {item['title'][:45]}")
            title, date, content = extract_detail(item["url"])
            if not title:
                title = item["title"]
            if not date:
                date = item["date"]
            if not content or len(re.sub(r"<[^>]+>", "", content).strip()) < 10:
                log(f"    正文过短({len(content) if content else 0}), 跳过")
                total_skip += 1
                continue
            summary = re.sub(r"<[^>]+>", "", content)[:200]
            summary = re.sub(r"\s+", " ", summary).strip()
            has_table = 1 if re.search(r"<table", content) else 0
            try:
                cur = c.execute(
                    "INSERT INTO gov_raw (title, summary, content, page_url, source_url, publish_date, site_name, script_name, group_name, has_table, date_rank) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
                    (title, summary, content, item["url"], item["url"], date, SITE, SCRIPT, GROUP, has_table, 0))
                rid = cur.lastrowid
                c.execute("INSERT INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)",
                          (rid, title, SITE, summary))
                conn.commit()
                total_new += 1
            except sqlite3.IntegrityError:
                total_dup += 1
            time.sleep(0.5)
    print(f"\n新增: {total_new}  重复: {total_dup}  过短: {total_skip}  总计: {total_new+total_dup+total_skip}")


if __name__ == "__main__":
    main()