#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""凤城市-部门动态爬虫 (lnfc.gov.cn fcszf/xwdt/bmdt)
站点: https://www.lnfc.gov.cn (辽宁丹东凤城市, TRS WCM)
栏目: /fcszf/xwdt/bmdt/glist.html (部门动态, 15条/页)
列表: div.time + div.name > a[href][title], 分页 glist.html → glist{N}.html
详情: div.center-title 标题 + div.center-info 正文 + 发布日期：YYYY-MM-DD
附件: 正文内 <a href="*.pdf/doc/xls..."> urljoin 绝对化
入库: INSERT OR IGNORE 直插 gov_raw (FTS 触发器自动同步 gov_search)
运行: python3 crawl_lnfc_bmdt.py [--pages=N]
"""
import json, os, re, sys, time, sqlite3, ssl, urllib.request
from urllib.parse import urljoin
from bs4 import BeautifulSoup

SITE_NAME = "凤城市-部门动态"
BASE_URL = "https://www.lnfc.gov.cn"
LIST_URL = BASE_URL + "/fcszf/xwdt/bmdt/glist.html"
TOTAL_PAGES = 50
CUTOFF = "2023-08-18"
OUT = "/tmp/lnfc_bmdt.jsonl"
DB_PATH = "/root/search.db"
CATEGORY = "部门动态"
GROUP_NAME = "辽宁"

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE


def parse_pages_arg():
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            return int(a.split("=")[1])
        if a.isdigit():
            return int(a)
    return 0


def http_get(url, timeout=20):
    req = urllib.request.Request(url, headers={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    })
    try:
        resp = urllib.request.urlopen(req, timeout=timeout, context=ctx)
        return resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        print(f"  [ERR] {url[:70]}: {e}", flush=True)
        return ""


def clean_title(t):
    t = re.sub(r"^[•··\s]+", "", t or "").strip()
    return t.replace("\u200b", "").replace("\ufeff", "")


def clean_content(html):
    """正文清洗: span/font unwrap + 嵌套p扁平化 + o:p清理 + 保留表格"""
    soup = BeautifulSoup(html, "html.parser")
    for tag in soup.find_all(lambda t: t.name and ":" in t.name):
        tag.decompose()
    for tag in soup.find_all(["span", "font"]):
        tag.unwrap()
    # 只提升含嵌套 p 的外层 p — 勿 unwrap 内层 p (永丰修复)
    while True:
        nested = [p for p in soup.find_all("p") if p.find("p")]
        if not nested:
            break
        nested[0].unwrap()
    for p in soup.find_all("p"):
        if p.get("style"):
            p.attrs = {"align": p.get("align")} if p.get("align") else {}
    return str(soup)


def extract_attachments(detail_html, detail_url):
    """正文内附件链接: <a href="*.pdf/doc/xls...">"""
    atts = []
    for m in re.finditer(r'<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', detail_html, re.I | re.S):
        href, name_html = m.group(1).strip(), m.group(2)
        if not href or href.startswith("javascript"):
            continue
        if re.search(r"\.(pdf|docx?|xlsx?|zip|rar|7z)$", href, re.I):
            name = re.sub(r"<[^>]+>", "", name_html).strip()[:100]
            full = urljoin(BASE_URL, href)
            atts.append({"name": name or full.split("/")[-1], "url": full})
    seen = set()
    uniq = []
    for a in atts:
        if a["url"] not in seen:
            seen.add(a["url"])
            uniq.append(a)
    return uniq


def extract_articles(html):
    articles = []
    pat = re.compile(
        r'<div class="time">([^<]+)</div>\s*<div class="name">\s*<a[^>]+href="([^"]+)"[^>]*title="([^"]*)"',
        re.IGNORECASE
    )
    for m in pat.finditer(html):
        date = m.group(1).strip()
        href = m.group(2)
        title = m.group(3).strip()
        url = urljoin(BASE_URL, href)
        if title:
            articles.append((url, title, date))
    return articles


def push_to_db(results):
    if not results:
        return 0, 0
    if not os.path.exists(DB_PATH):
        print(f"  未发现 {DB_PATH}, 跳过入库 (仅写 JSONL)", flush=True)
        return 0, 0
    conn = sqlite3.connect(DB_PATH, timeout=290)
    conn.execute("PRAGMA busy_timeout=290000")
    added = skipped = 0
    cur = conn.cursor()
    for r in results:
        try:
            cur.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name, attachments, group_name, industry)
                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?)""",
                (SITE_NAME, r["url"], r["url"], r["title"], r["pub_date"], r["content"],
                 (r["content"] or "")[:500], CATEGORY, "crawl_lnfc_bmdt.py", r["attachments"],
                 r.get("group_name", GROUP_NAME), r.get("industry", "other"))
            )
            if cur.rowcount > 0:
                added += 1
            else:
                skipped += 1
        except sqlite3.Error as e:
            print(f"  DB err: {e}", flush=True)
            skipped += 1
        conn.commit()
    conn.close()
    return added, skipped


def main():
    max_pages = parse_pages_arg()
    print(f"站点: {SITE_NAME} 最大页: {max_pages or '全量'} CUTOFF: {CUTOFF}", flush=True)

    results = []
    stats = {"list": 0, "detail_ok": 0, "empty": 0}
    cutoff_hit = False

    for page in range(1, TOTAL_PAGES + 1):
        if max_pages and page > max_pages:
            break
        n = page - 1
        url = LIST_URL if n == 0 else BASE_URL + f"/fcszf/xwdt/bmdt/glist{n}.html"
        html = http_get(url)
        if not html:
            print(f"  页{page} 抓取失败", flush=True)
            break
        articles = extract_articles(html)
        if not articles:
            print(f"  页{page} 无条目 (可能到头)", flush=True)
            break
        page_items = 0
        for item_url, title, date_str in articles:
            date_str = date_str[:10]
            if date_str < CUTOFF:
                print(f"    日期 {date_str} < CUTOFF 停止", flush=True)
                cutoff_hit = True
                break
            stats["list"] += 1
            page_items += 1
            dh = http_get(item_url)
            if not dh:
                print(f"    详情失败: {title[:30]}", flush=True)
                continue
            stats["detail_ok"] += 1
            dsoup = BeautifulSoup(dh, "html.parser")
            con = dsoup.select_one("div.center-info")
            content = clean_content(str(con)) if con else ""
            text = re.sub(r"<[^>]+>", "", content).strip()
            if len(text) < 10 and "<img" not in content:
                # 外链公告(微信等): 正文 = 标题 + URL 内嵌段 (纯PDF外链公告站同款处理)
                if item_url.startswith("http") and "lnfc.gov.cn" not in item_url:
                    content = f"<p>{clean_title(title)}</p>\n<p><a href=\"{item_url}\">{item_url}</a></p>"
                    text = re.sub(r"<[^>]+>", "", content).strip()
                    print(f"    外链公告: {title[:30]} -> {item_url[:50]}", flush=True)
                else:
                    print(f"    空正文跳过: {title[:30]}", flush=True)
                    stats["empty"] += 1
                    continue
            # img src 绝对化
            if "<img" in content:
                content = re.sub(
                    r'(<img[^>]+src=")([^"]+)(")',
                    lambda mm: mm.group(1) + (mm.group(2) if mm.group(2).startswith("http") else urljoin(BASE_URL, mm.group(2))) + mm.group(3),
                    content)
            # 日期: 列表优先, 详情"发布日期"兜底
            pub_date = date_str
            dm = re.search(r"发布日期[：:]\s*(\d{4}-\d{2}-\d{2})", dh)
            if dm:
                pub_date = dm.group(1)
            atts = extract_attachments(dh, item_url)
            results.append({
                "site_name": SITE_NAME,
                "title": clean_title(title),
                "pub_date": pub_date,
                "content": content,
                "source_url": item_url,
                "url": item_url,
                "group_name": GROUP_NAME,
                "industry": "other",
                "attachments": json.dumps(atts, ensure_ascii=False) if atts else "",
            })
            time.sleep(0.4)
        print(f"  页{page}/{TOTAL_PAGES} 列表 {page_items} 条 (累计 {len(results)})", flush=True)
        if cutoff_hit:
            break
        time.sleep(1.0)

    with open(OUT, "w", encoding="utf-8") as f:
        for r in results:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    print(f"已写 {len(results)} 条到 {OUT}", flush=True)

    added, skipped = push_to_db(results)
    print(f"=== 入库: 新增{added} 跳过{skipped} ===", flush=True)
    print(f"=== 完成: 列表{stats['list']} 详情OK{stats['detail_ok']} 空{stats['empty']} ===", flush=True)


if __name__ == "__main__":
    main()