#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""仙桃市生态环境局 — 公示公告爬虫 (修复版 v2)
修复: 
1. 日期从 span.date 提取(原取第一个span, 实为list_text标题→date错位+title被清空)
2. title 优先 a[title] 属性, 其次 list_text span
3. 写 gov_search FTS (原写废弃表 gov_search_v3 → 搜索不到)
4. 补 source_url/script_name/group_name
"""
import requests, sqlite3, re
from bs4 import BeautifulSoup

DB = "/root/search.db"
BASE = "https://www.xiantao.gov.cn"
LIST_TPL = BASE + "/bmxxgk/shbj/zfxxgk/fdzdgknr/qtzdgknr_37950/gsgg/index{}.shtml"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36"}
SITE = "xiantao_gsgg"
GROUP = "湖北"
SCRIPT = "crawl_xiantao.py"
# createPageHTML(20, 0, "index", "shtml", "pages clearfix", 311) -> 16 pages
MAX_PAGES = 20
seen_urls = set()

def extract_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
    except Exception:
        return None, None, None
    soup = BeautifulSoup(r.text, "html.parser")
    # 标题
    title = ""
    tt = soup.select_one("div.xl_tit")
    if tt:
        title = tt.get_text(strip=True)
    # 日期
    date = ""
    for td in soup.find_all("td"):
        if "发文日期" in td.get_text():
            nxt = td.find_next_sibling("td")
            if nxt:
                m = re.search(r"(\d{4}-\d{2}-\d{2})", nxt.get_text())
                if m: date = m.group(1); break
    # 正文 - TRS UEditor
    content_div = soup.find("div", class_="view") or soup.find("div", class_="TRS_Editor")
    content = ""
    if content_div:
        for tag in content_div.find_all(["p", "div", "table"]):
            txt = str(tag).strip()
            if txt: content += txt + "\n"
        content = content.strip()
    return title, date, content

def crawl_page(pg):
    if pg == 1:
        url = LIST_TPL.format("")
    else:
        url = LIST_TPL.format(f"_{pg}")
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
    except Exception:
        return [], 0
    soup = BeautifulSoup(r.text, "html.parser")
    items = []
    for a in soup.find_all("a", href=True):
        href = a["href"]
        if "/shbj/" not in href or "/gsgg/" not in href:
            continue
        # 跳过栏目首页链接
        if href.rstrip("/").endswith("/gsgg") or href.rstrip("/").endswith("/gsgg/"):
            continue
        if not href.startswith("http"):
            href = BASE + href
        if href in seen_urls:
            continue
        seen_urls.add(href)
        # 标题: 优先 a[title] 属性, 其次 list_text span, 最后纯文本
        title = (a.get("title") or "").strip()
        if not title:
            lt = a.select_one("span.list_text")
            if lt:
                title = lt.get_text(strip=True)
        if not title:
            title = a.get_text(strip=True)
        # 日期: 从 span.date 提取(修复: 原逻辑取第一个span实为list_text标题)
        date = ""
        sp = a.select_one("span.date")
        if sp:
            date = sp.get_text(strip=True)
        # 清理标题中的日期残留
        if sp and sp.get_text(strip=True) in title:
            title = title.replace(sp.get_text(strip=True), "").strip()
        items.append({"title": title.strip(), "url": href, "date": date})
    return items

def main():
    import argparse
    ap = argparse.ArgumentParser()
    ap.add_argument("--pages", type=int, default=1)
    args = ap.parse_args()
    pages = max(1, min(args.pages, MAX_PAGES))

    conn = sqlite3.connect(DB)
    conn.execute("PRAGMA journal_mode=WAL")
    c = conn.cursor()
    c.execute("CREATE TABLE IF NOT EXISTS gov_raw (id INTEGER PRIMARY KEY, site_name TEXT, source_url TEXT, page_url TEXT, title TEXT, publish_date TEXT, date_rank INTEGER DEFAULT 0, summary TEXT, status TEXT, category TEXT DEFAULT '', visits INTEGER DEFAULT 0, content TEXT DEFAULT '', tags TEXT DEFAULT '', industry TEXT DEFAULT 'other', attachments TEXT DEFAULT '', group_name TEXT, has_table INTEGER DEFAULT 0, script_name TEXT DEFAULT '')")
    c.execute("CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(title, site_name, summary, tokenize='trigram')")

    total_new = total_dup = total_skip = 0
    for pg in range(1, pages + 1):
        print(f"--- 第{pg}页 ---")
        items = crawl_page(pg)
        if not items:
            print("无结果，停止翻页"); break
        print(f"找到{len(items)}条")
        for item in items:
            c.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],))
            if c.fetchone():
                total_dup += 1; continue
            print(f"  [{item['date']}] {item['title'][:50]}")
            title, date, content = extract_detail(item["url"])
            if not title: title = item["title"]
            if not date: date = item["date"]
            if not content or len(content) < 200:
                print(f"    正文过短({len(content) if content else 0}), 跳过")
                total_skip += 1; continue
            summary = re.sub(r"<[^>]+>", "", content)[:200]
            summary = re.sub(r"\s+", " ", summary).strip()
            try:
                cur = c.execute(
                    "INSERT INTO gov_raw (title, summary, content, page_url, source_url, publish_date, site_name, script_name, group_name) VALUES (?,?,?,?,?,?,?,?,?)",
                    (title, summary, content, item["url"], item["url"], date, SITE, SCRIPT, GROUP))
                rid = cur.lastrowid
                c.execute("INSERT INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)",
                    (rid, title, SITE, summary))
                conn.commit()
                total_new += 1
            except sqlite3.IntegrityError:
                total_dup += 1
    print(f"\n新增: {total_new}  重复: {total_dup}  过短: {total_skip}  总计: {total_new+total_dup+total_skip}")

if __name__ == "__main__":
    main()