#!/usr/bin/env python3
"""
襄汾县人民政府 - 环境保护（重点领域信息公开）
URL: http://www.xiangfen.gov.cn/channels/4322.html
CMS: TRS WCM
Note: 该栏目为转载聚合页，前15页全部为外部链接（mee.gov.cn/shanxi.gov.cn/微信），
      第16页有本地链接但URL已失效（404）。本脚本仅保存标题+URL+日期。
"""
import sys, os, re, json, time, sqlite3
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

BASE = "http://www.xiangfen.gov.cn"
SITE = "襄汾县人民政府-环境保护"
COLUMN = "环境保护"
PROVINCE = "山西"
TOTAL_PAGES = 16  # 共16页

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
session = requests.Session()
session.headers.update(HEADERS)
session.verify = False


def log(msg):
    print(msg, file=sys.stderr, flush=True)


def fetch(url, retries=3):
    for attempt in range(retries):
        try:
            r = session.get(url, timeout=30)
            r.encoding = "utf-8"
            return r.text
        except Exception as e:
            log(f"  [WARN] 请求失败 (尝试 {attempt+1}/{retries}): {e}")
            time.sleep(2)
    return ""


# 常见正文容器选择器（按优先级排序）
CONTENT_SELECTORS = [
    ("div", "TRS_Editor", None),
    ("div", "Custom_UnionStyle", None),
    ("div", "neiright_JPZ_GK_CP", None),
    ("div", "neiright_Content", None),
    ("div", "innerBg", None),
    ("div", "content", None),
    ("div", "wenzhang", None),
    ("div", "pages_content", None),
    ("div", "UCAP-CONTENT", None),
    ("div", "detail", None),
    ("div", "article", None),
    ("div", "main-content", None),
    ("div", "text", None),
    ("div", "con", None),
    ("div", None, "content"),
    ("div", None, "main"),
    ("div", None, "article"),
    ("article", None, None),
]


def fetch_external_content(url):
    """尝试获取外部链接的正文内容"""
    try:
        r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = "utf-8"
    except Exception as e:
        log(f"    [WARN] 外部链接请求失败: {e}")
        return ""

    soup = BeautifulSoup(r.text, "html.parser")

    # 移除 script/style 标签
    for tag in soup(["script", "style", "nav", "header", "footer", "aside"]):
        tag.decompose()

    # 尝试各种正文容器
    content_div = None
    for tag, cls, id_ in CONTENT_SELECTORS:
        if cls:
            content_div = soup.find(tag, class_=cls)
        elif id_:
            content_div = soup.find(tag, id=id_)
        else:
            content_div = soup.find(tag)
        if content_div:
            break

    if not content_div:
        # 兜底：取 body 中所有 <p> 文本
        body = soup.find("body")
        if body:
            paras = [p.get_text("", strip=True) for p in body.find_all("p") if len(p.get_text("", strip=True)) > 5]
            content_div = body  # 用于下面的提取逻辑

    if content_div:
        # 简单可靠：取容器内所有段落文本
        paras = content_div.find_all("p")
        if paras:
            parts = [p.get_text("", strip=True) for p in paras if len(p.get_text("", strip=True)) > 5]
        else:
            # 无<p>标签，直接用换行分割全文
            text = content_div.get_text("", strip=True)
            parts = [s.strip() for s in text.split("\n") if len(s.strip()) > 5]
        # 过滤掉footer/版权等非正文行
        clean_parts = []
        for p in parts:
            if re.match(r'^(版权所有|ICP备案|网站标识码|京公网安备|网站声明|网站地图|联系我们|手机版|中国政府网|国务院部门|打印|字号|[\[\]【】大中小])', p):
                continue
            if '版权所有' in p and 'ICP备案' in p:
                continue
            if '来源：' in p and len(p) < 30:
                continue
            if p in clean_parts:
                continue
            clean_parts.append(p)
        if clean_parts:
            content = "\n\n".join(clean_parts)
            if len(content) > 50:
                return content

    # 兜底：获取所有非空 <p> 文本（任何位置）
    paras = [p.get_text("", strip=True) for p in soup.find_all("p") if len(p.get_text("", strip=True)) > 15]
    if paras:
        return "\n\n".join(paras)

    return ""


def parse_list_page(html):
    """从列表页解析标题、日期、URL"""
    items = []
    soup = BeautifulSoup(html, "html.parser")
    con = soup.find("div", class_="con")
    if not con:
        return items
    for ul in con.find_all("ul"):
        li_title = ul.find("li", class_="title")
        if not li_title or not li_title.find("a"):
            continue
        a = li_title.find("a")
        li_time = ul.find("li", class_="time")
        title = a.get("title") or a.get_text(strip=True)
        href = a.get("href", "")
        date_str = li_time.get_text(strip=True) if li_time else ""
        if title and href:
            items.append({
                "url": urljoin(BASE, href),
                "title": title.strip(),
                "date": date_str,
            })
    return items


def import_to_db(record):
    try:
        db = sqlite3.connect(DB_PATH, timeout=10)
        title = (record.get("title") or "")[:500]
        page_url = (record.get("page_url") or "")[:1000]
        content = record.get("content") or ""
        publish_date = (record.get("publish_date") or "")[:20]
        site_name = (record.get("site_name") or "unknown")[:100]
        attachments_str = json.dumps(record.get("attachments") or [], ensure_ascii=False)

        old_rowids = db.execute("SELECT rowid FROM gov_raw WHERE page_url = ?", (page_url,)).fetchall()
        for (rid,) in old_rowids:
            db.execute("DELETE FROM gov_search WHERE rowid = ?", (rid,))

        db.execute(
            "INSERT OR REPLACE INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments, script_name) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?, 'crawl_xiangfen_hb.py')",
            (title, page_url, content, publish_date, site_name, page_url, attachments_str)
        )
        new_rowid = db.execute("SELECT last_insert_rowid()").fetchone()[0]
        summary = content[:500] if content else title[:500]
        # 2026-09-22: 先提交 gov_raw —— 库上触发器已维护 FTS，下面这条手动写入会因
        #   rowid 重复而 IntegrityError；不先 commit 会把 gov_raw 那条一并回滚（静默丢数据）
        db.commit()
        db.execute("INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)",
                   (new_rowid, title, site_name, summary))
        db.commit()
        db.close()
        log(f"  ✅ {title[:30]}")
        return True
    except Exception as e:
        log(f"  [ERR] DB import failed: {e}")
        return False


def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--pages", type=int, default=5,
                        help="爬取前N页（共16页，每页约20条）")
    args = parser.parse_args()

    total_pages = min(args.pages, TOTAL_PAGES)
    count = 0

    for page in range(1, total_pages + 1):
        page_url = f"{BASE}/channels/4322.html" if page == 1 else f"{BASE}/channels/4322_{page}.html"
        log(f"📄 列表[page={page}]: {page_url}")
        html = fetch(page_url)
        if not html:
            break
        items = parse_list_page(html)
        if not items:
            log(f"  → 无更多数据")
            break
        log(f"  → {len(items)} 条")

        for idx, item in enumerate(items, 1):
            url = item["url"]
            log(f"  ({idx}/{len(items)}) {item['title'][:40]}")

            # 判断是否本地链接
            is_local = "xiangfen.gov.cn" in url

            if not is_local:
                # 外部链接：尝试获取正文
                content = fetch_external_content(url)
                if not content or len(content) < 50:
                    content = f"[转载] 原文链接：{url}"
                record = {
                    "title": item["title"],
                    "page_url": url,
                    "publish_date": item["date"],
                    "content": content,
                    "attachments": [],
                    "site_name": SITE,
                    "column": COLUMN,
                    "province": PROVINCE,
                }
            else:
                # 本地链接：尝试获取正文
                detail_html = fetch(url)
                if not detail_html or "404" in detail_html[:50] or "Page Not Found" in detail_html:
                    content = f"[本地链接已失效] 原文链接：{url}"
                else:
                    soup = BeautifulSoup(detail_html, "html.parser")
                    content_div = soup.find("div", class_="content")
                    if content_div:
                        parts = [p.get_text("", strip=True) for p in content_div.find_all(["p", "div"], recursive=False)]
                        content = "\n\n".join(p for p in parts if len(p) > 3)
                    else:
                        content = f"[无法解析正文] 原文链接：{url}"

                record = {
                    "title": item["title"],
                    "page_url": url,
                    "publish_date": item["date"],
                    "content": content,
                    "attachments": [],
                    "site_name": SITE,
                    "column": COLUMN,
                    "province": PROVINCE,
                }

            ok = import_to_db(record)
            if ok:
                count += 1
            time.sleep(0.5)

    log(f"\n✅ {SITE} 爬取完成，共入库 {count} 条")


if __name__ == "__main__":
    main()