#!/usr/bin/env python3
"""
crawl_pingba_tzgg.py - 平坝区人民政府-通知公告
===============================================
站点: https://www.pingba.gov.cn/xwzx/tzgg/
CMS:  Custom CMS + TRS Editor
列表: ul.NewsList > li > a[title][href] + span (15条/页, 静态分页)
        index.html → index_1.html → index_2.html ... index_18.html
详情: div.Article_Con.aBox > div.trs_editor_view, meta ArticleTitle/PubDate

用法:
  python3 crawl_pingba_tzgg.py              # 默认5页
  python3 crawl_pingba_tzgg.py --pages 5    # 指定页数
  python3 crawl_pingba_tzgg.py --dry-run    # 仅预览
"""

import sys, os, re, json, requests, time, subprocess
from datetime import datetime
from bs4 import BeautifulSoup

BASE_URL = "https://www.pingba.gov.cn"
LIST_PATH = "/xwzx/tzgg"
DEFAULT_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
}

requests.packages.urllib3.disable_warnings()


def log(msg):
    ts = datetime.now().strftime("%H:%M:%S")
    print(f"[{ts}] {msg}", flush=True)


def clean_title(title):
    if not title:
        return ""
    title = title.replace("·", "·").replace(" ", " ").replace("&", "&")
    return title.strip("·. \t\n\r")


def fetch_page(url, timeout=20):
    try:
        r = requests.get(url, headers=HEADERS, timeout=timeout, verify=False)
        if r.status_code == 200:
            r.encoding = "utf-8"
            return r.text
        return None
    except Exception as e:
        log(f"请求失败 {url}: {e}")
        return None


def extract_list_items(html):
    items = []
    soup = BeautifulSoup(html, "html.parser")
    ul = soup.find("ul", class_="NewsList")
    if not ul:
        log("  ⚠ 未找到 NewsList")
        return items
    for li in ul.find_all("li", recursive=True):
        a = li.find("a")
        date_span = li.find("span")
        if a and date_span:
            href = a.get("href", "").strip()
            title = a.get("title", "").strip() or a.get_text(strip=True)
            date_text = date_span.get_text(strip=True)
            if href and title:
                if href.startswith("/"):
                    href = BASE_URL + href
                elif not href.startswith("http"):
                    href = BASE_URL + "/" + href.lstrip("/")
                items.append({"title": clean_title(title), "url": href, "date": date_text})
    return items


def fetch_list(pages=DEFAULT_PAGES):
    all_items = []
    for page_idx in range(1, pages + 1):
        if page_idx == 1:
            url = f"{BASE_URL}{LIST_PATH}/index.html"
        else:
            url = f"{BASE_URL}{LIST_PATH}/index_{page_idx-1}.html"
        log(f"获取第{page_idx}页...")
        html = fetch_page(url, timeout=15)
        if not html:
            log(f"  ⚠ 第{page_idx}页获取失败，停止")
            break
        items = extract_list_items(html)
        if not items:
            log(f"  第{page_idx}页: 0条，停止")
            break
        log(f"  第{page_idx}页: {len(items)} 条")
        all_items.extend(items)

    # Deduplicate by URL
    seen = set()
    unique = []
    for item in all_items:
        if item["url"] not in seen:
            seen.add(item["url"])
            unique.append(item)
    log(f"去重后共 {len(unique)} 条")
    return unique


def fetch_detail(url):
    html = fetch_page(url, timeout=20)
    if not html:
        return "", "", ""

    soup = BeautifulSoup(html, "html.parser")

    # Title: meta ArticleTitle (h1 not always present)
    meta_t = soup.find("meta", attrs={"name": "ArticleTitle"})
    title = meta_t["content"].strip() if meta_t and meta_t.get("content") else ""
    if not title:
        h1 = soup.find("h1")
        if h1:
            title = h1.get_text(strip=True)
    title = clean_title(title)

    # Date: meta PubDate
    meta_d = soup.find("meta", attrs={"name": "PubDate"})
    date_text = meta_d["content"].strip() if meta_d and meta_d.get("content") else ""
    if not date_text:
        m = re.search(r"发布时间[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})", html)
        if m:
            date_text = m.group(1)
    dm = re.search(r"(\d{4})[-/](\d{1,2})[-/](\d{1,2})", str(date_text))
    if dm:
        date_text = f"{dm.group(1)}-{dm.group(2).zfill(2)}-{dm.group(3).zfill(2)}"

    # Content: div.Article_Con or div.trs_editor_view
    content_div = soup.find("div", class_="Article_Con")
    if not content_div:
        content_div = soup.find("div", class_=lambda c: c and "trs_editor_view" in str(c))
    if not content_div:
        content_div = soup.find("div", class_=lambda c: c and "ContentPageBox" in str(c))

    content_html = str(content_div) if content_div else ""
    return title, date_text, content_html


def clean_html(content_html):
    if not content_html:
        return ""
    soup = BeautifulSoup(content_html, "html.parser")
    for tag in soup.find_all(["style", "script"]):
        tag.decompose()
    for br in soup.find_all("br"):
        br.replace_with("\n")
    for tag in soup.find_all(["o:p"]):
        tag.decompose()

    # Unwrap nested container divs
    changed = True
    while changed:
        changed = False
        for tag in list(soup.find_all(["div", "p"])):
            kids = [c for c in tag.children if not (c.name is None and not str(c).strip())]
            if kids and all(c.name == "p" for c in kids):
                tag.unwrap()
                changed = True
                break

    def extract(element):
        parts = []
        for child in element.children:
            if child.name is None:
                text = str(child).strip()
                if text:
                    parts.append(text)
            elif child.name == "table":
                parts.append(str(child))
            elif child.name in ("p", "div", "td", "th", "li", "section",
                                "h1", "h2", "h3", "h4", "h5", "h6"):
                inner = extract(child)
                if inner.strip():
                    parts.append(inner)
            elif child.name in ("span", "font", "b", "i", "strong", "em", "a",
                                "u", "sub", "sup", "small", "label"):
                inner = extract(child)
                if inner.strip():
                    parts.append(inner)
            else:
                text = child.get_text(strip=True)
                if text:
                    parts.append(text)
        return "\n\n".join(p for p in parts if p.strip())

    result = extract(soup)

    # Embed images/attachments for short content
    if len(result) < 500:
        extra = []
        imgs = []
        for img in soup.find_all("img"):
            src = img.get("src", "").strip()
            if src and not src.startswith("data:"):
                if src.startswith("/"):
                    src = BASE_URL + src
                elif not src.startswith("http"):
                    src = BASE_URL + "/" + src.lstrip("/")
                alt = img.get("alt", "").strip()
                imgs.append(f"[图片: {alt}] {src}" if alt else f"[图片] {src}")
        if imgs:
            extra.append("[页面图片]")
            extra.extend(imgs)

        atts = []
        for a in soup.find_all("a"):
            href = a.get("href", "").strip()
            if any(href.lower().endswith(ext) for ext in [".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar"]):
                text = a.get_text(strip=True) or href.split("/")[-1]
                if href.startswith("/"):
                    href = BASE_URL + href
                elif not href.startswith("http"):
                    href = BASE_URL + "/" + href.lstrip("/")
                atts.append(f"[附件: {text}] {href}")
        if atts:
            extra.append("[页面附件]")
            extra.extend(atts)

        if extra:
            result = result + "\n\n" + "\n".join(extra)

    result = re.sub(r'\n{3,}', '\n\n', result)
    result = re.sub(r'\s+\n', '\n', result)
    result = re.sub(r'\n\s+', '\n', result)
    return result.strip()


def push_to_searchdb(items, source_name="pingba_tzgg", group_name="贵州省",
                     industry="政府公告"):
    if not items:
        log("无数据可入库")
        return
    db_path = "/mnt/data/search.db"
    inserted = 0
    for item in items:
        title = item.get("title", "")
        date = item.get("date", "")
        url = item.get("url", "")
        content = item.get("content", "")
        if not title or not url:
            continue
        esc = lambda s: s.replace("'", "''") if s else ""
        sql = (
            "INSERT OR IGNORE INTO gov_raw "
            "(site_name, source_url, page_url, title, publish_date, summary, content, group_name, industry) "
            "VALUES ("
            f"'{esc(source_name)}', '{esc(url)}', '{esc(url)}', "
            f"'{esc(title)}', '{esc(date)}', '{esc(title)}', "
            f"'{esc(content)}', '{esc(group_name)}', '{esc(industry)}'"
            ");"
        )
        try:
            r = subprocess.run(
                ["sqlite3", "-cmd", ".timeout 60000", db_path, f"PRAGMA busy_timeout=15000; {sql}"],
                capture_output=True, text=True, timeout=30
            )
            if r.returncode == 0:
                inserted += 1
        except Exception as e:
            log(f"入库失败: {e}")
    log(f"入库完成: {inserted}/{len(items)} 条")

    # FTS sync
    log("同步FTS...")
    fts_sql = (
        "INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary "
        "FROM gov_raw r "
        f"WHERE r.site_name='{esc(source_name)}' "
        "AND NOT EXISTS (SELECT 1 FROM gov_search s WHERE s.rowid=r.id);"
    )
    try:
        subprocess.run(["sqlite3", "-cmd", ".timeout 60000", db_path, fts_sql], capture_output=True, text=True, timeout=30)
    except Exception as e:
        log(f"FTS同步失败: {e}")


def main():
    import argparse
    parser = argparse.ArgumentParser(description="平坝区-通知公告爬虫")
    parser.add_argument("--pages", type=int, default=DEFAULT_PAGES, help="爬取页数")
    parser.add_argument("--dry-run", action="store_true", help="仅预览")
    args = parser.parse_args()
    pages = args.pages
    log(f"🚀 爬虫启动: pingba_tzgg, pages={pages}, dry_run={args.dry_run}")

    all_items = fetch_list(pages=pages)
    log(f"列表共 {len(all_items)} 条")

    if args.dry_run:
        for item in all_items[:5]:
            print(f"  [{item['date']}] {item['title'][:50]}")
            print(f"  URL: {item['url']}")
        print(f"  ... 共 {len(all_items)} 条")
        log("✅ 预览完成")
        return

    for i, item in enumerate(all_items):
        log(f"  [{i+1}/{len(all_items)}] {item['title'][:40]}...")
        dt, dd, dc = fetch_detail(item["url"])
        item["title"] = clean_title(dt or item["title"])
        item["date"] = dd or item["date"]
        item["content"] = clean_html(dc) if dc else ""
        time.sleep(0.3)

    log(f"\n=== 爬取完成: {len(all_items)} 条 ===")
    valid = [it for it in all_items if it.get("content")]
    log(f"含正文: {len(valid)} 条 / 空正文: {len(all_items) - len(valid)} 条")
    for item in valid[:3]:
        print(f"  [{item['date']}] {item['title'][:50]}")
        print(f"  URL: {item['url']}")
        print(f"  正文: {len(item.get('content', ''))} chars")
        print()

    if all_items:
        push_to_searchdb(all_items)
    log("✅ 全部完成")


if __name__ == "__main__":
    main()