#!/usr/bin/env python3
"""
柏乡县人民政府 - 生态环境分局信息公开专版 爬虫
CMS: Topway CMS (orglist.jsp 政务公开目录)
列表: orglist.jsp?orgCode=001001130&code=125 (20条/页, pn=1..170 分页)
详情: /single/{cid}/{id}.html, 标题 h1, 正文 div#content.detail-2
用法: python3 crawl_baixiang_sthjfj.py [--pages=N]  (默认 5 页)
"""
import os, sys, re, time, sqlite3, requests
from datetime import date
from urllib.parse import urljoin
from bs4 import BeautifulSoup

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "柏乡县-生态环境分局信息公开"
BASE = "https://www.baixiangxian.gov.cn"
LIST_URL = "https://www.baixiangxian.gov.cn/xxgk/list/orglist.jsp?orgCode=001001130&code=125"
ORG_CODE = "001001130"
CUTOFF_DATE = date(2023, 8, 15)
MAX_PAGES_DEFAULT = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
    "Referer": f"https://www.baixiangxian.gov.cn/xxgk/list/orglist.jsp?orgCode={ORG_CODE}&code=125",
}


def fetch_list_page(pn):
    """Fetch one orglist page. Returns list of dicts {url, title, pub_date}."""
    if pn == 1:
        url = LIST_URL
    else:
        url = f"{BASE}/xxgk/list/orglist.jsp?pn={pn}&orgCode={ORG_CODE}&code=125"
    r = requests.get(url, headers=HEADERS, timeout=30)
    r.encoding = "utf-8"
    soup = BeautifulSoup(r.text, "lxml")
    items = []
    ul = soup.find("ul", class_=re.compile(r"info-list-xxgk"))
    if not ul:
        return items
    for li in ul.find_all("li"):
        a = li.find("a")
        if not a:
            continue
        href = a.get("href", "")
        if not href:
            continue
        full = urljoin(BASE, href)
        # 标题: a 文本（可能截断带省略号，详情页再取完整标题）
        title = a.get_text(strip=True)
        # 日期: li 内 span/文本
        pub_date = ""
        m = re.search(r"(\d{4}-\d{2}-\d{2})", li.get_text())
        if m:
            pub_date = m.group(1)
        if full and title:
            items.append({"url": full, "title": title, "pub_date": pub_date})
    return items


def fetch_detail(url):
    """Fetch detail page: return (full_title, content_html)."""
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "lxml")
        h1 = soup.find("h1")
        title = h1.get_text(strip=True) if h1 else ""
        # 正文容器
        content_div = soup.find("div", id="content", class_="detail-2")
        if not content_div:
            content_div = soup.find("div", class_="detail-2")
        if not content_div:
            return title, ""
        # 去脚本/样式
        for tag in content_div.find_all(["script", "style"]):
            tag.decompose()
        # 链接/图片绝对化
        for a in content_div.find_all("a", href=True):
            href = a["href"]
            if href and not href.startswith("http"):
                a["href"] = urljoin(BASE, href)
        for img in content_div.find_all("img"):
            src = img.get("src", "")
            if src and not src.startswith("http"):
                img["src"] = urljoin(BASE, src)
        return title, str(content_div)
    except Exception as e:
        print(f"  [detail] {url} ERR {e}", file=sys.stderr)
        return "", ""


def main():
    max_pages = MAX_PAGES_DEFAULT
    args = sys.argv[1:]
    for i, a in enumerate(args):
        if a == "--pages" and i + 1 < len(args):
            max_pages = int(args[i + 1])
        elif a.startswith("--pages="):
            max_pages = int(a.split("=")[1])

    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    all_new = 0
    all_skip = 0
    reached_cutoff = False

    for page in range(1, max_pages + 1):
        items = fetch_list_page(page)
        if not items:
            print(f"  第{page}页: 空 -> 结束", file=sys.stderr)
            break
        print(f"  第{page}页: {len(items)} 条", file=sys.stderr)
        for it in items:
            pub = it["pub_date"]
            if pub and pub < CUTOFF_DATE.strftime("%Y-%m-%d"):
                reached_cutoff = True
                print(f"  已达截断日 {CUTOFF_DATE} (当前 {pub})，停止", file=sys.stderr)
                break
            url = it["url"]
            c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (url,))
            if c.fetchone():
                all_skip += 1
                continue
            detail_title, content = fetch_detail(url)
            if not content or len(content) < 20:
                print(f"    ! 空正文: {it['title'][:40]}", file=sys.stderr)
                all_skip += 1
                continue
            title = detail_title or it["title"]
            pub_date = pub or ""
            c.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name, page_url, title, content, publish_date, source_url, status, script_name) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
                (SITE_NAME, url, title[:500], content, pub_date[:20], url, "done", "crawl_baixiang_sthjfj.py"),
            )
            if c.rowcount > 0:
                all_new += 1
                print(f"    + {title[:50]} ({pub_date}) body:{len(content)}B", file=sys.stderr)
            else:
                all_skip += 1
            conn.commit()
            time.sleep(0.3)
        if reached_cutoff:
            break

    print(f"\n[{SITE_NAME}] 完成! 新增 {all_new} 条, 跳过 {all_skip} 条", file=sys.stderr)
    if all_new > 0:
        c.execute("""INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary)
                      SELECT rowid, title, site_name, substr(content,1,500) FROM gov_raw
                      WHERE site_name=? AND rowid NOT IN (SELECT rowid FROM gov_search)""", (SITE_NAME,))
        conn.commit()
    conn.close()


if __name__ == "__main__":
    main()