#!/usr/bin/env python3
"""
邢台市生态环境局 - 柏乡县分局 爬虫
CMS: 邢台市生态环境局站群 (html/{cid}/ 目录式)
列表: /html/1483/ 首页列表 (无分页, 全量在首页)
详情: /html/1483/{yyyy-mm-dd}/content-{id}.html, 标题 div.intr_title, 正文 div.conTxt
用法: python3 crawl_stj_xingtai_bx.py [--pages=N]  (默认 1 页, 首页即全量)
"""
import os, sys, re, time, sqlite3, requests
from datetime import date
from urllib.parse import urljoin
from bs4 import BeautifulSoup

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "邢台市生态环境局-柏乡县分局"
BASE = "http://stj.xingtai.gov.cn"
LIST_URL = "http://stj.xingtai.gov.cn/html/1483/"
CID = "1483"
CUTOFF_DATE = date(2023, 8, 15)
MAX_PAGES_DEFAULT = 1

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
    "Referer": LIST_URL,
}


def fetch_list_page(pn):
    """Fetch list page. Items = content links under /html/1483/."""
    if pn > 1:
        return []
    r = requests.get(LIST_URL, headers=HEADERS, timeout=30)
    r.encoding = "utf-8"
    soup = BeautifulSoup(r.text, "lxml")
    items = []
    for a in soup.find_all("a", href=True):
        href = a["href"]
        # 详情链接模式: /html/1483/{date}/content-{id}.html
        m = re.search(rf"/html/{CID}/(\d{{4}}-\d{{2}}-\d{{2}})/content-\d+\.html", href)
        if not m:
            continue
        full = urljoin(BASE, href)
        title = a.get_text(strip=True)
        pub_date = m.group(1)
        if full and title:
            items.append({"url": full, "title": title, "pub_date": pub_date})
    # 去重保序
    seen = set()
    uniq = []
    for it in items:
        if it["url"] not in seen:
            seen.add(it["url"])
            uniq.append(it)
    return uniq


def fetch_detail(url):
    """Fetch detail page: return (full_title, content_html)."""
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "lxml")
        title = ""
        t = soup.find("div", class_="intr_title")
        if t:
            title = t.get_text(strip=True)
        # 正文容器
        content_div = soup.find("div", class_="conTxt")
        if not content_div:
            return title, ""
        # 去脚本/样式
        for tag in content_div.find_all(["script", "style"]):
            tag.decompose()
        # 链接/图片绝对化
        for a in content_div.find_all("a", href=True):
            href = a["href"]
            if href and not href.startswith("http"):
                a["href"] = urljoin(BASE, href)
        for img in content_div.find_all("img"):
            src = img.get("src", "")
            if src and not src.startswith("http"):
                img["src"] = urljoin(BASE, src)
        return title, str(content_div)
    except Exception as e:
        print(f"  [detail] {url} ERR {e}", file=sys.stderr)
        return "", ""


def main():
    max_pages = MAX_PAGES_DEFAULT
    args = sys.argv[1:]
    for i, a in enumerate(args):
        if a == "--pages" and i + 1 < len(args):
            max_pages = int(args[i + 1])
        elif a.startswith("--pages="):
            max_pages = int(a.split("=")[1])

    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    all_new = 0
    all_skip = 0

    for page in range(1, max_pages + 1):
        items = fetch_list_page(page)
        if not items:
            print(f"  第{page}页: 空 -> 结束", file=sys.stderr)
            break
        print(f"  第{page}页: {len(items)} 条", file=sys.stderr)
        for it in items:
            pub = it["pub_date"]
            if pub and pub < CUTOFF_DATE.strftime("%Y-%m-%d"):
                print(f"  已达截断日 {CUTOFF_DATE} (当前 {pub})，停止", file=sys.stderr)
                break
            url = it["url"]
            c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (url,))
            if c.fetchone():
                all_skip += 1
                continue
            detail_title, content = fetch_detail(url)
            if not content or len(content) < 20:
                print(f"    ! 空正文: {it['title'][:40]}", file=sys.stderr)
                all_skip += 1
                continue
            title = detail_title or it["title"]
            pub_date = pub or ""
            c.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name, page_url, title, content, publish_date, source_url, status, script_name) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
                (SITE_NAME, url, title[:500], content, pub_date[:20], url, "done", "crawl_stj_xingtai_bx.py"),
            )
            if c.rowcount > 0:
                all_new += 1
                print(f"    + {title[:50]} ({pub_date}) body:{len(content)}B", file=sys.stderr)
            else:
                all_skip += 1
            conn.commit()
            time.sleep(0.3)

    print(f"\n[{SITE_NAME}] 完成! 新增 {all_new} 条, 跳过 {all_skip} 条", file=sys.stderr)
    if all_new > 0:
        c.execute("""INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary)
                      SELECT rowid, title, site_name, substr(content,1,500) FROM gov_raw
                      WHERE site_name=? AND rowid NOT IN (SELECT rowid FROM gov_search)""", (SITE_NAME,))
        conn.commit()
    conn.close()


if __name__ == "__main__":
    main()