#!/usr/bin/env python3
"""唐山中浩化工-新闻动态 爬虫 (cookie反爬 + 隐藏div列表)"""
import re, sys, sqlite3
import requests
from bs4 import BeautifulSoup
import os

SITE_NAME = "唐山中浩-新闻动态"
BASE = "https://www.tszhcc.com"
LIST = "/news/3/"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
MAX_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}


def get_session():
    """处理cookie反爬"""
    s = requests.Session()
    s.headers.update(HEADERS)
    r = s.get(BASE + LIST, timeout=20)
    md5 = re.search(r"md5_hash = '(.+?)'", r.text)
    if md5:
        s.cookies.set("visitToken", md5.group(1), domain=".tszhcc.com")
    return s


def fetch(session, url):
    try:
        r = session.get(url, timeout=30)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"  ⚠️ 请求失败 {url}: {e}")
        return None


def parse_list(html):
    soup = BeautifulSoup(html, "lxml")
    items = []
    news_list = soup.find("div", class_="xypg-left-news")
    if not news_list:
        return items
    for li in news_list.find_all("li"):
        a = li.find("a")
        if not a:
            continue
        href = a.get("href", "")
        title = a.get("title") or ""
        if not title:
            h3 = a.find("h3")
            if h3:
                title = h3.get_text(strip=True)
        date_div = a.find("div", class_="date")
        date_str = date_div.get_text(strip=True) if date_div else ""
        if href and title:
            items.append({"title": title, "url": href, "date": date_str[:10]})
    return items


def parse_detail(html):
    soup = BeautifulSoup(html, "lxml")
    # 标题
    h3 = soup.find("h3") or soup.find("h1") or soup.find("h2")
    title = h3.get_text(strip=True) if h3 else ""

    # 日期
    date = ""
    date_div = soup.find("div", class_="date")
    if date_div:
        date = date_div.get_text(strip=True)[:10]
    if not date:
        m = re.search(r"(\d{4}-\d{2}-\d{2})", str(soup))
        if m:
            date = m.group(1)

    # 正文
    content = ""
    for cls in ["xypg-detail-content", "news-content", "content", "article-content"]:
        div = soup.find("div", class_=cls)
        if div:
            for tag in div(["script", "style", "iframe"]):
                tag.decompose()
            content = str(div)
            break
    if not content:
        best, best_count = None, 0
        for div in soup.find_all("div"):
            ps = div.find_all("p")
            if len(ps) > best_count:
                best, best_count = div, len(ps)
        if best:
            for tag in best(["script", "style", "iframe"]):
                tag.decompose()
            content = str(best)

    content = re.sub(r'\s*style="[^"]*"', "", content)
    content = re.sub(r'\s*class="[^"]*"', "", content)
    return title, date, content


def run(max_pages=None):
    if max_pages is None:
        max_pages = MAX_PAGES

    db = sqlite3.connect(SEARCH_DB, timeout=60)
    known = set(
        r[0] for r in db.execute(
            "SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)
        ).fetchall()
    )
    db.close()

    session = get_session()

    all_items = []
    for page in range(1, max_pages + 1):
        if page == 1:
            url = BASE + LIST
        else:
            url = f"{BASE}{LIST}p{page}.html"
        print(f"  📄 第 {page} 页...", end=" ")
        html = fetch(session, url)
        if not html:
            print("❌")
            break

        items = parse_list(html)
        new_items = [it for it in items if it["url"] not in known]
        print(f"✅ {len(items)} 条 (新增 {len(new_items)})")
        if not items:
            break
        all_items.extend(new_items)

    if not all_items:
        print("  ⏭ 无新数据")
        return

    print(f"  🔍 获取 {len(all_items)} 条详情...")
    results = []
    for i, item in enumerate(all_items):
        html = fetch(session, item["url"])
        if not html:
            continue
        title, date_str, content = parse_detail(html)
        if not title:
            title = item["title"]
        if not date_str:
            date_str = item["date"]
        results.append({
            "site_name": SITE_NAME,
            "source_url": item["url"][:500],
            "page_url": item["url"],
            "title": title[:500],
            "publish_date": date_str[:10] if date_str else "",
            "summary": title[:500],
            "content": content,
            "status": "active",
            "category": "",
            "tags": "",
        })
        if (i + 1) % 10 == 0:
            print(f"    [{i+1}/{len(all_items)}]")

    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in results:
        try:
            db.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                (
                    item["site_name"][:200], item["source_url"], item["page_url"],
                    item["title"], item["publish_date"], item["summary"],
                    item["content"], item["status"], item["category"], item["tags"],
                ),
            )
            if db.total_changes > 0: ok += 1
            else: skip += 1
        except: skip += 1

    db.commit()
    db.execute(
        "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,),
    )
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}, FTS已同步")


if __name__ == "__main__":
    max_p = int(sys.argv[1]) if len(sys.argv) > 1 and sys.argv[1].isdigit() else None
    run(max_p)