#!/usr/bin/env python3
import os
"""万华化学 — 环评公示 爬虫 (Playwright 分页版本)
用法:
  python3 crawl_whchem.py              # 全量爬取
  python3 crawl_whchem.py 1            # 只爬第1页 (测试)
  python3 crawl_whchem.py --incremental # 增量模式
"""
import sys, re, sqlite3, time, json
import requests
from datetime import datetime, timedelta
from urllib.parse import urljoin

try:
    from playwright.sync_api import sync_playwright
except ImportError:
    print("需要 playwright: pip install playwright && playwright install chromium")
    sys.exit(1)

SITE_NAME = "万华化学"
BASE = "https://www.whchem.com"
LIST_URL = f"{BASE}/column/123/"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
MAX_PAGES = 23

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "Chrome/125.0.0.0 Safari/537.36",
}

CUTOFF = (datetime.now() - timedelta(days=365 * 3)).strftime("%Y-%m-%d")


def get_detail(url):
    """Fetch detail page with requests (plain HTML)."""
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        return r.text
    except:
        return None


def parse_detail(html, fallback_title=""):
    """Extract title, date, content from detail page."""
    title = ""
    # Try big_title class first
    tm = re.search(r'<h1[^>]*class="big_title[^"]*"[^>]*>(.*?)</h1>', html, re.DOTALL)
    if tm:
        title = re.sub(r'<[^>]+>', '', tm.group(1)).strip()
    if not title:
        tm = re.search(r'<h1[^>]*>(.*?)</h1>', html, re.DOTALL)
        if tm:
            title = re.sub(r'<[^>]+>', '', tm.group(1)).strip()
    if not title:
        title = fallback_title

    # Date - try pubTime (millisecond timestamp in JS)
    date_str = ""
    dm = re.search(r'pubTime=(\d{13})', html)
    if dm:
        ts = int(dm.group(1)) / 1000
        date_str = datetime.fromtimestamp(ts).strftime("%Y-%m-%d")
    if not date_str:
        dm = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', html)
        if dm:
            date_str = dm.group(1).replace("/", "-")
    if not date_str:
        for p in [r'发布时间[：:]\s*(\d{4}-\d{1,2}-\d{1,2})',
                  r'发布日期[：:]\s*(\d{4}-\d{1,2}-\d{1,2})',
                  r'<em class="date">(\d{4}-\d{1,2}-\d{1,2})']:
            m = re.search(p, html)
            if m:
                date_str = m.group(1)
                break

    # Content - try article class
    content = ""
    for cls in ["article fnt_16", "article"]:
        cm = re.search(f'<div[^>]*class="{cls}"[^>]*>(.*?)</div>\\s*</div>\\s*</div>', html, re.DOTALL)
        if cm:
            content = cm.group(1).strip()
            break
    if not content:
        cm = re.search(r'<div[^>]*class="article[^"]*"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
        if cm:
            content = cm.group(1).strip()
    if not content:
        # Fallback to ty-content (金昌站模式)
        cm = re.search(r'<div[^>]*class="ty-content"[^>]*>', html)
        if cm:
            cm = re.search(r'<div[^>]*class="ty-content"[^>]*>(.*?)</div>\s*<!--\s*正文结束', html, re.DOTALL)
            if cm:
                content = cm.group(1).strip()

    if content:
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL | re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL | re.I)
        content = re.sub(r'<p>\s*<br\s*/?>\s*</p>', '', content)
        content = re.sub(r'\s*style="[^"]*"', '', content)
        content = re.sub(r'<!--.*?-->', '', content)
        content = content.strip()

    return title, date_str, content


def scrape_list(pw_page, page_num):
    """Scrape article list from a page using Playwright."""
    hash_url = f"{LIST_URL}#c_portalResGroupNewsNew_list-16139644893080064-{page_num}"
    print(f"    Loading page {page_num}: {hash_url}")

    if page_num == 1:
        pw_page.goto(LIST_URL, wait_until="networkidle", timeout=30000)
    else:
        pw_page.goto(hash_url, wait_until="networkidle", timeout=30000)

    time.sleep(2)  # Let JS rendering settle

    # Try to get the list component content
    # The component might be in the #c_portalResGroupNewsNew_list-16139644893080064 div
    articles = []
    try:
        # Method 1: Look for direct article links in the list
        links = pw_page.query_selector_all("#c_portalResGroupNewsNew_list-16139644893080064 .p_news a[href*='/cmscontent/']")
        if not links:
            # Method 2: Look for component div more broadly
            links = pw_page.query_selector_all("#c_portalResGroupNewsNew_list-16139644893080064 a[href*='/cmscontent/']")
        if not links:
            # Method 3: Look for any article links in the main content area
            links = pw_page.query_selector_all("a[href*='/cmscontent/']")

        for link in links:
            href = link.get_attribute("href") or ""
            title = link.inner_text().strip()
            if not href or not title:
                continue
            if title in ("法律声明", "隐私保护", "网站声明", "关于我们", "联系我们"):
                continue
            if not href.startswith("http"):
                href = urljoin(BASE, href)
            articles.append({"url": href, "title": title})

    except Exception as e:
        print(f"    Error scraping page {page_num}: {e}")

    return articles


def push_to_db(db, records):
    inserted = 0
    skipped = 0
    for item in records:
        try:
            db.execute(
                """INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags)
                VALUES (?,?,?,?,?,?,?,?,?,?)""",
                (
                    item["site_name"][:200],
                    item["source_url"],
                    item["page_url"],
                    item["title"],
                    item["publish_date"],
                    item["summary"],
                    item["content"],
                    item["status"],
                    item["category"],
                    item["tags"],
                ),
            )
            if db.total_changes > 0:
                inserted += 1
            else:
                skipped += 1
        except:
            skipped += 1
    return inserted, skipped


def run(max_pages=None, incremental=False):
    if max_pages is None:
        max_pages = MAX_PAGES

    db = sqlite3.connect(SEARCH_DB)
    db.execute("PRAGMA journal_mode=WAL")

    known = set(
        r[0]
        for r in db.execute(
            "SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)
        ).fetchall()
    )
    print(f"  Known in DB: {len(known)}")

    all_items = []

    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True, args=["--no-sandbox", "--disable-setuid-sandbox"])
        context = browser.new_context(
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"
        )
        page = context.new_page()

        try:
            for pg in range(1, max_pages + 1):
                articles = scrape_list(page, pg)
                # Filter out duplicates and known URLs
                seen_urls = set()
                for art in articles:
                    if art["url"] in seen_urls or art["url"] in known:
                        continue
                    seen_urls.add(art["url"])
                print(f"    Found {len(articles)} articles, new: {len(seen_urls)}")

                if not articles:
                    print(f"    No articles on page {pg}, stopping")
                    break

                all_items.extend(
                    {"url": art["url"], "title": art["title"]}
                    for art in articles
                    if art["url"] not in known
                )

                if incremental and len(articles) > 0:
                    # Check if all articles on this page are known
                    if all(art["url"] in known for art in articles):
                        print(f"    All articles on page {pg} already in DB, stopping")
                        break

                # If we got fewer articles than expected, it's the last page
                if len(articles) < 8:
                    print(f"    Last page reached ({len(articles)} < 8)")
                    break

        finally:
            browser.close()

    if not all_items:
        print("  No new items to process")
        db.close()
        return

    print(f"  Fetching details for {len(all_items)} articles...")

    results = []
    for i, item in enumerate(all_items):
        html = get_detail(item["url"])
        if not html:
            print(f"    [{i+1}/{len(all_items)}] SKIP (fetch failed)")
            continue
        title, date_str, content = parse_detail(html, item["title"])
        results.append(
            {
                "site_name": SITE_NAME,
                "source_url": item["url"][:500],
                "page_url": item["url"],
                "title": (title or item["title"])[:500],
                "publish_date": date_str[:10] if date_str else "",
                "summary": (title or item["title"])[:500],
                "content": content,
                "status": "active",
                "category": "",
                "tags": "",
            }
        )
        if (i + 1) % 10 == 0:
            print(f"    [{i+1}/{len(all_items)}]")

    # Filter by date
    results = [r for r in results if (r["publish_date"] and r["publish_date"] >= CUTOFF) or not r["publish_date"]]

    inserted, skipped = push_to_db(db, results)
    db.commit()

    # Update FTS
    db.execute(
        """INSERT INTO gov_search(rowid,title,site_name,summary)
        SELECT r.id, r.title, r.site_name, r.summary
        FROM gov_raw r
        WHERE r.id NOT IN (SELECT rowid FROM gov_search)
        AND r.site_name=?""",
        (SITE_NAME,),
    )
    db.commit()
    db.close()

    print(f"  Done: new={inserted}, skip={skipped}, date_filtered={len(results)}")


if __name__ == "__main__":
    incremental = "--incremental" in sys.argv
    mp = None
    for a in sys.argv[1:]:
        if a.isdigit():
            mp = int(a)
            break
        if a == "--test":
            mp = 1
            break

    run(max_pages=mp, incremental=incremental)