#!/usr/bin/env python3
"""
crawl_zibo_epb.py — 淄博市生态环境局·受理
=========================================
CMS: 淄博政务公开平台（同高新区同一CMS）
?open=fdzdgknr&nr_page=N&nr_per_page=10 分页
fdzdgknr_total 变量获取总数

列表：<td class="zfxxgk-list-title"><a> → title + <td class="zfxxgk-list-time"> date
详情：<div class="details-content" id="details-content"> 正文
      <meta name="ArticleTitle"> / <meta name="PubDate">

用法:
    python3 crawl_zibo_epb.py               # 全量（近3年）
    python3 crawl_zibo_epb.py 1             # 增量（仅最新页）
"""

import os
import re
import sys
import time
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed

DB_PATH = os.environ.get("SEARCH_DB", os.environ.get("GOV_DB_PATH", "/root/search.db"))

BASE_URL = "https://epb.zibo.gov.cn"
LIST_PATH = "/gongkai/channel_c_5f9fa491ab327f36e4c13066_n_1605682663.2363/"
SITE_NAME = "淄博市生态环境局"
SOURCE = "淄博市生态环境局-受理"
PER_PAGE = 10

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/120.0.0.0 Safari/537.36",
}

MAX_WORKERS = 20
MAX_PAGES = 200


def log(msg):
    print(f"[zibo_epb] {msg}")


def fetch_total_count():
    url = f"{BASE_URL}{LIST_PATH}?open=fdzdgknr&nr_page=1&nr_per_page={PER_PAGE}&_t={int(time.time())}"
    for attempt in range(4):
        try:
            resp = requests.get(url, headers=HEADERS, timeout=30)
            resp.encoding = "utf-8"
            html = resp.text
        except Exception as e:
            log(f"ERROR fetching first page (try {attempt+1}): {e}")
            time.sleep(2 + attempt * 2)
            continue
        m = re.search(r'fdzdgknr_total\s*=\s*parseInt\("(\d+)"\)', html)
        if m:
            return int(m.group(1))
        log(f"  total not found (try {attempt+1}), retrying")
        time.sleep(2 + attempt * 2)
    return 0


def fetch_list_page(page):
    url = f"{BASE_URL}{LIST_PATH}?open=fdzdgknr&nr_page={page}&nr_per_page={PER_PAGE}&_t={int(time.time())}"
    for attempt in range(4):
        try:
            resp = requests.get(url, headers=HEADERS, timeout=30)
            resp.encoding = "utf-8"
            html = resp.text
        except Exception as e:
            log(f"  ERROR page {page} (try {attempt+1}): {e}")
            time.sleep(2 + attempt * 2)
            continue
        soup = BeautifulSoup(html, "html.parser")
        rows = soup.select("td.zfxxgk-list-title")
        if rows:
            break
        log(f"  Page {page} empty rows (try {attempt+1}), retrying")
        time.sleep(2 + attempt * 2)
    else:
        return []

    dates = soup.select("td.zfxxgk-list-time")
    items = []
    for i, td in enumerate(rows):
        a = td.find("a")
        if not a or not a.get("href"):
            continue
        href = a["href"]
        title = a.get("title") or a.get_text(strip=True) or ""
        title = title.strip()
        date_str = dates[i].get_text(strip=True) if i < len(dates) else ""
        full_url = urljoin(BASE_URL, href)
        items.append((full_url, title, date_str))
    return items


def fetch_detail(url):
    try:
        resp = requests.get(url, headers=HEADERS, timeout=30)
        resp.encoding = "utf-8"
        html = resp.text
    except Exception as e:
        log(f"  ERROR {url}: {e}")
        return None, None, None
    soup = BeautifulSoup(html, "html.parser")
    meta_title = soup.find("meta", attrs={"name": "ArticleTitle"})
    title = meta_title["content"].strip() if meta_title and meta_title.get("content") else ""
    date_str = ""
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date and meta_date.get("content"):
        m = re.match(r"(\d{4}-\d{2}-\d{2})", meta_date["content"].strip())
        if m:
            date_str = m.group(1)
    content_div = soup.find("div", class_=lambda c: c and "details-content" in (c if isinstance(c, str) else " ".join(c)))
    if not content_div:
        content_div = soup.find("div", id="details-content")
    if content_div:
        # 清理干扰元素: 面包屑/元数据表/重复标题/字号条/分享/扫码/相关文章
        for sel in [".sk-breadcrumb", "table.details-info", "h1.details-title",
                    ".share-details-row", ".share", ".details-erweima", ".related-item",
                    "li.details-info-fontsize", "li.details-change-fontsize"]:
            for el in content_div.select(sel):
                el.decompose()
        # 删除"打印"按钮 (字号条/打印工具)
        for el in content_div.find_all(string=re.compile(r"打印")):
            node = el.find_parent(["a", "li", "span", "button"])
            if node:
                node.decompose()
        # 内层 details-content (真正正文) 优先; 外层清理后无内层则用外层
        inner_list = content_div.find_all("div", class_=lambda c: c and "details-content" in (c if isinstance(c, str) else " ".join(c)))
        if inner_list:
            content_div = inner_list[-1]
        content_html = str(content_div)
    else:
        content_html = ""
    if not content_html:
        for cls in ["zfxxgk-text", "nr_right", "content", "article"]:
            content_div = soup.find("div", class_=lambda c: c and cls in (c if isinstance(c, str) else " ".join(c)))
            if content_div:
                content_html = str(content_div)
                break
    if not content_html:
        log(f"  WARNING: no content for {url}")
        return None, date_str, title
    return content_html, date_str, title


def fetch_detail_batch(urls):
    results = {}
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex:
        fut_map = {ex.submit(fetch_detail, url): url for url in urls}
        for fut in as_completed(fut_map):
            url = fut_map[fut]
            try:
                results[url] = fut.result()
            except Exception as e:
                results[url] = (None, None, None)
    return results


def save_to_db(items):
    import sqlite3
    if not items:
        return 0
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted = 0
    for url, title, date_str, content_html in items:
        try:
            c.execute(
                """INSERT OR REPLACE INTO gov_raw (site_name, page_url, title, publish_date, content, category, script_name) VALUES (?, ?, ?, ?, ?, ?, 'crawl_zibo_epb.py')""",
                (SITE_NAME, url, title, date_str, content_html, SOURCE),
            )
            inserted += 1
        except Exception as e:
            log(f"  DB error for {url}: {e}")
    conn.commit()
    conn.close()
    return inserted


def check_existing(urls):
    """返回已入库的 URL 集合 (增量停止条件)"""
    import sqlite3
    if not urls:
        return set()
    conn = sqlite3.connect(DB_PATH, timeout=60)
    try:
        ph = ",".join("?" * len(urls))
        rows = conn.execute(
            f"SELECT page_url FROM gov_raw WHERE page_url IN ({ph})", urls
        ).fetchall()
        return {r[0] for r in rows}
    except Exception:
        return set()
    finally:
        conn.close()


def main():
    incremental = len(sys.argv) > 1 and sys.argv[1] == "1"
    three_years_ago = (datetime.now() - timedelta(days=365 * 3)).strftime("%Y-%m-%d")
    log(f"3-year cutoff: {three_years_ago}")

    total_count = fetch_total_count()
    if total_count == 0:
        log("ERROR: could not detect total article count")
        return
    total_pages = (total_count + PER_PAGE - 1) // PER_PAGE
    log(f"Total: {total_count} articles, {total_pages} pages")

    all_articles = []
    if incremental:
        max_pages = total_pages  # 增量翻页, 遇到全部已入库即停
    else:
        max_pages = min(total_pages, MAX_PAGES)

    for page in range(1, max_pages + 1):
        items = fetch_list_page(page)
        if not items:
            log(f"Page {page}: no data, stopping")
            break
        filtered = [(u, t, d) for u, t, d in items if d and d >= three_years_ago]
        all_articles.extend(filtered)
        if incremental:
            existing = check_existing([u for u, _, _ in filtered])
            new_ones = [(u, t, d) for u, t, d in filtered if u not in existing]
            if not new_ones:
                log(f"Page {page}: all already in DB, stopping")
                break
            log(f"  Page {page}: {len(new_ones)} new, cumulative {len(all_articles)}")
            continue
        if items and not filtered:
            log(f"Page {page}: all before cutoff, stopping")
            break
        if page % 20 == 0:
            log(f"  Page {page}/{max_pages}: {len(all_articles)} so far")

    log(f"Total filtered: {len(all_articles)}")
    if not all_articles:
        return

    results = []
    batch_size = MAX_WORKERS * 3
    for batch_start in range(0, len(all_articles), batch_size):
        batch = all_articles[batch_start:batch_start + batch_size]
        batch_urls = [u for u, _, _ in batch]
        log(f"  Details [{batch_start+1}-{batch_start+len(batch)}/{len(all_articles)}]...")
        detail_map = fetch_detail_batch(batch_urls)
        for url, title, date_str in batch:
            content_html, detail_date, detail_title = detail_map.get(url, (None, None, None))
            final_title = detail_title or title
            final_date = detail_date or date_str
            results.append((url, final_title, final_date, content_html or ""))

    saved = save_to_db(results)
    log(f"\n{'=' * 50}")
    log(f"Done. Total: {len(results)} (saved: {saved})")
    if results:
        dates = sorted([r[2] for r in results if r[2]])
        log(f"Date range: {dates[0]} ~ {dates[-1]}")

    try:
        import sqlite3
        conn = sqlite3.connect(DB_PATH, timeout=60)
        conn.execute("""
            INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
            SELECT rowid, title, site_name,
                   CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
            FROM gov_raw WHERE category = ?
        """, (SOURCE,))
        conn.commit()
        conn.close()
        log("FTS index rebuilt")
    except Exception as e:
        log(f"FTS rebuild note: {e}")


if __name__ == "__main__":
    main()