#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Crawler for 迁安市人民政府 - 生态环境 (qianan.gov.cn)
CMS: 秦皇岛统一平台 (same as qhd.gov.cn)
List: ul.info-list-xxgk.mt15 > li > a[href] + span.time (date)
Pagination: /channel/list/298.html (p1), /channel/list/298_2.html (p2-11), 20/page, 11 pages ~220 items
Detail: div.detail#zoom > p + table (direct children)
         Title: <h1> or <title>
         Date: "发布日期：YYYY-MM-DD" in text or meta
         Table support: extract <table> as HTML

Usage:
    python3 /root/gov_crawler/crawl_qianan_sthj.py [max_pages]
"""

import requests, re, json, sqlite3, time, os, sys
from datetime import datetime
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = "/root/search.db"
SITE_NAME = "迁安市人民政府-生态环境"
CATEGORY = "生态环境"
GROUP = "河北"
BASE_URL = "http://www.qianan.gov.cn"
TOTAL_PAGES = 11

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}


# ---------- List ----------

def get_list_url(page_num):
    if page_num == 1:
        return f"{BASE_URL}/channel/list/298.html"
    return f"{BASE_URL}/channel/list/298_{page_num}.html"


def extract_list_page(page_num):
    url = get_list_url(page_num)
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  Error fetching {url}: {e}", file=sys.stderr)
        return []

    soup = BeautifulSoup(r.text, "html.parser")
    ul = soup.find("ul", class_="info-list-xxgk")
    if not ul:
        return []

    items = []
    for li in ul.find_all("li"):
        a = li.find("a")
        span = li.find("span", class_="time")
        if not a or not a.get("href"):
            continue

        href = a["href"].strip()
        title = a.get_text(strip=True)
        page_url = urljoin(url, href)
        date_str = span.get_text(strip=True) if span else ""

        if title and page_url:
            items.append((title, page_url, date_str))

    return items


# ---------- Detail ----------

def fetch_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  fetch_detail error {url}: {e}", file=sys.stderr)
        return None, None, None, []

    soup = BeautifulSoup(r.text, "html.parser")

    # --- Title ---
    title = ""
    h1 = soup.find("h1")
    if h1:
        title = h1.get_text(strip=True)
    if not title:
        t = soup.find("title")
        if t:
            title = t.get_text(strip=True).replace("迁安市人民政府", "").replace("- 迁安市人民政府", "").replace(" - 迁安市人民政府", "").strip().rstrip("_- ")

    # --- Date ---
    date_str = ""
    for tag in soup.find_all(["span", "div", "p"]):
        txt = tag.get_text(strip=True)
        if "发布日期" in txt or "发布时间" in txt:
            m = re.search(r"(\d{4}-\d{2}-\d{2})", txt)
            if m:
                date_str = m.group(1)
                break
    if not date_str:
        meta = soup.find("meta", attrs={"name": re.compile(r"(pubdate|PubDate|publishdate)", re.I)})
        if meta and meta.get("content"):
            m = re.search(r"(\d{4}-\d{2}-\d{2})", meta["content"])
            if m:
                date_str = m.group(1)

    # --- Content ---
    detail = soup.find("div", class_="detail", id="zoom")
    if not detail:
        detail = soup.find("div", id="zoom")

    content = ""
    if detail:
        for s in detail.find_all(["script", "style"]):
            s.decompose()
        parts = []
        # Iterate direct children for clean extraction
        for child in detail.children:
            if not child.name:
                continue
            if child.name == "table":
                parts.append(str(child))
            elif child.name == "p":
                if child.find(["div", "p"], recursive=False):
                    # Wrapper p (invalid HTML) - extract inner content
                    for inner in child.find_all(["p", "table"], recursive=True):
                        if inner.name == "table":
                            parts.append(str(inner))
                        elif not inner.find_parent("table"):
                            t = inner.get_text(strip=True)
                            if t:
                                parts.append(t)
                else:
                    t = child.get_text(strip=True)
                    if t:
                        parts.append(t)
            elif child.name == "div" and child.find(["p", "table"], recursive=False):
                for inner in child.find_all(["p", "table"], recursive=True):
                    if inner.name == "table":
                        parts.append(str(inner))
                    elif not inner.find_parent("table"):
                        t = inner.get_text(strip=True)
                        if t:
                            parts.append(t)
            else:
                t = child.get_text(strip=True)
                if t:
                    parts.append(t)
        content = "\n\n".join(parts)

    # --- Attachments ---
    attachments = []
    seen = set()
    for a_tag in soup.find_all("a", href=True):
        href = a_tag["href"]
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$", href, re.I):
            full_url = urljoin(url, href)
            name = a_tag.get_text(strip=True) or href.split("/")[-1].split("?")[0]
            if full_url not in seen:
                seen.add(full_url)
                attachments.append({"name": name, "url": full_url})
                if content:
                    content += f'\n\n附件：<p><a href="{full_url}">{name}</a></p>'

    return title, date_str, content, attachments


# ---------- DB ----------

def with_retry(fn, desc="DB op", max_attempts=10, delay=5):
    for attempt in range(1, max_attempts + 1):
        try:
            return fn()
        except sqlite3.OperationalError as e:
            if "locked" in str(e) and attempt < max_attempts:
                print(f"  {desc}: locked (attempt {attempt}/{max_attempts}), retry in {delay}s...", file=sys.stderr)
                time.sleep(delay)
            else:
                print(f"  {desc} failed: {e}", file=sys.stderr)
                return False
    return False


def init_db():
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=30000")
    return conn


def save_article(conn, title, page_url, publish_date, content, attachments):
    summary = content[:200] if content else title
    summary = re.sub(r"\s+", " ", summary).strip()
    att_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]"
    try:
        cur = conn.execute(
            """INSERT OR IGNORE INTO gov_raw
               (site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (SITE_NAME, page_url, page_url, title.strip(), publish_date,
             summary, content, CATEGORY, att_json, GROUP),
        )
        if cur.rowcount > 0:
            # ⚠️ FTS sync: gov_search rowid MUST match gov_raw.id, else article invisible to search
            rid = conn.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,)).fetchone()
            if rid:
                conn.execute(
                    "INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)",
                    (rid[0], title.strip(), SITE_NAME, summary),
                )
            return True
        return False
    except Exception as e:
        print(f"  DB error: {e}", file=sys.stderr)
        return False


# ---------- Main ----------

def main():
    # ⚠️ support --pages=N / --pages N (scheduler passes args); default 5 pages
    max_pages = 5
    if len(sys.argv) > 1:
        for i, a in enumerate(sys.argv):
            if a.startswith("--pages="):
                try:
                    max_pages = int(a.split("=", 1)[1])
                except ValueError:
                    pass
            elif a == "--pages" and i + 1 < len(sys.argv):
                try:
                    max_pages = int(sys.argv[i + 1])
                except ValueError:
                    pass
        # legacy: bare number arg
        if not any(a.startswith("--") for a in sys.argv[1:]):
            try:
                max_pages = int(sys.argv[1])
            except ValueError:
                pass

    print(f"Scraping {SITE_NAME}, max pages: {max_pages}", file=sys.stderr)
    conn = init_db()
    total_new = 0
    total_found = 0
    start_time = time.time()

    for pn in range(1, max_pages + 1):
        articles = extract_list_page(pn)
        if not articles:
            print(f"  Page {pn}: 0 items (end)", file=sys.stderr)
            break

        total_found += len(articles)
        batch_new = 0

        for title, page_url, date in articles:
            if date and date < "2015-01-01":
                continue

            cur = conn.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,))
            if cur.fetchone():
                continue

            det_title, det_date, det_content, det_atts = fetch_detail(page_url)
            final_title = det_title or title
            final_date = det_date or date
            final_content = det_content or f"[{final_title}]({page_url})"
            final_atts = det_atts or []

            if save_article(conn, final_title, page_url, final_date, final_content, final_atts):
                total_new += 1
                batch_new += 1

            time.sleep(0.3)

        elapsed = int(time.time() - start_time)
        print(f"  Page {pn}/{max_pages}: +{batch_new} new, {total_new}/{total_found} total, {elapsed//60}:{elapsed%60:02d}", file=sys.stderr)

        if pn % 5 == 0:
            conn.commit()

    conn.commit()
    elapsed = int(time.time() - start_time)
    conn.close()
    # ⚠️ scheduler extracts new_count from stdout "新增: N"
    print(f"新增: {total_new}", flush=True)
    print(f"跳过: {total_found - total_new}", flush=True)
    print(f"Done! Found: {total_found}, New: {total_new}, Time: {elapsed//60}m{elapsed%60:02d}s", file=sys.stderr)


if __name__ == "__main__":
    main()