#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Crawler for 启东市人民政府 - 公告公示 (qidong.gov.cn)
CMS: TrueCMS with jPage client-side pagination
List: div#initData (inline) + API /truecms/messageController/getMessage.do
Detail: div#zoom.content > p, h2 title, div.tips date

Usage:
    python3 /root/gov_crawler/crawl_qidong_gggs.py             # Full crawl
    python3 /root/gov_crawler/crawl_qidong_gggs.py --max-items 15  # Incremental
"""

import requests, re, json, sqlite3, time, os, sys
from datetime import datetime
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = "/root/search.db"
SITE_NAME = "启东市人民政府-公告公示"
CATEGORY = "公告公示"
GROUP = "江苏"
LIST_URL = "http://www.qidong.gov.cn/qdsrmzf/gggs/gggs.html"
API_HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": "http://www.qidong.gov.cn/qdsrmzf/gggs/gggs.html",
}

API_URL = "http://www.qidong.gov.cn/truecms/messageController/getMessage.do"
API_COLUMN_ID = "485473dc-7534-4359-be92-06cc7252f718"
BASE_DOMAIN = "www.qidong.gov.cn"
TOTAL_ITEMS = 119
PER_PAGE = 15

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}


# ---------- List ----------

def parse_items_from_html(html_fragment, base_url):
    items = []
    # Remove CDATA markers that prevent BS from seeing nested tags
    cleaned = html_fragment.replace("<![CDATA[", "").replace("]]>", "")
    soup = BeautifulSoup(cleaned, "html.parser")
    # Try UL format (inline data)
    for ul in soup.find_all("ul", class_="list-ul"):
        li = ul.find("li")
        if not li:
            continue
        a = li.find("a")
        date_i = li.find("i", class_="timedate")
        if not a or not a.get("href"):
            continue
        href = a["href"].strip()
        title = a.get_text(strip=True)
        title = re.sub(r"\.{3,}\s*$", "", title).strip()
        page_url = urljoin(base_url, href)
        date_str = date_i.get_text(strip=True) if date_i else ""
        if title and page_url:
            items.append((title, page_url, date_str))
    # Try LI format (API response - CDATA wrapped)
    for li in soup.find_all("li"):
        a = li.find("a")
        if not a or not a.get("href"):
            continue
        # Check if we already got this from UL parsing
        href = a["href"].strip()
        page_url = urljoin(base_url, href)
        if any(i[1] == page_url for i in items):
            continue
        title = a.get_text(strip=True)
        title = re.sub(r"\.{3,}\s*$", "", title).strip()
        # Date can be in <i> or <span>
        date_tag = li.find("i") or li.find("span")
        date_str = date_tag.get_text(strip=True) if date_tag else ""
        if title and page_url:
            items.append((title, page_url, date_str))
    return items


def fetch_api_page(page_num):
    startrecord = (page_num - 1) * PER_PAGE
    endrecord = startrecord + PER_PAGE - 1
    params = {
        "startrecord": startrecord,
        "endrecord": endrecord,
        "perpage": PER_PAGE,
        "columnId": API_COLUMN_ID,
    }
    try:
        r = requests.get(API_URL, params=params, headers=API_HEADERS, timeout=20)
        if r.status_code == 200:
            data = r.json()
            html = data.get("result", "")
            if html:
                return parse_items_from_html(html, LIST_URL)
    except Exception as e:
        print(f"  Error fetching API page {page_num}: {e}", file=sys.stderr)
    return []


def extract_all_items():
    items = []

    # Page 1: inline data
    try:
        r = requests.get(LIST_URL, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  Error fetching list: {e}", file=sys.stderr)
        return items

    soup = BeautifulSoup(r.text, "html.parser")
    init_data = soup.find("div", id="initData")
    if init_data:
        items.extend(parse_items_from_html(str(init_data), LIST_URL))

    # Pages 2+: API
    total_loaded = len(items)
    page_num = 2
    while total_loaded < TOTAL_ITEMS:
        api_items = fetch_api_page(page_num)
        if not api_items:
            break
        items.extend(api_items)
        total_loaded = len(items)
        page_num += 1
        time.sleep(0.3)

    return items


# ---------- Detail ----------

def fetch_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  fetch_detail error {url}: {e}", file=sys.stderr)
        return None, None, None, []

    soup = BeautifulSoup(r.text, "html.parser")

    # Title: <h2> before #zoom
    title = ""
    zoom = soup.find("div", id="zoom")
    h2 = None
    if zoom:
        h2 = zoom.find_previous_sibling("h2")
    if not h2:
        h2 = soup.find("h2")
    if h2:
        title = h2.get_text(strip=True)

    # Date: div.tips
    date_str = ""
    tips = soup.find("div", class_="tips")
    if tips:
        m = re.search(r"(\d{4}-\d{2}-\d{2})", tips.get_text())
        if m:
            date_str = m.group(1)

    # Content: div#zoom
    content = ""
    if zoom:
        for s in zoom.find_all(["script", "style"]):
            s.decompose()
        parts = []
        for p in zoom.find_all("p"):
            t = p.get_text(strip=True)
            if t:
                parts.append(t)
        content = "\n\n".join(parts)

    # Attachments
    attachments = []
    seen = set()
    for a in soup.find_all("a", href=True):
        href = a["href"]
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$", href, re.I):
            full_url = urljoin(url, href)
            name = a.get_text(strip=True) or href.split("/")[-1].split("?")[0]
            if full_url not in seen:
                seen.add(full_url)
                attachments.append({"name": name, "url": full_url})
                if content:
                    content += f'\n\n附件：<p><a href="{full_url}">{name}</a></p>'

    return title, date_str, content, attachments


# ---------- DB ----------

def with_retry(fn, desc="DB op", max_attempts=10, delay=5):
    for attempt in range(1, max_attempts + 1):
        try:
            return fn()
        except sqlite3.OperationalError as e:
            if "locked" in str(e) and attempt < max_attempts:
                print(f"  {desc}: locked (attempt {attempt}/{max_attempts}), retry in {delay}s...", file=sys.stderr)
                time.sleep(delay)
            else:
                print(f"  {desc} failed: {e}", file=sys.stderr)
                return False
    return False


def init_db():
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=30000")
    return conn


def save_article(conn, title, page_url, publish_date, content, attachments):
    summary = content[:200] if content else title
    summary = re.sub(r"\s+", " ", summary).strip()
    att_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]"
    try:
        cur = conn.execute(
            """INSERT OR IGNORE INTO gov_raw
               (site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (SITE_NAME, page_url, page_url, title.strip(), publish_date,
             summary, content, CATEGORY, att_json, GROUP),
        )
        return cur.rowcount > 0
    except Exception as e:
        print(f"  DB error: {e}", file=sys.stderr)
        return False


# ---------- Main ----------

def main():
    max_items = TOTAL_ITEMS
    if len(sys.argv) > 1:
        try:
            max_items = int(sys.argv[1])
        except ValueError:
            if sys.argv[1] in ("--max-items",) and len(sys.argv) > 2:
                try:
                    max_items = int(sys.argv[2])
                except ValueError:
                    pass

    print(f"Scraping {SITE_NAME}, max items: {max_items}", file=sys.stderr)
    conn = init_db()
    total_new = 0

    def do_clean():
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)         conn.execute("DELETE FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
        conn.commit()
    with_retry(do_clean, desc=f"Clean {SITE_NAME}")
    print(f"  Cleaned old data for {SITE_NAME}", file=sys.stderr)

    all_items = extract_all_items()
    if not all_items:
        print("  No items found!", file=sys.stderr)
        return

    total_found = len(all_items)
    print(f"  Found {total_found} items", file=sys.stderr)

    for idx, (title, page_url, date) in enumerate(all_items[:max_items]):
        if date and date < "2020-01-01":
            continue
        cur = conn.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,))
        if cur.fetchone():
            continue

        det_title, det_date, det_content, det_atts = fetch_detail(page_url)
        final_title = det_title or title
        final_date = det_date or date
        final_content = det_content or f"[{final_title}]({page_url})"
        final_atts = det_atts or []

        if save_article(conn, final_title, page_url, final_date, final_content, final_atts):
            total_new += 1
            if total_new <= 5:
                print(f"  + {final_date} {final_title[:50]}", file=sys.stderr)

        if (idx + 1) % 10 == 0:
            conn.commit()
        time.sleep(0.5)

    conn.commit()
    conn.close()
    print(f"\nDone! Found: {total_found}, New: {total_new}", file=sys.stderr)


if __name__ == "__main__":
    main()