#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
crawl_kmyl.py — 昆明宜良县人民政府-公示公告
http://www.kmyl.gov.cn/zfxxgk/fdzdgknr/gsgg/
CMS: 自定义政府CMS
Pagination: index_N.shtml (1-indexed)
"""
import requests
import re
import json
import sys
import os
import time
from bs4 import BeautifulSoup

SITE_NAME = "kmyl_gsgg"
BASE = "http://www.kmyl.gov.cn"
LIST_PATH = "/zfxxgk/fdzdgknr/gsgg"
SEARCH_DB = "/mnt/data/search.db"
GROUP = "云南"

session = requests.Session()
session.headers.update({
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml",
})

def fetch_list(page):
    if page == 1:
        url = f"{BASE}{LIST_PATH}/"
    else:
        url = f"{BASE}{LIST_PATH}/index_{page}.shtml"
    r = session.get(url, timeout=30)
    r.encoding = "utf-8"
    return r.text

def parse_list(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for item_div in soup.find_all("div", class_=re.compile(r"data-table-item")):
        url_div = item_div.find("div", class_="item-url")
        if not url_div:
            continue
        a = url_div.find("a")
        if not a or not a.get("href"):
            continue
        href = a["href"].strip()
        title = a.get_text(strip=True)
        if not href.startswith("/c/"):
            continue
        # Extract date from URL: /c/YYYY-MM-DD/ID.shtml
        m = re.search(r"/c/(\d{4}-\d{2}-\d{2})/", href)
        pub_date = m.group(1) if m else ""
        items.append({
            "url": href if href.startswith("http") else f"{BASE}{href}",
            "title": title,
            "pub_date": pub_date,
        })
    return items

def get_total_pages(html):
    soup = BeautifulSoup(html, "html.parser")
    links = soup.select("div.page a")
    max_n = 0
    for a in links:
        href = a.get("href", "")
        m = re.search(r"index_(\d+)\.shtml", href)
        if m:
            n = int(m.group(1))
            if n > max_n:
                max_n = n
    return max_n

def parse_detail(url):
    r = session.get(url, timeout=30)
    r.encoding = "utf-8"
    html = r.text
    soup = BeautifulSoup(html, "html.parser")

    title = ""
    m = re.search(r'<meta[^>]*ArticleTitle[^>]*content=["\']([^"\']*)["\']', html)
    if m:
        title = m.group(1)

    pub_date = ""
    m = re.search(r'<meta[^>]*PubDate[^>]*content=["\']([^"\']*)["\']', html)
    if m:
        raw = m.group(1)
        m2 = re.search(r"(\d{4}-\d{2}-\d{2})", raw)
        if m2:
            pub_date = m2.group(1)

    source = ""
    m = re.search(r'<meta[^>]*ContentSource[^>]*content=["\']([^"\']*)["\']', html)
    if m:
        source = m.group(1)

    content_html = ""
    summary = ""
    content_div = soup.find("div", class_="content")
    if content_div:
        content_html = str(content_div)
        summary = content_div.get_text(strip=True)[:200]
    else:
        # Fallback
        body = soup.find("body")
        if body:
            for s in body.find_all(["script", "style"]):
                s.decompose()
            txt = body.get_text(strip=True)
            txt = re.sub(r"[\s\n]{3,}", "\n", txt)
            if txt:
                content_html = txt
                summary = txt[:200]

    return {
        "title": title,
        "pub_date": pub_date,
        "content": content_html,
        "summary": summary,
        "source": source,
    }

def push_to_searchdb(items, batch_label):
    import subprocess
    sql_parts = []
    for item in items:
        title = item["title"].replace("'", "''")
        site_name = SITE_NAME
        page_url = item["url"].replace("'", "''")
        source_url = item.get("source_url", page_url).replace("'", "''")
        pub_date = (item.get("pub_date") or "").replace("'", "''")
        summary = (item.get("summary") or "").replace("'", "''")
        content = (item.get("content") or "").replace("'", "''")
        script_name = "kmyl"

        sql = (
            f"INSERT OR IGNORE INTO gov_raw"
            f"(site_name, source_url, page_url, title, publish_date, summary, content, status, script_name) "
            f"VALUES('{site_name}','{source_url}','{page_url}','{title}','{pub_date}','{summary}','{content}','active','{script_name}');\n"
        )
        sql_parts.append(sql)

    full_sql = "BEGIN;\n" + "".join(sql_parts) + "COMMIT;\n"
    r = subprocess.run(["sqlite3", "-cmd", ".timeout 60000", SEARCH_DB], input=full_sql, capture_output=True, text=True, timeout=60)
    return len(sql_parts), r.stderr

def sync_fts(batch_urls):
    import subprocess
    for url in batch_urls:
        url_esc = url.replace("'", "''")
        sql = (
            f"INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) "
            f"SELECT rowid, title, site_name, summary FROM gov_raw "
            f"WHERE page_url='{url_esc}' AND rowid NOT IN (SELECT rowid FROM gov_search);\n"
        )
        subprocess.run(["sqlite3", "-cmd", ".timeout 60000", SEARCH_DB], input=sql, capture_output=True, text=True, timeout=30)

def main():
    max_pages = 5
    if "--pages" in sys.argv:
        idx = sys.argv.index("--pages")
        if idx + 1 < len(sys.argv):
            max_pages = int(sys.argv[idx + 1])

    incremental = "--incremental" in sys.argv

    print(f"爬取: {SITE_NAME}")
    print(f"最大页数: {max_pages}")
    print(f"增量模式: {incremental}")

    # Fetch page 1 to get total pages
    html = fetch_list(1)
    total_pages = get_total_pages(html)
    if total_pages == 0:
        # No pagination found, just page 1
        total_pages = 1
    print(f"总页数: {total_pages}")

    if max_pages and max_pages < total_pages:
        total_pages = max_pages

    all_items = []
    for page in range(1, total_pages + 1):
        print(f"  列表页 {page}/{total_pages}...", end=" ", flush=True)
        html = fetch_list(page)
        items = parse_list(html)
        all_items.extend(items)
        print(f"{len(items)} 条")
        time.sleep(0.3)

    print(f"\n共获取列表项: {len(all_items)} 条")

    if incremental:
        # Check existing URLs in DB
        import sqlite3
        conn = sqlite3.connect(SEARCH_DB, timeout=60)
        existing = set()
        try:
            rows = conn.execute("SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchall()
            existing = {r[0] for r in rows}
        except:
            pass
        conn.close()
        all_items = [it for it in all_items if it["url"] not in existing]
        print(f"增量后剩余: {len(all_items)} 条")

    # Crawl details
    new_count = 0
    batch = []
    for i, item in enumerate(all_items):
        print(f"  [{i+1}/{len(all_items)}] {item['title'][:40]}...", end=" ", flush=True)
        try:
            detail = parse_detail(item["url"])
            item["pub_date"] = item["pub_date"] or detail["pub_date"]
            item["content"] = detail["content"]
            item["summary"] = detail["summary"]
            item["source_url"] = item["url"]
            batch.append(item)
            print(f"✅ ({len(detail.get('content',''))}b)")
        except Exception as e:
            print(f"❌ {e}")
        time.sleep(0.2)

        if len(batch) >= 20:
            n, err = push_to_searchdb(batch, SITE_NAME)
            new_count += n
            sync_fts([it["url"] for it in batch])
            if err:
                print(f"  DB Error: {err[:100]}")
            batch = []

    if batch:
        n, err = push_to_searchdb(batch, SITE_NAME)
        new_count += n
        sync_fts([it["url"] for it in batch])
        if err:
            print(f"  DB Error: {err[:100]}")

    print(f"\n✅ 完成！新增入库: {new_count} 条")

if __name__ == "__main__":
    main()