#!/usr/bin/env python3
import os
"""衢州市柯城区人民政府 — 公告公示 爬虫 (JPAAS TRS CMS)
用法:
  python3 crawl_kecheng.py              # 全量爬取
  python3 crawl_kecheng.py 5            # 爬前5页
  python3 crawl_kecheng.py --incremental # 增量模式
"""
import sys, re, sqlite3, json
import requests
from urllib.parse import urljoin
from datetime import datetime, timedelta

SITE_NAME = "柯城区人民政府-公告公示"
BASE = "https://www.kecheng.gov.cn"
COLUMN = "/col/col1229068908/index.html"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
MAX_PAGES = 29  # 435条, 15/page ≈ 29页

API_URL = f"{BASE}/api-gateway/jpaas-publish-server/front/page/build/unit"
API_PARAMS = {
    "webId": "3017",
    "pageId": "1229068908",
    "parseType": "bulidstatic",
    "pageType": "column",
    "tagId": "当前栏目_list",
    "editType": "null",
    "tplSetId": "8LeihoYLP84FLuUsPt4wo",
}

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Referer": BASE + COLUMN,
}

CUTOFF = (datetime.now() - timedelta(days=365 * 3)).strftime("%Y-%m-%d")


def fetch_list(session, page):
    """Fetch one page of list via GET API with paramJson pagination."""
    params = dict(API_PARAMS)
    params["paramJson"] = json.dumps({"pageNo": page, "pageSize": 15}, ensure_ascii=False)
    try:
        r = session.get(API_URL, params=params, headers=HEADERS, timeout=30)
        if r.status_code != 200:
            return None, 0
        data = r.json()
        if not data.get("success"):
            return None, 0
        html = data["data"]["html"]
        # Extract total count
        m = re.search(r'count="(\d+)"', html)
        total = int(m.group(1)) if m else 0
        return html, total
    except Exception as e:
        print(f"    API error: {e}")
        return None, 0


def parse_list(html):
    """Extract article items from list HTML.
    Format: <li><a href="..." title="...">title</a><span>2026-06-12</span></li>
    """
    items = []
    pattern = re.compile(
        r'<li>.*?<a[^>]*href="([^"]+)"[^>]*title="([^"]*)".*?<span>([^<]+)</span>',
        re.DOTALL,
    )
    for m in pattern.finditer(html):
        url = m.group(1)
        if not url.startswith("http"):
            url = urljoin(BASE, url)
        title = m.group(2).strip()
        date_str = m.group(3).strip()
        items.append({"url": url, "title": title, "date": date_str})
    return items


def fetch_page(session, url):
    try:
        r = session.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        return r.text
    except:
        return None


def parse_detail(html, fallback_title=""):
    """Extract title, date, content from detail page."""
    # Title from meta ArticleTitle
    title = ""
    m = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html)
    if m:
        title = m.group(1).strip()
    if not title:
        # Try main_title div area
        m = re.search(r'<div[^>]*class="main_title"[^>]*>(.*?)</div>', html, re.DOTALL)
        if m:
            title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    if not title:
        title = fallback_title

    # Date from meta PubDate
    date_str = ""
    m = re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{1,2}-\d{1,2})', html)
    if m:
        date_str = m.group(1)
    if not date_str:
        m = re.search(r"发布时间[：:]\s*(\d{4}-\d{1,2}-\d{1,2})", html)
        if m:
            date_str = m.group(1)

    # Content from <div class="main_section">
    # There are two: first is artic_main header, second is the actual content
    content = ""
    idx = html.find('<div class="main_section">')
    if idx >= 0:
        # Skip first occurrence (artic_main header), use second one
        idx2 = html.find('<div class="main_section">', idx + 30)
        if idx2 >= 0:
            start = idx2 + len('<div class="main_section">')
        else:
            start = idx + len('<div class="main_section">')
        # Extract content
        depth, i = 0, start
        while i < len(html):
            if html[i:i+4] == '<!--':
                e = html.find('-->', i+4)
                i = e+3 if e > i else i+1
                continue
            if html[i:i+6] == '</div>':
                if depth == 0:
                    content = html[start:i]
                    break
                depth -= 1
                i += 6
                continue
            if html[i] == '<' and html[i+1:i+4] in ('div', 'sec'):
                depth += 1
                i += 3
                continue
            i += 1

    if content:
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL | re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL | re.I)
        content = re.sub(r'\s*style="[^"]*"', '', content)
        content = re.sub(r'<!--.*?-->', '', content)
        content = content.strip()

    return title, date_str, content


def push_to_db(db, records):
    inserted = 0
    skipped = 0
    for item in records:
        try:
            db.execute(
                """INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags)
                VALUES (?,?,?,?,?,?,?,?,?,?)""",
                (
                    item["site_name"][:200],
                    item["source_url"],
                    item["page_url"],
                    item["title"],
                    item["publish_date"],
                    item["summary"],
                    item["content"],
                    item["status"],
                    item["category"],
                    item["tags"],
                ),
            )
            if db.total_changes > 0:
                inserted += 1
            else:
                skipped += 1
        except:
            skipped += 1
    return inserted, skipped


def run(max_pages=None, incremental=False):
    if max_pages is None:
        max_pages = MAX_PAGES

    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")

    known = set(
        r[0]
        for r in db.execute(
            "SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)
        ).fetchall()
    )
    print(f"  Known in DB: {len(known)}")

    session = requests.Session()
    session.headers.update(HEADERS)

    # Get page 1 to find total count
    print(f"  Fetching page 1...", end=" ")
    html, total = fetch_list(session, 1)
    if not html:
        print("FAIL")
        return
    items = parse_list(html)
    total_pages = min(max_pages, (total + 14) // 15 if total else max_pages)
    print(f"OK - {len(items)} items, {total_pages} pages (total: {total})")

    # Check if newest article is already known (incremental)
    all_items = []
    newest_url = items[0]["url"] if items else ""
    if incremental and newest_url in known:
        print(f"  No new data (newest URL already in DB)")
        db.close()
        return

    all_items.extend(items)

    # Fetch remaining pages
    for page in range(2, total_pages + 1):
        print(f"  Page {page}...", end=" ")
        html, _ = fetch_list(session, page)
        if not html:
            print("FAIL")
            break
        page_items = parse_list(html)
        print(f"{len(page_items)} items")
        if not page_items:
            break

        if incremental:
            all_known = all(item["url"] in known for item in page_items)
            if all_known:
                print(f"    All known, stopping")
                break
            new_items = [i for i in page_items if i["url"] not in known]
            if not new_items:
                break
            all_items.extend(page_items)
        else:
            all_items.extend(page_items)

    # Filter by date
    all_items = [i for i in all_items if i["date"] >= CUTOFF or not i["date"]]

    if not all_items:
        print("  No items to process")
        db.close()
        return

    print(f"  Fetching {len(all_items)} details...")

    results = []
    for i, item in enumerate(all_items):
        html = fetch_page(session, item["url"])
        if not html:
            continue
        title, date_str, content = parse_detail(html, item["title"])
        results.append(
            {
                "site_name": SITE_NAME,
                "source_url": item["url"][:500],
                "page_url": item["url"],
                "title": (title or item["title"])[:500],
                "publish_date": (date_str or item["date"])[:10],
                "summary": (title or item["title"])[:500],
                "content": content,
                "status": "active",
                "category": "",
                "tags": "",
            }
        )
        if (i + 1) % 20 == 0:
            print(f"    [{i+1}/{len(all_items)}]")

    inserted, skipped = push_to_db(db, results)
    db.commit()

    # Update FTS
    db.execute(
        """INSERT INTO gov_search(rowid,title,site_name,summary)
        SELECT r.id, r.title, r.site_name, r.summary
        FROM gov_raw r
        WHERE r.id NOT IN (SELECT rowid FROM gov_search)
        AND r.site_name=?""",
        (SITE_NAME,),
    )
    db.commit()
    db.close()

    print(f"  Done: new={inserted}, skip={skipped}")


if __name__ == "__main__":
    incremental = "--incremental" in sys.argv
    mp = None
    for a in sys.argv[1:]:
        if a.isdigit():
            mp = int(a)
            break
        if a == "--test":
            mp = 1
            break

    run(max_pages=mp, incremental=incremental)