#!/usr/bin/env python3
"""Crawler for 赣榆区人民政府 - 公示公告 (gsgg)
URL: https://www.ganyu.gov.cn/gyqzf/gsgg/gsgg.html
Also crawls: xwzx (新闻中心)
TrueCMS with jiasule WAF - only first page data extractable via curl
Content: <div class="art-main" id="zoom"> with full HTML
"""
import requests
import re
import sqlite3
import time
import random
import os

SITE_NAME = "赣榆区人民政府-公示公告"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
}
DELAY = (0.3, 0.5)

SECTIONS = {
    "xwzx": "https://www.ganyu.gov.cn/gyqzf/xwzx/xwzx.html",
    "gsgg": "https://www.ganyu.gov.cn/gyqzf/gsgg/gsgg.html",
}


def log(msg):
    print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)


def extract_items(url):
    """Extract article items from a list page."""
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
    except Exception as e:
        log(f"ERROR fetching {url}: {e}")
        return []
    
    html = r.text
    items = re.findall(
        r'<a href="([^"]+)"[^>]*title="([^"]*)"[^>]*>.*?</a>.*?<label>([^<]+)</label>',
        html, re.DOTALL
    )
    
    results = []
    for href, title, date_str in items:
        if href.startswith("http"):
            full_url = href
        else:
            full_url = f"https://www.ganyu.gov.cn{href}"
        results.append({
            "url": full_url,
            "title": title.strip(),
            "date": date_str.strip(),
        })
    
    return results


def crawl_detail(url):
    """Crawl detail page for full content."""
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
    except Exception as e:
        log(f"ERROR fetch detail: {e}")
        return "", "", ""
    
    html = r.text
    
    title_m = re.search(r'<meta name="[Aa]rticle[Tt]itle" content="([^"]*)"', html)
    date_m = re.search(r'<meta name="[Pp]ub[Dd]ate" content="([^"]*)"', html)
    src_m = re.search(r'<meta name="[Cc]ontent[Ss]ource" content="([^"]*)"', html)
    
    title = title_m.group(1).strip() if title_m else ""
    pub_date = date_m.group(1).strip()[:10] if date_m else ""
    source = src_m.group(1).strip() if src_m else ""
    
    # Extract zoom content via depth counting
    m = re.search(r'id="zoom"', html)
    if not m:
        return title, pub_date, source, ""
    
    gt_pos = html.find(">", m.start())
    if gt_pos == -1:
        return title, pub_date, source, ""
    
    depth = 1
    i = gt_pos + 1
    while i < len(html) and depth > 0:
        if html[i:i+6] == "</div>":
            depth -= 1
            i += 6
        elif html[i:i+4] == "<div" and i + 4 < len(html) and html[i+4] in (' ', '>', '\n', '\t', '\r', "'", '"'):
            depth += 1
            i += 4
        else:
            i += 1
    
    content = html[m.start():i+6]
    return title, pub_date, source, content


def save_to_db(items):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=5000")
    cur = conn.cursor()
    
    inserted = 0
    for item in items:
        try:
            cur.execute(
                """INSERT OR IGNORE INTO gov_raw 
                   (site_name, title, page_url, publish_date, source_url, content)
                   VALUES (?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, item["title"], item["url"], item["date"], item["source"], item["content"]),
            )
            if cur.rowcount > 0:
                inserted += 1
        except Exception as e:
            log(f"ERROR insert: {e}")
    
    conn.commit()
    conn.close()
    return inserted


def main():
    log(f"Starting crawl for {SITE_NAME}")
    log(f"DB: {DB_PATH}")
    
    seen_urls = set()
    all_items = []
    
    for sec_name, sec_url in SECTIONS.items():
        log(f"Section: {sec_name}")
        items = extract_items(sec_url)
        log(f"  Found {len(items)} items")
        
        for item in items:
            if item["url"] not in seen_urls:
                seen_urls.add(item["url"])
                all_items.append(item)
    
    log(f"Total unique items: {len(all_items)}")
    
    saved = 0
    for i, item in enumerate(all_items):
        time.sleep(random.uniform(*DELAY))
        result = crawl_detail(item["url"])
        
        title, pub_date, source, content = result
        item["title"] = title or item["title"]
        item["date"] = pub_date or item["date"]
        item["source"] = source or "赣榆区人民政府"
        item["content"] = content or ""
        
        if (i + 1) % 10 == 0:
            log(f"  Detail {i+1}/{len(all_items)}")
    
    saved = save_to_db(all_items)
    log(f"\n=== SUMMARY ===")
    log(f"Total unique items: {len(all_items)}")
    log(f"Saved: {saved} new items")
    log(f"Done!")


if __name__ == "__main__":
    main()