#!/usr/bin/env python3
import os
"""Crawl 郓城县人民政府 - 通知公告"""
import sys, re, os, requests
from datetime import datetime, timedelta
import sqlite3

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "郓城县-通知公告"
BASE_URL = "http://www.cnyc.gov.cn"
API_URL = BASE_URL + "/els-service/article"
DQ = "2c90808483d171790183e4d0219b0002"
CATAS = ["yc1590299564160716800"]
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
           "Content-Type": "application/json;charset=utf-8"}

def get_list(page):
    try:
        r = requests.post("%s/%d/15" % (API_URL, page), json={
            "dq": DQ, "catas": CATAS, "subject": "",
            "fwzt": 3, "highlight": "1"
        }, headers=HEADERS, timeout=20)
        data = r.json()
        if data.get("success"):
            contents = data["data"].get("contents", [])
            total = data["data"].get("elementsTotal", 0)
            return contents, total
        return [], 0
    except Exception as e:
        print("  API error page %d: %s" % (page, e), flush=True)
        return [], 0

def extract_content(html):
    """Extract content from the embedded var memo JS string"""
    idx = html.find("var memo = ")
    if idx < 0:
        return ""
    rest = html[idx+10:].lstrip()
    if not rest.startswith('"'):
        return ""
    raw = []
    i = 1
    while i < len(rest):
        ch = rest[i]
        if ch == "\\" and i+1 < len(rest):
            raw.append(ch)
            raw.append(rest[i+1])
            i += 2
        elif ch == '"':
            # Unescaped quote = end of JS string
            break
        else:
            raw.append(ch)
            i += 1
    raw_str = "".join(raw)
    raw_str = raw_str.replace('\\"', '"').replace("\\n", "\n").replace("\\t", "\t").replace("\\/", "/")
    try:
        raw_str = raw_str.encode().decode("unicode_escape")
    except:
        pass
    raw_str = re.sub(r'<script[^>]*>.*?</script>', "", raw_str, flags=re.DOTALL|re.I)
    raw_str = re.sub(r'<style[^>]*>.*?</style>', "", raw_str, flags=re.DOTALL|re.I)
    return raw_str.strip()

def fetch_detail(xxid, dwid):
    url = "%s/%s/%s/%s.html" % (BASE_URL, DQ, dwid, xxid)
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        html = r.text
        title = ""
        ar = re.search(r'<meta name="ArticleTitle" content="(.*?)"', html)
        if ar:
            title = ar.group(1).strip()[:200]
        if not title:
            t = re.search(r'<title>(.*?)</title>', html)
            if t:
                title = t.group(1).strip()[:200]
        date = ""
        pd = re.search(r'<meta name="PubDate" content="(.*?)"', html)
        if pd:
            date = pd.group(1).strip()[:10]
        content = extract_content(html)
        if not content or len(content.strip()) < 30:
            content = ""
        return title, date, content, url
    except Exception as e:
        print("  Detail error %s: %s" % (xxid, e), flush=True)
        return "", "", "", url

def main():
    print("Starting: %s" % SITE_NAME, flush=True)
    
    print("Phase 1: Getting list counts...", flush=True)
    first_page, total = get_list(1)
    if total == 0:
        print("  No data found!", flush=True)
        return
    
    total_pages = (total + 14) // 15
    print("  Total items: %d, Pages: %d" % (total, total_pages), flush=True)
    
    print("Phase 2: Collecting list items...", flush=True)
    all_items = []
    for page in range(1, min(total_pages + 1, 3*365)):
        items, _ = get_list(page) if page > 1 else (first_page, total)
        if not items:
            break
        for item in items:
            date = item.get("fwdate", "")
            if date and date < CUTOFF:
                continue
            all_items.append({
                "xxid": item["xxid"],
                "dwid": item["dwid"],
                "title": re.sub(r'<[^>]+>', '', item.get("subject", "")).strip()[:200],
                "date": date,
            })
        if items and items[-1].get("fwdate", "") < CUTOFF:
            break
        if page % 20 == 0:
            print("  Collected page %d/%d, items so far: %d" % (page, total_pages, len(all_items)), flush=True)
    
    print("  Total items within 3 years: %d" % len(all_items), flush=True)
    if not all_items:
        print("  Nothing to crawl!", flush=True)
        return
    
    print("Phase 3: Fetching details and saving...", flush=True)
    conn = sqlite3.connect(SEARCH_DB)
    conn.execute("PRAGMA journal_mode=WAL")
    c = conn.cursor()
    c.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        title TEXT, content TEXT, summary TEXT,
        source_url TEXT UNIQUE, site_name TEXT,
        publish_date TEXT, category TEXT
    )""")
    
    saved = 0
    for i, item in enumerate(all_items):
        detail_url = "%s/%s/%s/%s.html" % (BASE_URL, DQ, item["dwid"], item["xxid"])
        title, date, content, _ = fetch_detail(item["xxid"], item["dwid"])
        title = title or item["title"]
        date = date or item["date"]
        if not content or not content.strip():
            content = ""
        
        try:
            c.execute("""INSERT OR REPLACE INTO gov_raw
                (title, content, summary, source_url, site_name, publish_date, category)
                VALUES (?,?,?,?,?,?,?)""",
                (title, content, title[:200], detail_url, SITE_NAME, date, ""))
            if c.rowcount > 0:
                saved += 1
        except Exception as e:
            print("  DB error: %s" % e, flush=True)
        
        if (i + 1) % 20 == 0:
            conn.commit()
            print("  Saved %d/%d items..." % (i+1, len(all_items)), flush=True)
    
    conn.commit()
    conn.close()
    print("  Total saved: %d" % saved, flush=True)
    
    print("Phase 4: Updating FTS...", flush=True)
    conn = sqlite3.connect(SEARCH_DB)
    c = conn.cursor()
    try:
        c.execute("""INSERT OR REPLACE INTO gov_search (rowid, title, site_name, summary)
            SELECT r.id, r.title, r.site_name, r.summary
            FROM gov_raw r WHERE r.site_name = ? AND r.content != ''
            AND NOT EXISTS (SELECT 1 FROM gov_search s WHERE s.rowid = r.id)""", (SITE_NAME,))
        conn.commit()
        print("  FTS updated", flush=True)
    except Exception as e:
        print("  FTS error: %s" % e, flush=True)
    conn.close()
    
    print("Done!", flush=True)

if __name__ == "__main__":
    main()