#!/usr/bin/env python3
"""嵩县人民政府-公告公示 (hnsongxian.gov.cn)"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests
from bs4 import BeautifulSoup
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "嵩县人民政府-公告公示"
BASE_URL = "https://www.hnsongxian.gov.cn"
LIST_BASE = "https://www.hnsongxian.gov.cn/zfxxgk/gggs"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
MAX_PAGES = 5  # 874 items / 24 per page ≈ 37

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0 Safari/537.36"}

def fetch_page(page_num):
    if page_num == 1:
        url = f"{LIST_BASE}/"
    else:
        url = f"{LIST_BASE}/index_{page_num-1}.html"
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"  [ERROR] page {page_num}: {e}")
        return None

def parse_items(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for a in soup.find_all("a", class_="item"):
        href = a.get("href", "").strip()
        if not href:
            continue
        if not href.startswith("http"):
            href = BASE_URL + href
        p = a.find("p")
        title = p.get_text(strip=True) if p else ""
        time_div = a.find("div", class_="time")
        date_text = time_div.get_text(strip=True) if time_div else ""
        items.append((title, href, date_text))
    return items

def fetch_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        html = r.text
    except:
        return None, None, None

    soup = BeautifulSoup(html, "html.parser")

    # Title from <title>
    title_tag = soup.find("title")
    title = title_tag.get_text(strip=True) if title_tag else ""
    title = re.sub(r"\s*_公告公示_嵩县人民政府$", "", title).strip()

    # Content: try newscontents, then wzxq, then ny_content
    content_div = (soup.find("div", class_="newscontents")
                   or soup.find("div", class_="wzxq")
                   or soup.find("div", class_="ny_content"))
    if content_div:
        content = str(content_div)
        content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL|re.I)
        content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.DOTALL|re.I)
        return title, content, ""

    return None, None, None

def main(incremental=False, limit=None):
    print(f"\n{'='*50}\n{SITE_NAME}\n{'='*50}")
    all_items = []
    seen = set()
    max_pages = 2 if incremental else MAX_PAGES

    for page in range(1, max_pages + 1):
        print(f"\n--- Page {page} ---", end=" ", flush=True)
        html = fetch_page(page)
        if not html:
            break
        items = parse_items(html)
        if not items:
            print("empty")
            break
        print(f"{len(items)} items")

        exhausted = True
        for title, url, date_text in items:
            if url in seen:
                continue
            seen.add(url)

            if date_text and date_text < THREE_YEARS_AGO:
                continue
            exhausted = False

            if incremental:
                import sqlite3 as s3
                conn = s3.connect("/root/search.db")
                exists = conn.execute("SELECT 1 FROM gov_raw WHERE page_url=?", (url,)).fetchone()
                conn.close()
                if exists:
                    continue

            all_items.append((title, url, date_text))

        if exhausted:
            print("  All past 3yr cutoff, stop")
            break

        if limit and len(all_items) >= limit:
            all_items = all_items[:limit]
            break

        time.sleep(0.3)

    print(f"\n📋 Total: {len(all_items)} items")

    if not all_items:
        print("  No new items")
        return

    results = []
    for i, (title, url, list_date) in enumerate(all_items):
        print(f"  [{i+1}/{len(all_items)}] {title[:50]}...", end=" ", flush=True)
        dt, content, _ = fetch_detail(url)
        ftitle = dt or title
        summary = re.sub(r"<[^>]+>", " ", content or "").strip()[:300]
        summary = re.sub(r"\s+", " ", summary)
        results.append({
            "site_name": SITE_NAME, "title": ftitle, "url": url,
            "source_url": url, "content": content or "",
            "pub_date": list_date, "summary": summary, "tags": SITE_NAME,
        })
        print("OK")
        time.sleep(0.3)

    if results:
        push_to_searchdb(results, "hnsongxian")
    print(f"\nDone! {len(results)} records")

if __name__ == "__main__":
    t0 = time.time()
    inc = len(sys.argv) > 1 and sys.argv[1] in ("1", "--incremental")
    limit = None
    if "--limit" in sys.argv:
        limit = int(sys.argv[sys.argv.index("--limit") + 1])
    main(incremental=inc, limit=limit)
    print(f"Time: {time.time()-t0:.1f}s")