#!/usr/bin/env python3
"""乌鲁木齐甘泉堡经开区-生态环境 (gqp.gov.cn)"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests
from bs4 import BeautifulSoup
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "乌鲁木齐甘泉堡经开区-生态环境"
BASE_URL = "http://www.gqp.gov.cn"
LIST_URL = "http://www.gqp.gov.cn/gqbkfq/c120324/zfxxgk_list.shtml"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
MAX_PAGES = 5  # 125 items / 9 per page

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0 Safari/537.36"}

def fetch_page(page_num):
    if page_num == 1:
        url = LIST_URL
    else:
        url = f"{BASE_URL}/gqbkfq/c120324/zfxxgk_list_{page_num}.shtml"
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"  [ERROR] page {page_num}: {e}")
        return None

def parse_items(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for li in soup.find_all("li"):
        span = li.find("span", class_="fr")
        a = li.find("a", href=lambda h: h and "/gqbkfq/c120324/" in h)
        if span and a:
            href = a["href"].strip()
            if not href.startswith("http"):
                href = BASE_URL + href
            title = a.get_text(strip=True)
            date_text = span.get_text(strip=True)
            items.append((title, href, date_text))
    return items

def fetch_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        html = r.text
    except:
        return None, None, None

    soup = BeautifulSoup(html, "html.parser")

    # Title from p.detail-title
    title_p = soup.find("p", class_="detail-title")
    title = title_p.get_text(strip=True) if title_p else ""

    # Date from detail-meta
    date_match = re.search(r"发布日期：(\d{4}-\d{2}-\d{2})", html)
    pub_date = date_match.group(1) if date_match else ""

    # Content from detail-content
    content_div = soup.find("div", class_="detail-content")
    if content_div:
        content = str(content_div)
        content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL|re.I)
        content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.DOTALL|re.I)
        return title, content, pub_date

    return None, None, pub_date

def main(incremental=False, limit=None):
    print(f"\n{'='*50}\n{SITE_NAME}\n{'='*50}")
    all_items = []
    seen = set()
    max_pages = 2 if incremental else MAX_PAGES

    for page in range(1, max_pages + 1):
        print(f"\n--- Page {page} ---", end=" ", flush=True)
        html = fetch_page(page)
        if not html:
            break
        items = parse_items(html)
        if not items:
            print("empty")
            break
        print(f"{len(items)} items")

        exhausted = True
        for title, url, date_text in items:
            if url in seen:
                continue
            seen.add(url)

            if date_text and date_text < THREE_YEARS_AGO:
                continue
            exhausted = False

            if incremental:
                import sqlite3 as s3
                conn = s3.connect("/root/search.db")
                exists = conn.execute("SELECT 1 FROM gov_raw WHERE page_url=?", (url,)).fetchone()
                conn.close()
                if exists:
                    continue

            all_items.append((title, url, date_text))

        if exhausted:
            print("  All past 3yr cutoff, stop")
            break

        if limit and len(all_items) >= limit:
            all_items = all_items[:limit]
            break

        time.sleep(0.3)

    print(f"\n📋 Total: {len(all_items)} items")

    if not all_items:
        print("  No new items")
        return

    results = []
    for i, (title, url, list_date) in enumerate(all_items):
        print(f"  [{i+1}/{len(all_items)}] {title[:50]}...", end=" ", flush=True)
        dt, content, pub_date = fetch_detail(url)
        ftitle = dt or title
        fdate = pub_date or list_date
        summary = re.sub(r"<[^>]+>", " ", content or "").strip()[:300]
        summary = re.sub(r"\s+", " ", summary)
        results.append({
            "site_name": SITE_NAME, "title": ftitle, "url": url,
            "source_url": url, "content": content or "",
            "pub_date": fdate, "summary": summary, "tags": SITE_NAME,
        })
        print("OK")
        time.sleep(0.3)

    if results:
        push_to_searchdb(results, "gqp")
    print(f"\nDone! {len(results)} records")

if __name__ == "__main__":
    t0 = time.time()
    inc = len(sys.argv) > 1 and sys.argv[1] in ("1", "--incremental")
    limit = None
    if "--limit" in sys.argv:
        limit = int(sys.argv[sys.argv.index("--limit") + 1])
    main(incremental=inc, limit=limit)
    print(f"Time: {time.time()-t0:.1f}s")