#!/usr/bin/env python3
"""甘肃环评信息网 (gshpxx.com) - 信息公开 + 验收公示"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests
from bs4 import BeautifulSoup
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "甘肃环评信息网-信息公开"
BASE_URL = "http://www.gshpxx.com"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")

# 两个栏目
CATEGORIES = {
    "xxgk": {"name": "信息公开", "pages": 65, "site_name": "甘肃环评信息网-信息公开"},
    "ysgs": {"name": "验收公示", "pages": 13, "site_name": "甘肃环评信息网-验收公示"},
}

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0 Safari/537.36"}

def fetch_page(cat, page_num):
    if page_num == 1:
        url = f"{BASE_URL}/category/{cat}.html"
    else:
        url = f"{BASE_URL}/category/{cat}.html?page={page_num}"
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"  [ERROR] {url}: {e}")
        return None

def parse_items(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for a in soup.find_all("a", href=lambda h: h and "/show/" in h):
        href = a["href"].strip()
        title = a.get_text(strip=True)
        if not title or len(title) < 5:
            continue
        url = BASE_URL + href if href.startswith("/") else href
        items.append((title, url))
    # Dedup by URL
    seen = set()
    unique = []
    for t, u in items:
        if u not in seen:
            seen.add(u)
            unique.append((t, u))
    return unique

def fetch_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        html = r.text
    except:
        return None, None, None

    soup = BeautifulSoup(html, "html.parser")

    # Title from <title>
    title_tag = soup.find("title")
    title = title_tag.get_text(strip=True) if title_tag else ""
    title = re.sub(r"\s*[_-].*", "", title).strip()

    # Date: look for YYYY-MM-DD
    dm = re.search(r"(\d{4}-\d{2}-\d{2})", html)
    pub_date = dm.group(1) if dm else ""

    # Content: page_cont > cont
    content_div = soup.find("div", class_="page_cont")
    if not content_div:
        content_div = soup.find("div", class_="cont")
    if content_div:
        content = str(content_div)
        content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL|re.I)
        content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.DOTALL|re.I)
        return title, content, pub_date

    return None, None, pub_date

def main(incremental=False, limit=None):
    print(f"\n{'='*50}\n{SITE_NAME} (+验收公示)\n{'='*50}")
    all_items = []
    seen_urls = set()

    for cat_key, cat_info in CATEGORIES.items():
        max_pages = 2 if incremental else cat_info["pages"]
        print(f"\n📁 {cat_info['name']} ({max_pages} pages)")

        for page in range(1, max_pages + 1):
            print(f"  Page {page}...", end=" ", flush=True)
            html = fetch_page(cat_key, page)
            if not html:
                break
            items = parse_items(html)
            if not items:
                print("empty")
                break
            print(f"{len(items)} items")

            for title, url in items:
                if url in seen_urls:
                    continue
                seen_urls.add(url)

                # Extract date from title area - check text around the link
                # Date is in format YYYY-MM-DD near the link
                date_match = re.search(r"(\d{4}-\d{2}-\d{2})", html[html.find(url.split('/')[-1]):html.find(url.split('/')[-1])+200] if url.split('/')[-1] in html else "")
                date_text = date_match.group(1) if date_match else ""

                if date_text and date_text < THREE_YEARS_AGO:
                    continue

                if incremental:
                    import sqlite3 as s3
                    conn = s3.connect("/root/search.db")
                    exists = conn.execute("SELECT 1 FROM gov_raw WHERE page_url=?", (url,)).fetchone()
                    conn.close()
                    if exists:
                        continue

                all_items.append((title, url, date_text, cat_info["site_name"]))

            if limit and len(all_items) >= limit:
                all_items = all_items[:limit]
                break

    print(f"\n📋 Total: {len(all_items)} items")

    if not all_items:
        print("  No new items")
        return

    results = []
    for i, (title, url, list_date, site_name) in enumerate(all_items):
        print(f"  [{i+1}/{len(all_items)}] {title[:50]}...", end=" ", flush=True)
        dt, content, pub_date = fetch_detail(url)
        ftitle = dt or title
        fdate = pub_date or list_date
        summary = re.sub(r"<[^>]+>", " ", content or "").strip()[:300]
        summary = re.sub(r"\s+", " ", summary)
        results.append({
            "site_name": site_name, "title": ftitle, "url": url,
            "source_url": url, "content": content or "",
            "pub_date": fdate, "summary": summary, "tags": site_name,
        })
        print("OK")
        time.sleep(0.3)

    if results:
        push_to_searchdb(results, "gshpxx")
    print(f"\nDone! {len(results)} records")

if __name__ == "__main__":
    t0 = time.time()
    inc = len(sys.argv) > 1 and sys.argv[1] in ("1", "--incremental")
    limit = None
    if "--limit" in sys.argv:
        limit = int(sys.argv[sys.argv.index("--limit") + 1])
    main(incremental=inc, limit=limit)
    print(f"Time: {time.time()-t0:.1f}s")