#!/usr/bin/env python3
"""
南雄市人民政府 - 建设项目环境影响评价信息
https://www.gdnx.gov.cn/zdly/hjbhh/zxcx/
WAF防护较强（创宇云盾），详情页可能被屏蔽。
先从列表页提取标题+日期，正文待WAF解封后自动补充。
"""
import sys, os, re, time, json
sys.path.insert(0, '/root/gov_crawler')
from crawler_lib import fetch_page, clean_html
import sqlite3
import os

SITE_NAME = "南雄市-建设项目环评"
BASE = "https://www.gdnx.gov.cn"
LIST_URL = "https://www.gdnx.gov.cn/zdly/hjbhh/zxcx/index.html"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
THREE_YEARS_AGO = "2023-06-01"

def parse_list(html):
    """从列表页提取 (title, url, date_str)"""
    items = []
    for m in re.finditer(r'<li>(.*?)</li>', html, re.DOTALL):
        li = m.group(1)
        if 'content/post' not in li:
            continue
        url_m = re.search(r'href="(https://www\.gdnx\.gov\.cn/zdly/hjbhh/zxcx/content/[^"]+)"', li)
        title_m = re.search(r'title="([^"]+)"', li)
        if not url_m or not title_m:
            continue
        url = url_m.group(1).strip()
        title = title_m.group(1).strip()
        date_str = ""
        date_m = re.search(r'(20\d{2}-\d{2}-\d{2})', li)
        if date_m:
            date_str = date_m.group(1)
        items.append((title, url, date_str))
    return items

def insert_to_db(items):
    if not items:
        return
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in items:
        try:
            db.execute("INSERT OR IGNORE INTO gov_raw "
                "(site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?,?,?,?,?,?,?,?,?,?)", (
                item["site_name"][:200], item["url"], item["url"],
                item["title"][:500], item["pub_date"][:10],
                item["summary"][:500], item["content"],
                "active", "", item["tags"][:100],
            ))
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except:
            skip += 1
    db.commit()
    db.execute("INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?", (SITE_NAME,))
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}, FTS已同步")

def main():
    print(f"\n{'='*50}")
    print(f"🏠 {SITE_NAME}")
    print(f"{'='*50}")

    all_items = []
    seen_urls = set()

    if len(sys.argv) > 1 and sys.argv[1] == "inc":
        pages = [1]
        print("📋 增量模式：仅第1页")
    else:
        pages = range(1, 8)
        print("📋 全量模式：7页从列表页导入（跳过WAF封锁的详情页）")

    for page_no in pages:
        if page_no == 1:
            url = LIST_URL
        else:
            url = f"{BASE}/zdly/hjbhh/zxcx/index_{page_no}.html"
        print(f"  📄 第 {page_no} 页...", end=" ", flush=True)
        html = fetch_page(url, encoding='utf-8')
        if not html:
            print("❌ 无响应")
            break
        items = parse_list(html)
        if not items:
            print("0 条")
            break
        print(f"✅ {len(items)} 条")

        for title, href, date_str in items:
            if href in seen_urls:
                continue
            seen_urls.add(href)
            if date_str and date_str < THREE_YEARS_AGO:
                continue
            all_items.append({
                "site_name": SITE_NAME,
                "title": title,
                "url": href,
                "content": "",
                "pub_date": date_str,
                "summary": title[:300],
                "tags": "环评审批",
            })
        time.sleep(0.5)

    if all_items:
        insert_to_db(all_items)
    print(f"\n✅ 完成! 共 {len(all_items)} 条")

if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")