#!/usr/bin/env python3
"""
涪城区-环评审批 (www.myfc.gov.cn)
==============================
政府信息公开平台
列表: /fcqrmzf/c122189/zwgkly.shtml
详情: /fcqrmzf/c122189/YYYYMM/hash.shtml
正文: div.content

用法:
    python3 crawl_myfc.py          # 全量
    python3 crawl_myfc.py --test   # 测试 5 条
"""

import re, sys, os, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
import os
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# ── 配置 ──
SITE_NAME  = "涪城区-环评审批"
BASE_URL   = "https://www.myfc.gov.cn"
LIST_URL   = "https://www.myfc.gov.cn/fcqrmzf/c122189/zwgkly.shtml"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
CUTOFF     = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS    = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f"  ⚠ fetch fail: {e}")
        return None

def parse_list(html):
    """提取列表条目 (url, title, approx_date)"""
    items = []
    # Find the article-list UL
    start = html.find('class="article-list"')
    if start < 0:
        start = html.find("class='article-list'")
    if start < 0:
        return items
    end = html.find('</ul>', start)
    if end < 0:
        return items
    section = html[start:end]

    # Extract all links with titles
    hrefs = re.findall(r'href="([^"]+)"', section)
    titles = re.findall(r"title='([^']+)'", section)

    for href, title in zip(hrefs, titles):
        title = title.strip()
        if not title or not href:
            continue
        if not href.startswith('http'):
            href = BASE_URL + href
        # Extract approx date from URL: /YYYYMM/
        ym = re.search(r'/(\d{4})(\d{2})/', href)
        approx_date = f"{ym.group(1)}-{ym.group(2)}-01" if ym else ""
        items.append({"title": title, "url": href, "pub_date": approx_date})
    return items

def fetch_detail(url):
    """获取详情页的 title, content, pub_date"""
    html = fetch(url)
    if not html:
        return {"title": "", "content": "", "pub_date": ""}

    # Title
    title = ""
    m = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html, re.I)
    if m:
        title = m.group(1).strip()

    # Date
    pub_date = ""
    m = re.search(r'<meta[^>]*name="PubDate"[^>]*content="([^"]+)', html, re.I)
    if m:
        pub_date = m.group(1)[:10]
    if not pub_date:
        m = re.search(r'发布日期[：:]\s*(\d{4}-\d{2}-\d{2})', html)
        if m:
            pub_date = m.group(1)

    # Content
    content = ""
    # Try j-fontContent content div (primary pattern for this site)
    m = re.search(r'id="zoom"[^>]*class="j-fontContent content"[^>]*>(.*?)</div>\s*</div>\s*<div class="others"', html, re.DOTALL)
    if m:
        content = m.group(1)
    if not content or len(content) < 50:
        m = re.search(r'class="j-fontContent content"[^>]*>(.*?)</div>', html, re.DOTALL)
        if m:
            content = m.group(1)
    if not content or len(content) < 50:
        m = re.search(r'class="TRS_Editor"[^>]*>(.*?)</div>', html, re.DOTALL)
        if m:
            content = m.group(1)

    if content:
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<link[^>]*>', '', content, flags=re.I)
        content = content.strip()

    return {"title": title, "content": content, "pub_date": pub_date}

def to_db(items):
    """批量写入 search.db"""
    if not items:
        print("  ⏭ 无数据")
        return 0, 0
    import sqlite3
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, fail = 0, 0
    for it in items:
        try:
            db.execute(
                "INSERT OR IGNORE INTO gov_raw "
                "(site_name, title, page_url, content, publish_date, summary, tags) "
                "VALUES (?,?,?,?,?,?,?)",
                (
                    SITE_NAME,
                    (it.get("title") or "")[:500],
                    it.get("url", ""),
                    it.get("content", ""),
                    (it.get("pub_date") or "")[:10],
                    "",
                    "环评审批",
                )
            )
            if db.total_changes > 0:
                ok += 1
            else:
                fail += 1
        except Exception as e:
            fail += 1
            print(f"    ⚠ DB error: {e}")
    # FTS sync
    # 2026-09-22: 先提交 gov_raw —— 库上触发器已维护 FTS，下面这条手动写入会因
    #   rowid 重复而 IntegrityError；不先 commit 会把 gov_raw 那条一并回滚（静默丢数据）
    db.commit()
    db.execute(
        "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary "
        "FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,)
    )
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{fail}, FTS已同步")
    return ok, fail

def crawl(test=False):
    """全量爬取"""
    html = fetch(LIST_URL)
    if not html:
        print("  ❌ 无法获取列表页")
        return 0, 0

    items = parse_list(html)
    if not items:
        print("  ❌ 列表页解析失败")
        return 0, 0

    # Filter by 3 years
    all_items = [it for it in items if not it["pub_date"] or it["pub_date"] >= CUTOFF]
    print(f"  📄 列表: {len(items)}条, 近3年: {len(all_items)}条")

    if test:
        all_items = all_items[:5]
        print(f"  🧪 测试模式: 只处理前 {len(all_items)} 条")

    if not all_items:
        return 0, 0

    # Crawl details
    results = []
    for i, item in enumerate(all_items, 1):
        detail = fetch_detail(item["url"])
        entry = {
            "title": detail["title"] or item["title"],
            "url": item["url"],
            "content": detail["content"],
            "pub_date": detail["pub_date"] or item.get("pub_date", ""),
        }
        results.append(entry)
        print(f"  [{i}/{len(all_items)}] {detail['pub_date']} {entry['title'][:40]}... ({len(detail['content'])}B)")
        time.sleep(0.5)  # Be polite

    # Insert to DB
    ok, fail = to_db(results)
    return ok, fail

if __name__ == "__main__":
    test = "--test" in sys.argv
    print(f"\n📡 [{SITE_NAME}] {'测试模式' if test else '全量'}")
    t0 = time.time()
    ok, fail = crawl(test=test)
    print(f"  ⏱ 耗时: {time.time()-t0:.1f}s")
    print(f"  ✅ 新增: {ok}  ❌ 跳过: {fail}")