#!/usr/bin/env python3
"""Crawl 修水县 - 项目环评 (TRS JS 列表 → data_<year>.xml + x-api-key)

2026-09-08 修复: 静态 index.html 列表已改前端 JS(data_<year>.xml + enc.js 异步渲染),
requests 拿到的 HTML 无行 → 改为直接抓同目录 data_<年份>.xml (需 x-api-key 头)。
"""
import requests, re, sqlite3, sys, time, hashlib, os
from datetime import datetime, timedelta
import xml.etree.ElementTree as ET

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = '修水县 - 项目环评'
BASE_URL = 'https://www.xiushui.gov.cn'
XML_DIR = '/fdzdxxgk/01/04/04/11/03/'
XML_API_KEY = '********^^^^^^^^'
CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime('%Y-%m-%d')

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'x-api-key': XML_API_KEY,
}
session = requests.Session()
session.headers.update(HEADERS)


def fetch_items_xml(year):
    url = f'{BASE_URL}{XML_DIR}data_{year}.xml'
    try:
        resp = session.get(url, timeout=30)
        resp.encoding = 'utf-8'
        raw = resp.text
    except Exception as e:
        print(f'  [ERROR] data_{year}.xml: {e}')
        return []
    try:
        root = ET.fromstring(raw)
    except Exception:
        try:
            s = re.sub(r'^[^<]*<', '<', raw, count=1)
            root = ET.fromstring(s)
        except Exception as e:
            print(f'  [ERROR] data_{year}.xml parse: {e}')
            return []
    items = []
    for it in root.iter('ITEM'):
        def g(tag):
            e = it.find(tag)
            return ''.join(e.itertext()).strip() if e is not None else ''
        title, pub, rel = g('TITLE'), g('PUBURL'), g('RELTIME')
        if not title or not pub:
            continue
        if not pub.startswith('http'):
            pub = BASE_URL + XML_DIR + pub.lstrip('./')
        items.append({'title': title, 'url': pub, 'date': rel[:10]})
    return items


def fetch_detail(url):
    try:
        resp = session.get(url, timeout=30)
        resp.encoding = 'utf-8'
        html = resp.text
    except Exception:
        return '', '', ''

    tm = re.search(r'<meta name="ArticleTitle" content="([^"]*)"', html)
    title = tm.group(1).strip() if tm else ''
    dm = re.search(r'<meta name="PubDate" content="([^"]*)"', html)
    date_str = dm.group(1)[:10] if dm else ''

    m = re.search(r'<div class="view TRS_UEDITOR[^>]* id="article-box">(.*?)</div>\s*</div>\s*</font>', html, re.DOTALL)
    if not m:
        m = re.search(r'<div[^>]* id="article-box"[^>]*>(.*?)</div>\s*</div>\s*</font>', html, re.DOTALL)
    if not m:
        m = re.search(r'<font[^>]* id="Zoom"[^>]*>(.*?)</font>', html, re.DOTALL)
    if not m:
        # 兜底: TRS_Editor/view 容器
        m2 = re.search(r'<div[^>]*class="[^"]*(?:view\s+TRS_UEDITOR|TRS_Editor|trs_editor_view)[^"]*"[^>]*>(.*?)</div>', html, re.DOTALL)
        if m2:
            m = m2

    if m:
        content_raw = m.group(1).strip()
        content_raw = re.sub(r'<script[^>]*>.*?</script>', '', content_raw, flags=re.DOTALL | re.I)
        content_raw = re.sub(r'<style[^>]*>.*?</style>', '', content_raw, flags=re.DOTALL | re.I)
        return title, date_str, content_raw.strip()
    return title, date_str, ''


def main():
    is_incremental = 'incremental' in sys.argv
    print(f'=== {SITE_NAME} ===')

    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted, skipped, old_total = 0, 0, 0
    start_time = time.time()

    now_year = datetime.now().year
    cutoff_year = int(CUTOFF_DATE[:4])
    years = list(range(now_year, cutoff_year - 1, -1)) if not is_incremental else [now_year]
    items_all = []
    for y in years:
        recs = fetch_items_xml(y)
        print(f'  {y}: {len(recs)} items from xml')
        items_all.extend(recs)
    print(f'  total items: {len(items_all)}')

    for item in items_all:
        if item['date'] and item['date'] < CUTOFF_DATE:
            old_total += 1
            continue

        has = c.execute("SELECT 1 FROM gov_raw WHERE page_url=? AND content IS NOT NULL AND content!=''", (item['url'],)).fetchone()
        if has:
            skipped += 1
            continue

        try:
            title, date_str, content = fetch_detail(item['url'])
        except Exception as e:
            print(f'  [ERROR] Detail: {e}')
            time.sleep(1)
            skipped += 1
            continue

        if not content and not title:
            skipped += 1
            print(f'  [EMPTY] {item["title"][:40]}')
            continue

        final_title = title or item['title']
        final_date = date_str or item['date']
        int_id = int(hashlib.md5(item['url'].encode()).hexdigest()[:15], 16) % (2**63)
        date_rank = int(final_date.replace('-', '')) if final_date else 0
        summary = re.sub(r'<[^>]+>', '', content)[:200] if content else final_title
        summary = re.sub(r'\s+', ' ', summary).strip()

        try:
            c.execute(
                'INSERT OR IGNORE INTO gov_raw(id, site_name, source_url, page_url, title, publish_date, content, summary, date_rank) VALUES(?,?,?,?,?,?,?,?,?)',
                (int_id, SITE_NAME, item['url'], item['url'], final_title, final_date, content, summary, date_rank)
            )
            if c.rowcount > 0:
                inserted += 1
        except Exception as e:
            print(f'  [DB] {e}')
            skipped += 1
        time.sleep(0.3)

    conn.commit()
    # FTS rebuild
    try:
        c2 = conn.cursor()
        c2.execute("DELETE FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE site_name=?)", (SITE_NAME,))
        rows = c2.execute("SELECT id, title, site_name, summary FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchall()
        for r in rows:
            c2.execute("INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES(?,?,?,?)", r)
        conn.commit()
        print(f'  FTS: {len(rows)} records rebuilt')
    except Exception as e:
        print(f'  FTS error: {e}')
    conn.close()
    elapsed = time.time() - start_time
    print(f'\nDone ({elapsed:.0f}s). Inserted {inserted}, Skipped {skipped}, Old {old_total}')


if __name__ == '__main__':
    main()