#!/usr/bin/env python3
"""crawl_ahgc_hjsp_gs.py - 池州市贵池区 建设项目环境影响评价审批 (145330栏目)
列表: /OpennessTarget/257/145330/page_N.html (td.bt > a + td.cwrq)
详情: /OpennessContent/show/{id}.html (meta ArticleTitle/PubDate + div#zoom)
"""
import sys, os, re, time, sqlite3, subprocess
from datetime import datetime, timedelta

BASE = 'https://www.ahgc.gov.cn'
LIST_TMPL = '/OpennessTarget/257/145330/page_{}.html'
DETAIL_TMPL = '/OpennessContent/show/{}.html'
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime('%Y-%m-%d')
SITE_NAME = '贵池区-环评审批公示'
MAX_PAGES = 9

def fetch(url, retries=3):
    for i in range(retries):
        try:
            r = subprocess.run(['curl', '-sL', '--max-time', '15', url],
                capture_output=True, timeout=20)
            if r.returncode == 0 and r.stdout:
                return r.stdout.decode('utf-8', errors='replace')
        except:
            pass
        time.sleep(2)
    return ''

def extract_list(html):
    items = re.findall(r'<td class="bt"><a href="/OpennessContent/show/(\d+)\.html"[^>]*>(.*?)</a>', html, re.DOTALL)
    dates = re.findall(r'<td class="cwrq">([^<]+)</td>', html)
    return [(item_id, dates[i] if i < len(dates) else '') for i, (item_id, _) in enumerate(items)]

def extract_detail(html):
    title = pub_date = ''
    m = re.search(r'<meta name="ArticleTitle"[^>]*content="([^"]*)"', html)
    if m: title = m.group(1).strip()
    m = re.search(r'<meta name="PubDate"[^>]*content="([^"]*)"', html)
    if m: pub_date = m.group(1).strip()
    content = ''
    m = re.search(r'<div[^>]*id="zoom"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m: content = m.group(1).strip()
    # 附件: a href 指向文件
    for fm in re.finditer(r'<a[^>]*href="([^"]*\.(?:pdf|docx?|xlsx?|zip))"[^>]*>([^<]+)</a>', html, re.DOTALL):
        furl = fm.group(1)
        fname = re.sub(r'<[^>]+>', '', fm.group(2)).strip()
        if not furl.startswith('http'): furl = BASE + furl
        content += '\n<p><a href="{}" target="_blank">[文件] {}</a></p>'.format(furl, fname)
    return title, pub_date, content

def main():
    total = 0
    db = sqlite3.connect(DB_PATH, timeout=30)
    for page in range(1, MAX_PAGES + 1):
        url = BASE + LIST_TMPL.format(page)
        html = fetch(url)
        if not html:
            print('  Page {}: empty'.format(page))
            break
        items = extract_list(html)
        if not items:
            print('  Page {}: no items, done'.format(page))
            break
        all_before = True
        for item_id, date_str in items:
            if not date_str:
                continue
            date_str = date_str.strip()[:10]
            if date_str < CUTOFF:
                continue
            all_before = False
            detail_html = fetch(BASE + DETAIL_TMPL.format(item_id))
            if not detail_html:
                continue
            title, pub_date, content = extract_detail(detail_html)
            if not title:
                continue
            if pub_date and len(pub_date) >= 10:
                pub_date = pub_date[:10]
            summary = re.sub(r'<[^>]+>', '', content)[:200].strip()
            detail_url = BASE + DETAIL_TMPL.format(item_id)
            sql = "INSERT OR REPLACE INTO gov_raw (id, title, content, publish_date, source_url, page_url, site_name, summary) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
            try:
                db.execute(sql, (int(item_id), title, content, pub_date, detail_url, detail_url, SITE_NAME, summary))
                total += 1
            except sqlite3.OperationalError as e:
                print(f'  DB lock, retry: {e}')
                time.sleep(3)
                try:
                    db.execute(sql, (int(item_id), title, content, pub_date, detail_url, detail_url, SITE_NAME, summary))
                    total += 1
                except Exception as e2:
                    print(f'  DB error: {e2}')
        db.commit()
        print('  Page {}: {} items, {} total'.format(page, len(items), total))
        if all_before:
            print('  All before cutoff, stopping')
            break
        time.sleep(0.3)
    db.close()
    # 同步 FTS 影子表 (gov_search rowid 对齐 gov_raw.id)
    try:
        db2 = sqlite3.connect(DB_PATH, timeout=30)
        db2.execute("INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) SELECT id, title, site_name, summary FROM gov_raw WHERE site_name=?", (SITE_NAME,))
        db2.commit()
        db2.close()
        print("FTS synced")
    except Exception as e:
        print(f"FTS sync error: {e}")
    print('\nDone! {} records'.format(total))

if __name__ == '__main__':
    main()