#!/usr/bin/env python3
"""crawl_wannian.py - 万年县人民政府-县生态环境局-公告公示"""
import sys, os, re, time, requests
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365*3)).strftime('%Y-%m-%d')
SITE_NAME = '万年县人民政府-公告公示'
BASE_URL = 'http://www.zgwn.gov.cn/BMGKSTHJ/db011/list{page}.shtml'
MAX_PAGES = 100

def extract_list_items(html):
    items = []
    m = re.search(r'<table[^>]*doclist[^>]*>.*?</table>', html, re.DOTALL)
    if not m:
        return items
    table_html = m.group()
    rows = re.findall(r'<tr>.*?</tr>', table_html, re.DOTALL)
    for row in rows:
        lm = re.search(r"<a[^>]*href=\"([^\"]+)\"[^>]*title='([^']*)'", row)
        if not lm:
            lm = re.search(r'<a[^>]*href="([^"]+)"[^>]*title="([^"]*)"', row)
        if not lm:
            continue
        url = lm.group(1)
        if not url.startswith('http'):
            url = 'http://www.zgwn.gov.cn' + url
        title = lm.group(2)
        dm = re.search(r'<td[^>]*width="18%"[^>]*>.*?<span>(.*?)</span>', row, re.DOTALL)
        date = dm.group(1).strip() if dm else ''
        items.append((title, url, date))
    return items

def extract_content(html):
    m = re.search(r'<UCAPCONTENT>(.*?)</UCAPCONTENT>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
        content = re.sub(r'>\s+<', '><', content)
        return content
    return ''

def run(incremental=False):
    records = []
    for pn in range(1, MAX_PAGES + 1):
        url = BASE_URL.format(page='' if pn == 1 else f'_{pn}')
        try:
            r = requests.get(url, timeout=15)
            r.encoding = 'utf-8'
        except Exception as e:
            print(f"  List error page {pn}: {e}")
            break
        items = extract_list_items(r.text)
        if not items:
            print(f"  Page {pn}: no items, stopping")
            break
        page_in_3y = 0
        for title, detail_url, date in items:
            if date < CUTOFF:
                continue
            page_in_3y += 1
            content = ''
            try:
                dr = requests.get(detail_url, timeout=15)
                dr.encoding = 'utf-8'
                content = extract_content(dr.text)
            except Exception as e:
                print(f"    Detail error: {detail_url} - {e}")
            records.append({
                'title': title.strip(),
                'url': detail_url,
                'pub_date': date,
                'site_name': SITE_NAME,
                'content': content,
                'summary': '',
            })
        print(f"  Page {pn}: {len(items)} items, {page_in_3y} in 3y")
        if incremental:
            break
        if page_in_3y == 0:
            break
        time.sleep(0.3)
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "wannian_gggs")
    return len(valid)

if __name__ == '__main__':
    inc = '--incremental' in sys.argv
    cnt = run(incremental=inc)
    print(f"Done: {cnt} records")