#!/usr/bin/env python3
"""crawl_cnjx_6119.py - 绩溪县人民政府-通知公告 (Epoint, 列表liststyle1 + 详情#zoom)
修复: print加flush防静默; fetch_detail重试收敛(3->2); 支持--pages N分页回填"""
import sys, os, requests, re
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365*3)).strftime('%Y-%m-%d')
SITE_NAME = '绩溪县-通知公告'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0'}
BASE = 'https://www.cnjx.gov.cn/News/showList/6119'


def fetch_urls_from_page(page_num):
    if page_num == 0:
        url = BASE + '/page_1.html'
    else:
        url = f'{BASE}/page_{page_num + 1}.html'
    try:
        r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
    except:
        return []
    items = re.findall(
        r'<span>(\d{4}-\d{2}-\d{2})</span>\s*<a href="([^"]+)" title="([^"]*)"',
        r.text, re.DOTALL
    )
    results = []
    for pub_date, href, title in items:
        if href.startswith('/'):
            full_url = 'https://www.cnjx.gov.cn' + href
        elif href.startswith('./'):
            full_url = BASE + href[1:]
        else:
            full_url = BASE + '/' + href
        results.append({'url': full_url, 'pub_date': pub_date, 'title': title})
    return results


def fetch_detail(url):
    r = None
    for _attempt in range(2):
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
            break
        except:
            r = None
    if r is None:
        return '', ''
    title = ''
    m = re.search(r'<title>([^<]*)</title>', r.text, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
        title = re.sub(r'\s*[-_|].*$', '', title).strip()
    content = ''
    idx = r.text.find('id="zoom"')
    if idx > 0:
        div_start = r.text.rfind('<div', 0, idx)
        if div_start >= 0:
            start = r.text.find('>', div_start) + 1
            depth = 1
            pos = start
            # 安全上限: 最多扫描 2MB, 防止异常超大页面卡死
            limit = min(len(r.text), start + 2_000_000)
            while pos < limit and depth > 0:
                if r.text[pos:pos+4] == '<div':
                    depth += 1
                    pos += 4
                elif r.text[pos:pos+6] == '</div>':
                    depth -= 1
                    pos += 6
                else:
                    pos += 1
            if depth == 0:
                content = r.text[start:pos-6].strip()
    return title, content


def run(pages=30):
    records = []
    seen = set()
    for pg in range(pages):
        items = fetch_urls_from_page(pg)
        if not items:
            break
        print(f"  Page {pg+1}: {len(items)} items", flush=True)
        for it in items:
            if it['pub_date'] < CUTOFF:
                continue
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            title, content = fetch_detail(it['url'])
            if not title or not content.strip():
                continue
            records.append({
                'title': title,
                'url': it['url'],
                'pub_date': it['pub_date'],
                'site_name': SITE_NAME,
                'content': content,
                'summary': '',
            })
    print(f"  Total: {len(records)} items", flush=True)
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "cnjx_6119")
    return len(valid)


if __name__ == '__main__':
    pages = 30
    if '--incremental' in sys.argv:
        pages = 1
    for i, a in enumerate(sys.argv):
        if a == '--pages' and i + 1 < len(sys.argv):
            pages = int(sys.argv[i + 1])
        elif a.startswith('--pages='):
            pages = int(a.split('=')[1])
    cnt = run(pages)
    print(f"Done: {cnt} records", flush=True)