#!/usr/bin/env python3
"""crawl_ahtlyaq_hjsp.py - 铜陵义安区-环评审批(OpennessContent, --col支持双栏目)"""
import sys, os, requests, re, argparse
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365*3)).strftime('%Y-%m-%d')
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0'}
SITES = {
    '53929': {'name': '义安区-环评审批决定', 'path': '/openness/OpennessContent/showList/1599/53929'},
    '53865': {'name': '义安区-环评受理公示', 'path': '/openness/OpennessContent/showList/1599/53865'},
}
BASE = 'https://www.ahtlyaq.gov.cn'

def get_site(col):
    return SITES.get(col, SITES['53929'])

def fetch_urls_from_page(site, page_num):
    url = f'{BASE}{site["path"]}/page_{page_num + 1}.html'
    try:
        r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
    except:
        return []
    # 列表: <a class="biaoti-title" href="/openness/OpennessContent/show/1160755.html" title="标题">
    items = re.findall(
        r'<a[^>]+href="([^"]*OpennessContent/show/\d+\.html)"[^>]*title="([^"]*)"',
        r.text
    )
    results = []
    for href, title in items:
        title = re.sub(r'\s+', '', title)
        if len(title) < 4:
            continue
        if href.startswith('/'):
            full_url = BASE + href
        elif href.startswith('http'):
            full_url = href
        else:
            full_url = BASE + '/' + href
        results.append({'url': full_url, 'pub_date': '', 'title': title})
    return results

def fetch_detail(url):
    for _attempt in range(2):
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
            break
        except:
            r = None
    if r is None:
        return '', ''
    title = ''
    m = re.search(r'<title>([^<]*)</title>', r.text, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
        title = re.sub(r'\s*[-_|]\s*义安区人民政府\s*$', '', title)
    title = re.sub(r'\s+', '', title).strip()
    # 详情页日期: meta PubDate 或页面日期
    pub_date = ''
    mp = re.search(r'<meta name="PubDate" content="(\d{4}-\d{2}-\d{2})', r.text)
    if mp:
        pub_date = mp.group(1)
    if not pub_date:
        md = re.search(r'(\d{4}-\d{2}-\d{2})', r.text[:4000])
        if md:
            pub_date = md.group(1)
    content = ''
    m = re.search(r'class="col-md-12"[^>]*>(.*?)</div>', r.text, re.DOTALL)
    if m:
        content = m.group(1).strip()
    return title, content, pub_date

def run(max_pages=5, col='53929'):
    site = get_site(col)
    records = []
    seen = set()
    for pg in range(max_pages):
        items = fetch_urls_from_page(site, pg)
        if not items:
            break
        print(f"  Page {pg+1}: {len(items)} items")
        for it in items:
            if it['pub_date'] and it['pub_date'] < CUTOFF:
                continue
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            title, content, pub_date = fetch_detail(it['url'])
            if not title:
                title = it['title']
            pd = pub_date or it['pub_date']
            if not content.strip():
                continue
            records.append({
                'title': title, 'url': it['url'], 'pub_date': pd,
                'site_name': site['name'], 'content': content, 'summary': '',
            })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, f"ahtlyaq_{col}")
    return len(valid)

if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=5)
    ap.add_argument('--col', default='53929')
    args = ap.parse_args()
    cnt = run(max_pages=args.pages, col=args.col)
    print(f"Done: {cnt} records")