#!/usr/bin/env python3
"""crawl_chongyang_hp.py - 崇阳县人民政府-建设项目环境影响评价信息 (TRS静态+createPage)"""
import sys, os, requests, re
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365*3)).strftime('%Y-%m-%d')
SITE_NAME = '崇阳县-建设项目环评'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0'}
BASE = 'http://www.chongyang.gov.cn/xxgk/xxgkml/gysyjs/hjbh/jsxmhjyxpjxx'

def fetch_urls_from_page(page_num):
    if page_num == 0:
        url = BASE + '/'
    else:
        url = f'{BASE}/index_{page_num + 1}.shtml'
    for _attempt in range(3):
        try:
            r = requests.get(url, timeout=(10, 20), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
            break
        except:
            r = None
    if r is None:
        return []
    items = re.findall(
        r'<a href="([^"]+)"[^>]*title="([^"]*)"[^>]*class="title"',
        r.text, re.DOTALL
    )
    results = []
    for href, title in items:
        if href.startswith('http'):
            full_url = href
        elif href.startswith('./'):
            full_url = BASE + href[1:]
        elif href.startswith('/'):
            full_url = 'http://www.chongyang.gov.cn' + href
        else:
            full_url = BASE + '/' + href
        results.append({'url': full_url, 'pub_date': '', 'title': title})
    return results

def fetch_detail(url):
    for _attempt in range(3):
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
            break
        except:
            r = None
    if r is None:
        return '', ''
    title = ''
    # 优先取页面标题元素, fallback <title> 标签
    # h1 内可含嵌套标签(<UCAPTITLE>等), 提取全部文本再剥标签
    m = re.search(r'<h1[^>]*>(.*?)</h1>', r.text, re.DOTALL)
    if m:
        cand = re.sub(r'<[^>]+>', '', m.group(1)).strip()
        cand = re.sub(r'^begin-->|end-->$', '', cand).strip()
        if cand and '404' not in cand:
            title = cand
    if not title:
        for pat in [r'class="[^"]*title[^"]*"[^>]*>\s*([^<]{5,200})\s*<',
                    r'class="[^"]*Title[^"]*"[^>]*>\s*([^<]{5,200})\s*<']:
            m = re.search(pat, r.text, re.DOTALL)
            if m:
                cand = re.sub(r'<[^>]+>', '', m.group(1)).strip()
                cand = re.sub(r'^begin-->|end-->$', '', cand).strip()
                if cand and '404' not in cand:
                    title = cand
                    break
    if not title:
        m = re.search(r'<title>([^<]*)</title>', r.text, re.DOTALL)
        if m:
            title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
            # 显式剥离站点名(不用字符类split, 标题内部连字符如2026-06会被误分割截断)
            title = re.sub(r'^\s*崇阳县政府\s*[-_|]\s*', '', title)
            title = re.sub(r'\s*[-_|]\s*崇阳县政府\s*$', '', title)
    title = re.sub(r'\s+', ' ', title).strip()
    # 日期
    pub_date = ''
    m = re.search(r'(\d{4}-\d{2}-\d{2})', r.text[:6000])
    if m:
        pub_date = m.group(1)
    content = ''
    for marker in ['<div class="view TRS_UEDITOR', '<div class="trs_editor_view TRS_UEDITOR']:
        idx = r.text.find(marker)
        if idx > 0:
            start = r.text.find('>', idx) + 1
            depth = 1
            pos = start
            while pos < len(r.text) and depth > 0:
                if r.text[pos:pos+4] == '<div':
                    depth += 1
                    pos += 4
                elif r.text[pos:pos+6] == '</div>':
                    depth -= 1
                    pos += 6
                else:
                    pos += 1
            content = r.text[start:pos-6].strip()
            if content:
                break
    if not content:
        m = re.search(r'<div class="inner">(.*?)</div>', r.text, re.DOTALL)
        if m:
            content = m.group(1).strip()
    if not content:
        m = re.search(r'<div id="zoom"[^>]*>(.*?)</div>', r.text, re.DOTALL)
        if m:
            content = m.group(1).strip()
    return title, content, pub_date

def run(incremental=False):
    records = []
    seen = set()
    max_pages = 1 if incremental else 10
    for pg in range(max_pages):
        items = fetch_urls_from_page(pg)
        if not items:
            break
        print(f"  Page {pg+1}: {len(items)} items")
        for it in items:
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            title, content, pub_date = fetch_detail(it['url'])
            if not title or not content.strip():
                continue
            if pub_date and pub_date < CUTOFF:
                continue
            records.append({
                'title': title,
                'url': it['url'],
                'pub_date': pub_date,
                'site_name': SITE_NAME,
                'content': content,
                'summary': '',
            })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "chongyang_hp")
    return len(valid)

if __name__ == '__main__':
    inc = '--incremental' in sys.argv
    cnt = run(incremental=inc)
    print(f"Done: {cnt} records")