#!/usr/bin/env python3
"""crawl_taz_zjzwfw.py - 台州市-建设项目环境影响评价信息公示 (浙江政务服务网)

站点: https://taz.zjzwfw.gov.cn/col/col1460296/index.html
CMS: 大汉版通 jcms1 经典版 (UTF-8, 服务端渲染, 无AJAX)
列表: a.bt_link[title=标题](href=/art/Y/M/D/art_1460296_ID.html) + div[style*=B4B4B4]日期
分页: 无分页, 固定9条(2023年已停更)
详情: <title>/div.article_main 蓝底td 标题 + meta[pubDate] + div#zoom 正文
"""
import sys, os, requests, re, argparse
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = '2020-01-01'  # 停更站: 抓全部历史
SITE_NAME = '台州市-建设项目环评公示'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0'}
BASE = 'https://taz.zjzwfw.gov.cn'
LIST_URL = BASE + '/col/col1460296/index.html'

def fetch_list():
    try:
        r = requests.get(LIST_URL, timeout=(5, 15), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
    except:
        return []
    items = re.findall(
        r"<a href='([^']+)' class='bt_link' title='([^']*)'[^>]*>",
        r.text
    )
    results = []
    for href, title in items:
        title = re.sub(r'\s+', '', title).strip()
        if len(title) < 4:
            continue
        url = href if href.startswith('http') else BASE + href
        # 日期在 a 之后的 div[style*=B4B4B4]
        m = re.search(r"<a href='" + re.escape(href) + r"'.*?</a>.*?<div style=\"[^\"]*B4B4B4[^\"]*\">(\d{4}-\d{2}-\d{2})</div>", r.text, re.DOTALL)
        pd = m.group(1) if m else ''
        results.append({'url': url, 'pub_date': pd, 'title': title})
    return results

def extract_balanced_div(html, open_tag_re):
    """平衡 div 匹配: 从匹配到的 <div...> 的 '>' 之后, 到对应 </div>, 返回内部 HTML。
    修复非贪婪 (.*?)</div> 在正文含嵌套 div(表格/附件区)时截断的问题"""
    m = re.search(open_tag_re, html)
    if not m:
        return ''
    gt = html.find('>', m.start())
    if gt == -1:
        return ''
    i = gt + 1
    depth = 1
    for mm in re.finditer(r'<div[\s>]|</div>', html[i:]):
        if mm.group(0).startswith('<div'):
            depth += 1
        else:
            depth -= 1
            if depth == 0:
                return html[i:i + mm.start()]
    return html[i:]

def fetch_detail(url):
    for _attempt in range(2):
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
            break
        except:
            r = None
    if r is None:
        return '', ''
    title = ''
    m = re.search(r'<meta name="pubDate" content="([^"]*)"', r.text)
    pd = m.group(1)[:10] if m else ''
    m = re.search(r'<title>\s*([^<]+?)\s*</title>', r.text, re.DOTALL)
    if m:
        title = re.sub(r'\s+', '', m.group(1)).strip()
        title = re.split(r'[-–—|_]', title)[0].strip()
    content = ''
    c = extract_balanced_div(r.text, r'<div[^>]*id="zoom"[^>]*>')
    if c.strip():
        content = c.strip()
    if not content.strip():
        c = extract_balanced_div(r.text, r'<div[^>]*class="article_main"[^>]*>')
        if c.strip():
            content = c.strip()
    content = re.sub(r'src="(?!https?://|//)(/[^"]+)"', lambda mm: f'src="{BASE}{mm.group(1)}"', content)
    return title, pd, content

def run(max_pages=5):
    records = []
    items = fetch_list()
    print(f"  Page 1: {len(items)} items")
    for it in items:
        if it['pub_date'] and it['pub_date'] < CUTOFF:
            continue
        title, pd, content = fetch_detail(it['url'])
        if not title:
            title = it['title']
        if not content.strip():
            continue
        records.append({
            'title': title, 'url': it['url'], 'source_url': it['url'],
            'pub_date': pd or it['pub_date'],
            'site_name': SITE_NAME, 'content': content, 'summary': '',
        })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "taz_zjzwfw")
    return len(valid)

if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=5)
    args = ap.parse_args()
    cnt = run(max_pages=args.pages)
    print(f"Done: {cnt} records")