#!/usr/bin/env python3
"""crawl_nbfh_zjzwfw.py - 宁波奉化-建设项目环评公示 (浙江政务服务网·宁波市奉化区)

站点: https://nbfh.zjzwfw.gov.cn/col/col1460362/index.html
CMS: 大汉版通 jcms1 经典版 (UTF-8, 服务端渲染, 无AJAX) — 同 taz_zjzwfw/qzly_zjzwfw 模板
列表: <a href='/art/Y/M/D/art_1460362_ID.html' class='bt_link' title='标题'> + <div style="color: #B4B4B4...">日期</div>
分页: 无分页, 固定约10条 (2023年停更)
详情: meta[pubDate] + meta[ArticleTitle] + div#zoom 正文(平衡div, 含表格)
样本: 奉化四明路快速化改造工程环评首次公示 (2023-07-05)
"""
import sys, os, requests, re, argparse
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = '2020-01-01'  # 停更站: 抓全部历史
SITE_NAME = '宁波奉化-建设项目环评公示'
GROUP_NAME = '浙江'
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Referer': 'https://nbfh.zjzwfw.gov.cn/',
}
BASE = 'https://nbfh.zjzwfw.gov.cn'
LIST_URL = BASE + '/col/col1460362/index.html'
COL_ID = '1460362'


def fetch(url):
    try:
        r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f"[WARN] fetch failed: {url} - {e}", file=sys.stderr)
        return ''


def fetch_list():
    html = fetch(LIST_URL)
    if not html:
        return []
    items = re.findall(
        r"<a href='([^']+)' class='bt_link' title='([^']*)'[^>]*>",
        html
    )
    results = []
    for href, title in items:
        # 只抓精准子栏目: /art/YYYY/M/D/art_1460362_ID.html
        if not re.search(r'/art/\d{4}/\d+/\d+/art_' + COL_ID + r'_\d+\.html', href):
            continue
        title = re.sub(r'\s+', '', title).strip()
        title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
        if len(title) < 4:
            continue
        url = href if href.startswith('http') else BASE + href
        # 日期在 a 之后 div[style*=B4B4B4]
        m = re.search(r"<a href='" + re.escape(href) + r"'.*?</a>.*?color: #B4B4B4[^>]*>(\d{4}-\d{2}-\d{2})</div>", html, re.DOTALL)
        pd = m.group(1) if m else ''
        results.append({'url': url, 'pub_date': pd, 'title': title})
    return results


def extract_balanced_div(html, open_tag_re):
    """平衡 div 匹配 (大小写不敏感)"""
    m = re.search(open_tag_re, html, re.I)
    if not m:
        return ''
    gt = html.find('>', m.start())
    if gt == -1:
        return ''
    i = gt + 1
    depth = 1
    for mm in re.finditer(r'<div[\s>]|</div>', html[i:], re.I):
        if mm.group(0).startswith('<div') or mm.group(0).startswith('<DIV'):
            depth += 1
        else:
            depth -= 1
            if depth == 0:
                return html[i:i + mm.start()]
    return html[i:]


def fetch_detail(url):
    html = fetch(url)
    if not html:
        return '', '', ''
    # 标题: meta ArticleTitle 优先, <title> 回退
    title = ''
    m = re.search(r'<meta name="ArticleTitle" content="([^"]*)"', html)
    if m:
        title = m.group(1).strip()
    if not title:
        m = re.search(r'<title>\s*([^<]+?)\s*</title>', html, re.DOTALL)
        if m:
            title = re.sub(r'\s+', '', m.group(1)).strip()
            title = re.split(r'[-–—|_]', title)[0].strip()
    title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
    # 日期: meta pubDate
    date_text = ''
    m = re.search(r'<meta name="pubDate" content="([^"]*)"', html)
    if m:
        md = re.search(r'(\d{4}-\d{2}-\d{2})', m.group(1))
        if md:
            date_text = md.group(1)
    # 正文: div#zoom 优先, div.article_main 回退
    content = ''
    c = extract_balanced_div(html, r'<div[^>]*id="zoom"[^>]*>')
    if c.strip():
        content = c.strip()
    if not content.strip():
        c = extract_balanced_div(html, r'<div[^>]*class="article_main"[^>]*>')
        if c.strip():
            content = c.strip()
    content = re.sub(r'<script.*?</script>', '', content, flags=re.DOTALL | re.I)
    content = re.sub(r'<style.*?</style>', '', content, flags=re.DOTALL | re.I)
    # 附件/图片相对路径 → 绝对化
    from urllib.parse import urljoin
    content = re.sub(r'href="(?!https?://|//|#|javascript:)([^"]+)"',
                     lambda mm: f'href="{urljoin(url, mm.group(1))}"', content)
    content = re.sub(r'src="(?!https?://|//)([^"]+)"',
                     lambda mm: f'src="{urljoin(url, mm.group(1))}"', content)
    # 清首尾空 p
    content = re.sub(r'^(<p[^>]*>(\s*(<br\s*/?>)?\s*)?</p>)+', '', content)
    content = re.sub(r'(<p[^>]*>(\s*(<br\s*/?>)?\s*)?</p>)+$', '', content)
    return title, date_text, content.strip()


def run(max_pages=5):
    records = []
    items = fetch_list()
    print(f"  Page 1: {len(items)} items")
    for it in items:
        if it['pub_date'] and it['pub_date'] < CUTOFF:
            continue
        title, date_text, content = fetch_detail(it['url'])
        if not title:
            title = it['title']
        if not content:
            print(f"    SKIP empty content: {it['url']}")
            continue
        pd = date_text or it['pub_date']
        # 附件提取 (从正文)
        atts = []
        for am in re.finditer(r'<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>', content, re.DOTALL):
            h, t = am.group(1), re.sub(r'<[^>]+>', '', am.group(2)).strip()
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|caj|wps)$', h, re.I) or 'download' in h.lower():
                atts.append({'title': t or '附件', 'url': h})
        records.append({
            'title': title, 'url': it['url'], 'source_url': it['url'],
            'pub_date': pd, 'site_name': SITE_NAME, 'content': content,
            'summary': '', 'attachments': json.dumps(atts, ensure_ascii=False) if atts else '[]',
            'group_name': GROUP_NAME,
        })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "nbfh_zjzwfw")
    return len(valid)


if __name__ == '__main__':
    import json
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=5)
    args = ap.parse_args()
    n = run(max_pages=args.pages)
    print(f"[RESULT] {SITE_NAME}: {n} items")