#!/usr/bin/env python3
"""crawl_lianxi_xzxk.py - 濂溪区人民政府-濂溪生态环境局-行政许可 (TRS 模板)

站点: https://www.lianxi.gov.cn/lxsthjj/fdzdgknr/xzglyfw/xzxk/
CMS: TRS (中国濂溪政府门户), 列表 td.info>a[title] + td.fbrq 日期, index_N.html 分页 (8页, P9空停)
详情: meta[ArticleTitle] + meta[PubDate] + div#content 平衡div (内含 trs_editor_view TRS_UEDITOR, 表格+<p>段落)
附件: ./P0...doc/pdf 相对路径 → urljoin(详情URL) 绝对化
CUTOFF 边界: P3 中段 (2023-08-10 起窗口内)
样本: 干法预浸料制备生产线建设项目环境影响报告表的批准公告 (2025-11-21)
"""
import sys, os, re, argparse
from datetime import datetime, timedelta
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=3 * 365)).strftime("%Y-%m-%d")
SITE_NAME = '濂溪区生态环境局-行政许可'
GROUP_NAME = '江西'
BASE = 'https://www.lianxi.gov.cn'
LIST_PATH = '/lxsthjj/fdzdgknr/xzglyfw/xzxk/'
LIST_URL = BASE + LIST_PATH + 'index.html'
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Referer': LIST_URL,
}


def fetch(url):
    import requests
    requests.packages.urllib3.disable_warnings()
    try:
        r = requests.get(url, timeout=(8, 20), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f'[WARN] fetch failed: {url} - {e}', file=sys.stderr)
        return ''


def extract_balanced_div(html, open_tag_re):
    """平衡 div 匹配 (大小写不敏感)"""
    m = re.search(open_tag_re, html, re.I)
    if not m:
        return ''
    gt = html.find('>', m.start())
    if gt == -1:
        return ''
    i = gt + 1
    depth = 1
    for mm in re.finditer(r'<div[\s>]|</div>', html[i:], re.I):
        if mm.group(0).startswith('<div'):
            depth += 1
        else:
            depth -= 1
            if depth == 0:
                return html[i:i + mm.start()]
    return html[i:]


def fetch_list(page_num):
    url = LIST_URL if page_num == 1 else f'{BASE}{LIST_PATH}index_{page_num}.html'
    html = fetch(url)
    if not html:
        return []
    items = re.findall(
        r'<a[^>]+href="([^"]*xzxk/\d{6}/t\d+_\d+\.html)"[^>]*title="([^"]*)"[^>]*>.*?<td class="fbrq">\s*([\d-]+)',
        html, re.DOTALL)
    results = []
    for href, title, d in items:
        title = re.sub(r'\s+', '', title).strip()
        title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
        if len(title) < 4:
            continue
        url = href if href.startswith('http') else BASE + href
        results.append({'url': url, 'pub_date': d, 'title': title})
    # 去重
    seen = set()
    uniq = []
    for r in results:
        if r['url'] not in seen:
            seen.add(r['url'])
            uniq.append(r)
    return uniq


def fetch_detail(url):
    html = fetch(url)
    if not html:
        return None
    m = re.search(r'<meta name="ArticleTitle" content="([^"]*)"', html)
    title = m.group(1).strip() if m else ''
    m = re.search(r'<meta name="PubDate" content="([^"]*)"', html)
    pub_date = m.group(1)[:10] if m else ''
    # 正文: div#content 平衡 div
    content = extract_balanced_div(html, r'<div[^>]*id="content"[^>]*>')
    if not content.strip():
        content = extract_balanced_div(html, r'<div[^>]*class="[^"]*TRS_EDITOR[^"]*"[^>]*>')
    if not content.strip():
        return None
    content = re.sub(r'<script.*?</script>', '', content, flags=re.DOTALL | re.I)
    content = re.sub(r'<style.*?</style>', '', content, flags=re.DOTALL | re.I)
    # href/src 相对路径转绝对 (按详情页 urljoin, 附件 ./P0... 同目录)
    content = re.sub(r'href="(?!https?://|//|#|javascript:)([^"]+)"',
                     lambda mm: f'href="{urljoin(url, mm.group(1))}"', content)
    content = re.sub(r'src="(?!https?://|//)([^"]+)"',
                     lambda mm: f'src="{urljoin(url, mm.group(1))}"', content)
    # 清首尾空 p
    content = re.sub(r'^(<p[^>]*>(\s*(<br\s*/?>)?\s*)?</p>)+', '', content)
    content = re.sub(r'(<p[^>]*>(\s*(<br\s*/?>)?\s*)?</p>)+$', '', content)
    content = content.strip()
    if not content:
        return None
    return {'title': title, 'pub_date': pub_date, 'content': content}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=8)
    ap.add_argument('--max-detail', type=int, default=0)
    args = ap.parse_args()

    print(f'=== {SITE_NAME} === 列表页数: {args.pages}, CUTOFF: {CUTOFF}')
    records = []
    seen = set()
    for pg in range(1, args.pages + 1):
        items = fetch_list(pg)
        if not items:
            print(f'  P{pg}: 0 条 → 实际末页 P{pg-1}')
            break
        print(f'  P{pg}: {len(items)} 条')
        for it in items:
            if it['pub_date'] and it['pub_date'] < CUTOFF:
                continue
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            d = fetch_detail(it['url'])
            if not d:
                continue
            title = d['title'] or it['title']
            if not d['title']:
                # 详情无 ArticleTitle 时用列表标题
                pass
            records.append({
                'title': title,
                'url': it['url'],
                'source_url': it['url'],
                'pub_date': d['pub_date'] or it['pub_date'],
                'site_name': SITE_NAME,
                'content': d['content'],
                'summary': '',
                'group_name': GROUP_NAME,
            })
        if args.max_detail and len(records) >= args.max_detail:
            break
    print(f'窗口内合计: {len(records)} 条')

    if records:
        push_to_searchdb(records, 'lianxi_xzxk')
    print(f'[RESULT] {SITE_NAME}: {len(records)} items')


if __name__ == '__main__':
    main()