#!/usr/bin/env python3
"""crawl_linxi_hjbh.py - 临西县-生态环境 (临西县人民政府-部门行政许可栏目59)

站点: https://www.linxi.gov.cn/channel/list/59.html
CMS: 临西县政府网站 (UTF-8)
列表: ul.info-list-xxgk > li > a(href=/single/59/ID.html) + span.time(YYYY-MM-DD)
分页: channel/list/59_N.html (首页=59.html, 第2页=59_2.html, 每页~20条)
详情: <title> 标题 + div 正文容器
修复(2026-08-08): 标题提取优先级 meta ArticleTitle > h1.c-h1 > 列表页标题。
原逻辑从<title> split 截取首个分隔符前内容 → 得到'临西县人民政府'(站点名), 真实标题丢失,
导致 FTS 搜项目名/公告标题搜不到。列表页标题实测完整可直接用。
"""
import sys, os, requests, re, argparse
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365*3)).strftime('%Y-%m-%d')
SITE_NAME = '临西县-生态环境(行政许可)'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0'}
BASE = 'https://www.linxi.gov.cn'

def fetch_list(page_idx):
    if page_idx == 1:
        url = f'{BASE}/channel/list/59.html'
    else:
        url = f'{BASE}/channel/list/59_{page_idx}.html'
    try:
        r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
    except:
        return []
    items = re.findall(
        r'<a href="(/single/\d+/\d+\.html)"[^>]*>(.*?)</a>\s*<span class="time">(\d{4}-\d{2}-\d{2})</span>',
        r.text, re.DOTALL
    )
    results = []
    for href, title_text, pd in items:
        title = re.sub(r'<[^>]+>', '', title_text).strip()
        title = re.sub(r'\s+', '', title).strip()
        if len(title) < 4:
            continue
        url = BASE + href
        results.append({'url': url, 'pub_date': pd, 'title': title})
    return results

def extract_title(html, list_title=''):
    """标题优先级: meta ArticleTitle > h1.c-h1 > 列表页标题。
    <title> 是 '站点名 - 公告标题' 格式, split 截取前半会得到站点名, 不可用"""
    m = re.search(r'<meta[^>]+name=["\']ArticleTitle["\'][^>]+content=["\']([^"\']+)["\']', html, re.I)
    if m:
        t = re.sub(r'\s+', '', m.group(1)).strip()
        if len(t) >= 4:
            return t
    m = re.search(r'<h1[^>]*class="[^"]*c-h1[^"]*"[^>]*>(.*?)</h1>', html, re.DOTALL | re.I)
    if m:
        t = re.sub(r'<[^>]+>', '', m.group(1)).strip()
        t = re.sub(r'\s+', '', t)
        if len(t) >= 4:
            return t
    return list_title

def extract_balanced_div(html, open_tag_re):
    """平衡 div 匹配: 从匹配到的 <div...> 的 '>' 之后, 到对应 </div>, 返回内部 HTML。
    修复非贪婪 (.*?)</div> 在正文含嵌套 div(表格/附件区)时截断的问题"""
    m = re.search(open_tag_re, html)
    if not m:
        return ''
    gt = html.find('>', m.start())
    if gt == -1:
        return ''
    i = gt + 1
    depth = 1
    for mm in re.finditer(r'<div[\s>]|</div>', html[i:]):
        if mm.group(0).startswith('<div'):
            depth += 1
        else:
            depth -= 1
            if depth == 0:
                return html[i:i + mm.start()]
    return html[i:]

def fetch_detail(url, list_title=''):
    for _attempt in range(2):
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
            break
        except:
            r = None
    if r is None:
        return '', ''
    title = extract_title(r.text, list_title)
    content = ''
    # 正文容器: 优先平衡匹配 c-detail (临西详情页实际容器, 含正文+表格+附件)
    # 修复: 原模式列表缺 c-detail + 非贪婪 (.*?)</div> 首遇</div>即截断 → 正文全丢
    for pat in [r'<div[^>]*class="[^"]*c-detail[^"]*"[^>]*>',
                r'<div[^>]*class="[^"]*article-content[^"]*"[^>]*>',
                r'<div[^>]*id="content"[^>]*>',
                r'<div[^>]*class="[^"]*TRS_Editor[^"]*"[^>]*>',
                r'<div[^>]*class="[^"]*content[^"]*"[^>]*>',
                r'<div[^>]*class="[^"]*article[^"]*"[^>]*>']:
        c = extract_balanced_div(r.text, pat)
        if c.strip():
            content = c.strip()
            if len(content) > 50:
                break
    content = re.sub(r'<script.*?</script>', '', content, flags=re.DOTALL | re.I)
    content = re.sub(r'<div[^>]*class="clx"[^>]*></div>', '', content)
    content = re.sub(r'href="(?!https?://|//|#)(/[^"]+)"', lambda mm: f'href="{BASE}{mm.group(1)}"', content)
    content = re.sub(r'src="(?!https?://|//)(/[^"]+)"', lambda mm: f'src="{BASE}{mm.group(1)}"', content)
    content = re.sub(r'(<p>\s*(<br\s*/?>)?\s*</p>)+$', '', content)
    return title, content.strip()

def run(max_pages=5):
    records = []
    seen = set()
    for pg in range(max_pages):
        items = fetch_list(pg + 1)
        if not items:
            print(f"  Page {pg+1}: 0 items (stop)")
            break
        print(f"  Page {pg+1}: {len(items)} items")
        for it in items:
            if it['pub_date'] and it['pub_date'] < CUTOFF:
                continue
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            title, content = fetch_detail(it['url'], it['title'])
            if not title:
                title = it['title']
            if not content.strip():
                continue
            records.append({
                'title': title, 'url': it['url'], 'source_url': it['url'],
                'pub_date': it['pub_date'],
                'site_name': SITE_NAME, 'content': content, 'summary': '',
            })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "linxi_hjbh")
    return len(valid)

if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=5)
    args = ap.parse_args()
    cnt = run(max_pages=args.pages)
    print(f"Done: {cnt} records")