#!/usr/bin/env python3
"""crawl_jiujiang_sthjj.py - 九江市生态环境局-公告公示

站点: https://sthjj.jiujiang.gov.cn/zwzx_215/gggs/
CMS: 九江市政府网站群 (sitefiles系, UTF-8)
列表: li > a[href='./YYYYMM/tYYYYMMDD_ID.html'] + span.list-text 标题 + span.timedate 日期, 每页20条
分页: 静态 index_N.html (首页 index.html 无后缀, 第2页=index_2.html, 共17页)
详情: div.m-article.w-1200 > h2 标题 + h3.tit-info 日期 + div.trs_editor_view 正文(平衡div匹配)
附件: 相对路径 ./P0...pdf → 按详情页目录 urljoin 拼接
陷阱: 页面导航/侧栏无正文类名冲突; 正文容器 trs_editor_view 内含嵌套 div(表格) → 平衡匹配
标题: strip ·  实体前缀; 详情 h2 优先, 列表 span.list-text 兜底
"""
import sys, os, re, argparse
import requests
from urllib.parse import urljoin
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365 * 3)).strftime('%Y-%m-%d')
SITE_NAME = '九江市-公告公示'
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36'
}
BASE = 'https://sthjj.jiujiang.gov.cn'
LIST_DIR = '/zwzx_215/gggs'


def fetch_list(page_idx):
    if page_idx == 1:
        url = f'{BASE}{LIST_DIR}/index.html'
    else:
        url = f'{BASE}{LIST_DIR}/index_{page_idx}.html'
    try:
        r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
    except Exception:
        return []
    # li > a[href='./YYYYMM/t....html'] + span.list-text 标题 + span.timedate 日期
    items = re.findall(
        r"<a[^>]+href='(\./\d{6}/t\d+_\d+\.html)'[^>]*>.*?<span class=\"list-text\">(.*?)</span>\s*<span class=\"timedate\">\s*([\d-]+)\s*</span>",
        r.text, re.DOTALL
    )
    results = []
    for href, title_text, pd in items:
        title = re.sub(r'<[^>]+>', '', title_text).strip()
        title = re.sub(r'\s+', ' ', title).strip()
        # strip &middot;&nbsp; 实体前缀
        title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
        if len(title) < 4:
            continue
        url = urljoin(f'{BASE}{LIST_DIR}/', href)
        results.append({'url': url, 'pub_date': pd.strip(), 'title': title})
    return results


def extract_balanced_div(html, open_tag_re):
    """平衡 div 匹配: 从匹配到的 <div...> 的 '>' 之后, 到对应 </div>, 返回内部 HTML。
    修复非贪婪 (.*?)</div> 在正文含嵌套 div(表格/附件区)时截断的问题"""
    m = re.search(open_tag_re, html)
    if not m:
        return ''
    gt = html.find('>', m.start())
    if gt == -1:
        return ''
    i = gt + 1
    depth = 1
    for mm in re.finditer(r'<div[\s>]|</div>', html[i:]):
        if mm.group(0).startswith('<div'):
            depth += 1
        else:
            depth -= 1
            if depth == 0:
                return html[i:i + mm.start()]
    return html[i:]


def fetch_detail(url):
    for _attempt in range(2):
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
            break
        except Exception:
            r = None
    if r is None:
        return '', ''
    # 定位 m-article 容器, 内部取 h2 标题 + trs_editor_view 正文
    art = extract_balanced_div(r.text, r'<div[^>]*class="m-article w-1200"[^>]*>')
    if not art.strip():
        art = r.text
    title = ''
    m = re.search(r'<h2[^>]*>(.*?)</h2>', art, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
        title = re.sub(r'\s+', ' ', title).strip()
        title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
    # 正文: trs_editor_view (TRS 编辑器容器, 含嵌套 div 表格)
    content = ''
    c = extract_balanced_div(art, r'<div[^>]*class="[^"]*trs_editor_view[^"]*"[^>]*>')
    if c.strip():
        content = c.strip()
    if not content.strip():
        c = extract_balanced_div(art, r'<div[^>]*class="[^"]*(?:article|content|zoom)[^"]*"[^>]*>')
        if c.strip():
            content = c.strip()
    # 附件区: div.appendix1 (在 trs_editor_view 之外, 含附件 PDF 链接)
    app = extract_balanced_div(art, r'<div[^>]*class="[^"]*appendix1[^"]*"[^>]*>')
    if app.strip():
        content = content + '\n' + app.strip()
    content = re.sub(r'<script.*?</script>', '', content, flags=re.DOTALL | re.I)
    content = re.sub(r'<style.*?</style>', '', content, flags=re.DOTALL | re.I)
    # href/src 相对路径转绝对 (按详情页目录 urljoin, 附件 ./P0...pdf 同目录)
    content = re.sub(r'href="(?!https?://|//|#|javascript:)([^"]+)"',
                     lambda mm: f'href="{urljoin(url, mm.group(1))}"', content)
    content = re.sub(r'src="(?!https?://|//)([^"]+)"',
                     lambda mm: f'src="{urljoin(url, mm.group(1))}"', content)
    # 清首尾空 p
    content = re.sub(r'^(<p[^>]*>(\s*(<br\s*/?>)?\s*)?</p>)+', '', content)
    content = re.sub(r'(<p[^>]*>(\s*(<br\s*/?>)?\s*)?</p>)+$', '', content)
    return title, content.strip()


def run(max_pages=5):
    records = []
    seen = set()
    for pg in range(max_pages):
        items = fetch_list(pg + 1)
        if not items:
            print(f"  Page {pg + 1}: 0 items (stop)")
            break
        print(f"  Page {pg + 1}: {len(items)} items")
        for it in items:
            if it['pub_date'] and it['pub_date'] < CUTOFF:
                continue
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            title, content = fetch_detail(it['url'])
            if not title:
                title = it['title']
            if not content.strip():
                print(f"    SKIP empty content: {it['url']}")
                continue
            records.append({
                'title': title, 'url': it['url'], 'source_url': it['url'],
                'pub_date': it['pub_date'],
                'site_name': SITE_NAME, 'content': content, 'summary': '',
            })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "jiujiang_sthjj")
    return len(valid)


if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=5)
    args = ap.parse_args()
    cnt = run(max_pages=args.pages)
    print(f"Done: {cnt} records")