#!/usr/bin/env python3
"""crawl_ncjk_gggs.py - 南昌经开区-公告公示 (江西南昌)

站点: https://ncjk.nc.gov.cn/ncjkq/gggs/list.shtml
CMS: TRS 系 (createPageHTML('page_div',13,1,'list','shtml',257) = 257条/13页)
列表: <a href="/ncjkq/gggs/YYYYMM/32hex.shtml" target="_blank">标题</a> + <span>日期</span>
分页: list_N.shtml (N=2..13)
⚠️ WAF: 瑞数 js 混淆 + 限速封IP (~30s) — requests/urllib3 全被拦 (TLS 指纹), 必须用 curl subprocess
      + Chrome UA + Referer + Accept; 请求间隔 1.5s; 403(7148B警告页)时 sleep 30 重试
详情: meta ArticleTitle + meta PubDate + div.zoom (内含 zoomcon)
"""
import sys, os, subprocess, re, argparse, json, time
from datetime import datetime, timedelta
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365 * 3)).strftime('%Y-%m-%d')
SITE_NAME = '南昌经开区-公告公示'
GROUP_NAME = '江西'
BASE = 'https://ncjk.nc.gov.cn'
LIST_URL = BASE + '/ncjkq/gggs/list.shtml'
UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
WAF_BLOCK_SIZE = 8000  # 7148B 警告页判定


def curl_get(url, referer, cj=None):
    if cj is None:
        cj = f'/tmp/ncjk_cj_{os.getpid()}_{int(time.time() * 1000) % 100000}.txt'
    cmd = [
        'curl', '-sk', '-m', '25',
        '-A', UA,
        '-H', f'Referer: {referer}',
        '-H', 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
        '-H', 'Accept-Language: zh-CN,zh;q=0.9',
        '-H', 'Connection: keep-alive',
        '-c', cj, '-b', cj,
        '-L', url,
    ]
    r = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
    return r.stdout


def fetch(url, referer=None):
    """带限速与 WAF 重试的抓取 (每次独立 cookie jar, 避免频率追踪)"""
    ref = referer or LIST_URL
    for attempt in range(3):
        html = curl_get(url, ref)
        if len(html) < WAF_BLOCK_SIZE or '页面未找到' in html:
            # WAF 拦截页 or 404
            print(f"    [WAF/blocked] {url} len={len(html)} attempt={attempt + 1}, sleep 30...", file=sys.stderr)
            time.sleep(30)
            continue
        return html
    return ''


def extract_items(html, page_no):
    """列表条目: /ncjkq/gggs/YYYYMM/32hex.shtml + 标题 + 日期"""
    items = []
    for m in re.finditer(r'<a href="(/ncjkq/gggs/\d{6}/[0-9a-f]{32}\.shtml)"[^>]*>(.*?)</a>', html, re.DOTALL):
        href, title_html = m.group(1), m.group(2)
        title = re.sub(r'<[^>]+>', '', title_html).strip()
        title = title.replace('\u200b', '').strip()
        title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
        if len(title) < 4:
            continue
        # 日期: 同 li 内 <span>YYYY-MM-DD</span>
        date_text = ''
        dm = re.search(r'<span>(\d{4}-\d{2}-\d{2})</span>', html[m.end():m.end() + 200])
        if dm:
            date_text = dm.group(1)
        items.append({'url': urljoin(BASE, href), 'title': title, 'date': date_text})
    # 去重
    seen = set()
    uniq = []
    for it in items:
        if it['url'] in seen:
            continue
        seen.add(it['url'])
        uniq.append(it)
    return uniq


def extract_balanced_div(html, open_tag_re):
    """平衡 div 匹配 (大小写不敏感)"""
    m = re.search(open_tag_re, html, re.I)
    if not m:
        return ''
    gt = html.find('>', m.start())
    if gt == -1:
        return ''
    i = gt + 1
    depth = 1
    for mm in re.finditer(r'<div[\s>]|</div>', html[i:], re.I):
        if mm.group(0).startswith('<div') or mm.group(0).startswith('<DIV'):
            depth += 1
        else:
            depth -= 1
            if depth == 0:
                return html[i:i + mm.start()]
    return html[i:]


def fetch_detail(url):
    html = fetch(url)
    if not html:
        return '', '', ''
    # 标题
    title = ''
    m = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html)
    if m:
        title = m.group(1).strip()
    if not title:
        m = re.search(r'<title>\s*([^<]+?)\s*</title>', html, re.DOTALL)
        if m:
            title = re.sub(r'\s+', '', m.group(1)).strip()
            title = re.split(r'_公告公示|_', title)[0].strip()
    title = title.replace('\u200b', '').strip()
    title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
    # 日期
    date_text = ''
    m = re.search(r'<meta[^>]*name="PubDate"[^>]*content="([^"]*)"', html)
    if m:
        md = re.search(r'(\d{4}-\d{2}-\d{2})', m.group(1))
        if md:
            date_text = md.group(1)
    if not date_text:
        m = re.search(r'(\d{4}-\d{2}-\d{2})\s+\d{2}:\d{2}', html)
        if m:
            date_text = m.group(1)
    # 正文: div#zoomcon (UCAP CMS, 含 <UCAPCONTENT> 标签包裹)
    content = ''
    c = extract_balanced_div(html, r'<div[^>]*id="zoomcon"[^>]*>')
    if c.strip():
        content = c.strip()
    if not content.strip():
        # 回退: class=zoom
        c = extract_balanced_div(html, r'<div[^>]*class="[^"]*zoom[^"]*"[^>]*>')
        if c.strip():
            content = c.strip()
    content = re.sub(r'<UCAPCONTENT>|</UCAPCONTENT>', '', content)
    content = re.sub(r'<script.*?</script>', '', content, flags=re.DOTALL | re.I)
    return title, date_text, content


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=13)
    args = ap.parse_args()

    items = []
    seen = set()
    for page_no in range(1, args.pages + 1):
        url = LIST_URL if page_no == 1 else LIST_URL.replace('list.shtml', f'list_{page_no}.shtml')
        html = fetch(url, referer=LIST_URL)
        if not html:
            print(f"  page {page_no}: fetch fail (stop)")
            break
        page_items = extract_items(html, page_no)
        before = len(items)
        for it in page_items:
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            items.append(it)
        print(f"  page {page_no}: +{len(items) - before} items (total {len(items)})")
        if not page_items:
            break
        time.sleep(1.5)  # WAF 限速

    records = []
    for idx, it in enumerate(items):
        title, date_text, content = fetch_detail(it['url'])
        if not title:
            title = it['title']
        if not content:
            print(f"    SKIP empty content: {it['url']}")
            continue
        pd = date_text or it.get('date', '')
        if pd and pd < CUTOFF:
            continue
        # 附件提取
        atts = []
        for am in re.finditer(r'<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>', content, re.DOTALL):
            h, t = am.group(1), re.sub(r'<[^>]+>', '', am.group(2)).strip()
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|caj|wps)$', h, re.I) or 'download' in h.lower():
                atts.append({'title': t or '附件', 'url': h})
        records.append({
            'title': title, 'url': it['url'], 'source_url': it['url'],
            'pub_date': pd, 'site_name': SITE_NAME, 'content': content,
            'summary': '', 'attachments': json.dumps(atts, ensure_ascii=False) if atts else '[]',
            'group_name': GROUP_NAME,
        })
        if idx % 5 == 4:
            time.sleep(1.5)
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "ncjk_gggs")
    return len(valid)


if __name__ == '__main__':
    main()