#!/usr/bin/env python3
"""crawl_funing_js.py - 阜宁县人民政府-通知公告 (Hanweb JPage + 创宇云防御WAF)"""

import sys, os, re, json, time, requests, urllib.parse
from datetime import datetime, timedelta

# Unbuffered output
sys.stdout.reconfigure(line_buffering=True)
sys.stderr.reconfigure(line_buffering=True)

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = '阜宁县人民政府-通知公告'
DOMAIN = 'https://www.funing.gov.cn'
COLUMN_ID = '6617'
UNIT_ID = '24217'
WEB_ID = '48'
PAGES = 5

# Dataproxy config
DATAPROXY_BASE = '/module/web/jpage/dataproxy.jsp'
DATAPROXY_PARAMS = {
    'appid': '1',
    'webid': WEB_ID,
    'path': '/',
    'columnid': COLUMN_ID,
    'unitid': UNIT_ID,
    'webname': '阜宁县人民政府',
    'permissiontype': '0',
}


def get_session():
    """创建带 WAF cookie 的 session"""
    s = requests.Session()
    s.headers.update({
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36',
        'Accept-Language': 'zh-CN,zh;q=0.9',
    })
    # 先访问首页获取 __jsluid_s cookie（绕创宇云防御）
    try:
        s.get(DOMAIN + '/', timeout=30)
    except Exception:
        pass
    return s


def fetch_list(session, page):
    """获取第N页列表数据，返回 [(href, title, pub_date), ...]"""
    params = {**DATAPROXY_PARAMS, 'page': str(page)}
    url = DOMAIN + DATAPROXY_BASE + '?' + urllib.parse.urlencode(params)
    try:
        r = session.get(url, timeout=30)
        r.encoding = 'utf-8'
        if r.status_code != 200:
            print(f'  List page {page}: HTTP {r.status_code}')
            return []
    except Exception as e:
        print(f'  List page {page}: Error {e}')
        return []

    # 两步法：先解 CDATA，再提取字段
    cdata_blocks = re.findall(
        r'<record><!\[CDATA\[(.*?)\]\]></record>', r.text, re.DOTALL
    )
    records = []
    seen_hrefs = set()
    for block in cdata_blocks:
        href_m = re.search(r'href="([^"]+)"', block)
        title_m = re.search(r'title="([^"]*)"', block)
        date_m = re.search(r'\[(\d{4}-\d{2}-\d{2})\]', block)
        if href_m and title_m:
            h = href_m.group(1)
            if h not in seen_hrefs:
                seen_hrefs.add(h)
                records.append((
                    h,
                    title_m.group(1).strip(),
                    date_m.group(1) if date_m else ''
                ))
    print(f'  Page {page}: {len(records)} new items')
    return records


def fetch_detail(session, url):
    """获取文章详情，返回 (title, pub_date, content_html)"""
    full_url = url if url.startswith('http') else DOMAIN + url
    try:
        r = session.get(full_url, timeout=30)
        r.encoding = 'utf-8'
    except Exception as e:
        print(f'    Detail error: {e}')
        return '', '', ''
    html = r.text

    # 标题
    title = ''
    m = re.search(r'<title>([^<]+)</title>', html)
    if m:
        parts = m.group(1).split()
        title = parts[-1] if len(parts) > 1 else parts[0]

    # 发布日期
    pub_date = ''
    m = re.search(r'发布日期[：:]\s*(\d{4}[-/]\d{2}[-/]\d{2})', html)
    if m:
        pub_date = m.group(1).replace('/', '-')

    # 正文：提取 <!--<$[信息内容]>begin--> 到 <!--<$[信息内容]>end-->
    content_m = re.search(
        r'<!--<\$\[信息内容\]>begin-->(.*?)<!--<\$\[信息内容\]>end-->',
        html, re.DOTALL
    )
    if not content_m:
        # 降级：取 zoom div
        zoom_m = re.search(r'id="zoom"[\s\S]*?</div>', html, re.DOTALL)
        if zoom_m:
            content_m = zoom_m

    if not content_m:
        return title, pub_date, ''

    raw = content_m.group(1) if content_m.lastindex else content_m.group()

    # 清理垃圾标记
    raw = re.sub(r'<!--ZJEG_RSS\.content\..*?-->', '', raw)
    raw = re.sub(r'<script[^>]*>[\s\S]*?</script>', '', raw)
    raw = re.sub(r'<style[^>]*>[\s\S]*?</style>', '', raw)
    raw = re.sub(r'<meta[^>]*>', '', raw)
    raw = re.sub(r'<!--.*?-->', '', raw)
    raw = re.sub(r'(?:begin|end)-->', '', raw)

    # 保留 <p>, <table>, <a>, <img> 标签（含自闭合）
    parts = []
    seen_keys = set()
    for tag_name in ['p', 'table', 'a', 'img']:
        for m in re.finditer(r'(?:<%s[^>]*/>|<%s[^>]*>.*?</%s>)' % (tag_name, tag_name, tag_name), raw, re.DOTALL):
            tag_html = m.group()
            key = re.sub(r'<[^>]+>', '', tag_html).strip()[:40]
            if key:
                if key not in seen_keys:
                    seen_keys.add(key)
                    parts.append(tag_html)
            elif '<img' in tag_html or '<a' in tag_html:
                # 纯图片/链接内容也保留
                parts.append(tag_html)

    if not parts:
        # 纯文本降级
        text = re.sub(r'<[^>]+>', '', raw)
        text = re.sub(r'\s+', '\n', text).strip()
        return title, pub_date, text

    body_html = '\n'.join(parts)

    # 相对URL转绝对URL
    body_html = re.sub(
        r'(href|src)="(/(?!http)[^"]*)"',
        lambda m: '%s="%s%s"' % (m.group(1), DOMAIN, m.group(2)),
        body_html
    )

    return title, pub_date, body_html


def run(incremental=False):
    session = get_session()
    all_records = []
    seen = set()  # 去重 by href
    max_pages = 1 if incremental else PAGES

    for page in range(1, max_pages + 1):
        items = fetch_list(session, page)
        if not items:
            break
        for href, title, pub_date in items:
            if href not in seen:
                seen.add(href)
                full_url = href if href.startswith('http') else DOMAIN + href
                all_records.append({
                    'title': title,
                    'url': full_url,
                    'pub_date': pub_date,
                    'site_name': SITE_NAME,
                })
        time.sleep(0.5)

    print(f'Total unique items: {len(all_records)}')

    # 获取详情
    for i, rec in enumerate(all_records):
        detail_title, detail_date, content = fetch_detail(session, rec['url'])
        rec['title'] = detail_title or rec['title']
        rec['pub_date'] = detail_date or rec['pub_date']
        if content:
            rec['content'] = content
            rec['summary'] = re.sub(r'<[^>]+>', '', content)[:200]
        else:
            rec['content'] = ''
            rec['summary'] = ''
        if (i + 1) % 20 == 0:
            print(f'  Detail {i+1}/{len(all_records)}')
        time.sleep(0.3)

    print(f'Detail fetched: {len(all_records)} items')
    if all_records:
        push_to_searchdb(all_records, 'funing_js_tzgg')
    return len(all_records)


if __name__ == '__main__':
    inc = '--incremental' in sys.argv
    cnt = run(incremental=inc)
    print(f'Done: {cnt} records')