#!/usr/bin/env python3
"""crawl_yjgx_gkmlpt.py - 阳江市生态环境局高新分局-政府信息公开平台 (NFCMS gkmlpt API)"""

import sys, os, re, json, time, requests
from bs4 import BeautifulSoup
from datetime import datetime

sys.stdout.reconfigure(line_buffering=True)
sys.stderr.reconfigure(line_buffering=True)

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = '阳江市生态环境局高新分局'
GROUP = '广东'
INDUSTRY = '生态环境'
DOMAIN = 'http://www.yjgx.gov.cn'
APP_PATH = '/yjgxyjshjbhjgxfj'
BASE_URL = DOMAIN + APP_PATH

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
    'Accept': 'application/json',
}

PAGE_SIZE = 100
MAX_PAGES = 10  # 843条, 100/page = 9页


def get_csrf():
    """从首页获取最新的CSRF token"""
    try:
        resp = requests.get(f'{BASE_URL}/gkmlpt/index', headers={'User-Agent': HEADERS['User-Agent']}, timeout=15)
        m = re.search(r"CSRF\s*:\s*'([^']+)'", resp.text)
        if m:
            return m.group(1)
    except Exception as e:
        print(f'  × CSRF获取失败: {e}')
    return ''


def fetch_list(csrf, page=1):
    """获取API列表页，返回文章列表"""
    url = f'{BASE_URL}/gkmlpt/api/all/0?page={page}&pageSize={PAGE_SIZE}'
    headers = {**HEADERS, 'X-CSRF-TOKEN': csrf}
    try:
        resp = requests.get(url, headers=headers, timeout=20)
        if resp.status_code != 200:
            print(f'  × API HTTP {resp.status_code}')
            return [], 0
        data = resp.json()
        articles = data.get('articles', [])
        total = data.get('total', 0)
        print(f'  √ 第{page}页: {len(articles)}条 (共{total}条)')
        return articles, total
    except Exception as e:
        print(f'  × API异常: {e}')
        return [], 0


def fetch_detail(csrf, url):
    """获取详情页内容"""
    headers = {**HEADERS, 'X-CSRF-TOKEN': csrf}
    try:
        resp = requests.get(url, headers=headers, timeout=20)
        resp.encoding = 'utf-8'
        if resp.status_code != 200:
            return '', '', ''
    except Exception as e:
        return '', '', ''

    html = resp.text
    soup = BeautifulSoup(html, 'html.parser')

    # 标题
    title = ''
    h1 = soup.select_one('.title.document-number, h1.title')
    if h1:
        title = h1.get_text(strip=True)
    if not title:
        t_tag = soup.find('title')
        if t_tag:
            title = t_tag.get_text(strip=True)
            title = re.sub(r'[-—|].*?$', '', title).strip()

    # 发布日期
    pub_date = ''
    date_row = soup.select_one('.date-row')
    if date_row:
        m = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', date_row.get_text())
        if m:
            pub_date = m.group(1).replace('/', '-')
    if not pub_date:
        for txt in soup.stripped_strings:
            m = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', txt)
            if m:
                pub_date = m.group(1).replace('/', '-')
                break

    # 正文: .article-content 或 .policy-article-content
    content_html = ''
    for sel in ['.article-content', '.policy-article-content', '.content', '#content', '.TRS_Editor']:
        el = soup.select_one(sel)
        if el and el.get_text(strip=True):
            content_html = str(el)
            break

    if not content_html:
        content_html = str(soup.find('body') or '')

    # 提取保留标签
    parts = []
    seen_keys = set()
    for tag_name in ['p', 'table', 'a', 'img']:
        for m in re.finditer(r'(?:<%s[^>]*/>|<%s[^>]*>.*?</%s>)' % (tag_name, tag_name, tag_name),
                             content_html, re.DOTALL):
            tag_html = m.group()
            key = re.sub(r'<[^>]+>', '', tag_html).strip()[:40]
            if key:
                if key not in seen_keys:
                    seen_keys.add(key)
                    parts.append(tag_html)
            elif '<img' in tag_html or '<a' in tag_html:
                parts.append(tag_html)

    if not parts:
        text = re.sub(r'<[^>]+>', '', content_html)
        text = re.sub(r'\s+', '\n', text).strip()
        return title, pub_date, text

    body_html = '\n'.join(parts)
    body_html = re.sub(r'(href|src)="(/(?!http)[^"]*)"',
                       lambda m: '%s="%s%s"' % (m.group(1), DOMAIN, m.group(2)), body_html)
    return title, pub_date, body_html


def run(incremental=False):
    csrf = get_csrf()
    if not csrf:
        print('× 无法获取CSRF token')
        return 0

    all_records = []
    seen = set()
    max_pages = 1 if incremental else MAX_PAGES

    for page in range(1, max_pages + 1):
        articles, total = fetch_list(csrf, page)
        if not articles:
            break
        for a in articles:
            art_url = a.get('url', '')
            if not art_url or art_url in seen:
                continue
            seen.add(art_url)
            all_records.append({
                'title': a.get('title', ''),
                'url': art_url,
                'pub_date': '',
                'site_name': SITE_NAME,
                'summary': a.get('description', '')[:200],
            })
        time.sleep(0.3)
        if page * PAGE_SIZE >= total:
            break

    print(f'Total unique items: {len(all_records)}')

    total_done = 0
    batch = []
    for i, rec in enumerate(all_records):
        detail_title, detail_date, content = fetch_detail(csrf, rec['url'])
        rec['title'] = detail_title or rec['title']
        rec['pub_date'] = detail_date or rec['pub_date']
        if content:
            rec['content'] = content
            rec['summary'] = re.sub(r'<[^>]+>', '', content)[:200]
        else:
            rec['content'] = ''
            rec['summary'] = ''
        batch.append(rec)
        
        if len(batch) >= 50:
            push_to_searchdb(batch, 'yjgx_gkmlpt')
            total_done += len(batch)
            print(f'  Committed {total_done}/{len(all_records)}')
            batch = []
        
        if (i + 1) % 20 == 0:
            print(f'  Detail {i+1}/{len(all_records)}')
        time.sleep(0.3)

    # 最后一批
    if batch:
        push_to_searchdb(batch, 'yjgx_gkmlpt')
        total_done += len(batch)

    print(f'Detail fetched: {total_done} items')
    return total_done


if __name__ == '__main__':
    inc = '--incremental' in sys.argv
    cnt = run(incremental=inc)
    print(f'Done: {cnt} records')