#!/usr/bin/env python3
"""crawl_yangzhou_tzgg.py - 扬州市生态环境局-通知公告

站点: https://sthj.yangzhou.gov.cn/zfxxgk/fdzdgk/tzgg/index.html
CMS: JPAAS 发布系统 (与 crawl_yangzhou.py 环评审批同源, API 动态加载)
列表: /api-gateway/jpaas-publish-server/front/page/build/unit
      webId=5biGb8ANMvmWd52WUVRQu tplSetId=Y3ymqFLmqouIDOIdPh2n8
      pageId=KlS5SScWzHtnTxPQ7g7lk tagId=列表列表 (注意: 不是 ajax分页!)
      paramJson={"pageNo":N,"pageSize":15}; count=171 共12页
      返回 HTML: li.clearfix > a[href=.../tzgg/art/YYYY/art_xxx.html] 标题 + span 日期
陷阱: tagId=列表列表 返回全站最新文章(含 ywgz/wrfz 等其他栏目)
      → 必须过滤 href 含 /zfxxgk/fdzdgk/tzgg/ 的条目, 只抓精准子栏目
详情: div.artTitle 标题 + ul.xxgk_head 生成日期 + div.artTxt 正文(回退 div.bt-content)
附件: /api-gateway/jpaas-web-server/front/document/download?fileUrl=... 相对根路径 → BASE+href
标题: strip ·  实体前缀
"""
import sys, os, re, json, argparse, urllib.parse
import requests

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
from datetime import datetime, timedelta

CUTOFF = (datetime.now() - timedelta(days=365 * 3)).strftime('%Y-%m-%d')
SITE_NAME = '扬州市-通知公告'
GROUP_NAME = '江苏'
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Referer': 'https://sthj.yangzhou.gov.cn/zfxxgk/fdzdgk/tzgg/index.html',
}
BASE = 'https://sthj.yangzhou.gov.cn'
API_URL = BASE + '/api-gateway/jpaas-publish-server/front/page/build/unit'
PAGE_ID = 'KlS5SScWzHtnTxPQ7g7lk'
WEB_ID = '5biGb8ANMvmWd52WUVRQu'
TPL_ID = 'Y3ymqFLmqouIDOIdPh2n8'
PAGE_SIZE = 15
TZGG_PREFIX = '/zfxxgk/fdzdgk/tzgg/'


def fetch_api(page_no, page_size=PAGE_SIZE):
    """Fetch list page via JPAAS API"""
    params = {
        'parseType': 'bulidstatic',
        'webId': WEB_ID,
        'tplSetId': TPL_ID,
        'pageType': 'column',
        'tagId': '列表列表',
        'editType': 'null',
        'pageId': PAGE_ID,
        'paramJson': json.dumps({'pageNo': page_no, 'pageSize': page_size}, ensure_ascii=False),
    }
    url = API_URL + '?' + urllib.parse.urlencode(params)
    try:
        r = requests.get(url, timeout=(5, 20), headers=HEADERS, verify=False)
        data = r.json()
        return data.get('data', {}).get('html', '')
    except Exception as e:
        print(f"[WARN] API fetch failed page {page_no}: {e}", file=sys.stderr)
        return ''


def extract_items(html):
    """Extract tzgg items from API HTML. 只抓精准 /tzgg/ 子栏目"""
    items = []
    for m in re.finditer(r'<li[^>]*>\s*<a href="([^"]+)"[^>]*>(.*?)</a>\s*<span>([^<]*)</span>', html, re.DOTALL):
        href, title_html, date_text = m.group(1).strip(), m.group(2), m.group(3).strip()
        # 过滤: 只留 tzgg 精准子栏目
        if TZGG_PREFIX not in href:
            continue
        title = re.sub(r'<[^>]+>', '', title_html)
        title = re.sub(r'\s+', ' ', title).strip()
        # strip &middot;&nbsp; 实体前缀
        title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
        if len(title) < 4:
            continue
        if not href.startswith('http'):
            href = BASE + href if href.startswith('/') else BASE + '/' + href
        md = re.search(r'(\d{4}-\d{2}-\d{2})', date_text)
        pd = md.group(1) if md else ''
        items.append({'url': href, 'pub_date': pd, 'title': title})
    return items


def extract_balanced_div(html, open_tag_re):
    """平衡 div 匹配: 从匹配到的 <div...> 的 '>' 之后, 到对应 </div>, 返回内部 HTML"""
    m = re.search(open_tag_re, html)
    if not m:
        return ''
    gt = html.find('>', m.start())
    if gt == -1:
        return ''
    i = gt + 1
    depth = 1
    for mm in re.finditer(r'<div[\s>]|</div>', html[i:]):
        if mm.group(0).startswith('<div'):
            depth += 1
        else:
            depth -= 1
            if depth == 0:
                return html[i:i + mm.start()]
    return html[i:]


def fetch_detail(url):
    """详情页: artTitle 标题 + xxgk_head 日期 + artTxt/bt-content 正文"""
    try:
        r = requests.get(url, timeout=(5, 20), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
        html = r.text
    except Exception:
        return '', '', ''
    # 标题
    title = ''
    m = re.search(r'<div[^>]*class="[^"]*artTitle[^"]*"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
        title = re.sub(r'\s+', ' ', title).strip()
    if not title:
        m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]+)"', html)
        if m:
            title = m.group(1).strip()
    # strip &middot;&nbsp; 实体前缀
    title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
    # 日期: xxgk_head 生成日期
    date_text = ''
    m = re.search(r'生成日期[^<]*<[^>]*>([^<]+)', html)
    if m:
        md = re.search(r'(\d{4}-\d{2}-\d{2})', m.group(1))
        if md:
            date_text = md.group(1)
    if not date_text:
        m = re.search(r'<meta\s+name="PubDate"\s+content="([^"]+)"', html)
        if m:
            md = re.search(r'(\d{4}-\d{2}-\d{2})', m.group(1))
            if md:
                date_text = md.group(1)
    # 正文: artTxt 优先, 回退 bt-content
    content = ''
    for cls in ('artTxt', 'bt-content'):
        c = extract_balanced_div(html, rf'<div[^>]*class="[^"]*{re.escape(cls)}[^"]*"[^>]*>')
        if c.strip():
            content = c.strip()
            break
    if not content.strip():
        c = extract_balanced_div(html, r'<div[^>]*class="[^"]*(?:TRS_Editor|article|content)[^"]*"[^>]*>')
        if c.strip():
            content = c.strip()
    content = re.sub(r'<script.*?</script>', '', content, flags=re.DOTALL | re.I)
    content = re.sub(r'<style.*?</style>', '', content, flags=re.DOTALL | re.I)
    # href/src 相对路径转绝对 (附件 /api-gateway/... 相对根路径)
    content = re.sub(r'href="(?!https?://|//|#|javascript:)([^"]+)"',
                     lambda mm: f'href="{BASE}{mm.group(1)}"' if mm.group(1).startswith('/') else f'href="{BASE}/{mm.group(1)}"',
                     content)
    content = re.sub(r'src="(?!https?://|//)([^"]+)"',
                     lambda mm: f'src="{BASE}{mm.group(1)}"' if mm.group(1).startswith('/') else f'src="{BASE}/{mm.group(1)}"',
                     content)
    # 清首尾空 p
    content = re.sub(r'^(<p[^>]*>(\s*(<br\s*/?>)?\s*)?</p>)+', '', content)
    content = re.sub(r'(<p[^>]*>(\s*(<br\s*/?>)?\s*)?</p>)+$', '', content)
    return title, date_text, content.strip()


def run(max_pages=5):
    records = []
    seen = set()
    for pg in range(max_pages):
        html = fetch_api(pg + 1)
        if not html:
            print(f"  Page {pg + 1}: empty (stop)")
            break
        items = extract_items(html)
        if not items:
            print(f"  Page {pg + 1}: 0 tzgg items (stop)")
            break
        print(f"  Page {pg + 1}: {len(items)} tzgg items")
        for it in items:
            if it['pub_date'] and it['pub_date'] < CUTOFF:
                continue
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            title, date_text, content = fetch_detail(it['url'])
            if not content:
                print(f"    SKIP empty content: {it['url']}")
                continue
            if not title:
                title = it['title']
            pd = date_text or it['pub_date']
            # 附件: 从正文提取 download 链接
            atts = []
            for am in re.finditer(r'<a[^>]*href="([^"]*(?:download|fileUrl)[^"]*)"[^>]*>(.*?)</a>', content, re.DOTALL):
                at = re.sub(r'<[^>]+>', '', am.group(2)).strip()
                at = re.sub(r'\s+', ' ', at).strip()
                atts.append({'title': at or '附件', 'url': am.group(1)})
            records.append({
                'title': title, 'url': it['url'], 'source_url': it['url'],
                'pub_date': pd, 'site_name': SITE_NAME, 'content': content,
                'summary': '', 'attachments': json.dumps(atts, ensure_ascii=False) if atts else '[]',
                'group_name': GROUP_NAME,
            })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "yangzhou_tzgg")
    return len(valid)


if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=5)
    args = ap.parse_args()
    n = run(args.pages)
    print(f"[RESULT] {SITE_NAME}: {n} items")