#!/usr/bin/env python3
"""crawl_sheyang_hjbh.py - 射阳县-生态环境

站点: https://www.sheyang.gov.cn/col/col24425/index.html?uid=32978
CMS: 江苏政府网站群 col/col + jpage 组件 (WCM)
列表: UL.xxgk_list > LI.cf > A[title] 标题 + SPAN 日期(YYYY-MM-DD); 数据在 <record><![CDATA[...]]></record>
分页: ⚠️ 无真实分页 — 首页 CDATA 一次性输出全部 75 条 (jpage dataproxy.jsp 返回空, totalRecord 为模板默认)
详情: p.con-title 标题(模板标记begin/end间) + div.main-txt#zoom 正文(ContentStart~ContentEnd) + meta Maketime 日期
附件: 正文内链接保留; div.xz 附件区为空(此栏目以公报/新闻为主)
"""
import sys, os, re, json, argparse
import requests
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
from datetime import datetime, timedelta

CUTOFF = (datetime.now() - timedelta(days=365 * 3)).strftime('%Y-%m-%d')
SITE_NAME = '射阳县-生态环境'
GROUP_NAME = '江苏'
BASE = 'https://www.sheyang.gov.cn'
LIST_URL = '/col/col24425/index.html?uid=32978'
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Referer': BASE + '/col/col24425/index.html',
}


def fetch(url):
    try:
        r = requests.get(url, timeout=(5, 25), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f"[WARN] fetch failed: {url} - {e}", file=sys.stderr)
        return ''


def fetch_list(page_idx):
    """列表页: 无分页, 首页 CDATA 即全部记录"""
    if page_idx > 0:
        return []
    html = fetch(BASE + LIST_URL)
    if not html:
        return []
    items = []
    for m in re.finditer(r'<record><!\[CDATA\[(.*?)\]\]></record>', html, re.DOTALL):
        rec = m.group(1)
        am = re.search(r'<a\s+href="([^"]+)"[^>]*title="([^"]*)"[^>]*>(.*?)</a>\s*<span>([^<]*)</span>', rec, re.DOTALL)
        if not am:
            continue
        href, title_attr, title_html, date_text = am.group(1), am.group(2), am.group(3), am.group(4).strip()
        title = (title_attr or re.sub(r'<[^>]+>', '', title_html)).strip()
        title = re.sub(r'\s+', ' ', title).strip()
        # strip &middot;&nbsp; 实体前缀
        title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
        if len(title) < 4:
            continue
        # 只抓精准子栏目 (art 详情链接须为本栏目 colid 24425)
        if '/art/' not in href or 'art_24425_' not in href:
            continue
        url = urljoin(BASE, href)
        md = re.search(r'(\d{4}-\d{2}-\d{2})', date_text)
        pd = md.group(1) if md else ''
        items.append({'url': url, 'pub_date': pd, 'title': title})
    return items


def fetch_detail(url):
    """详情页: con-title 标题 + main-txt#zoom 正文(ContentStart~ContentEnd) + Maketime 日期"""
    html = fetch(url)
    if not html:
        return '', '', ''
    # 标题: p.con-title (模板标记间)
    title = ''
    m = re.search(r'<p[^>]*class="[^"]*con-title[^"]*"[^>]*>(.*?)</p>', html, re.DOTALL)
    if m:
        seg = re.sub(r'<!--.*?-->', '', m.group(1), flags=re.DOTALL)
        title = re.sub(r'<[^>]+>', '', seg).strip()
        title = re.sub(r'\s+', ' ', title).strip()
    if not title:
        m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]+)"', html)
        if m:
            title = m.group(1).strip()
    title = re.sub(r'^\s*(?:&middot;|&nbsp;|·|\s)+', '', title)
    title = re.sub(r'_(射阳县人民政府|射阳县|射阳县人民政府网站)$', '', title)
    # 日期: meta Maketime
    date_text = ''
    m = re.search(r'<meta\s+name=[\'"]Maketime[\'"]\s+content=[\'"]([^\'"]+)[\'"]', html)
    if m:
        md = re.search(r'(\d{4}-\d{2}-\d{2})', m.group(1))
        if md:
            date_text = md.group(1)
    if not date_text:
        m = re.search(r'<meta\s+name=[\'"]PubDate[\'"]\s+content=[\'"]([^\'"]+)[\'"]', html)
        if m:
            md = re.search(r'(\d{4}-\d{2}-\d{2})', m.group(1))
            if md:
                date_text = md.group(1)
    # 正文: ContentStart ~ ContentEnd
    content = ''
    s = html.find('ContentStart')
    e = html.find('ContentEnd')
    if s != -1 and e != -1 and e > s:
        content = html[s + len('ContentStart'):e].strip()
    if not content.strip():
        m = re.search(r'<div[^>]*class="[^"]*main-txt[^"]*"[^>]*>(.*?)(?:<div[^>]*class="xz"|</div>\s*<!--编辑)', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
    content = re.sub(r'<script.*?</script>', '', content, flags=re.DOTALL | re.I)
    content = re.sub(r'<style.*?</style>', '', content, flags=re.DOTALL | re.I)
    # 图片/附件相对路径 → 详情页目录 urljoin
    content = re.sub(r'href="(?!https?://|//|#|javascript:)([^"]+)"',
                     lambda mm: f'href="{urljoin(url, mm.group(1))}"', content)
    content = re.sub(r'src="(?!https?://|//)([^"]+)"',
                     lambda mm: f'src="{urljoin(url, mm.group(1))}"', content)
    # 清首尾空 p
    content = re.sub(r'^(<p[^>]*>(\s*(<br\s*/?>)?\s*)?</p>)+', '', content)
    content = re.sub(r'(<p[^>]*>(\s*(<br\s*/?>)?\s*)?</p>)+$', '', content)
    return title, date_text, content.strip()


def run(max_pages=5):
    records = []
    seen = set()
    for pg in range(max_pages):
        items = fetch_list(pg)
        if not items:
            print(f"  Page {pg + 1}: 0 items (stop)")
            break
        print(f"  Page {pg + 1}: {len(items)} items (all-in-one list, no real pagination)")
        for it in items:
            if it['pub_date'] and it['pub_date'] < CUTOFF:
                continue
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            title, date_text, content = fetch_detail(it['url'])
            if not content:
                print(f"    SKIP empty content: {it['url']}")
                continue
            if not title:
                title = it['title']
            pd = date_text or it['pub_date']
            # 附件提取 (从正文)
            atts = []
            for am in re.finditer(r'<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>', content, re.DOTALL):
                h, t = am.group(1), re.sub(r'<[^>]+>', '', am.group(2)).strip()
                if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|caj|wps)$', h, re.I) or 'attachment' in h.lower() or '/module/download/' in h.lower():
                    atts.append({'title': t or '附件', 'url': h})
            records.append({
                'title': title, 'url': it['url'], 'source_url': it['url'],
                'pub_date': pd, 'site_name': SITE_NAME, 'content': content,
                'summary': '', 'attachments': json.dumps(atts, ensure_ascii=False) if atts else '[]',
                'group_name': GROUP_NAME,
            })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "sheyang_hjbh")
    return len(valid)


if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=5)
    args = ap.parse_args()
    n = run(args.pages)
    print(f"[RESULT] {SITE_NAME}: {n} items")