#!/usr/bin/env python3
"""crawl_xjyc_sthj.py - 叶城县人民政府-生态环境 (xjyc.gov.cn)

站点: https://www.xjyc.gov.cn/ycx/c109165/zfxxgk.shtml (叶城县-生态环境栏目)
CMS: 新疆政府网站集群 (UTF-8)
列表: ul.innerList > li > span.fr(日期 YYYY-MM-DD) + a(标题)  href=/ycx/c109165/{YYYYMM}/{hash}.shtml
分页: createPageHTML('page-div',14,1,'zfxxgk','shtml',200) → zfxxgk_{N}.shtml (共14页, 第14页5条, 第15页404)
详情: meta ArticleTitle + meta PubDate + div.xxgk-tt-content-body(id=zoomcon)
正文: 文字型(土壤调查公示等) / 图片型(检测报告, ewebeditor_doc内img)
"""
import sys, os, requests, re, argparse
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365*3)).strftime('%Y-%m-%d')
SITE_NAME = '叶城县-生态环境'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0'}
BASE = 'https://www.xjyc.gov.cn'
LIST_URL = BASE + '/ycx/c109165/zfxxgk.shtml'
TOTAL_PAGES = 14

def fetch_list(page_idx):
    """page_idx: 1-based. 返回 [{url, pub_date, title}]"""
    url = LIST_URL if page_idx == 1 else LIST_URL.replace('zfxxgk.shtml', f'zfxxgk_{page_idx}.shtml')
    try:
        r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
    except:
        return []
    items = re.findall(
        r'<li>\s*<span class="fr">(\d{4}-\d{2}-\d{2})</span>\s*<a href="([^"]+)"[^>]*>\s*([^<]{4,150}?)\s*</a>',
        r.text, re.DOTALL
    )
    results = []
    for pd, href, title in items:
        title = re.sub(r'\s+', '', title).strip()
        if len(title) < 4:
            continue
        url = href if href.startswith('http') else BASE + href
        results.append({'url': url, 'pub_date': pd, 'title': title})
    return results

def fetch_detail(url):
    for _attempt in range(2):
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
            break
        except:
            r = None
    if r is None:
        return '', ''
    title = ''
    m = re.search(r'<meta name="ArticleTitle" content="([^"]*)"', r.text)
    if m:
        title = m.group(1).strip()
    if not title:
        m = re.search(r'<title>\s*([^<]+?)\s*</title>', r.text, re.DOTALL)
        if m:
            title = re.sub(r'\s+', '', m.group(1)).strip()
            # 去掉 " - 生态环境 - 叶城县人民政府" 后缀
            title = re.split(r'[-–—]', title)[0].strip()
    content = ''
    # 正文容器 div.xxgk-tt-content-body (id=zoomcon)
    m = re.search(r'<div class="xxgk-tt-content[^"]*"[^>]*>(.*?)</div>\s*<!--\s*关联稿件', r.text, re.DOTALL)
    if not m:
        m = re.search(r'<div class="xxgk-tt-content[^"]*"[^>]*>(.*?)</div>\s*<div class="glgj"', r.text, re.DOTALL)
    if not m:
        m = re.search(r'<div class="xxgk-tt-content[^"]*"[^>]*>(.*?)</div>', r.text, re.DOTALL)
    if m:
        content = m.group(1).strip()
    if not content.strip():
        m = re.search(r'id="zoomcon"[^>]*>(.*?)</div>', r.text, re.DOTALL)
        if m:
            content = m.group(1).strip()
    # img 相对路径 → 完整域名
    content = re.sub(r'src="(?!https?://|//)(/[^"]+)"', lambda mm: f'src="{BASE}{mm.group(1)}"', content)
    return title, content

def run(max_pages=5):
    records = []
    seen = set()
    pages = min(max_pages, TOTAL_PAGES)
    for pg in range(pages):
        items = fetch_list(pg + 1)
        if not items:
            break
        print(f"  Page {pg+1}: {len(items)} items")
        for it in items:
            if it['pub_date'] and it['pub_date'] < CUTOFF:
                continue
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            title, content = fetch_detail(it['url'])
            if not title:
                title = it['title']
            if not content.strip():
                continue
            records.append({
                'title': title, 'url': it['url'], 'source_url': it['url'], 'pub_date': it['pub_date'],
                'site_name': SITE_NAME, 'content': content, 'summary': '',
            })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "xjyc_sthj")
    return len(valid)

if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=5)
    args = ap.parse_args()
    cnt = run(max_pages=args.pages)
    print(f"Done: {cnt} records")