#!/usr/bin/env python3
"""crawl_ahhuoshan_gsgg.py - 霍山县-通知公告 (LS CMS Ls.ajax接口, 自动提取siteId)"""
import sys, os, requests, re, time, argparse
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365*3)).strftime('%Y-%m-%d')
SITE_NAME = '霍山县-通知公告'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0',
           'X-Requested-With': 'XMLHttpRequest',
           'Referer': 'https://www.ahhuoshan.gov.cn/public/column/6597791?type=4&catId=7364194&action=list&nav=3'}
API = 'https://www.ahhuoshan.gov.cn/site/label/8888'
ORGAN_ID = '6597791'
CAT_ID = '7364194'
SITE_ID = None  # 运行时从页面提取
FILE = '/c2/hs/publicInfoList_newest2020'

def get_site_id():
    """从栏目页 JS 提取 siteId 和 file 路径"""
    global SITE_ID, FILE
    if SITE_ID:
        return SITE_ID
    try:
        r = requests.get(
            f'https://www.ahhuoshan.gov.cn/public/column/{ORGAN_ID}?type=4&catId={CAT_ID}&action=list&nav=3',
            timeout=(5, 15), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
        m = re.search(r"siteId\s*:\s*(\d+)", r.text)
        if m:
            SITE_ID = m.group(1)
        else:
            m = re.search(r"siteId['\"]?\s*[:=]\s*['\"]?(\d+)", r.text)
            if m:
                SITE_ID = m.group(1)
        # file 路径: file: "/c3/jh/publicInfoList_newest2020" 或 file = "..."(冒号/等号都兼容)
        mf = re.search(r'file\s*[:=]\s*"([^"]*publicInfoList_newest2020[^"]*)"', r.text)
        if mf:
            FILE = mf.group(1)
    except:
        pass
    if not SITE_ID:
        SITE_ID = '0'
    print(f"  [siteId] {SITE_ID} [file] {FILE}")
    return SITE_ID

def fetch_list(page_idx):
    site_id = get_site_id()
    data = {
        'labelName': 'publicInfoList', 'siteId': site_id, 'organId': ORGAN_ID,
        'pageSize': 20, 'catId': CAT_ID, 'type': 4,
        'dateFormat': 'yyyy年MM月dd日', 'pdfAndWord': 'false', 'file': FILE,
    }
    if page_idx > 1:
        data['pageIndex'] = page_idx
    try:
        r = requests.post(API, timeout=(5, 15), headers=HEADERS, data=data, verify=False)
        r.encoding = 'utf-8'
    except:
        return []
    items = re.findall(
        r'<a href="([^"]+)"[^>]*>([^<]{4,120})</a>.*?(\d{4})年(\d{2})月(\d{2})日',
        r.text, re.DOTALL
    )
    results = []
    for href, title, y, m, d in items:
        title = re.sub(r'\s+', '', title)
        if len(title) < 4:
            continue
        results.append({'url': href, 'pub_date': f'{y}-{m}-{d}', 'title': title})
    return results

def fetch_detail(url):
    for _attempt in range(3):
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
            break
        except:
            r = None
            time.sleep(0.5)
    if r is None:
        return '', ''
    if '/404/' in r.url or r.status_code == 404:
        return '', ''
    title = ''
    m = re.search(r'<h1[^>]*>(.*?)</h1>', r.text, re.DOTALL)
    if m:
        cand = re.sub(r'<[^>]+>', '', m.group(1)).strip()
        if cand and '404' not in cand:
            title = cand
    if not title:
        m = re.search(r'<title>([^<]*)</title>', r.text, re.DOTALL)
        if m:
            title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    title = re.sub(r'\s+', '', title).strip()
    content = ''
    m = re.search(r'class="wenzhang"[^>]*>(.*?)</div>', r.text, re.DOTALL)
    if m:
        content = m.group(1).strip()
    if not content:
        m = re.search(r'class="clearfix xxgkcontent minh500"[^>]*>(.*?)</div>', r.text, re.DOTALL)
        if m:
            content = m.group(1).strip()
    # 意见征集模板 (URL /content/article/): 正文在 zjnr_content（征集公告 tab）
    if not content:
        m = re.search(r'class="zjnr_content"[^>]*>(.*?)</div>', r.text, re.DOTALL)
        if m:
            content = m.group(1).strip()
    return title, content

def run(max_pages=5):
    records = []
    seen = set()
    for pg in range(max_pages):
        items = fetch_list(pg + 1)
        if not items:
            break
        print(f"  Page {pg+1}: {len(items)} items")
        for it in items:
            if it['pub_date'] and it['pub_date'] < CUTOFF:
                continue
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            title, content = fetch_detail(it['url'])
            if not title:
                title = it['title']
            if not content.strip():
                continue
            records.append({
                'title': title, 'url': it['url'], 'source_url': it['url'],
                'pub_date': it['pub_date'],
                'site_name': SITE_NAME, 'content': content, 'summary': '',
            })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "ahhuoshan_gsgg")
    return len(valid)

if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=5)
    args = ap.parse_args()
    cnt = run(max_pages=args.pages)
    print(f"Done: {cnt} records")