#!/usr/bin/env python3
"""crawl_akss_gsgg.py - 阿克苏市-公示公告 (博山CMS: 首页JSON dataList + index_N静态页)"""
import sys, os, requests, re, json, argparse
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365*3)).strftime('%Y-%m-%d')
SITE_NAME = '阿克苏市-公示公告'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0'}
BASE = 'https://www.akss.gov.cn/qwfb/gsgg'

def ts_to_date(ts):
    try:
        return datetime.fromtimestamp(int(ts) / 1000).strftime('%Y-%m-%d')
    except:
        return ''

def fetch_urls_from_page(page_num):
    if page_num == 0:
        url = BASE + '/index.html'
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
        except:
            return []
        # 首页: 内嵌 JSON dataList
        m = re.search(r'var dataList=(\[.*?\]);', r.text, re.DOTALL)
        if not m:
            return []
        try:
            data = json.loads(m.group(1))
        except:
            return []
        infolist = data[0]['infolist'] if data and isinstance(data[0], dict) and 'infolist' in data[0] else []
        results = []
        for it in infolist:
            title = it.get('title') or it.get('infotitle', '')
            url = it.get('url', '')
            pd = ts_to_date(it.get('releasetime') or it.get('releaseTime') or 0)
            title = re.sub(r'\s+', '', title)
            if title and url and pd:
                results.append({'url': url, 'pub_date': pd, 'title': title})
        return results
    else:
        # 后续页: index_N.html 静态表格
        url = f'{BASE}/index_{page_num + 1}.html'
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
        except:
            return []
        # <a href=".../YYYYMMDD/iID.html" target="_blank" title="标题">标题</a> ... <td width="110">YYYY-MM-DD</td>
        items = re.findall(
            r'<a href="([^"]+)"[^>]*target="_blank"[^>]*title="([^"]*)"[^>]*>.*?</a>.*?<td[^>]*>(\d{4}-\d{2}-\d{2})</td>',
            r.text, re.DOTALL
        )
        results = []
        for href, title, pub_date in items:
            title = re.sub(r'\s+', '', title)
            if len(title) < 4 or '/i' not in href.split('/')[-1]:
                continue
            if href.startswith('http'):
                full_url = href
            else:
                full_url = 'https://www.akss.gov.cn' + href if href.startswith('/') else BASE + '/' + href
            results.append({'url': full_url, 'pub_date': pub_date, 'title': title})
        return results

def fetch_detail(url):
    for _attempt in range(2):
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
            break
        except:
            r = None
    if r is None:
        return '', ''
    title = ''
    m = re.search(r'<title>([^<]*)</title>', r.text, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
        title = re.sub(r'\s*[-_|]\s*阿克苏市\s*$', '', title)
        title = re.sub(r'^\s*阿克苏市\s*[-_|]\s*', '', title)
    title = re.sub(r'\s+', '', title).strip()
    content = ''
    m = re.search(r'class="table-a"[^>]*>(.*?)</div>', r.text, re.DOTALL)
    if m:
        content = m.group(1).strip()
    return title, content

def run(max_pages=5):
    records = []
    seen = set()
    for pg in range(max_pages):
        items = fetch_urls_from_page(pg)
        if not items:
            break
        print(f"  Page {pg+1}: {len(items)} items")
        for it in items:
            if it['pub_date'] and it['pub_date'] < CUTOFF:
                continue
            if it['url'] in seen:
                continue
            seen.add(it['url'])
            title, content = fetch_detail(it['url'])
            if not title:
                title = it['title']
            if not content.strip():
                continue
            records.append({
                'title': title, 'url': it['url'], 'pub_date': it['pub_date'],
                'site_name': SITE_NAME, 'content': content, 'summary': '',
            })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "akss_gsgg")
    return len(valid)

if __name__ == '__main__':
    ap = argparse.ArgumentParser()
    ap.add_argument('--pages', type=int, default=5)
    args = ap.parse_args()
    cnt = run(max_pages=args.pages)
    print(f"Done: {cnt} records")