#!/usr/bin/env python3
"""crawl_tianchang_hpgs.py - 天长市人民政府-环评公示 (滁州平台 label AJAX POST /czxxgk/site/label/8888)"""
import sys, os, requests, re
from datetime import datetime, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365*3)).strftime('%Y-%m-%d')
SITE_NAME = '天长市-环评公示'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0'}
LABEL_URL = 'https://www.tianchang.gov.cn/czxxgk/site/label/8888'
LABEL_DATA = {
    'labelName': 'publicInfoList',
    'siteId': '2653861',
    'organId': '161054621',
    'catId': '170140492',
    'type': '4',
    'dateFormat': 'yyyy-MM-dd',
    'file': '/c1/chuzhou/publicInfoList_newest2020',
}

def fetch_urls_from_page(page_num):
    try:
        r = requests.post(LABEL_URL, data=LABEL_DATA, timeout=(5, 15), headers=HEADERS, verify=False)
        r.encoding = 'utf-8'
    except:
        return []
    results = []
    trs = re.findall(r'<tr[^>]*>(.*?)</tr>', r.text, re.DOTALL)
    for tr in trs:
        a = re.search(r'href="([^"]+\.htm[l]?)"[^>]*>(.*?)</a>', tr, re.DOTALL)
        if not a:
            continue
        title = re.sub(r'<[^>]+>', '', a.group(2)).strip()
        if len(title) < 4:
            continue
        d = re.search(r'(\d{4}-\d{2}-\d{2})', tr)
        pub_date = d.group(1) if d else ''
        href = a.group(1)
        if href.startswith('/'):
            full_url = 'https://www.tianchang.gov.cn' + href
        elif href.startswith('http'):
            full_url = href
        else:
            full_url = 'https://www.tianchang.gov.cn/' + href
        results.append({'url': full_url, 'pub_date': pub_date, 'title': title})
    return results

def fetch_detail(url):
    for _attempt in range(3):
        try:
            r = requests.get(url, timeout=(5, 15), headers=HEADERS, verify=False)
            r.encoding = 'utf-8'
            break
        except:
            r = None
    if r is None:
        return '', ''
    title = ''
    m = re.search(r'<title>([^<]*)</title>', r.text, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
        title = re.sub(r'\s*[-_|_].*$', '', title).strip()
    content = ''
    idx = r.text.find('<div class="gk_container"')
    if idx > 0:
        start = r.text.find('>', idx) + 1
        depth = 1
        pos = start
        while pos < len(r.text) and depth > 0:
            if r.text[pos:pos+4] == '<div':
                depth += 1
                pos += 4
            elif r.text[pos:pos+6] == '</div>':
                depth -= 1
                pos += 6
            else:
                pos += 1
        content = r.text[start:pos-6].strip()
    # 滁州平台详情页有PC/移动双版本索引元数据表(table_suoyin), 内容相同, 删移动版(hidden-lg hidden-md)保留PC版
    content = re.sub(r'<table class="table_suoyin[^"]*hidden-lg[^"]*"[^>]*>.*?</table>', '', content, flags=re.S | re.I)
    if not content:
        m = re.search(r'<div id="zoom"[^>]*>(.*?)</div>', r.text, re.DOTALL)
        if m:
            content = m.group(1).strip()
    return title, content

def run(incremental=False):
    records = []
    seen = set()
    items = fetch_urls_from_page(0)
    print(f"  Page 1: {len(items)} items")
    for it in items:
        if it['pub_date'] < CUTOFF:
            continue
        if it['url'] in seen:
            continue
        seen.add(it['url'])
        title, content = fetch_detail(it['url'])
        if not title or not content.strip():
            continue
        records.append({
            'title': title,
            'url': it['url'],
            'pub_date': it['pub_date'],
            'site_name': SITE_NAME,
            'content': content,
            'summary': '',
        })
    print(f"  Total: {len(records)} items")
    valid = [r for r in records if r['content'].strip()]
    if valid:
        push_to_searchdb(valid, "tianchang_hpgs")
    return len(valid)

if __name__ == '__main__':
    cnt = run(incremental='--incremental' in sys.argv)
    print(f"Done: {cnt} records")