#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""crawl_tengzhou_nshz.py — 滕州市南沙河镇人民政府-通知公告"""

import os, re, sqlite3, time, sys, random, urllib.parse
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed

DB_PATH = os.environ.get('SEARCH_DB', '/root/search.db')
SITE_NAME = '滕州市南沙河镇人民政府-通知公告'
LIST_URL = 'http://www.tengzhou.gov.cn/govsearch/searPageNewTengZhou.jsp'
PAGE_SIZE = 20
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime('%Y-%m-%d')
MAX_WORKERS = 5

POST_DATA = {
    'siteid': '189', 'classinfoid': '4162', 'channelid': '12634',
    'pubURL': '', 'indexPa': '', 'schn': '', 'curpos': '', 'sinfo': '', 'surl': '',
}

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
}

import requests
sess = requests.Session()
sess.headers.update(HEADERS)


def fetch_list(page):
    """Fetch one page of list via POST"""
    data = POST_DATA.copy()
    data['page'] = str(page)
    for _ in range(3):
        try:
            r = sess.post(LIST_URL, data=data, timeout=15)
            r.encoding = 'utf-8'
            return r.text
        except Exception as e:
            time.sleep(1)
    return None


def parse_list(html):
    """Parse articles from list HTML"""
    items = []
    for li in re.finditer(r'<li>(.*?)</li>', html, re.DOTALL):
        content = li.group(1)
        a = re.search(r'<a\s+href="([^"]+)"[^>]*title="([^"]*)"', content)
        b = re.search(r'<b>(\d{4})年(\d{2})月(\d{2})日</b>', content)
        if a and b:
            items.append({
                'url': a.group(1),
                'title': a.group(2).strip(),
                'date': f'{b.group(1)}-{b.group(2)}-{b.group(3)}',
            })
    return items


def parse_detail(html):
    """Extract title and content"""
    # Title from <h2> inside #jdlf
    t = re.search(r'<div\s+id="jdlf">\s*<h2>(.*?)</h2>', html, re.DOTALL)
    title = t.group(1).strip() if t else ''
    
    # Content from <div class="zwnr">
    m = re.search(r'<div class="zwnr">(.*?)</div>\s*<script', html, re.DOTALL)
    if not m:
        m = re.search(r'<div class="zwnr">(.*?)</div>\s*</div>\s*<div\s+class="seg-line"', html, re.DOTALL)
    if not m:
        m = re.search(r'<div class="zwnr">(.*?)</div>', html, re.DOTALL)
    content = m.group(1).strip() if m else ''
    
    # Clean
    content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL)
    content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL)
    
    return title, content.strip()


def process_item(item):

    resp = requests.get(item['url'], headers=HEADERS, timeout=15)
    resp.encoding = 'utf-8'
    html = resp.text
    if not html:
        return None, 'FETCH_ERR'
    dt, content = parse_detail(html)
    return {
        'site_name': SITE_NAME,
        'title': dt or item['title'],
        'page_url': item['url'],
        'publish_date': item['date'],
        'source_url': item['url'],
        'content': content,
    }, None


def main():
    print(f'[{datetime.now().strftime("%H:%M:%S")}] {SITE_NAME}')
    
    # Fetch first page to get total
    html = fetch_list(1)
    if not html:
        print('[ERROR] Failed page 1')
        return
    
    # Find total records
    total = 0
    m = re.search(r'm_nRecordCount\s*=\s*(\d+)', html)
    if m:
        total = int(m.group(1))
    print(f'Total: {total} records')
    
    total_pages = (total + PAGE_SIZE - 1) // PAGE_SIZE
    
    # Fetch all pages
    all_items = parse_list(html)
    for p in range(2, total_pages + 1):
        h = fetch_list(p)
        if h:
            all_items.extend(parse_list(h))
        time.sleep(random.uniform(0.2, 0.4))
    
    print(f'Collected: {len(all_items)}')
    
    recent = [i for i in all_items if i['date'] >= CUTOFF]
    print(f'Within 3 years: {len(recent)} (filtered: {len(all_items)-len(recent)})')
    
    conn = sqlite3.connect(DB_PATH, timeout=30)
    c = conn.cursor()
    existing = set(r[0] for r in c.execute('SELECT page_url FROM gov_raw WHERE site_name=?', (SITE_NAME,)))
    
    to_fetch = [i for i in recent if i['url'] not in existing]
    print(f'New items: {len(to_fetch)}')
    
    new = err = 0
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        futures = {executor.submit(process_item, item): item for item in to_fetch}
        for i, future in enumerate(as_completed(futures), 1):
            item = futures[future]
            result, error = future.result()
            if error or not result:
                print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... {error or "FAIL"}')
                err += 1
                continue
            try:
                c.execute('''INSERT OR IGNORE INTO gov_raw
                    (site_name, title, page_url, publish_date, source_url, content)
                    VALUES (?,?,?,?,?,?)''', (
                        result['site_name'], result['title'], result['page_url'],
                        result['publish_date'], result['source_url'], result['content']))
                if c.rowcount:
                    new += 1
                    print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... OK ({len(result["content"])}B)')
                else:
                    print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... DUP')
            except Exception as e:
                print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... ERR: {e}')
                err += 1
            conn.commit()
    
    conn.close()
    print(f'\n=== Done === New: {new}, Errors: {err}')


if __name__ == '__main__':
    main()