#!/usr/bin/env python3
"""公安县-公示公告 爬虫 (荆州政务信息公开系统 API)
URL: http://zwgk.gongan.gov.cn/list.shtml?column_id=38554
API: POST /api/content_center/document/list-data-new
"""
import requests
import sqlite3
import sys
import re
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor, as_completed

DB_PATH = '/root/search.db'
API_URL = 'http://zwgk.gongan.gov.cn/api/content_center/document/list-data-new'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
THREE_YEARS_AGO = (datetime.now() - timedelta(days=3*365)).strftime('%Y-%m-%d')
SITE_NAME = '公安县-公示公告'
PAGE_SIZE = 20
MAX_WORKERS = 8  # 并发抓详情页

def get_conn():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=60000")
    return conn

def fetch_api_page(page):
    payload = {"column_id": 38554, "dept_id": 8, "page_index": page, "page_size": PAGE_SIZE}
    try:
        resp = requests.post(API_URL, json=payload, headers=HEADERS, timeout=15)
        if resp.status_code != 200:
            return [], 0
        data = resp.json()
        if data.get('code') != 200:
            return [], 0
        items = []
        for item in data['data'].get('list', []):
            title = item.get('title', '')
            pub_date = item.get('rel_time', '')[:10]
            page_url = item.get('pub_url', '')
            doc_id = item.get('id', 0)
            items.append((page_url, title, pub_date, doc_id))
        return items, data['data'].get('total', 0)
    except Exception as e:
        print(f'[ERROR] API page {page}: {e}', file=sys.stderr)
        return [], 0

def get_detail_content(page_url):
    try:
        resp = requests.get(page_url, headers=HEADERS, timeout=15)
        resp.encoding = 'utf-8'
        if resp.status_code != 200:
            return ''
        soup = BeautifulSoup(resp.text, 'html.parser')
        content_el = soup.select_one('.zw-center-txt')
        return str(content_el) if content_el else ''
    except Exception as e:
        return ''

def main():
    daily_mode = '--daily' in sys.argv
    conn = get_conn()
    cur = conn.cursor()
    total_added = 0
    total_skipped = 0
    total_skip_date = 0

    items, total = fetch_api_page(1)
    if not items:
        print('[ERROR] 无法获取数据')
        return

    total_pages = (total + PAGE_SIZE - 1) // PAGE_SIZE
    print(f'总条数: {total}, 总页数: {total_pages}')

    all_items = items
    if not daily_mode:
        for page in range(2, total_pages + 1):
            more_items, _ = fetch_api_page(page)
            all_items.extend(more_items)
            print(f'[API] 第{page}页 -> {len(more_items)} 条')

    # 先过滤已存在的URL
    to_fetch = []
    for page_url, title, pub_date, doc_id in all_items:
        if pub_date < THREE_YEARS_AGO:
            total_skip_date += 1
            continue
        cur.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,))
        if cur.fetchone():
            total_skipped += 1
            continue
        to_fetch.append((page_url, title, pub_date))

    print(f'需抓取详情页: {len(to_fetch)} 条')

    # 并发抓详情页
    if to_fetch:
        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
            future_map = {executor.submit(get_detail_content, url): (url, title, date) 
                         for url, title, date in to_fetch}
            for i, future in enumerate(as_completed(future_map), 1):
                url, title, date = future_map[future]
                content = future.result()
                summary = BeautifulSoup(content or '', 'html.parser').get_text(strip=True)[:500] if content else ''
                cur.execute(
                    "INSERT OR IGNORE INTO gov_raw (site_name, page_url, title, content, publish_date, summary) VALUES (?, ?, ?, ?, ?, ?)",
                    (SITE_NAME, url, title, content, date, summary)
                )
                total_added += 1
                label = f'[{total_added}/{len(to_fetch)}]'
                if daily_mode:
                    label = '  [ADD]'
                print(f'  {label} {date} {title[:40]}')

    conn.commit()
    print(f'\n同步 FTS ({total_added} 条新增)...')
    cur.execute(
        "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary "
        "FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,)
    )
    conn.commit()
    conn.close()
    print(f'\n=== 完成 ===')
    print(f'新增: {total_added}, 跳过重复: {total_skipped}, 超过3年: {total_skip_date}')

if __name__ == '__main__':
    main()