#!/usr/bin/env python3
"""柞水县人民政府 - 公示公告 爬虫 v2 (并发版)
VSB9 反向分页: gsgg.htm → gsgg/73.htm → ... → gsgg/1.htm
"""

import sys, os, re, time, json, logging, argparse, sqlite3
from datetime import datetime, timedelta
from urllib.parse import urljoin
from concurrent.futures import ThreadPoolExecutor, as_completed

import requests
from bs4 import BeautifulSoup

BASE_URL = "https://www.snzs.gov.cn"
LIST_URL = "/zwdt/gsgg"
SITE_NAME = "柞水县人民政府-公示公告"
TOTAL_PAGES = 74
PAGE_SIZE = 15
NUM_THREADS = 5
DATE_CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
DB_PATH = "/root/search.db"

logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(levelname)s %(message)s', datefmt='%H:%M:%S')
log = logging.getLogger(__name__)
HEADERS = {'User-Agent': 'Mozilla/5.0'}

def init_db():
    db = sqlite3.connect(DB_PATH)
    db.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT NOT NULL,
        title TEXT NOT NULL,
        page_url TEXT NOT NULL UNIQUE,
        publish_date TEXT,
        source_url TEXT,
        content TEXT,
        crawl_time TEXT DEFAULT (datetime('now','localtime'))
    )""")
    db.commit()
    return db

def fetch(url):
    try:
        r = requests.get(url, timeout=20, headers=HEADERS)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        log.error(f"请求失败 {url[:60]}: {e}")
        return None

def parse_list_items(html):
    soup = BeautifulSoup(html, 'html.parser')
    items = []
    for li in soup.find_all('li'):
        span = li.find('span')
        a = li.find('a', href=True)
        if not (span and a and 'info/1071' in a.get('href','')):
            continue
        date = span.get_text(strip=True)
        if not re.match(r'\d{4}-\d{2}-\d{2}', date):
            continue
        href = a['href']
        if not href.startswith('http'):
            href = urljoin(BASE_URL, href)
        title = a.get_text(strip=True)
        items.append({'title': title, 'page_url': href, 'publish_date': date})
    return items

def get_page_url(page_num):
    if page_num == 1:
        return f"{BASE_URL}{LIST_URL}.htm"
    page_id = TOTAL_PAGES + 1 - page_num
    return f"{BASE_URL}{LIST_URL}/{page_id}.htm"

def fetch_detail(item):
    try:
        html = fetch(item['page_url'])
        if not html: return None
        soup = BeautifulSoup(html, 'html.parser')
        h1 = soup.find('h1')
        title = h1.get_text(strip=True) if h1 else ''
        m = re.search(r'(\d{4}-\d{2}-\d{2})', html)
        pub_date = m.group(1) if m else ''
        content_div = soup.select_one('.v_news_content')
        content_html = ''
        if content_div:
            for s in content_div.select('script,style'):
                s.decompose()
            content_html = str(content_div).strip()
        item['title'] = title or item['title']
        item['publish_date'] = pub_date or item['publish_date']
        item['content'] = content_html
        item['site_name'] = SITE_NAME
        item['source_url'] = "柞水县人民政府"
        return item
    except Exception as e:
        log.error(f"详情失败 {item['page_url'][:50]}: {e}")
        return None

def save_batch(db, items):
    saved = 0
    for item in items:
        db.execute("INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, source_url, content) VALUES (?,?,?,?,?,?)",
                   (item['site_name'], item['title'], item['page_url'], item.get('publish_date',''), item.get('source_url',''), item.get('content','')))
        if db.cursor().rowcount:
            saved += 1
    db.commit()
    return saved

def crawl(full=True):
    db = init_db()
    total_pages = TOTAL_PAGES if full else 1

    # 列表收集
    all_items = []
    for page in range(1, total_pages + 1):
        url = get_page_url(page)
        time.sleep(0.3)
        html = fetch(url)
        if not html: break
        items = parse_list_items(html)
        if not items: break
        dates = [i['publish_date'] for i in items if i['publish_date']]
        if dates and max(dates) < DATE_CUTOFF:
            log.info(f"第{page}页 [{items[0]['publish_date']}→{items[-1]['publish_date']}] → 终止")
            break
        log.info(f"第{page}页: {len(items)} 条 [{items[0]['publish_date']}→{items[-1]['publish_date']}]")
        all_items.extend(items)

    # 去重 + 过滤
    seen = set()
    to_crawl = []
    for item in all_items:
        if item['page_url'] in seen: continue
        seen.add(item['page_url'])
        if item['publish_date'] and item['publish_date'] < DATE_CUTOFF: continue
        if db.execute("SELECT 1 FROM gov_raw WHERE page_url=?", (item['page_url'],)).fetchone(): continue
        to_crawl.append(item)

    log.info(f"待爬详情: {len(to_crawl)}/{len(all_items)}")

    # 并发详情
    saved_count = 0
    with ThreadPoolExecutor(max_workers=NUM_THREADS) as executor:
        futures = {executor.submit(fetch_detail, item): i for i, item in enumerate(to_crawl)}
        batch = []
        for i, future in enumerate(as_completed(futures)):
            result = future.result()
            if result:
                batch.append(result)
            if (i+1) % 20 == 0:
                log.info(f"详情 [{i+1}/{len(to_crawl)}]")

            if len(batch) >= 20:
                saved_count += save_batch(db, batch)
                batch = []
        if batch:
            saved_count += save_batch(db, batch)

    log.info(f"完成: crawled={len(to_crawl)}, saved={saved_count}")
    return {"total": len(to_crawl), "saved": saved_count}

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--full', action='store_true')
    args = parser.parse_args()
    result = crawl(full=args.full)
    print(json.dumps(result, ensure_ascii=False))