#!/usr/bin/env python3
"""靖西市-建设项目环境影响评价审批 爬虫 (TRS CMS)
URL: http://www.jingxi.gov.cn/xxgk/fdzdgkmr/zdxxgk/shgysy/hjbh/jcxmhjyx/
分页: index.shtml (第1页), index_{1..12}.shtml (第2-13页)
共13页376条
"""
import requests
import re
import sqlite3
import sys
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor, as_completed

DB_PATH = '/root/search.db'
BASE_URL = 'http://www.jingxi.gov.cn'
LIST_DIR = '/xxgk/fdzdgkmr/zdxxgk/shgysy/hjbh/jcxmhjyx/'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
THREE_YEARS_AGO = (datetime.now() - timedelta(days=3*365)).strftime('%Y-%m-%d')
SITE_NAME = '靖西市-建设项目环境影响评价审批'
TOTAL_PAGES = 13
MAX_WORKERS = 8

def get_conn():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=60000")
    return conn

def crawl_page(page_url):
    articles = []
    try:
        resp = requests.get(page_url, headers=HEADERS, timeout=15)
        resp.encoding = 'utf-8'
        if resp.status_code != 200:
            return articles
        soup = BeautifulSoup(resp.text, 'html.parser')
        items = soup.select('.more-list li')
        for li in items:
            a = li.select_one('a')
            if not a:
                continue
            href = a.get('href', '')
            if not href.startswith('http'):
                href = BASE_URL + LIST_DIR.rstrip('/') + '/' + href.lstrip('./')
            title = a.get('title', '') or a.get_text(strip=True)
            date_span = li.select_one('span')
            list_date = date_span.get_text(strip=True) if date_span else ''
            articles.append((href, title, list_date))
    except Exception as e:
        print(f'[ERROR] 列表页抓取失败: {page_url} - {e}', file=sys.stderr)
    return articles

def get_detail(url):
    try:
        resp = requests.get(url, headers=HEADERS, timeout=15)
        resp.encoding = 'utf-8'
        if resp.status_code != 200:
            return '', ''
        soup = BeautifulSoup(resp.text, 'html.parser')
        h1 = soup.select_one('.article h1')
        title = h1.get_text(strip=True) if h1 else ''
        # 日期
        pub_date = ''
        inf_left = soup.select_one('.article-inf-left')
        if inf_left:
            m = re.search(r'(\d{4}-\d{2}-\d{2})', inf_left.get_text())
            if m:
                pub_date = m.group(1)
        content_el = soup.select_one('.article-con')
        content = str(content_el) if content_el else ''
        return title, content, pub_date
    except Exception as e:
        print(f'[ERROR] 详情页失败: {url} - {e}', file=sys.stderr)
        return '', '', ''

def main():
    daily_mode = '--daily' in sys.argv
    conn = get_conn()
    cur = conn.cursor()

    if daily_mode:
        pages = [BASE_URL + LIST_DIR + 'index.shtml']
    else:
        pages = [BASE_URL + LIST_DIR + 'index.shtml']
        for i in range(1, TOTAL_PAGES):
            pages.append(f'{BASE_URL}{LIST_DIR}index_{i}.shtml')

    all_articles = []
    for page_url in pages:
        arts = crawl_page(page_url)
        pname = page_url.split('/')[-1]
        print(f'[列表页] {pname} -> {len(arts)} 条')
        all_articles.extend(arts)

    to_fetch = []
    total_skip_date = 0
    total_skipped = 0

    for href, title, list_date in all_articles:
        if list_date < THREE_YEARS_AGO:
            total_skip_date += 1
            continue
        cur.execute("SELECT id FROM gov_raw WHERE page_url = ?", (href,))
        if cur.fetchone():
            total_skipped += 1
            continue
        to_fetch.append((href, title, list_date))

    print(f'\n需抓取详情页: {len(to_fetch)} 条')

    total_added = 0
    if to_fetch:
        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
            future_map = {executor.submit(get_detail, url): (url, title, date)
                         for url, title, date in to_fetch}
            for i, future in enumerate(as_completed(future_map), 1):
                url, title, date = future_map[future]
                detail_title, content, detail_date = future.result()
                final_title = detail_title or title
                final_date = detail_date or date
                summary = BeautifulSoup(content or '', 'html.parser').get_text(strip=True)[:500] if content else ''
                cur.execute(
                    "INSERT OR IGNORE INTO gov_raw (site_name, page_url, title, content, publish_date, summary) VALUES (?, ?, ?, ?, ?, ?)",
                    (SITE_NAME, url, final_title, content, final_date, summary)
                )
                total_added += 1
                label = f'[{total_added}/{len(to_fetch)}]' if not daily_mode else '  [ADD]'
                print(f'  {label} {final_date} {final_title[:40]}')

    conn.commit()
    print(f'\n同步 FTS ({total_added} 条新增)...')
    cur.execute(
        "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary "
        "FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,)
    )
    conn.commit()
    conn.close()
    print(f'\n=== 完成 ===')
    print(f'新增: {total_added}, 跳过重复: {total_skipped}, 超过3年: {total_skip_date}')

if __name__ == '__main__':
    main()