#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
爬取 西藏自治区人民政府驻格尔木办事处 - 园区动态
https://gem.xizang.gov.cn/xwzx/yqdt/
TRS系统，index.html分页 + 详情页TRS_UEDITOR正文
"""

import requests
import re
import sys
import os
import time
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "https://gem.xizang.gov.cn/xwzx/yqdt/"
SITE_NAME = "西藏自治区人民政府驻格尔木办事处-园区动态"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}

session = requests.Session()
session.headers.update(HEADERS)

DB_PATH = "/root/search.db"
import sqlite3

def get_existing_urls(cursor):
    cursor.execute("SELECT page_url FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
    return {row[0] for row in cursor.fetchall()}

def insert_article(cursor, title, publish_date, page_url, content):
    # 清理过长的正文内容
    if content and len(content) > 500000:
        content = content[:500000]
    cursor.execute(
        "INSERT OR IGNORE INTO gov_raw (site_name, page_url, title, publish_date, content, date_rank, status) "
        "VALUES (?, ?, ?, ?, ?, ?, 'normal')",
        (SITE_NAME, page_url, title, publish_date, content, 0)
    )

def extract_content(html_text, base):
    """从详情页提取正文HTML"""
    soup = BeautifulSoup(html_text, 'html.parser')
    
    # 标题
    title = ""
    h3 = soup.find('h3')
    if h3:
        title = h3.get_text(strip=True)
    if not title:
        mt = soup.find('meta', attrs={'name': 'ArticleTitle'})
        if mt and mt.get('content'):
            title = mt['content'].strip()
    if not title:
        title_tag = soup.find('title')
        if title_tag:
            t = title_tag.get_text(strip=True)
            if '_' in t:
                title = t.split('_')[0].strip()
            else:
                title = t
    
    # 发布时间
    date_str = ""
    p_sub = soup.find('p', class_='atl-sub')
    if p_sub:
        span = p_sub.find('span')
        if span:
            date_str = span.get_text(strip=True).replace('年', '-').replace('月', '-').replace('日', '')
    
    # 来源
    source = ""
    if p_sub:
        spans = p_sub.find_all('span')
        if len(spans) >= 2:
            source = spans[1].get_text(strip=True)
    
    # 正文 - TRS_UEDITOR
    content_div = soup.find('div', class_=re.compile(r'TRS_UEDITOR|trs_paper_default|view'))
    if not content_div:
        content_div = soup.find('div', class_='xl-atl-plists')
    if not content_div:
        content_div = soup.find('div', class_=re.compile(r'xl-atl-plists|TRS_Editor'))
    
    content_html = ""
    attachments_list = []
    
    if content_div:
        # 处理图片相对路径
        for img in content_div.find_all('img'):
            src = img.get('src', '')
            if src and not src.startswith('http'):
                img['src'] = urljoin(base, src)
        
        # 处理附件链接
        for a in content_div.find_all('a'):
            href = a.get('href', '')
            if href and not href.startswith('http') and not href.startswith('javascript'):
                abs_url = urljoin(base, href)
                a['href'] = abs_url
                if not re.search(r'\.(jpg|jpeg|png|gif|webp|bmp)$', href, re.I):
                    attachments_list.append(abs_url)
        
        content_html = str(content_div)
    
    attachments = ','.join(attachments_list) if attachments_list else ""
    
    return title, date_str, source, content_html, attachments


def crawl_page(page_num):
    """爬取指定页码的列表页"""
    if page_num == 0:
        url = urljoin(BASE_URL, "index.html")
    else:
        url = urljoin(BASE_URL, f"index_{page_num}.html")
    
    resp = session.get(url, timeout=60, verify=False)
    resp.encoding = 'utf-8'
    soup = BeautifulSoup(resp.text, 'html.parser')
    
    items = []
    for ul in soup.find_all('ul', class_='gl-cons-rgs'):
        for li in ul.find_all('li', class_='clearfix'):
            a_tag = li.find('a')
            span_tag = li.find('span')
            if a_tag and a_tag.get('href'):
                href = a_tag['href'].strip()
                detail_url = urljoin(resp.url, href)
                title = a_tag.get_text(strip=True)
                date_str = span_tag.get_text(strip=True) if span_tag else ""
                items.append((title, date_str, detail_url))
    
    return items


def crawl_detail(detail_url):
    """爬取详情页"""
    try:
        resp = session.get(detail_url, timeout=60, verify=False)
        resp.encoding = 'utf-8'
        title, date_str, source, content_html, attachments = extract_content(resp.text, detail_url)
        return title, date_str, source, content_html, attachments
    except Exception as e:
        print(f"  [ERROR] 详情页异常 {detail_url}: {e}")
        return "", "", "", "", ""


def main():
    incremental = False
    incremental_days = 30
    for arg in sys.argv[1:]:
        if arg == '--incremental':
            incremental = True
        elif arg.isdigit():
            incremental_days = int(arg)
    
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cursor = conn.cursor()
    
    cursor.execute("""
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            title TEXT,
            date_str TEXT,
            page_url TEXT UNIQUE,
            content TEXT,
            source TEXT,
            site_name TEXT,
            attachments TEXT
        )
    """)
    conn.commit()
    
    existing_urls = get_existing_urls(cursor)
    print(f"已有 {len(existing_urls)} 条记录")
    
    cutoff_date = ""
    if incremental:
        from datetime import datetime, timedelta
        cutoff = datetime.now() - timedelta(days=incremental_days)
        cutoff_date = cutoff.strftime('%Y-%m-%d')
        print(f"增量模式：仅爬取 {cutoff_date} 之后的文章")
    
    page_count = 20
    all_items = []
    for p in range(page_count):
        print(f"获取第 {p+1}/{page_count} 页...")
        try:
            items = crawl_page(p)
            all_items.extend(items)
            print(f"  本页 {len(items)} 条")
        except Exception as e:
            print(f"  [ERROR] 第 {p+1} 页异常: {e}")
        time.sleep(0.5)
    
    print(f"\n列表共 {len(all_items)} 条")
    
    new_count = 0
    skip_count = 0
    error_count = 0
    
    for i, (title, date_str, detail_url) in enumerate(all_items, 1):
        if incremental and cutoff_date and date_str and date_str < cutoff_date:
            skip_count += 1
            if skip_count == 1:
                print(f"  跳过旧数据（早于 {cutoff_date}），首个: {title[:30]} ({date_str})")
            continue
        
        if detail_url in existing_urls:
            skip_count += 1
            continue
        
        print(f"  [{i}/{len(all_items)}] {title[:30]}...")
        dt_title, dt_date, source, content_html, attachments = crawl_detail(detail_url)
        
        final_title = dt_title if dt_title else title
        final_date = dt_date if dt_date else date_str
        
        if not content_html or len(content_html.strip()) < 50:
            print(f"    ⚠ 正文过短或为空，跳过")
            error_count += 1
            continue
        insert_article(cursor, final_title, final_date, detail_url, content_html)
        new_count += 1
        
        if new_count % 10 == 0:
            conn.commit()
            print(f"    ✓ 已提交 {new_count} 条")
        
        time.sleep(0.3)
    
    conn.commit()
    
    # FTS 由 search.db 触发器 trg_gov_raw_fts_* 统一维护, 不再自建 gov_fts / 手工重建
    print("\nFTS 由触发器维护, 跳过本地重建")
    
    conn.close()
    
    print(f"\n=== 完成 ===")
    print(f"新增: {new_count} 条")
    print(f"跳过: {skip_count} 条")
    print(f"错误: {error_count} 条")


if __name__ == '__main__':
    main()