#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
肇庆市生态环境局 - 建设项目环境影响评价信息公开平台 爬虫
系统: Vue SPA + REST API
API: GET /zqhjj/gkmlpt/api/all/21022?page=N
返回JSON含articles[].{id,title,url,date,publisher,classify_main_name}
"""

import re
import sys
import json
import time
import subprocess
from datetime import datetime

import requests
from bs4 import BeautifulSoup

# ── 配置 ──────────────────────────────────────────
BASE_URL = "http://www.zhaoqing.gov.cn"
API_URL = f"{BASE_URL}/zqhjj/gkmlpt/api/all/21022"
SITE_NAME = "肇庆市生态环境局"
GROUP = "广东省"
DB_PATH = "/mnt/data/search.db"
TOTAL_PAGES = 3  # 100+100+63=263条

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "application/json, text/plain, */*",
}

session = requests.Session()
session.headers.update(HEADERS)


# ── 工具函数 ──────────────────────────────────────
def clean_text(text):
    if not text:
        return ""
    return re.sub(r'\s+', ' ', text).strip()


def timestamp_to_date(ts):
    """Convert UNIX timestamp to YYYY-MM-DD"""
    return datetime.fromtimestamp(ts).strftime("%Y-%m-%d")


def clean_content_html(html_content):
    """提取正文，只处理直接子元素避免重复，表格保持HTML"""
    if not html_content:
        return ""
    soup = BeautifulSoup(html_content, 'html.parser')
    for tag in soup(['script', 'style']):
        tag.decompose()
    
    parts = []
    for child in soup.children:
        if not child.name:
            continue
        if child.name == 'p':
            t = child.get_text(strip=True)
            if t:
                parts.append(t)
        elif child.name == 'table':
            # 表格保留完整HTML，让搜索App渲染为漂亮表格
            parts.append(str(child))
        elif child.name in ('div', 'section', 'article'):
            # 提取div内的段落文本（排除表格内的p，避免重复）
            for p in child.find_all('p', recursive=True):
                # 跳过表格内部的p（表格会整体保留为HTML）
                if p.find_parent('table'):
                    continue
                t = p.get_text(strip=True)
                if t:
                    parts.append(t)
            # div中的表格保持HTML
            for tbl in child.find_all('table', recursive=True):
                parts.append(str(tbl))
        elif child.name == 'img':
            src = child.get('src', '')
            alt = child.get('alt', '')
            if src:
                # 相对路径转绝对
                if src.startswith('/'):
                    src = BASE_URL + src
                parts.append(f'![{alt}]({src})')
    return '\n\n'.join(parts)


# ── 列表获取（API JSON） ───────────────────────────
def fetch_list(max_pages=3):
    """从API获取所有文章列表"""
    all_articles = []
    for page in range(1, max_pages + 1):
        print(f"  获取列表 第{page}页...")
        try:
            r = session.get(API_URL, params={'page': page}, timeout=30)
            if r.status_code != 200:
                print(f"    HTTP {r.status_code}")
                continue
            data = r.json()
            articles = data.get('articles', [])
            if not articles:
                print(f"    无数据")
                break
            all_articles.extend(articles)
            print(f"    获取{len(articles)}条")
            time.sleep(0.3)
        except Exception as e:
            print(f"    [ERR] {e}")
            break
    return all_articles


# ── 详情页解析 ────────────────────────────────────
def parse_detail(html, url):
    """解析详情页，返回 (title, publish_date, source, content)"""
    soup = BeautifulSoup(html, 'html.parser')
    title = ''
    publish_date = ''
    source = ''
    content = ''

    # 1. 标题 - 取最后一个h1（第一个是"政府信息公开", 第二个才是文章标题）
    h1s = soup.find_all('h1')
    for h1 in h1s:
        t = clean_text(h1.get_text())
        if t and '政府信息' not in t:
            title = t
            break
    if not title and h1s:
        title = clean_text(h1s[-1].get_text())

    # 2. 发布日期和来源 - 从表格找到的信息行
    date_match = re.search(r'发布日期[：:]\s*(\d{4}-\d{2}-\d{2})', html)
    if date_match:
        publish_date = date_match.group(1)
    source_match = re.search(r'发布机构[：:]\s*([^<\n]+)', html)
    if source_match:
        source = source_match.group(1).strip()

    # 3. 正文 - 从 div.article-content
    article_div = soup.find('div', class_='article-content')
    if not article_div:
        article_div = soup.find('div', class_='content-container')
    if article_div:
        content = clean_content_html(str(article_div))
    if not content:
        paragraphs = soup.find_all('p')
        body_texts = [clean_text(p.get_text()) for p in paragraphs if len(clean_text(p.get_text())) > 30]
        content = '\n\n'.join(body_texts)

    return title, publish_date, source, content


# ── 入库 ──────────────────────────────────────────
def insert_article(article, detail_title, pub_date, source, content):
    """写入gov_raw + FTS gov_search"""
    page_url = article['url']
    title = (detail_title or article['title']).replace("'", "''")
    publish_date = pub_date or timestamp_to_date(article.get('date', 0))
    source_name = source or article.get('publisher', SITE_NAME)
    source_name = source_name.replace("'", "''")
    content = content.replace("'", "''")
    summary = title[:200]
    site_name = SITE_NAME

    sql_raw = f"""INSERT OR IGNORE INTO gov_raw 
        (page_url, title, content, publish_date, source_url, site_name, summary, group_name, industry)
    VALUES 
        ('{page_url}', '{title}', '{content}', '{publish_date}', '{page_url}', '{site_name}', '{summary}', '{GROUP}', '政府公告');
"""

    # 单进程INSERT+取rowid
    combined_sql = sql_raw + """
SELECT CASE WHEN changes() > 0 THEN last_insert_rowid() ELSE 0 END;
"""
    try:
        result = subprocess.run(
            ['sqlite3', "-cmd", ".timeout 60000", DB_PATH],
            input=combined_sql,
            capture_output=True, text=True, timeout=30
        )
        if result.returncode == 0:
            out = result.stdout.strip()
            if out:
                try:
                    rowid = int(out.strip())
                except ValueError:
                    rowid = 0
            else:
                rowid = 0

            if rowid > 0:
                fts_sql = f"""INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary)
SELECT rowid, title, site_name, summary FROM gov_raw WHERE rowid = {rowid};
"""
                subprocess.run(
                    ['sqlite3', "-cmd", ".timeout 60000", DB_PATH],
                    input=fts_sql,
                    capture_output=True, text=True, timeout=30
                )
                return True
        else:
            if 'UNIQUE constraint' not in result.stderr:
                print(f"    [DB] {result.stderr[:200]}")
    except Exception as e:
        print(f"    [DB] 错误: {e}")
    return False


# ── 主流程 ────────────────────────────────────────
def main(max_pages=TOTAL_PAGES):
    print(f"=== 肇庆市生态环境局 环评信息公开爬虫 ===")

    # 1. 获取列表
    articles = fetch_list(max_pages)
    if not articles:
        print("  无文章列表")
        return
    print(f"\n列表总计: {len(articles)} 条")

    # 2. 获取已知URL
    known_urls = set()
    try:
        result = subprocess.run(
            ['sqlite3', "-cmd", ".timeout 60000", DB_PATH, "SELECT page_url FROM gov_raw WHERE page_url LIKE '%zhaoqing.gov.cn%'"],
            capture_output=True, text=True, timeout=30
        )
        if result.returncode == 0 and result.stdout.strip():
            known_urls = set(result.stdout.strip().split('\n'))
    except Exception:
        pass
    print(f"已知URL: {len(known_urls)}")

    # 3. 遍历详情
    added = 0
    skipped = 0
    for a in articles:
        url = a['url']
        if url in known_urls:
            skipped += 1
            continue

        title_api = a['title']
        print(f"  抓取: {title_api[:40]}...")

        try:
            r = session.get(url, timeout=30)
            r.encoding = 'utf-8'
        except Exception as e:
            print(f"    [ERR] {e}")
            time.sleep(1)
            continue

        detail_title, pub_date, source, content = parse_detail(r.text, url)
        if not detail_title:
            detail_title = title_api
        if not pub_date:
            pub_date = timestamp_to_date(a.get('date', 0))

        # 正文质量检查
        content_len = len(content.strip())
        if content_len < 20:
            print(f"    [SKIP] 正文过短({content_len}字)")
            continue

        if insert_article(a, detail_title, pub_date, source, content):
            added += 1
        else:
            skipped += 1

        time.sleep(0.3)

    print(f"\n结果: 新增 {added} 条, 跳过 {skipped} 条")
    print("=== 完成 ===")


if __name__ == '__main__':
    pages = TOTAL_PAGES
    if len(sys.argv) > 1:
        for arg in sys.argv[1:]:
            if arg.startswith('--pages='):
                try:
                    pages = int(arg.split('=')[1])
                except ValueError:
                    pass
    main(max_pages=pages)