#!/usr/bin/env python3
"""
新会区-建设项目环评审批受理公告(jsxmhpspslgg)
URL: http://www.xinhui.gov.cn/zwgk/zdlyxxgk/hjbhxxgk/jsxmhjyxpjxx/jsxmhpspslgg/
分页: index.html / index_2.html ... index_N.html (20条/页)
"""

import requests
import sqlite3
import json
import os
import argparse
from bs4 import BeautifulSoup

DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
SITE_NAME = "新会区-环评审批受理公告"
BASE_URL = "http://www.xinhui.gov.cn"
LIST_PATH = "/zwgk/zdlyxxgk/hjbhxxgk/jsxmhjyxpjxx/jsxmhpspslgg"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}


def get_soup(url, timeout=20):
    r = requests.get(url, headers=HEADERS, timeout=timeout)
    r.encoding = "utf-8"
    return BeautifulSoup(r.text, "html.parser")


def parse_list(soup):
    """提取列表页: li > span.li-left > a[title] + span.li-right"""
    items = []
    for li in soup.select("li"):
        a = li.find("a")
        if not a:
            continue
        href = a.get("href", "")
        if "post_" not in href:
            continue
        if not href.startswith("http"):
            href = BASE_URL + href

        # 完整标题优先从 title 属性取
        title = a.get("title", "") or a.get_text(strip=True)

        # 日期从 span.li-right
        date_span = li.find("span", class_="li-right")
        pub_date = date_span.get_text(strip=True) if date_span else ""

        items.append((href, title, pub_date))
    return items


def extract_detail(soup, url):
    """提取详情页: h1标题 + .xl-article正文 + PDF附件"""
    title = ""
    pub_date = ""
    content = ""
    attachments = []

    # Title - h1 第一行
    h1 = soup.select_one("h1")
    if h1:
        title = h1.get_text(strip=True)

    # Meta date
    meta_pd = soup.find("meta", attrs={"name": "PubDate"})
    if meta_pd and meta_pd.get("content"):
        pub_date = meta_pd["content"].strip()

    # Content - .xl-article
    article = soup.select_one(".xl-article")
    if article:
        # 获取内容，保留 table 原始 HTML
        text_parts = []
        for child in article.children:
            if child.name == "table":
                text_parts.append(str(child))
                text_parts.append("")
            elif child.name in ("p", "div"):
                # 如果内嵌 <table>，取原始 table HTML
                inner_table = child.find("table")
                if inner_table:
                    text_parts.append(str(inner_table))
                    text_parts.append("")
                    continue
                txt = child.get_text(strip=True)
                if txt:
                    text_parts.append(txt)
            elif isinstance(child, str):
                txt = child.strip()
                if txt:
                    text_parts.append(txt)

        content = "\n".join(text_parts)

        # 如果上面的提取不够完整，fallback 到纯文本
        if len(content.strip()) < 50:
            content = article.get_text(strip=True, separator="\n")

        # 附件
        for a_tag in article.find_all("a"):
            a_href = a_tag.get("href", "")
            if a_href.endswith((".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar")):
                if not a_href.startswith("http"):
                    a_href = BASE_URL + a_href
                attachments.append({
                    "name": a_tag.get_text(strip=True) or os.path.basename(a_href),
                    "url": a_href,
                })

    return title, pub_date, content, attachments


def crawl(max_pages=None):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cur = conn.cursor()

    all_items = []
    pages_to_crawl = max_pages if max_pages else 20  # 默认最多20页

    for page in range(1, pages_to_crawl + 1):
        if page == 1:
            url = f"{BASE_URL}{LIST_PATH}/index.html"
        else:
            url = f"{BASE_URL}{LIST_PATH}/index_{page}.html"

        print(f"[列表] 第{page}页: {url}")
        try:
            soup = get_soup(url)
            items = parse_list(soup)
            print(f"  -> {len(items)} 条")
            if not items:
                print("  -> 空页，停止")
                break
            all_items.extend(items)
        except Exception as e:
            print(f"  -> 失败: {e}")
            break

    new_count = 0
    error_count = 0

    for idx, (page_url, list_title, list_date) in enumerate(all_items, 1):
        cur.execute(
            "SELECT id FROM gov_raw WHERE page_url=? AND site_name=?",
            (page_url, SITE_NAME),
        )
        if cur.fetchone():
            print(f"  [{idx}/{len(all_items)}] 跳过: {list_title[:30]}...")
            continue

        print(f"  [{idx}/{len(all_items)}] {list_title[:40]}...")
        try:
            soup = get_soup(page_url)
            title, pub_date, content, attachments = extract_detail(soup, page_url)
            if not title:
                title = list_title
            if not pub_date:
                pub_date = list_date

            att_json = json.dumps(attachments, ensure_ascii=False) if attachments else ""

            cur.execute(
                """INSERT OR REPLACE INTO gov_raw (page_url, site_name, title, publish_date, source_url, content, attachments, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, 'crawl_xinhui.py')""",
                (page_url, SITE_NAME, title, pub_date, page_url, content, att_json),
            )
            conn.commit()
            new_count += 1
            print(f"    -> 新增")
        except Exception as e:
            error_count += 1
            print(f"    -> 异常: {e}")

    conn.close()
    return new_count, len(all_items), error_count


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--max-pages", type=int, default=None)
    args = parser.parse_args()

    new, total, errors = crawl(args.max_pages)
    print(f"\n完成: 新增 {new}, 列表 {total}, 异常 {errors}")