#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
爬取 长兴县人民政府 - 乡镇、部门非规范性文件
https://www.zjcx.gov.cn/col/col1229518369/index.html
JPAAS信息公开系统，通过API获取列表
支持附件(docx/xlsx/pdf)正文提取
"""

import json
import re
import sys
import time
import os
import sqlite3
import subprocess
import tempfile
from urllib.parse import urljoin, unquote

import requests
from bs4 import BeautifulSoup

# ========== 配置 ==========
SITE_NAME = "长兴县人民政府-乡镇部门非规范性文件"
BASE_URL = "https://www.zjcx.gov.cn"
API_URL = "https://www.zjcx.gov.cn/api-gateway/jpaas-publish-server/front/page/build/unit"
API_PARAMS = {
    "parseType": "bulidstatic",
    "webId": "3645",
    "pageId": "1229518369",
    "pageType": "column",
    "tagId": "组配分类list",
    "tplSetId": "QIrUapMnq9Avhahnnyp8M",
}
SEARCH = {"xxgkId": "A001-005", "xxgkType": "xxgk_combination", "nodeId": "330522000000", "className": ""}
DB_PATH = "/root/search.db"
PAGE_SIZE = 15
ATTACH_DIR = "/tmp/zjcx_attachments"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}

session = requests.Session()
session.headers.update(HEADERS)
os.makedirs(ATTACH_DIR, exist_ok=True)


def get_existing_urls(cursor):
    cursor.execute("SELECT page_url FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
    return {row[0] for row in cursor.fetchall()}


def fetch_article_list(page_no, page_size=PAGE_SIZE):
    """通过JPAAS API获取文章列表"""
    search_str = json.dumps(SEARCH, ensure_ascii=False)
    param_json = json.dumps({"pageNo": page_no, "pageSize": page_size, "search": search_str}, ensure_ascii=False)

    params = {**API_PARAMS, "paramJson": param_json}
    resp = session.get(API_URL, params=params, timeout=60, verify=False)
    data = resp.json()
    html = data["data"]["html"]

    total = 0
    for m in re.finditer(r'count="(\d+)"', html):
        total = int(m.group(1))
        break

    items = []
    for li in re.finditer(r'<li[^>]*class="cf"[^>]*>(.*?)</li>', html, re.DOTALL):
        li_html = li.group(1)
        href_m = re.search(r'href="([^"]+)"', li_html)
        title_m = re.search(r'title="([^"]*)"', li_html)
        date_m = re.search(r'<span[^>]*class="fr"[^>]*>(\d{4}[-/]\d{2}[-/]\d{2})', li_html)
        if href_m:
            href = href_m.group(1)
            title = title_m.group(1).strip() if title_m else ""
            date_str = date_m.group(1) if date_m else ""
            full_url = urljoin(BASE_URL, href)
            items.append((title, date_str, full_url))

    return items, total


def download_attachment(url):
    """下载附件到临时目录，返回本地路径"""
    try:
        r = session.get(url, timeout=60, verify=False, stream=True)
        if r.status_code != 200:
            return None
        # 从URL提取文件名
        url_path = unquote(url.split('?')[0].split('/')[-1])
        if not url_path:
            url_path = f"attach_{int(time.time())}"
        local_path = os.path.join(ATTACH_DIR, url_path)
        with open(local_path, 'wb') as f:
            for chunk in r.iter_content(8192):
                f.write(chunk)
        return local_path
    except:
        return None


def extract_text_from_docx(path):
    """用python-docx提取文本"""
    try:
        import docx
        doc = docx.Document(path)
        return '\n'.join(p.text for p in doc.paragraphs if p.text.strip())
    except:
        return ""


def extract_text_from_xlsx(path):
    """用openpyxl提取文本"""
    try:
        import openpyxl
        wb = openpyxl.load_workbook(path, read_only=True, data_only=True)
        texts = []
        for ws in wb.worksheets:
            for row in ws.iter_rows():
                for cell in row:
                    if cell.value is not None:
                        texts.append(str(cell.value))
        return '\n'.join(texts)
    except:
        return ""


def extract_text_from_pdf(path):
    """用pdftotext提取文本"""
    try:
        result = subprocess.run(['pdftotext', path, '-'], capture_output=True, text=True, timeout=30)
        return result.stdout.strip()
    except:
        return ""


def extract_attachment_text(attach_url):
    """下载并提取附件正文"""
    local_path = download_attachment(attach_url)
    if not local_path:
        return ""

    text = ""
    ext = os.path.splitext(local_path)[1].lower()

    if ext == '.docx':
        text = extract_text_from_docx(local_path)
    elif ext == '.doc':
        try:
            result = subprocess.run(['catdoc', local_path], capture_output=True, text=True, timeout=30)
            text = result.stdout.strip()
        except:
            try:
                result = subprocess.run(['antiword', local_path], capture_output=True, text=True, timeout=30)
                text = result.stdout.strip()
            except:
                pass
    elif ext in ('.xlsx', '.xls'):
        text = extract_text_from_xlsx(local_path)
    elif ext == '.pdf':
        text = extract_text_from_pdf(local_path)

    # 清理临时文件
    try:
        os.remove(local_path)
    except:
        pass

    return text


def crawl_detail(detail_url):
    """爬取详情页，返回 (title, date, content_html, attachments_str, attach_texts)"""
    try:
        resp = session.get(detail_url, timeout=60, verify=False)
        resp.encoding = "utf-8"
        soup = BeautifulSoup(resp.text, "html.parser")
    except Exception as e:
        return None, None, None, None, "", str(e)

    # 标题
    title = ""
    mt = soup.find("meta", attrs={"name": "ArticleTitle"})
    if mt and mt.get("content"):
        title = mt["content"].strip()
    if not title:
        title_div = soup.find("div", class_="title1")
        if title_div:
            title = title_div.get_text(strip=True)

    # 日期
    date_str = ""
    mt2 = soup.find("meta", attrs={"name": "PubDate"})
    if mt2 and mt2.get("content"):
        date_str = mt2["content"].strip()[:10]

    # 正文区域 - div.zhengw
    content_html = ""
    attachments_list = []
    zw = soup.find("div", class_="zhengw")
    if zw:
        # 处理附件链接（脚本中的pdfurl）
        for script in zw.find_all("script"):
            for m in re.finditer(r'pdfurl\s*=\s*"([^"]+)"', script.string or ""):
                attach_url = m.group(1)
                full_attach = urljoin(BASE_URL, attach_url)
                attachments_list.append(full_attach)

        # 处理 <a> 标签中的附件链接（JPAAS下载API或直接文件链接）
        for a_tag in zw.find_all("a", href=True):
            href = a_tag["href"]
            # JPAAS下载API或文件扩展名
            if "document/download" in href or re.search(r'\.(docx?|xlsx?|pdf)$', href, re.I):
                full_url = urljoin(BASE_URL, href)
                if full_url not in attachments_list:
                    attachments_list.append(full_url)

        # 移除script标签
        for s in zw.find_all("script"):
            s.decompose()

        # 提取正文HTML（含<a>标签，保留附件链接信息）
        content_html_raw = str(zw)
        text_content = zw.get_text(strip=True)
        if len(text_content) > 50:
            content_html = content_html_raw

    attachments = ",".join(attachments_list) if attachments_list else ""

    # 如果正文为空但有附件，提取附件正文
    attach_text = ""
    if not content_html and attachments_list:
        print(f"     提取附件({len(attachments_list)}个)...")
        for au in attachments_list:
            txt = extract_attachment_text(au)
            if txt:
                attach_text += txt + "\n\n"
        if attach_text:
            # 用<p>包裹attachment文本放入content
            content_html = "<div class=\"zhengw\"><p>" + attach_text.replace("\n", "</p><p>") + "</p></div>"

    return title, date_str, content_html, attachments, attach_text, None


def main():
    incremental = False
    incremental_days = 30
    for arg in sys.argv[1:]:
        if arg == "--incremental":
            incremental = True
        elif arg.isdigit():
            incremental_days = int(arg)

    conn = sqlite3.connect(DB_PATH, timeout=60)
    cursor = conn.cursor()

    cursor.execute("""
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            title TEXT,
            date_str TEXT,
            page_url TEXT UNIQUE,
            content TEXT,
            source TEXT,
            site_name TEXT,
            attachments TEXT
        )
    """)
    conn.commit()

    existing_urls = get_existing_urls(cursor)
    print(f"已有 {len(existing_urls)} 条记录")

    cutoff_date = ""
    if incremental:
        from datetime import datetime, timedelta
        cutoff = datetime.now() - timedelta(days=incremental_days)
        cutoff_date = cutoff.strftime("%Y-%m-%d")
        print(f"增量模式：仅爬取 {cutoff_date} 之后的文章")

    # 先获取第一页，确定总数
    first_items, total = fetch_article_list(1)
    print(f"总条数: {total}")

    if not total:
        print("[ERROR] 无法获取总条数")
        conn.close()
        return

    total_pages = (total + PAGE_SIZE - 1) // PAGE_SIZE
    print(f"总页数: {total_pages}")

    # 获取所有列表项
    all_items = []
    for page in range(1, total_pages + 1):
        print(f"获取第 {page}/{total_pages} 页...")
        try:
            items, _ = fetch_article_list(page)
            all_items.extend(items)
            print(f"  本页 {len(items)} 条")
        except Exception as e:
            print(f"  [ERROR] 第 {page} 页异常: {e}")
        time.sleep(0.5)

    print(f"\n列表共 {len(all_items)} 条")

    # 爬取详情
    new_count = 0
    skip_count = 0
    error_count = 0
    attach_extracted = 0

    for i, (title, date_str, detail_url) in enumerate(all_items, 1):
        # 增量过滤
        if incremental and cutoff_date and date_str and date_str < cutoff_date:
            skip_count += 1
            if skip_count == 1:
                print(f"  跳过旧数据（早于 {cutoff_date}），首个: {title[:30]} ({date_str})")
            continue

        if detail_url in existing_urls:
            skip_count += 1
            continue

        print(f"  [{i}/{len(all_items)}] {title[:30]}...")
        dt_title, dt_date, content_html, attachments, attach_text, err = crawl_detail(detail_url)

        final_title = dt_title or title
        final_date = dt_date or date_str

        has_attach = attachments.strip() != ""
        has_content = content_html and len(content_html.strip()) > 50

        if not has_content and not has_attach:
            print(f"    ⚠ 无正文也无附件，跳过")
            error_count += 1
            continue

        summary_text = ""
        if not has_content and has_attach:
            summary_text = "【附件】" + attachments
        elif has_content:
            summary_text = ""

        if attach_text:
            attach_extracted += 1

        cursor.execute(
            "INSERT OR IGNORE INTO gov_raw (title, publish_date, page_url, content, summary, site_name, date_rank, status) "
            "VALUES (?, ?, ?, ?, ?, ?, 0, 'normal')",
            (final_title, final_date, detail_url, content_html or "", summary_text, SITE_NAME)
        )
        new_count += 1

        if new_count % 10 == 0:
            conn.commit()
            print(f"    ✓ 已提交 {new_count} 条")

        time.sleep(0.3)

    conn.commit()

    # FTS 由 search.db 触发器 trg_gov_raw_fts_* 统一维护, 不再自建 gov_fts / 手工重建
    print("\nFTS 由触发器维护, 跳过本地重建")

    conn.close()

    print(f"\n=== 完成 ===")
    print(f"新增: {new_count} 条")
    print(f"附件提取: {attach_extracted} 条")
    print(f"跳过: {skip_count} 条")
    print(f"错误: {error_count} 条")


if __name__ == "__main__":
    main()