#!/usr/bin/env python3
"""
第五师双河市 - 招标公告
https://www.xjshs.gov.cn/xxgk-list-zbggm.html

CMS: 自定义（ZDCMS-like AJAX API）
列表: POST api-ajax_list-{page}.html  (ajax_type=["18_xxgk",288,18,"xxgk","Y-m-d",35,20,...])
分页: 每页20条，共81页(1616条)
详情标题: <meta name="ArticleTitle">
详情日期: <meta name="PubDate">
正文容器: <div class="xxgk_content" id="xxgk_content"> -> <p><span> 结构
"""

import sys
import os
import re
import subprocess
import json
from bs4 import BeautifulSoup
import urllib.parse

# ---------- 数据库配置 ----------
DB_PATH = "/mnt/data/search.db" if os.path.exists("/mnt/data/search.db") else os.path.expanduser("~/data/search.db")

# ---------- 站点配置 ----------
BASE_URL = "https://www.xjshs.gov.cn"
DOMAIN = "www.xjshs.gov.cn"
API_URL = BASE_URL + "/api-ajax_list"
PAGES = 5  # 默认每5页
GROUP_NAME = "新疆生产建设兵团"
INDUSTRY = "政府公告"
SITE_NAME = "第五师双河市"

# AJAX请求参数
AJAX_TYPE_BASE = ["18_xxgk", 288, 18, "xxgk", "Y-m-d", 35, 20,
                   ["inputtime DESC", "displayorder DESC", "is_top DESC"], None]


def fetch_api(page=1):
    """调用AJAX API获取列表数据"""
    cmd = [
        "curl", "-sS", "--connect-timeout", "10", "-m", "30",
        "-X", "POST",
        "-H", "Content-Type: application/x-www-form-urlencoded; charset=UTF-8",
        "-H", "X-Requested-With: XMLHttpRequest",
        "-H", "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        "-H", "Accept: application/json, text/javascript, */*; q=0.01",
        "-H", "Referer: " + BASE_URL + "/xxgk-list-zbggm.html",
    ]
    url = f"{API_URL}-{page}.html"
    cmd.append(url)

    # 构建POST数据
    for item in AJAX_TYPE_BASE:
        if isinstance(item, list):
            for sub in item:
                cmd.extend(["--data-urlencode", f"ajax_type[7][]={sub}"])
        elif item is None:
            cmd.extend(["--data-urlencode", "ajax_type[]=null"])
        else:
            cmd.extend(["--data-urlencode", f"ajax_type[]={item}"])
    cmd.extend(["-d", "is_ds=1"])

    try:
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=35)
        if result.returncode != 0:
            print(f"  [WARN] curl返回码 {result.returncode}")
            return None, 0
        data = json.loads(result.stdout)
        items = data.get("data", [])
        total = data.get("total", 0)
        return items, total
    except (subprocess.TimeoutExpired, json.JSONDecodeError) as e:
        print(f"  [WARN] API错误: {e}")
        return None, 0


def fetch_page(url):
    """获取详情页HTML"""
    # 确保使用HTTPS
    url = url.replace("http://", "https://")
    cmd = [
        "curl", "-sL", "--connect-timeout", "10", "-m", "30",
        "-H", "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        url
    ]
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=35)
        if result.returncode != 0:
            return None
        return result.stdout
    except subprocess.TimeoutExpired:
        return None


def fetch_detail(detail_url):
    """获取详情页并提取标题、日期、正文"""
    html = fetch_page(detail_url)
    if not html:
        return None, None, None

    soup = BeautifulSoup(html, "html.parser")

    # 标题: meta ArticleTitle
    title = ""
    for m in soup.select('meta[name=ArticleTitle]'):
        title = m.get("content", "").strip()
        break
    if not title:
        t = soup.find("title")
        if t:
            title = t.get_text(strip=True).replace(" - 第五师双河市人民政府", "").strip()

    # strip &middot; &nbsp; 实体
    title = title.replace("&middot;", "").replace("&nbsp;", "").replace("\xa0", "").replace("\u3000", "").strip()

    # 日期: meta PubDate
    date_str = ""
    for m in soup.select('meta[name=PubDate]'):
        date_str = m.get("content", "").strip()
        break
    if date_str:
        date_str = date_str[:10]

    # 正文: div.xxgk_content
    content_div = soup.select_one("div.xxgk_content") or soup.select_one("#xxgk_content")

    text = ""
    if content_div:
        paragraphs = []
        for child in content_div.children:
            name = getattr(child, "name", None)
            if name == "p":
                # 检查p内是否有table
                inner_table = child.find("table")
                if inner_table:
                    for node in child.children:
                        nname = getattr(node, "name", None)
                        if nname is None:
                            t = str(node).strip()
                            if t:
                                paragraphs.append(t)
                        elif nname == "table":
                            paragraphs.append(str(node))
                        elif nname == "br":
                            pass
                        else:
                            t2 = node.get_text(strip=True)
                            if t2:
                                paragraphs.append(t2)
                else:
                    p_text = child.get_text(strip=True)
                    if p_text:
                        paragraphs.append(p_text)
            elif name == "table":
                paragraphs.append(str(child))
            elif name is None:
                t = str(child).strip()
                if t:
                    paragraphs.append(t)
        if paragraphs:
            text = "\n\n".join(paragraphs)

    # 附件/图片/iframe嵌入：正文过短时扫描
    if len(text) < 500 and content_div:
        links = []
        for img in content_div.select("img[src]"):
            src = img.get("src", "")
            if src and not src.startswith("http"):
                src = BASE_URL + src if src.startswith("/") else src
            links.append(f"[图片] {src}")
        for iframe in content_div.select("iframe[src]"):
            src = iframe.get("src", "")
            if src and not src.startswith("http"):
                src = BASE_URL + src if src.startswith("/") else src
            links.append(f"[附件PDF] {src}")
        for a in content_div.select("a[href]"):
            href = a.get("href", "")
            ext = href.lower().split("?")[0].rsplit(".", 1)[-1] if "." in href else ""
            if ext in ("pdf", "doc", "docx", "xls", "xlsx", "jpg", "png", "gif", "zip", "rar"):
                if not href.startswith("http"):
                    href = BASE_URL + href if href.startswith("/") else href
                fname = a.get_text(strip=True) or href.split("/")[-1]
                links.append(f"[附件: {fname}] {href}")
        if links:
            text = "\n".join(links) if not text else text + "\n" + "\n".join(links)

    return title, date_str, text


def main():
    import argparse
    parser = argparse.ArgumentParser(description="第五师双河市-招标公告爬虫")
    parser.add_argument("--pages", type=int, default=PAGES, help=f"爬取页数(每页20条,默认{PAGES})")
    args = parser.parse_args()
    max_pages = args.pages

    # ---------- 1. 获取API列表数据 ----------
    print(f"🌐 爬取API列表页 1-{max_pages}...")
    all_items = []
    total_all = 0
    for page in range(1, max_pages + 1):
        items, total = fetch_api(page)
        if items is None:
            print(f"  [ERROR] 无法获取第{page}页")
            continue
        total_all = total if total > 0 else total_all
        # API返回完整数据，包含title/url/inputtime
        for item in items:
            all_items.append({
                "title": item.get("title", ""),
                "url": item.get("url", ""),
                "date": item.get("inputtime", ""),
                "description": item.get("description", ""),
            })
        print(f"  第{page}页: {len(items)}条 (共{total}条)")

    print(f"\n📋 共获取 {len(all_items)} 条列表数据（总计{total_all}条）")

    # ---------- 2. 去重 ----------
    seen_urls = set()
    unique_items = []
    for item in all_items:
        url = item["url"]
        if url not in seen_urls:
            seen_urls.add(url)
            unique_items.append(item)
    print(f"🔍 去重后 {len(unique_items)} 条")

    # ---------- 3. 爬取详情 ----------
    print("\n📄 爬取详情页...")
    results = []
    for idx, item in enumerate(unique_items, 1):
        url = item["url"]
        list_title = item["title"]
        list_date = item["date"]

        print(f"  [{idx}/{len(unique_items)}] {list_title[:30]}...", end=" ")
        sys.stdout.flush()

        title, date_str, text = fetch_detail(url)

        if not title:
            title = list_title
            title = title.replace("&middot;", "").replace("&nbsp;", "").replace("\xa0", "").replace("\u3000", "").strip()
        if not date_str:
            date_str = list_date[:10] if list_date else ""

        if text:
            print(f"✅ {len(text)}字")
        else:
            print(f"⚠️ 正文为空")

        results.append({
            "title": title,
            "date": date_str,
            "content": text,
            "url": url,
            "source_url": url,
        })

    # ---------- 4. 入库 ----------
    print(f"\n💾 入库 {len(results)} 条到 {DB_PATH}...")
    if not os.path.exists(DB_PATH):
        print(f"  [ERROR] 数据库不存在: {DB_PATH}")
        print("\n📊 结果预览:")
        for r in results[:5]:
            print(f"  {r['date']} | {r['title'][:40]} | {len(r['content'])}字")
        print(f"\n  ...共 {len(results)} 条 (未入库)")
        return

    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()

    added = 0
    for r in results:
        try:
            c.execute(
                "INSERT OR IGNORE INTO gov_raw (title, content, publish_date, page_url, source_url, site_name, group_name, industry) VALUES (?,?,?,?,?,?,?,?)",
                (r["title"], r["content"], r["date"], r["url"], r["source_url"], SITE_NAME, GROUP_NAME, INDUSTRY)
            )
            if c.rowcount > 0:
                added += 1
        except Exception as e:
            print(f"  [ERROR] 入库失败: {e}")

    conn.commit()

    # ---------- 5. FTS同步 ----------
    if added > 0:
        print(f"\n🔍 FTS同步 {added} 条...")
        script = """
import sqlite3, sys
db = sys.argv[1]
conn = sqlite3.connect(db, timeout=60)
c = conn.cursor()
added = int(sys.argv[2])
c.execute("SELECT id, title, content FROM gov_raw ORDER BY id DESC LIMIT ?", (added,))
rows = c.fetchall()
site = '""" + SITE_NAME + """'
for rowid, title, content in rows:
    c.execute("INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)",
              (rowid, title, site, (content or '')[:500]))
conn.commit()
c.execute("SELECT COUNT(*) FROM gov_search")
print(f"FTS条数: {c.fetchone()[0]}")
conn.close()
"""
        result = subprocess.run(["python3", "-c", script, DB_PATH, str(added)],
                                capture_output=True, text=True, timeout=30)
        print(f"  FTS: {result.stdout.strip()}")
        if result.stderr:
            print(f"  FTS stderr: {result.stderr[:100]}")
    else:
        print("\n  ⚠️ 无新增记录，跳过FTS同步")

    conn.close()

    # ---------- 6. 输出结果 ----------
    print(f"\n📊 结果总览:")
    empty_count = sum(1 for r in results if not r["content"])
    print(f"  总条数: {len(results)}")
    print(f"  新增入库: {added}")
    print(f"  空正文: {empty_count}")
    print(f"  站点: {SITE_NAME}")
    print(f"  栏目: 招标公告")

    print(f"\n{'─' * 60}")
    print(f"{'日期':<14} {'标题':<40} {'字数':>6}")
    print(f"{'─' * 60}")
    for r in results[:5]:
        title_short = r['title'][:38] if len(r['title']) > 38 else r['title']
        print(f"{r['date']:<14} {title_short:<40} {len(r['content']):>6}")
    if len(results) > 5:
        print(f"  ... 共 {len(results)} 条")
    print(f"{'─' * 60}")


if __name__ == "__main__":
    main()