#!/usr/bin/env python3
"""
泰和县人民政府 - 政府采购与招标投标 (jxth.gov.cn)
AJAX API: /api-ajax_list-{page}.html  (数融CMS)
"""
import sys, os, re, time, json
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "泰和县人民政府-政府采购"
BASE_URL = "http://www.jxth.gov.cn"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {
    "User-Agent": "Mozilla/5.0",
    "X-Requested-With": "XMLHttpRequest",
}

AJAX_TYPE = "9_xxgk"
CATID = "107854"
NUM = 50


def fetch_api(page):
    """调用数融CMS AJAX API (用curl，requests发不对)"""
    import subprocess
    url = f"{BASE_URL}/api-ajax_list-{page}.html"
    site_id = AJAX_TYPE.split("_")[0] if "_" in AJAX_TYPE else "9"
    data_str = (
        f"ajax_type%5B%5D={AJAX_TYPE}&ajax_type%5B%5D={CATID}"
        f"&ajax_type%5B%5D={site_id}&ajax_type%5B%5D=xxgk"
        f"&ajax_type%5B%5D=Y-m-d&ajax_type%5B%5D={NUM}"
        f"&ajax_type%5B%5D=20"
        f"&ajax_type%5B%5D%5B%5D=is_top+DESC"
        f"&ajax_type%5B%5D%5B%5D=displayorder+DESC"
        f"&ajax_type%5B%5D%5B%5D=inputtime+DESC"
        f"&ajax_type%5B%5D=&is_ds=1"
    )
    for retry in range(3):
        try:
            cmd = ["curl", "-s", url, "-A", "Mozilla/5.0",
                   "-H", "X-Requested-With: XMLHttpRequest",
                   "--data", data_str, "--max-time", "15"]
            result = subprocess.run(cmd, capture_output=True, text=True, timeout=20)
            if result.returncode == 0 and result.stdout:
                return json.loads(result.stdout)
        except:
            time.sleep(2)
    return None


def fetch_detail(url):
    """抓取详情页内容"""
    for retry in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
            if r.status_code == 200:
                r.encoding = "utf-8"
                return r.text
        except:
            time.sleep(2)
    return None


def extract_content(html):
    """从详情页提取正文"""
    patterns = [
        'id="zoom"', 'class="xxgk_content"', 'class="zwxxgk_box"',
        'class="zwxxgk_bd"', 'class="nr_content"', 'class="content"',
        'class="articlecon"', 'class="info-content"',
        'class="cont"', 'class="fullscreen-layout-padding__content"'
    ]
    for p in patterns:
        i = html.find(p)
        if i < 0:
            continue
        ds = html.rfind("<div", 0, i)
        if ds < 0:
            continue
        s = html[ds:]
        d = 0
        for j in range(len(s)):
            if s[j:j+4] == "<div" and (j+4 >= len(s) or s[j+4] in " >\n\r\t"):
                d += 1
            elif s[j:j+6] == "</div>":
                d -= 1
                if d == 0:
                    gt = s.find(">", 0, j)
                    c = s[gt+1:j] if gt > 0 else s[7:j]
                    c = re.sub(r'<(script|style)[^>]*>.*?</\1>', '', c, flags=re.DOTALL|re.I)
                    if c.strip():
                        return c.strip()
    return ""


def extract(html):
    t = (re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html) or [None, ""])[1]
    if not t:
        t = re.sub(r'_[^_]+$', '', (re.search(r'<title>(.*?)<', html) or [None, ""])[1]).strip()
    pd = (re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{2}-\d{2})', html) or [None, ""])[1]
    if not pd:
        pd = (re.findall(r'(\d{4}-\d{2}-\d{2})', html) or [""])[0]
    return t, pd, extract_content(html)


def crawl(incremental=False, limit=None):
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")

    all_items = []
    max_pages = 1 if incremental else 3

    for page in range(1, max_pages + 1):
        data = fetch_api(page)
        if not data or not data.get("data"):
            print(f"  ❌ 第{page}页: API无数据")
            break
        items = data["data"]
        print(f"  📄 第{page}页: {len(items)}条", end=" ", flush=True)

        page_new = 0
        for item in items:
            title = (item.get("title") or "").strip()
            if not title:
                continue
            inp = item.get("inputtime", "")[:10]
            if inp and inp < THREE_YEARS_AGO:
                continue
            item_id = item.get("id")
            url = f"{BASE_URL}/xxgk-show-{item_id}.html" if item_id else ""
            all_items.append({"title": title, "url": url, "date": inp})
            page_new += 1

        print(f"+{page_new}条，累计{len(all_items)}条")
        if len(data) < 20:
            break

        if limit and len(all_items) >= limit:
            all_items = all_items[:limit]
            break

    print(f"\n📊 列表总计: {len(all_items)} 条")

    parsed = []
    for i, item in enumerate(all_items):
        print(f"  [{i+1}/{len(all_items)}] {item['title'][:50]}...", end=" ", flush=True)
        html = fetch_detail(item["url"])
        if not html:
            print("❌ 详情失败")
            continue
        t, pd, content = extract(html)
        if not content:
            print("⚠ 无正文")
            continue

        final_title = t or item["title"]
        final_date = pd or item["date"]
        summary = re.sub(r'<[^>]+>', ' ', content).strip()[:300]
        summary = re.sub(r'\s+', ' ', summary)

        parsed.append({
            "site_name": SITE_NAME,
            "title": final_title,
            "url": item["url"],
            "content": content,
            "pub_date": final_date,
            "summary": summary,
            "tags": SITE_NAME,
        })
        print(f"✅ ({len(content)}字)")
        time.sleep(0.3)

    if parsed:
        push_to_searchdb(parsed, "jxth")
        print(f"\n✅ 完成! 入库 {len(parsed)} 条")
    else:
        print("\n⏭ 无新数据")


if __name__ == "__main__":
    incremental = len(sys.argv) > 1 and sys.argv[1] in ("1", "--incremental")
    limit = None
    for i, a in enumerate(sys.argv):
        if a == "--limit" and i + 1 < len(sys.argv):
            limit = int(sys.argv[i + 1])
    t0 = time.time()
    crawl(incremental=incremental, limit=limit)
    print(f"⏱ 耗时: {time.time() - t0:.1f}s")