#!/usr/bin/env python3
"""
吉水县-污染防治 独立爬虫 (requests版, 修复版)
站点: www.jishui.gov.cn
栏目: 污染防治 (catid=165920, ajax_type=4_xxgk)
列表: AJAX API POST (requests, 非curl)
"""

import requests, re, sqlite3, time, sys
from datetime import datetime, timedelta
import os

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "吉水县-污染防治"
DOMAIN = "www.jishui.gov.cn"
CATID = "165920"
AJAX_TYPE = "4_xxgk"

CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

# 已知缺失的 1022xxxx 文章（不在API列表中）
MISSING_ARTICLES = [
    "http://www.jishui.gov.cn/xxgk-show-10239261.html",
]


def fetch_api(page, num=35):
    """通过requests调用API获取列表"""
    site_id = AJAX_TYPE.split("_")[0]
    url = f"http://{DOMAIN}/api-ajax_list-{page}.html"
    api_headers = {
        "User-Agent": HEADERS["User-Agent"],
        "X-Requested-With": "XMLHttpRequest",
        "Referer": f"http://{DOMAIN}/xxgk-list-wuranfangzhi.html",
    }
    form_data = {
        "ajax_type[]": [AJAX_TYPE, CATID, site_id, "xxgk", "Y-m-d", str(num), "20",
                        ["is_top DESC", "displayorder DESC", "inputtime DESC"], ""],
        "is_ds": "1"
    }
    for retry in range(3):
        try:
            r = requests.post(url, data=form_data, headers=api_headers, timeout=30)
            if r.status_code == 200:
                d = r.json()
                return d.get("data", []), d.get("total", 0)
        except Exception as e:
            if retry < 2:
                time.sleep(2)
    return [], 0


def clean_title(title):
    for p in ['吉水县人民政府信息公开-', '吉水县人民政府-', '吉水县-']:
        if title.startswith(p):
            title = title[len(p):]
    idx = title.find('displayorder')
    if idx > 0:
        title = title[:idx].rstrip()
    return title.strip()


def extract_content(html):
    """精准提取正文 - 优先xxgk_content"""
    patterns = [
        ('class="xxgk_content"', 'div'),
        ('id="barrierfree_container', 'div'),
        ('class="zwxxgk_box"', 'div'),
        ('id="zoom"', 'div'),
        ('class="articlecon"', 'div'),
    ]
    for pat, tag in patterns:
        i = html.find(pat)
        if i < 0:
            continue
        ds = html.rfind(f"<{tag}", 0, i)
        if ds < 0:
            continue
        s = html[ds:]
        d = 0
        for j in range(len(s)):
            if s[j:j+4] == f"<{tag}" and (j+4 >= len(s) or s[j+4] in " >\n\r\t"):
                d += 1
            elif s[j:j+3+len(tag)] == f"</{tag}>":
                d -= 1
                if d == 0:
                    gt = s.find(">", 0, j)
                    c = s[gt+1:j] if gt > 0 else s[7:j]
                    c = re.sub(r'<(script|style)[^>]*>.*?</\1>', '', c, flags=re.DOTALL|re.I)
                    return c.strip()
    return ""


def extract_meta(html):
    """提取标题和日期"""
    title = ""
    tm = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html)
    if tm:
        title = tm.group(1)
    if not title:
        ttm = re.search(r'<title>(.*?)<', html)
        if ttm:
            title = re.sub(r'_[^_]+$', '', ttm.group(1)).strip()
    
    date = ""
    for pm in re.finditer(r'PubDate[^>]*content="([^"]*)"', html):
        d = pm.group(1).strip()
        cm = re.match(r'(\d{4})-(\d{1,2})-(\d{1,2})', d)
        if cm:
            date = f"{cm.group(1)}-{cm.group(2).zfill(2)}-{cm.group(3).zfill(2)}"
            break
    if not date:
        dm = re.search(r'(\d{4}-\d{2}-\d{2})', html)
        if dm:
            date = dm.group(1)
    
    return title, date


def fetch_url(url):
    for retry in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=30)
            if r.status_code == 200:
                r.encoding = "utf-8"
                return r.text
        except Exception as e:
            if retry < 2:
                time.sleep(2)
    return None


def insert_article(url, title, date, content):
    """插入一条文章到search.db"""
    db = sqlite3.connect(SEARCH_DB)
    db.execute("PRAGMA journal_mode=WAL")
    try:
        db.execute(
            "INSERT OR IGNORE INTO gov_raw(site_name,source_url,page_url,title,publish_date,content,summary,status,category) VALUES(?,?,?,?,?,?,?,?,?)",
            (SITE_NAME, url, url, title, date, content, title[:200], "active", "污染防治"))
        affected = db.total_changes
        db.commit()
        if affected:
            db.execute(
                "INSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.page_url=? AND r.id NOT IN (SELECT rowid FROM gov_search)",
                (url,))
            db.commit()
        db.close()
        return affected
    except Exception as e:
        db.close()
        print(f"  ❌ DB error: {e}")
        return 0


def crawl_api(max_pages=50):
    """从API爬取列表文章"""
    print(f"\n📋 API列表爬取 ({DOMAIN})")
    all_items = []
    total_count = 0
    for pg in range(1, max_pages + 1):
        data, total = fetch_api(pg)
        if not data:
            if pg == 1:
                print("  ⚠ API无返回")
            else:
                print(f"  第{pg}页: 空 -> 结束")
            break
        if total_count == 0:
            total_count = total
        print(f"  第{pg}页: {len(data)} 条 (共{total}条)")
        for item in data:
            all_items.append({
                "url": item.get("url", f"http://{DOMAIN}/xxgk-show-{item.get('id', '')}.html"),
                "title": item.get("title", ""),
                "date": str(item.get("inputtime", ""))[:10],
            })
        time.sleep(0.3)

    print(f"\n📊 API共 {len(all_items)} 条")
    new = skip = no_body = err = 0
    for i, item in enumerate(all_items, 1):
        item_url = item["url"]
        item_title_api = item["title"]
        item_date = item["date"]

        if item_date and item_date < CUTOFF:
            skip += 1
            continue

        html = fetch_url(item_url)
        if not html:
            print(f"  ⚠ 取不到: {item_title_api[:30]}")
            err += 1
            continue

        pt, pd = extract_meta(html)
        real_title = clean_title(pt) if pt else clean_title(item_title_api)
        real_date = pd if pd else item_date

        if real_date and real_date < CUTOFF:
            skip += 1
            continue

        content = extract_content(html)
        text_len = len(re.sub(r'<[^>]+>', '', content).strip()) if content else 0
        if text_len < 10:
            no_body += 1
            if text_len == 0:
                print(f"  ⚠ 空正文: {real_title[:30]}")
            continue

        if insert_article(item_url, real_title, real_date, content):
            new += 1
        else:
            skip += 1

        if i % 5 == 0:
            print(f"  ...{i}/{len(all_items)}")
        time.sleep(0.3)

    print(f"  API文章: 新增{new}, 跳过{skip}, 空正文{no_body}, 错误{err}")
    return new


def crawl_missing():
    """补充爬取不在API列表中的文章"""
    print(f"\n📋 补充缺失文章 ({len(MISSING_ARTICLES)}条)")
    new = 0
    for url in MISSING_ARTICLES:
        html = fetch_url(url)
        if not html:
            print(f"  ⚠ 取不到: {url}")
            continue
        pt, pd = extract_meta(html)
        real_title = clean_title(pt) if pt else clean_title(url.split("/")[-1])
        real_date = pd if pd else ""
        content = extract_content(html)
        text_len = len(re.sub(r'<[^>]+>', '', content).strip()) if content else 0
        print(f"  {real_title[:40]}... ({real_date}, {text_len}字)")
        if text_len < 10:
            print(f"    ⚠ 空正文，跳过")
            continue
        if insert_article(url, real_title, real_date, content):
            new += 1
            print(f"    ✅ 入库")
        else:
            print(f"    ⏭ 已存在")
    print(f"  补充: 新增{new}")
    return new


def run(max_pages=50):
    print(f"\n{'='*50}")
    print(f"🚀 {SITE_NAME}")
    print(f"{'='*50}")

    n1 = crawl_api(max_pages)
    n2 = crawl_missing()

    # FTS完全同步
    print(f"\n📊 FTS同步...")
    db = sqlite3.connect(SEARCH_DB)
    db.execute("DELETE FROM gov_search WHERE site_name=?", (SITE_NAME,))
    db.execute("INSERT INTO gov_search(rowid,title,site_name,summary) SELECT rowid,title,site_name,summary FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    db.commit()
    c = db.execute("SELECT COUNT(*) FROM gov_search WHERE site_name=?", (SITE_NAME,))
    cnt = c.fetchone()[0]
    db.close()
    print(f"  FTS: {cnt} 条")

    print(f"\n✅ {SITE_NAME}: 总计新增{n1+n2}, FTS共{cnt}条")


if __name__ == "__main__":
    mp = int(sys.argv[1]) if len(sys.argv) > 1 else 50
    run(mp)