#!/usr/bin/env python3
"""
永丰县-基础建设 独立爬虫 (requests版)
站点: www.jxyongfeng.gov.cn
栏目: 基础建设 (catid=134486, ajax_type=5_xxgk)
"""

import requests, re, sqlite3, time, sys
from datetime import datetime, timedelta
import os
import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "永丰县-基础建设"
DOMAIN = "www.jxyongfeng.gov.cn"
CATID = "134486"
AJAX_TYPE = "5_xxgk"

CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

MISSING_ARTICLES = []


def fetch_api(page, num=50):
    """通过requests dict-form data调用API (修正: 替代curl子进程)"""
    site_id = AJAX_TYPE.split("_")[0]
    url = f"http://{DOMAIN}/api-ajax_list-{page}.html"
    api_headers = {
        "User-Agent": HEADERS["User-Agent"],
        "X-Requested-With": "XMLHttpRequest",
        "Referer": f"http://{DOMAIN}/xxgk-list-jichujianshe.html",
    }
    form_data = {
        "ajax_type[]": [AJAX_TYPE, CATID, site_id, "xxgk", "Y-m-d", str(num), "20",
                        ["is_top DESC", "displayorder DESC", "inputtime DESC"], ""],
        "is_ds": "1"
    }
    for retry in range(3):
        try:
            r = requests.post(url, data=form_data, headers=api_headers, timeout=30)
            if r.status_code == 200:
                d = r.json()
                return d.get("data", []), d.get("total", 0)
        except Exception as e:
            if retry < 2:
                time.sleep(2)
    return [], 0


def clean_title(title):
    """清理标题中的前缀和displayorder垃圾"""
    for p in ['永丰县人民政府信息公开-', '永丰县人民政府-', '永丰县-']:
        if title.startswith(p):
            title = title[len(p):]
    idx = title.find('displayorder')
    if idx > 0:
        title = title[:idx].rstrip()
    return title.strip()


def extract_content(html):
    """精准提取正文 - 优先xxgk_content"""
    patterns = [
        ('class="xxgk_content"', 'div'),
        ('class="xxgk_cont"', 'div'),
        ('id="barrierfree_container', 'div'),
        ('class="articlecon"', 'div'),
        ('id="zoom"', 'div'),
    ]
    for pat, tag in patterns:
        i = html.find(pat)
        if i < 0:
            continue
        ds = html.rfind(f"<{tag}", 0, i)
        if ds < 0:
            continue
        s = html[ds:]
        d = 0
        for j in range(len(s)):
            if s[j:j+4] == f"<{tag}" and (j+4 >= len(s) or s[j+4] in " >\n\r\t"):
                d += 1
            elif s[j:j+3+len(tag)] == f"</{tag}>":
                d -= 1
                if d == 0:
                    gt = s.find(">", 0, j)
                    c = s[gt+1:j] if gt > 0 else s[7:j]
                    c = re.sub(r'<(script|style)[^>]*>.*?</\1>', '', c, flags=re.DOTALL|re.I)
                    text = re.sub(r'<[^>]+>', '', c).strip()
                    if len(text) > 50:
                        return c.strip()
    return ""


def extract_meta(html):
    """提取标题和日期"""
    title = ""
    tm = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html)
    if tm:
        title = tm.group(1)
    if not title:
        ttm = re.search(r'<title>(.*?)<', html)
        if ttm:
            title = re.sub(r'_[^_]+$', '', ttm.group(1)).strip()
    date = ""
    for pm in re.finditer(r'PubDate[^>]*content="([^"]*)"', html):
        d = pm.group(1).strip()
        cm = re.match(r'(\d{4})-(\d{1,2})-(\d{1,2})', d)
        if cm:
            date = f"{cm.group(1)}-{cm.group(2).zfill(2)}-{cm.group(3).zfill(2)}"
            break
    if not date:
        dm = re.search(r'(\d{4}-\d{2}-\d{2})', html)
        if dm:
            date = dm.group(1)
    return title, date


def fetch_url(url):
    for retry in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=30)
            if r.status_code == 200:
                r.encoding = "utf-8"
                return r.text
        except Exception as e:
            if retry < 2:
                time.sleep(2)
    return None


def insert_article(url, title, date, content):
    """插入一条文章到search.db"""
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    try:
        db.execute(
            "INSERT OR IGNORE INTO gov_raw(site_name,source_url,page_url,title,publish_date,content,summary,status,category) VALUES(?,?,?,?,?,?,?,?,?)",
            (SITE_NAME, url, url, title, date, content, title[:200], "active", "基础建设"))
        affected = db.total_changes
        db.commit()
        if affected:
            db.execute(
                "INSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.page_url=? AND r.id NOT IN (SELECT rowid FROM gov_search)",
                (url,))
            db.commit()
        db.close()
        return affected
    except Exception as e:
        db.close()
        print(f"  ❌ DB error: {e}")
        return 0


def crawl_api(max_pages=50):
    """从API爬取列表文章，从详情页提取真实标题"""
    print(f"\n📋 API列表爬取 ({DOMAIN})")
    all_items = []
    total_count = 0
    for pg in range(1, min(max_pages, _MAX_PG or max_pages) + 1):
        data, total = fetch_api(pg)
        if not data:
            if pg == 1:
                print("  ⚠ API无返回")
            else:
                print(f"  第{pg}页: 空 -> 结束")
            break
        if total_count == 0:
            total_count = total
        print(f"  第{pg}页: {len(data)} 条 (共{total}条)")
        for item in data:
            all_items.append({
                "url": f"http://{DOMAIN}/xxgk-show-{item.get('id', '')}.html",
                "title": item.get("title", ""),
                "date": str(item.get("inputtime", ""))[:10],
            })
        time.sleep(0.3)

    print(f"\n📊 API共 {len(all_items)} 条")
    new = skip = no_body = err = 0
    for i, item in enumerate(all_items, 1):
        item_url = item["url"]
        item_title_api = item["title"]
        item_date = item["date"]

        if item_date and item_date < CUTOFF:
            skip += 1
            continue

        html = fetch_url(item_url)
        if not html:
            print(f"  ⚠ 取不到: {item_title_api[:30]}")
            err += 1
            continue

        # 从详情页提取真实标题和日期
        pt, pd = extract_meta(html)
        real_title = clean_title(pt) if pt else clean_title(item_title_api)
        real_date = pd if pd else item_date

        if real_date and real_date < CUTOFF:
            skip += 1
            continue

        content = extract_content(html)
        text_len = len(re.sub(r'<[^>]+>', '', content).strip()) if content else 0
        if text_len < 10:
            no_body += 1
            if text_len == 0:
                print(f"  ⚠ 空正文: {real_title[:30]}")
            continue

        if insert_article(item_url, real_title, real_date, content):
            new += 1
        else:
            skip += 1

        if i % 5 == 0:
            print(f"  ...{i}/{len(all_items)}")
        time.sleep(0.3)

    print(f"  API文章: 新增{new}, 跳过{skip}, 空正文{no_body}, 错误{err}")
    return new


def crawl_missing():
    """补充爬取不在API列表中的文章"""
    if not MISSING_ARTICLES:
        return 0
    print(f"\n📋 补充缺失文章 ({len(MISSING_ARTICLES)}条)")
    new = 0
    for path in MISSING_ARTICLES:
        url = f"http://{DOMAIN}/{path}"
        html = fetch_url(url)
        if not html:
            print(f"  ⚠ 取不到: {url}")
            continue
        pt, pd = extract_meta(html)
        real_title = clean_title(pt) if pt else clean_title(path)
        real_date = pd if pd else ""
        content = extract_content(html)
        text_len = len(re.sub(r'<[^>]+>', '', content).strip()) if content else 0
        print(f"  {real_title[:40]}... ({real_date}, {text_len}字)")
        if text_len < 10:
            print(f"    ⚠ 空正文，跳过")
            continue
        if insert_article(url, real_title, real_date, content):
            new += 1
            print(f"    ✅ 入库")
        else:
            print(f"    ⏭ 已存在")
    print(f"  补充: 新增{new}")
    return new


def run(max_pages=50):
    print(f"\n{'='*50}")
    print(f"🚀 {SITE_NAME}")
    print(f"{'='*50}")

    n1 = crawl_api(max_pages)
    n2 = crawl_missing()

    # FTS完全同步
    print(f"\n📊 FTS同步...")
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("DELETE FROM gov_search WHERE site_name=?", (SITE_NAME,))
    db.execute("INSERT INTO gov_search(rowid,title,site_name,summary) SELECT rowid,title,site_name,summary FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    db.commit()
    c = db.execute("SELECT COUNT(*) FROM gov_search WHERE site_name=?", (SITE_NAME,))
    cnt = c.fetchone()[0]
    db.close()
    print(f"  FTS: {cnt} 条")

    print(f"\n✅ {SITE_NAME}: 总计新增{n1+n2}, FTS共{cnt}条")


if __name__ == "__main__":
    mp = int(sys.argv[1]) if len(sys.argv) > 1 else 50
    run(mp)