#!/usr/bin/env python3
"""
金塔县人民政府 - 民意征集 (建设项目用地预选/设计方案批前公示)
API: /common/search/{channelId}?_isAgg=false&_isJson=true&_pageSize=15
Detail: div.pages_content#UCAP-CONTENT
直接写入 search.db gov_raw 表 (2026-08-08 从temp库方案改回)
"""

import os, sys, re, json, time, requests
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
TEMP_DB = os.getenv("TEMP_SEARCH_DB", "/root/search.db")
SITE_NAME = "jtxzf.gov.cn-民意征集"
BASE_URL = "https://www.jtxzf.gov.cn"
CHANNEL_ID = "3730d74518314de4bbb641b2add0f85a"
PAGE_SIZE = 15
CUTOFF_DATE = "2023-06-16"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": "https://www.jtxzf.gov.cn/jintaxian/c101622/common_list1.shtml",
}


def fix_url(raw_url):
    """API returns http:// with double slash, fix to https://"""
    url = raw_url.strip()
    if url.startswith("http://"):
        url = "https://" + url[7:]
    elif url.startswith("/"):
        url = "https://www.jtxzf.gov.cn" + url
    url = re.sub(r"https://([^/]+)//", r"https://\1/", url)
    return url


def get_detail(url):
    """Fetch detail page and extract content."""
    try:
        full_url = fix_url(url)
        resp = requests.get(full_url, headers=HEADERS, timeout=15)
        resp.encoding = "utf-8"
        if resp.status_code != 200:
            return None, None
        soup = BeautifulSoup(resp.text, "html.parser")
        
        # Title from h1
        title_el = soup.find("h1")
        title = title_el.get_text(strip=True) if title_el else ""
        
        # Content from UCAP-CONTENT
        content_div = soup.find("div", id="UCAP-CONTENT")
        if not content_div:
            content_div = soup.find("div", class_="pages_content")
        if not content_div:
            content_div = soup.find("div", class_="detailcon")
        
        if content_div:
            base_for_attachments = full_url.rsplit("/", 1)[0] + "/"
            for a in content_div.find_all("a", href=True):
                href = a["href"]
                if not href.startswith("http") and not href.startswith("//") and not href.startswith("data:"):
                    a["href"] = urljoin(base_for_attachments, href)
            for img in content_div.find_all("img", src=True):
                src = img["src"]
                if not src.startswith("http") and not src.startswith("//") and not src.startswith("data:"):
                    img["src"] = urljoin(base_for_attachments, src)
            content = str(content_div)
        else:
            content = ""
        
        return title, content
    except Exception as e:
        print(f"[jtxzf] 详情页请求失败 {url}: {e}", flush=True)
        return None, None


def ensure_gov_raw_table(conn):
    """Create gov_raw table matching sync_temp_to_main.py expected schema."""
    c = conn.cursor()
    c.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY, site_name TEXT, source_url TEXT,
        page_url TEXT, title TEXT, publish_date TEXT,
        date_rank INTEGER DEFAULT 0, summary TEXT, status TEXT,
        category TEXT DEFAULT '', visits INTEGER DEFAULT 0,
        content TEXT DEFAULT '', tags TEXT DEFAULT ''
    )""")
    # Create unique index on page_url (for dedup)
    try:
        c.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_temp_page_url ON gov_raw(page_url)")
    except Exception:
        pass
    for idx_name, cols in [("idx_temp_sn", "site_name"), ("idx_temp_pubdate", "publish_date")]:
        try:
            c.execute(f"CREATE INDEX IF NOT EXISTS {idx_name} ON gov_raw({cols})")
        except Exception:
            pass
    conn.commit()


def crawl_full():
    """Full crawl - all pages."""
    print(f"[jtxzf] 全量模式，截止 {CUTOFF_DATE}", flush=True)
    
    conn = __import__("sqlite3").connect(TEMP_DB)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=30000")
    ensure_gov_raw_table(conn)
    c = conn.cursor()
    
    new_count = 0
    skip_count = 0
    nocontent_count = 0
    page = 1
    
    while True:
        api_url = f"{BASE_URL}/common/search/{CHANNEL_ID}?_isAgg=false&_isJson=true&_pageSize={PAGE_SIZE}&_template=index&_rangeTimeGte=&_channelName=&page={page}"
        try:
            resp = requests.get(api_url, headers=HEADERS, timeout=15)
            data = resp.json()
            results = data.get("data", {}).get("results", [])
            total = data.get("data", {}).get("total", 0)
        except Exception as e:
            print(f"[jtxzf] API请求失败 page={page}: {e}", flush=True)
            break
        
        if not results:
            break
        
        total_pages = (total + PAGE_SIZE - 1) // PAGE_SIZE
        print(f"[jtxzf] 第{page}页 ({len(results)}条, 共{total_pages}页)", flush=True)
        
        for item in results:
            raw_url = item.get("url", "")
            title = item.get("title", "")
            date_str = (item.get("publishedTimeStr") or "")[:10]
            
            if not raw_url:
                continue
            
            if date_str < CUTOFF_DATE:
                print(f"[jtxzf] 达到截止日期 {CUTOFF_DATE}，停止", flush=True)
                conn.commit()
                print(f"[jtxzf] 完成！新增 {new_count}, 跳过 {skip_count}, 无内容 {nocontent_count}", flush=True)
                conn.close()
                return
            
            page_url = fix_url(raw_url)
            
            # Check if exists by page_url
            c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,))
            if c.fetchone():
                skip_count += 1
                continue
            
            # Fetch detail
            detail_title, content = get_detail(raw_url)
            if not detail_title:
                detail_title = title
            if not content or len(content.strip()) < 200:
                nocontent_count += 1
                skip_count += 1
                continue
            
            # Insert into gov_raw (matching sync script schema)
            try:
                c.execute(
                    """INSERT OR IGNORE INTO gov_raw 
                    (site_name, source_url, page_url, title, publish_date, content, status, category)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
                    (SITE_NAME, page_url, page_url, detail_title, date_str,
                     content, "published", "环评公示"),
                )
                if c.rowcount > 0:
                    new_count += 1
                    conn.commit()
                else:
                    skip_count += 1
            except Exception as e:
                print(f"[jtxzf] 插入失败 {page_url}: {e}", flush=True)
        
        page += 1
    
    conn.commit()
    conn.close()
    print(f"[jtxzf] 完成！新增 {new_count}, 跳过 {skip_count}, 无内容 {nocontent_count}", flush=True)


def crawl_incremental(days=7):
    """Incremental crawl for daily config - last N days."""
    cutoff = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d")
    print(f"[jtxzf] 增量模式，{days}天 ({cutoff})", flush=True)
    
    conn = __import__("sqlite3").connect(TEMP_DB)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=30000")
    ensure_gov_raw_table(conn)
    c = conn.cursor()
    
    new_count = 0
    page = 1
    
    while True:
        api_url = f"{BASE_URL}/common/search/{CHANNEL_ID}?_isAgg=false&_isJson=true&_pageSize={PAGE_SIZE}&_template=index&_rangeTimeGte=&_channelName=&page={page}"
        try:
            resp = requests.get(api_url, headers=HEADERS, timeout=15)
            data = resp.json()
            results = data.get("data", {}).get("results", [])
        except Exception as e:
            print(f"[jtxzf] API请求失败 page={page}: {e}", flush=True)
            break
        
        if not results:
            break
        
        for item in results:
            raw_url = item.get("url", "")
            title = item.get("title", "")
            date_str = (item.get("publishedTimeStr") or "")[:10]
            
            if not raw_url or date_str < cutoff:
                continue
            
            page_url = fix_url(raw_url)
            
            c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,))
            if c.fetchone():
                continue
            
            _, content = get_detail(raw_url)
            if not content or len(content.strip()) < 200:
                continue
            
            try:
                c.execute(
                    """INSERT OR IGNORE INTO gov_raw 
                    (site_name, source_url, page_url, title, publish_date, content, status, category)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
                    (SITE_NAME, page_url, page_url, title, date_str,
                     content, "published", "环评公示"),
                )
                if c.rowcount > 0:
                    new_count += 1
                    conn.commit()
            except Exception as e:
                print(f"[jtxzf] 插入失败 {page_url}: {e}", flush=True)
        
        page += 1
    
    conn.commit()
    conn.close()
    print(f"[jtxzf] 增量完成！新增 {new_count}", flush=True)


if __name__ == "__main__":
    mode = sys.argv[1] if len(sys.argv) > 1 else "full"
    if mode == "full":
        crawl_full()
    elif mode == "incremental":
        crawl_incremental()
    else:
        print("Usage: python3 crawl_jtxzf.py [full|incremental]")