#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
六安市生态环境局 - 行政许可结果 + 公示公告 (EpointWebBuilder CMS)
v2: 使用正确的 label/8888 API (pageIndex 有效分页)
"""
import sys, os, re, time, requests, sqlite3, json
from concurrent.futures import ThreadPoolExecutor, as_completed

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
API_LABEL = "https://sthjj.luan.gov.cn/luan/site/label/8888"

def get_xzxk_list():
    """行政许可结果 - 使用 label/8888 JSON API"""
    items = []
    payload = {
        "labelName": "publicInfoList",
        "siteId": "6789941",
        "organId": "6608281",
        "pageSize": "20",
        "isDate": "true",
        "dateFormat": "yyyy-MM-dd",
        "length": "50",
        "type": "4",
        "catId": "7053922",
        "action": "list",
        "result": "",
        "isJson": "true",
        "isSetValue": "true"
    }
    
    # First page to get total/pageCount
    payload["pageIndex"] = "1"
    r = requests.post(API_LABEL, data=payload, headers=HEADERS, timeout=30)
    j = r.json()
    page_count = int(j.get("pageCount", 1))
    total = int(j.get("total", 0))
    print(f"XZXK API: total={total}, pages={page_count}")
    
    for page in range(1, page_count + 1):
        payload["pageIndex"] = str(page)
        try:
            r = requests.post(API_LABEL, data=payload, headers=HEADERS, timeout=30)
            j = r.json()
            data = j.get("data", [])
            for item in data:
                items.append({
                    "url": item.get("link", ""),
                    "title": (item.get("title", "") or "").strip(),
                    "date": (item.get("publishDate", "") or ""),
                    "site": "luan_xzxk",
                    "summarize": (item.get("summarize", "") or ""),
                    "file_num": (item.get("fileNum", "") or ""),
                    "content_id": str(item.get("id", "")),
                })
            print(f"  Page {page}/{page_count}: {len(data)} items")
        except Exception as e:
            print(f"  Page {page} error: {e}")
        time.sleep(0.3)
    
    return items


def get_xzxk_items_from_public():
    """Fallback: 直接从公开页面获取列表（备用方案）"""
    items = []
    for page in range(1, 33):  # 32 pages total
        r = requests.post("https://sthjj.luan.gov.cn/luan/site/label/481565", data={
            "pageSize": "20", "platformCode": "luan_gova", "isAllSite": "false",
            "sort": "desc", "withOutDomain": "true", "typeCode": "public_content",
            "columnId": "6608281", "catIds": "7053922",
            "fromCode": "title", "orderType": "1", "isLimitScore": "0",
            "fuzzySearch": "false", "file": "/c1/luan/searchDataList-gk-2020",
            "keywords": "", "page": str(page),
        }, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        html = r.text
        ids = re.findall(r"/public/6608281/(\d+)\.html", html)
        titles = re.findall(r'title="([^"]*)"', html)
        dates = re.findall(r"<td>(\d{4}-\d{2}-\d{2})</td>", html)
        
        if not ids:
            break
        
        for i, id_ in enumerate(ids):
            title = titles[i] if i < len(titles) else ""
            date = dates[i] if i < len(dates) else ""
            items.append({
                "url": "https://sthjj.luan.gov.cn/public/6608281/{}.html".format(id_),
                "title": title.strip(),
                "date": date,
                "site": "luan_xzxk"
            })
        
        if len(ids) < 20:
            break
        time.sleep(0.3)
    
    return items


def check_tzgg_full():
    """检查通知公告是否已全量"""
    # We already have 706 records. Let me verify by checking the tzgg list pages
    tzgg_items = []
    for page in range(1, 200):
        try:
            r = requests.get("https://sthjj.luan.gov.cn/content/column/6806171?pageIndex={}".format(page), headers=HEADERS, timeout=30)
            r.encoding = "utf-8"
            html = r.text
        except Exception as e:
            print("  tzgg list error:", e)
            break

        urls = re.findall(r'href="(https://sthjj\.luan\.gov\.cn/zwzx/tzgg/\d+\.html)"', html)
        if not urls:
            break
        for u in urls:
            tzgg_items.append(u)
        if len(urls) < 19:
            break
        time.sleep(0.3)
    
    return tzgg_items


def fetch_detail(item):
    """抓取详情页正文"""
    url = item["url"]
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        html = r.text
    except Exception:
        return (url, item["title"], "", item["date"], item["site"])

    # Title from meta
    title = item.get("title", "")
    m = re.search(r'<meta name="ArticleTitle"[^>]*content="([^"]*)"', html)
    if m:
        t = m.group(1).strip()
        if t:
            title = t

    # Content - try xzxk pattern first, then tzgg
    content = ""
    m = re.search(r'class="gkwz_contnet j-fontContent"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if not m:
        m = re.search(r'class="j-fontContent newscontnet"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        raw = m.group(1).strip()
        raw = re.sub(r'<script[^>]*>.*?</script>', "", raw, flags=re.DOTALL|re.I)
        raw = re.sub(r'<style[^>]*>.*?</style>', "", raw, flags=re.DOTALL|re.I)
        content = raw

    # Date
    date = item.get("date", "")
    if not date:
        for pat in [r'发布时间[：:]\s*(\d{4}[-/]\d{2}[-/]\d{2})', r'<meta name="PubDate"[^>]*content="([^"]*)"']:
            m = re.search(pat, html)
            if m:
                date = m.group(1)
                break

    return (url, title, content, date, item["site"])


def main():
    total_new = 0
    
    # 1. 行政许可结果 - 使用正确的 API
    print("=" * 50)
    print("Step 1: Fetching xzxk (行政许可) list via label/8888 API...")
    xzxk = get_xzxk_list()
    print(f"  Total xzxk items from API: {len(xzxk)}")
    
    # 2. 通知公告 - 已有706条，检查是否有新的
    print("=" * 50)
    print("Step 2: Checking tzgg (通知公告) list...")
    # tzgg already has 706 records, check if more pages exist
    tzgg_urls = check_tzgg_full()
    print(f"  TZGG pages found: {len(tzgg_urls)} items")
    
    # Deduplicate tzgg against existing DB
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute("PRAGMA busy_timeout=30000")
    
    existing_tzgg = set(r[0] for r in conn.execute(
        "SELECT page_url FROM gov_raw WHERE site_name='luan_tzgg'"
    ).fetchall())
    
    existing_xzxk = set(r[0] for r in conn.execute(
        "SELECT page_url FROM gov_raw WHERE site_name='luan_xzxk'"
    ).fetchall())
    conn.close()
    
    print(f"  Existing: xzxk={len(existing_xzxk)}, tzgg={len(existing_tzgg)}")
    
    # Filter xzxk items
    new_xzxk = [it for it in xzxk if it["url"] not in existing_xzxk]
    print(f"  New xzxk items to fetch: {len(new_xzxk)}")
    
    # Filter tzgg items
    new_tzgg = []
    for url in tzgg_urls:
        if url not in existing_tzgg:
            # Need to fetch title and date too - simplified for now
            new_tzgg.append({"url": url, "title": "", "date": "", "site": "luan_tzgg"})
    print(f"  New tzgg items to fetch: {len(new_tzgg)}")
    
    all_new = new_xzxk + new_tzgg
    if not all_new:
        print("\nNo new items to fetch. All up to date!")
        return
    
    print(f"\nTotal new items: {len(all_new)}, fetching details (5 threads)...")
    
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute("PRAGMA busy_timeout=30000")
    
    done = 0
    with ThreadPoolExecutor(max_workers=5) as pool:
        fut_map = {pool.submit(fetch_detail, item): item for item in all_new}
        for f in as_completed(fut_map):
            url, title, content, date, site = f.result()
            try:
                c = conn.execute(
                    "INSERT OR IGNORE INTO gov_raw (title, page_url, source_url, content, publish_date, site_name) VALUES (?, ?, ?, ?, ?, ?)",
                    (title, url, url, content, date, site)
                )
                conn.commit()
                if c.rowcount > 0:
                    total_new += 1
            except Exception as e:
                print(f"  DB insert error for {url}: {e}")
            done += 1
            if done % 50 == 0 or done == len(all_new):
                print(f"  {done}/{len(all_new)} processed, new={total_new}")
    
    print(f"\n{'=' * 50}")
    print(f"Done! Processed {len(all_new)} new items, inserted {total_new}.")
    conn.close()


if __name__ == "__main__":
    main()