#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
丽江市古城区 - 重大项目信息
http://www.ljgucheng.gov.cn/xljgcq/c100041/zfxxgk_nrz.shtml
"""
import sys, os, re, time, requests, sqlite3
from concurrent.futures import ThreadPoolExecutor, as_completed

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
LIST_PREFIX = "http://www.ljgucheng.gov.cn/xljgcq/c100041/zfxxgk_nrz"
DOMAIN = "http://www.ljgucheng.gov.cn"
SITE_NAME = "ljgucheng_zdxm"


def get_list_items():
    items = []
    page = 1
    while True:
        url = f"{LIST_PREFIX}.shtml" if page == 1 else f"{LIST_PREFIX}_{page}.shtml"
        try:
            r = requests.get(url, headers=HEADERS, timeout=30)
            r.encoding = "utf-8"
            html = r.text
        except Exception as e:
            print(f"  Page {page} error: {e}")
            break

        # Extract only from zfxxgk_zdgkc container to avoid navigation links
        idx = html.find('zfxxgk_zdgkc')
        if idx < 0:
            break
        list_area = html[idx:html.find('</ul>', idx)]

        entries = re.findall(
            r'<a[^>]*href="([^"]+)"[^>]*>([^<]+)</a>.*?<b[^>]*>(\d{4}-\d{2}-\d{2})</b>',
            list_area, re.DOTALL
        )
        if not entries:
            break

        for href, title, date in entries:
            if href.endswith('.pdf'):
                continue
            title = title.strip()
            full_url = href if href.startswith("http") else DOMAIN + href
            items.append({
                "url": full_url,
                "title": title,
                "date": date.strip(),
                "site": SITE_NAME
            })

        print(f"  Page {page}: {len(entries)} items")
        if len(entries) < 15:
            break
        page += 1
        time.sleep(0.3)

    return items


def fetch_detail(item):
    url = item["url"]
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        html = r.text
    except Exception:
        return (url, item["title"], "", item["date"], item["site"])

    # Title
    title = item["title"]
    m = re.search(r'<meta name="ArticleTitle"[^>]*content="([^"]*)"', html)
    if m:
        t = m.group(1).strip()
        if t:
            title = t

    # Content - try scroll_cont first (c100041), then data-article (c101528)
    content = ""
    m = re.search(r'<div[^>]*class="scroll_cont"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if m:
        raw = m.group(1).strip()
        raw = re.sub(r'<script[^>]*>.*?</script>', "", raw, flags=re.DOTALL | re.I)
        raw = re.sub(r'<style[^>]*>.*?</style>', "", raw, flags=re.DOTALL | re.I)
        content = raw
    else:
        m = re.search(r'data-article="content">(.*?)</div>', html, re.DOTALL)
        if m:
            raw = m.group(1).strip()
            raw = re.sub(r'<script[^>]*>.*?</script>', "", raw, flags=re.DOTALL | re.I)
            raw = re.sub(r'<style[^>]*>.*?</style>', "", raw, flags=re.DOTALL | re.I)
            content = raw

    # Date
    date = item["date"]
    if not date:
        m = re.search(r'发布时间[：:]\s*(\d{4}[-/]\d{2}[-/]\d{2})', html)
        if m:
            date = m.group(1)

    return (url, title, content, date, item["site"])


def main():
    total_new = 0

    print("Fetching list...")
    items = get_list_items()
    print(f"  Total: {len(items)} items")

    if not items:
        print("No items found!")
        return

    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute("PRAGMA busy_timeout=30000")
    existing = set(
        r[0] for r in conn.execute(
            "SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)
        ).fetchall()
    )

    new_items = [it for it in items if it["url"] not in existing]
    print(f"  Existing: {len(existing)}, New: {len(new_items)}")

    if not new_items:
        print("All up to date!")
        conn.close()
        return

    print(f"Fetching {len(new_items)} details (5 threads)...")
    done = 0
    with ThreadPoolExecutor(max_workers=5) as pool:
        fut_map = {pool.submit(fetch_detail, item): item for item in new_items}
        for f in as_completed(fut_map):
            url, title, content, date, site = f.result()
            c = conn.execute(
                "INSERT OR IGNORE INTO gov_raw (title, page_url, source_url, content, publish_date, site_name) VALUES (?, ?, ?, ?, ?, ?)",
                (title, url, url, content, date, site)
            )
            conn.commit()
            if c.rowcount > 0:
                total_new += 1
            done += 1
            if done % 20 == 0 or done == len(new_items):
                print(f"  {done}/{len(new_items)} processed, new={total_new}")

    print(f"\nDone! {len(new_items)} processed, {total_new} new.")
    conn.close()


if __name__ == "__main__":
    main()