#!/usr/bin/env python3
"""金昌市生态环境局-环评批复 爬虫 (JPaaS CMS API)"""
import re, sys, sqlite3, json
import requests
from bs4 import BeautifulSoup
import os

SITE_NAME = "金昌市生态环境局-环评批复"
BASE = "https://sthj.jcs.gov.cn"
API_URL = f"{BASE}/api-gateway/jpaas-publish-server/front/page/build/unit"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
MAX_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Referer": f"{BASE}/zwgk/fdzdgknr/hjxx/jsxmhjyxpg/index.html",
}

API_PARAMS = {
    "parseType": "bulidstatic",
    "webId": "47f74ec6fa054832961ddb79491f6065",
    "tplSetId": "a7ce57f894724fb1970fb836e7b631ff",
    "pageType": "column",
    "tagId": "法定主动公开内容第一个栏目list",
    "editType": "null",
    "pageId": "c3594f9e0ce64fe38759159c9d07d002",
    "rows": "15",
    "count": "0",
    "pageNo": "1",
}


def fetch_page(session, page_no):
    params = dict(API_PARAMS)
    params["pageNo"] = str(page_no)
    try:
        r = session.get(API_URL, params=params, headers=HEADERS, timeout=30)
        if r.status_code == 200:
            data = r.json()
            if data.get("success"):
                return data["data"]["html"]
    except:
        pass
    return None


def parse_list(html):
    items = []
    soup = BeautifulSoup(html, "lxml")
    for li in soup.find_all("li", class_="cf"):
        a_tag = li.find("a")
        if not a_tag:
            continue
        href = a_tag.get("href", "")
        title = a_tag.get("title") or a_tag.get_text(strip=True)
        if not href or not title:
            continue
        full_url = href if href.startswith("http") else BASE + href
        span = li.find("span", class_="fr")
        date_str = span.get_text(strip=True) if span else ""
        items.append({"title": title, "url": full_url, "date": date_str})
    return items


def fetch_detail(session, url):
    try:
        r = session.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        html = r.text
    except:
        return None, None, None

    soup = BeautifulSoup(html, "lxml")

    title = ""
    for tag in ["h1", "h2"]:
        el = soup.find(tag)
        if el:
            title = el.get_text(strip=True)
            break

    date = ""
    for meta in soup.find_all("meta"):
        if meta.get("name", "").lower() in ("pubdate", "publish_date"):
            date = meta.get("content", "")
            break
    if not date:
        m = re.search(r"(\d{4}[-/]\d{1,2}[-/]\d{1,2})", str(soup))
        if m:
            date = m.group(1)

    content = ""
    for cls in ["article-content", "content", "TRS_Editor", "zoom", "NewsContent", "pages_content"]:
        div = soup.find("div", class_=cls)
        if div:
            for tag in div(["script", "style", "iframe"]):
                tag.decompose()
            content = str(div)
            break
    if not content:
        best, best_count = None, 0
        for div in soup.find_all("div"):
            ps = div.find_all("p")
            if len(ps) > best_count:
                best, best_count = div, len(ps)
        if best:
            for tag in best(["script", "style", "iframe"]):
                tag.decompose()
            content = str(best)

    content = re.sub(r'\s*style="[^"]*"', "", content)
    content = re.sub(r'\s*class="[^"]*"', "", content)
    return title, date, content


def run(max_pages=None):
    if max_pages is None:
        max_pages = MAX_PAGES
    session = requests.Session()

    db = sqlite3.connect(SEARCH_DB, timeout=60)
    known = set(r[0] for r in db.execute(
        "SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchall())
    db.close()

    all_items = []
    for page in range(1, max_pages + 1):
        print(f"  Page {page}...", end=" ", flush=True)
        html = fetch_page(session, page)
        if not html:
            print("FAIL")
            break
        items = parse_list(html)
        new_items = [it for it in items if it["url"] not in known]
        print(f"OK {len(items)} items (new: {len(new_items)})")
        all_items.extend(new_items)
        if len(items) < 10:
            break

    if not all_items:
        print("  no new data")
        return

    print(f"  Fetching {len(all_items)} details...")
    results = []
    for i, item in enumerate(all_items):
        ret = fetch_detail(session, item["url"])
        if not ret or not ret[0]:
            detail_title = detail_date = detail_content = None
        else:
            detail_title, detail_date, detail_content = ret
        title = detail_title if detail_title else item["title"]
        date_str = detail_date if detail_date else item["date"]
        content = detail_content if detail_content else ""

        results.append({
            "site_name": SITE_NAME,
            "source_url": item["url"][:500],
            "page_url": item["url"],
            "title": title[:500],
            "publish_date": date_str[:10] if date_str else "",
            "summary": title[:500],
            "content": content,
            "status": "active",
            "category": "",
            "tags": "",
        })
        if (i + 1) % 10 == 0:
            print(f"    [{i+1}/{len(all_items)}]")

    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in results:
        try:
            db.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name,source_url,page_url,title,publish_date,summary,content,status,category,tags) VALUES (?,?,?,?,?,?,?,?,?,?)",
                (item["site_name"][:200], item["source_url"], item["page_url"], item["title"],
                 item["publish_date"], item["summary"], item["content"], item["status"],
                 item["category"], item["tags"]),
            )
            if db.total_changes > 0: ok += 1
            else: skip += 1
        except: skip += 1
    db.commit()
    db.execute(
        "INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,),
    )
    db.commit()
    db.close()
    print(f"  Saved: new={ok}, skip={skip}, FTS synced")


if __name__ == "__main__":
    max_p = int(sys.argv[1]) if len(sys.argv) > 1 and sys.argv[1].isdigit() else None
    run(max_p)