#!/usr/bin/env python3
import os
"""江门市生态环境局-建设项目环评 爬虫"""
import re, sys, sqlite3
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

SITE_NAME = "江门市生态环境局-环评信息"
BASE = "https://www.jiangmen.gov.cn"
LIST = "/bmpd/jmssthjj/zdlyxxgk/jsxmhjyxpjxx/"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
MAX_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": BASE + "/",
}


def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"  WARN {url}: {e}")
        return None


def parse_list(html, base_url):
    soup = BeautifulSoup(html, "lxml")
    items = []
    for li in soup.find_all("li"):
        a = li.find("a")
        if not a or "/content/post_" not in a.get("href", ""):
            continue
        href = a.get("href", "")
        title = a.get("title") or a.get_text(strip=True)
        span = li.find("span")
        date_str = span.get_text(strip=True) if span else ""
        full_url = urljoin(base_url, href)
        items.append({"title": title, "url": full_url, "date": date_str})
    return items


def parse_detail(html):
    soup = BeautifulSoup(html, "lxml")
    title = ""
    for tag in ["h1", "h2"]:
        el = soup.find(tag)
        if el:
            title = el.get_text(strip=True)
            break
    date = ""
    for meta in soup.find_all("meta"):
        if meta.get("name", "").lower() in ("pubdate", "publish_date"):
            date = meta.get("content", "")
            break
    if not date:
        m = re.search(r"(\d{4}[-/]\d{1,2}[-/]\d{1,2})", str(soup))
        if m:
            date = m.group(1)

    content = ""
    for cls in ["article-content", "content", "TRS_Editor", "zoom", "newscontent"]:
        div = soup.find("div", class_=cls)
        if div:
            for tag in div(["script", "style", "iframe"]):
                tag.decompose()
            content = str(div)
            break
    if not content:
        best, best_count = None, 0
        for div in soup.find_all("div"):
            ps = div.find_all("p")
            if len(ps) > best_count:
                best, best_count = div, len(ps)
        if best:
            for tag in best(["script", "style", "iframe"]):
                tag.decompose()
            content = str(best)
    content = re.sub(r'\s*style="[^"]*"', "", content)
    content = re.sub(r'\s*class="[^"]*"', "", content)
    return title, date, content


def run(max_pages=None):
    if max_pages is None:
        max_pages = MAX_PAGES
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    known = set(r[0] for r in db.execute(
        "SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchall())
    db.close()

    all_items = []
    for page in range(1, max_pages + 1):
        if page == 1:
            url = BASE + LIST
        else:
            url = f"{BASE}{LIST}index_{page}.html"
        print(f"  Page {page}...", end=" ")
        html = fetch(url)
        if not html:
            print("FAIL")
            break
        items = parse_list(html, url)
        new_items = [it for it in items if it["url"] not in known]
        print(f"OK {len(items)} items (new: {len(new_items)})")
        all_items.extend(new_items)
        if len(items) < 10:
            break

    if not all_items:
        print("  no new data")
        return

    print(f"  Fetching {len(all_items)} details...")
    results = []
    for i, item in enumerate(all_items):
        html = fetch(item["url"])
        if not html:
            continue
        title, date_str, content = parse_detail(html)
        if not title:
            title = item["title"]
        if not date_str:
            date_str = item["date"]
        results.append({
            "site_name": SITE_NAME, "source_url": item["url"][:500], "page_url": item["url"],
            "title": title[:500], "publish_date": date_str[:10] if date_str else "",
            "summary": title[:500], "content": content, "status": "active", "category": "", "tags": "",
        })
        if (i + 1) % 10 == 0:
            print(f"    [{i+1}/{len(all_items)}]")

    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in results:
        try:
            db.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name,source_url,page_url,title,publish_date,summary,content,status,category,tags) VALUES (?,?,?,?,?,?,?,?,?,?)",
                (item["site_name"][:200], item["source_url"], item["page_url"], item["title"],
                 item["publish_date"], item["summary"], item["content"], item["status"],
                 item["category"], item["tags"]),
            )
            if db.total_changes > 0: ok += 1
            else: skip += 1
        except: skip += 1
    db.commit()
    db.execute(
        "INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,),
    )
    db.commit()
    db.close()
    print(f"  Saved: new={ok}, skip={skip}, FTS synced")


if __name__ == "__main__":
    max_p = int(sys.argv[1]) if len(sys.argv) > 1 and sys.argv[1].isdigit() else None
    run(max_p)