#!/usr/bin/env python3
"""
宜阳县人民政府-建设项目环境影响评价信息
列表: https://www.yyzfw.gov.cn/zwgk/zdly/hjbh/jsxmhjyxpjxx/index.html
分页: index_N.html (N=2..54)
详情: div.mailbox_content_wznrs
"""
import requests
import os
import sys
import sqlite3
import re
from bs4 import BeautifulSoup
from urllib.parse import urljoin

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "宜阳县-环评信息"
LIST_BASE = "https://www.yyzfw.gov.cn/zwgk/zdly/hjbh/jsxmhjyxpjxx/index.html"
DAYS = int(sys.argv[1]) if len(sys.argv) > 1 and sys.argv[1].isdigit() else 365

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
session = requests.Session()
session.headers.update(HEADERS)


def parse_list_page(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    tbl = soup.find("table")
    if not tbl:
        return items
    for tr in tbl.find_all("tr")[1:]:
        tds = tr.find_all("td")
        if len(tds) < 2:
            continue
        a = tds[0].find("a")
        if not a:
            continue
        href = a.get("href", "").strip()
        title = a.get_text(strip=True)
        title = re.sub(r"^[-\u00b7\s]+", "", title)
        date_str = tds[1].get_text(strip=True) if len(tds) > 1 else ""
        date_str = date_str.replace("/", "-")
        if href and not href.startswith("http"):
            href = urljoin(LIST_BASE, href)
        if title and href:
            items.append({"page_url": href, "title": title, "publish_date": date_str})
    return items


def fetch_detail(page_url):
    try:
        r = session.get(page_url, timeout=15)
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "html.parser")
        content = ""
        content_div = soup.find("div", class_="mailbox_content_wznrs")
        if content_div:
            for tag in content_div.find_all(["script", "style"]):
                tag.decompose()
            content = str(content_div)
        title = ""
        title_el = soup.find("div", class_="content_top")
        if title_el:
            h3 = title_el.find("h3")
            if h3:
                title = h3.get_text(strip=True)
        return title, content
    except Exception as e:
        print("  [ERROR] " + str(e), flush=True)
        return "", ""


def get_page_url(page_num):
    if page_num == 1:
        return LIST_BASE
    return LIST_BASE.replace("index.html", "index_{}.html".format(page_num))


def find_total_pages():
    for page in range(2, 200):
        url = get_page_url(page)
        try:
            r = session.get(url, timeout=10)
            r.encoding = "utf-8"
            if len(r.text) < 500:
                return page - 1
            items = parse_list_page(r.text)
            if len(items) == 0:
                return page - 1
        except:
            return page - 1
    return 54


def main():
    print("[{}] Starting crawl (days={})".format(SITE_NAME, DAYS), flush=True)

    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    c = conn.cursor()
    c.execute("SELECT page_url FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
    existing = set(row[0] for row in c.fetchall())

    total_pages = find_total_pages()
    print("  Total pages: {}".format(total_pages), flush=True)

    seen_urls = set()
    all_items = []
    for page in range(1, total_pages + 1):
        url = get_page_url(page)
        try:
            r = session.get(url, timeout=15)
            r.encoding = "utf-8"
            items = parse_list_page(r.text)
            print("  Page {}/{}: {} items".format(page, total_pages, len(items)), flush=True)
            for item in items:
                if item["page_url"] not in seen_urls:
                    seen_urls.add(item["page_url"])
                    all_items.append(item)
        except Exception as e:
            print("  Page {} ERROR: {}".format(page, e), flush=True)

    print("  Unique items: {}".format(len(all_items)), flush=True)

    new_count = 0
    for i, item in enumerate(all_items):
        if item["page_url"] in existing:
            continue
        title, content = fetch_detail(item["page_url"])
        if title:
            item["title"] = title
        item["content"] = content
        c.execute(
            "INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content) VALUES (?, ?, ?, ?, ?, ?)",
            (SITE_NAME, LIST_BASE, item["page_url"], item["title"], item["publish_date"], item["content"])
        )
        if c.rowcount > 0:
            new_count += 1
        conn.commit()
        if (i + 1) % 20 == 0:
            print("  Progress: {}/{} processed, {} new".format(i + 1, len(all_items), new_count), flush=True)

    conn.close()
    total = sqlite3.connect(SEARCH_DB, timeout=60).execute(
        "SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)
    ).fetchone()[0]
    print("  New: {}, Total in DB: {}".format(new_count, total), flush=True)
    print("[{}] Done!".format(SITE_NAME), flush=True)


if __name__ == "__main__":
    main()