#!/usr/bin/env python3
"""
安徽淮南潘集经济开发区（安徽淮南现代煤化工产业园）-通知公告
列表API: https://ahccci.huainan.gov.cn/content/column/12494462?pageIndex=N (N=1..7)
详情: div.secnr 或 div#wenzhang
总计: 125条, 7页
"""
import requests
import os
import sys
import sqlite3
import re
from bs4 import BeautifulSoup

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "淮南煤化工园区-通知公告"
API_URL = "https://ahccci.huainan.gov.cn/content/column/12494462"
DAYS = int(sys.argv[1]) if len(sys.argv) > 1 and sys.argv[1].isdigit() else 365

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
session = requests.Session()
session.headers.update(HEADERS)


def parse_list_page(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    ul = soup.find("ul", class_=re.compile(r"doc_list"))
    if not ul:
        return items
    for li in ul.find_all("li"):
        a = li.find("a")
        if not a:
            continue
        href = a.get("href", "").strip()
        # Title from the span inside a, or from a title attribute
        span = a.find("span")
        title = span.get_text(strip=True) if span else a.get_text(strip=True)
        if not title:
            title = a.get("title", "")
        # Date from span.right.date
        date_span = li.find("span", class_=re.compile(r"date"))
        date_str = date_span.get_text(strip=True) if date_span else ""
        # Ensure absolute URL
        if href and not href.startswith("http"):
            href = "https://ahccci.huainan.gov.cn" + ("" if href.startswith("/") else "/") + href
        if title and href:
            items.append({"page_url": href, "title": title, "publish_date": date_str})
    return items


def fetch_detail(page_url):
    try:
        r = session.get(page_url, timeout=15)
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "html.parser")
        content = ""
        # Try secnr first, then wenzhang
        for sel in [("class_", "secnr"), ("id", "wenzhang")]:
            d = soup.find(sel[0], sel[1])
            if d:
                for tag in d.find_all(["script", "style"]):
                    tag.decompose()
                content = str(d)
                break
        # Title from meta
        title = ""
        meta = soup.find("meta", attrs={"name": "ArticleTitle"})
        if meta:
            title = meta.get("content", "")
        return title, content
    except Exception as e:
        print("  [ERROR] " + str(e), flush=True)
        return "", ""


def find_total_pages():
    for page in range(1, 50):
        url = API_URL + "?pageIndex=" + str(page)
        try:
            r = session.get(url, timeout=10)
            r.encoding = "utf-8"
            items = parse_list_page(r.text)
            if len(items) == 0:
                return page - 1
        except:
            return page - 1
    return 7


def main():
    print("[{}] Starting crawl (days={})".format(SITE_NAME, DAYS), flush=True)

    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    c = conn.cursor()

    total_pages = find_total_pages()
    print("  Total pages: {}".format(total_pages), flush=True)

    seen = set()
    all_items = []
    for page in range(1, total_pages + 1):
        url = API_URL + "?pageIndex=" + str(page)
        try:
            r = session.get(url, timeout=15)
            r.encoding = "utf-8"
            items = parse_list_page(r.text)
            print("  Page {}/{}: {} items".format(page, total_pages, len(items)), flush=True)
            for item in items:
                if item["page_url"] not in seen:
                    seen.add(item["page_url"])
                    all_items.append(item)
        except Exception as e:
            print("  Page {} ERROR: {}".format(page, e), flush=True)

    print("  Unique items: {}".format(len(all_items)), flush=True)

    c.execute("SELECT page_url FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
    existing = set(row[0] for row in c.fetchall())

    new_count = 0
    for i, item in enumerate(all_items):
        if item["page_url"] in existing:
            continue
        title, content = fetch_detail(item["page_url"])
        if title:
            item["title"] = title
        item["content"] = content
        c.execute(
            "INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content) VALUES (?, ?, ?, ?, ?, ?)",
            (SITE_NAME, API_URL, item["page_url"], item["title"], item["publish_date"], item["content"])
        )
        if c.rowcount > 0:
            new_count += 1
        conn.commit()
        if (i + 1) % 20 == 0:
            print("  Progress: {}/{} processed, {} new".format(i + 1, len(all_items), new_count), flush=True)

    conn.close()
    total = sqlite3.connect(SEARCH_DB, timeout=60).execute(
        "SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)
    ).fetchone()[0]
    print("  New: {}, Total in DB: {}".format(new_count, total), flush=True)
    print("[{}] Done!".format(SITE_NAME), flush=True)


if __name__ == "__main__":
    main()