#!/usr/bin/env python3
import os
"""平定县—生态环境分局公告公示爬虫
URL: http://xxgk.pd.gov.cn/bmtzgg/xzdw/pdsthjfj/pdsthjjtzgg/
"""

import requests, re, sqlite3
from datetime import datetime, timedelta
from bs4 import BeautifulSoup

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36"}
BASE = "http://xxgk.pd.gov.cn/bmtzgg/xzdw/pdsthjfj/pdsthjjtzgg/"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
SITE = "平定县生态环境分局"

def list_url(page):
    return BASE + ("index.shtml" if page == 0 else "index_%d.shtml" % page)

def parse_list(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for li in soup.select("ul.infodisc-con_cntitem--list li"):
        a = li.find("a")
        em = li.find("em")
        if a and em:
            items.append({
                "title": a.get("title", "") or a.text.strip(),
                "date": em.text.strip(),
                "url": a.get("href", "")
            })
    return items

def get_detail(url):
    full = url if url.startswith("http") else BASE + url
    try:
        r = requests.get(full, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
    except:
        return "", ""
    soup = BeautifulSoup(r.text, "html.parser")
    div = soup.select_one("div.view.TRS_UEDITOR") or soup.select_one("div.infodisc-con_cntinner")
    if not div:
        return "", ""
    return str(div), full

def main():
    today = datetime.now().strftime("%Y-%m-%d")
    count = 0
    conn = sqlite3.connect(os.getenv("SEARCH_DB", "/root/search.db"))
    c = conn.cursor()

    for page in range(20):
        url = list_url(page)
        try:
            r = requests.get(url, headers=HEADERS, timeout=30)
            r.encoding = "utf-8"
        except:
            break

        items = parse_list(r.text)
        if not items:
            pg = re.findall(r"createPageHTML\((\d+)", r.text)
            if page >= (int(pg[0]) if pg else 0):
                break
            continue

        for it in items:
            if it["date"] < CUTOFF:
                continue
            head = it["title"][:45].replace("\n", " ")
            print("  [%s] %s... " % (it["date"], head), end="", flush=True)
            content_html, full_url = get_detail(it["url"])
            if not content_html:
                print("无正文")
                continue
            c.execute("""INSERT OR REPLACE INTO gov_raw
                (page_url, title, content, publish_date, site_name, source_url, summary, category)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
                (it["url"], it["title"], content_html, it["date"], SITE, full_url, "", ""))
            count += 1
            print("OK")

    conn.commit()
    conn.close()
    print("\n✅ 平定县：共入库 %d 条" % count)

if __name__ == "__main__":
    main()