#!/usr/bin/env python3
"""
新晃侗族自治县 - 产业开发区文件通知（环评类）
https://www.xinhuang.gov.cn/xinhuang/c133675/list2021.shtml
分页: list2021.shtml (page1), list2021_{N}.shtml
列表: ul.ul_list1 > li > a + span
详情: div.p_box
"""
import requests, sys, os, sqlite3, re, time
from datetime import datetime
from bs4 import BeautifulSoup

BASE = "https://www.xinhuang.gov.cn"
DB = "/root/search.db"
SITE_NAME = "新晃产业开发区"
SITE_SOURCE = "https://www.xinhuang.gov.cn/xinhuang/c133675/list2021.shtml"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
    "Referer": SITE_SOURCE,
}

def get_total_pages(html):
    m = re.search(r"createPageHTML\('paging',(\d+),", html)
    return int(m.group(1)) if m else 1

def fetch_list(page=1):
    url = SITE_SOURCE if page == 1 else f"{BASE}/xinhuang/c133675/list2021_{page}.shtml"
    r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
    r.encoding = "utf-8"
    return r.text

def parse_list(html):
    items = []
    soup = BeautifulSoup(html, "html.parser")
    ul = soup.find("ul", class_="ul_list1")
    if not ul:
        return items
    for li in ul.find_all("li"):
        a = li.find("a")
        span = li.find("span")
        if not a or not a.get("href"):
            continue
        title = a.get("title", "").strip() or a.get_text(strip=True)
        href = a["href"]
        if href.startswith("/"):
            href = BASE + href
        date = span.get_text(strip=True) if span else ""
        items.append({"title": title, "url": href, "date": date})
    return items

def fetch_detail(url):
    r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
    r.encoding = "utf-8"
    return r.text

def extract_content(html):
    soup = BeautifulSoup(html, "html.parser")
    p_box = soup.find("div", class_="p_box")
    if not p_box:
        return "", ""
    parts = []

    # Attachment links
    for a in p_box.find_all("a", href=True):
        href = a["href"]
        fname = a.get_text(strip=True) or href.split("/")[-1]
        if any(x in href.lower() for x in [".pdf",".doc",".docx",".xls",".xlsx",".zip"]):
            parts.append("[附件：{}]({})".format(fname, BASE + href if href.startswith("/") else href))

    # Tables (HTML)
    for table in p_box.find_all("table"):
        tbl = re.sub(r">\s+<", "><", str(table))
        parts.append(re.sub(r"\s{2,}", " ", tbl))
        table.decompose()

    # Unwrap spans, extract p/li
    for span in p_box.find_all("span"):
        span.unwrap()
    seen = set()
    for el in p_box.find_all(["p", "li"]):
        text = el.get_text(strip=True)
        if text and text not in seen:
            parts.append(text)
            seen.add(text)

    text = "\n\n".join(parts) if parts else ""
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip(), text[:500].strip()

def crawl(incremental=False):
    conn = sqlite3.connect(DB, timeout=60)
    c = conn.cursor()
    existing = set()
    if incremental:
        for row in c.execute("SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)):
            existing.add(row[0])

    html = fetch_list(1)
    total_pages = get_total_pages(html)
    print("Total pages: {}".format(total_pages))

    all_items = parse_list(html)
    for p in range(2, total_pages + 1):
        h = fetch_list(p)
        if h:
            all_items.extend(parse_list(h))
            print("  Page {}/{}: {} items".format(p, total_pages, len(parse_list(h))))
        time.sleep(0.5)

    print("Total items: {}".format(len(all_items)))

    inserted = skipped = 0
    for item in all_items:
        url = item["url"]
        if url in existing:
            skipped += 1
            continue
        try:
            detail_html = fetch_detail(url)
        except:
            skipped += 1
            continue
        text, summary = extract_content(detail_html)
        if not text or len(text) < 10:
            skipped += 1
            continue
        date_str = re.sub(r"[^\d-]", "", item.get("date",""))[:10] or "2020-01-01"
        dr = int(date_str.replace("-","")) if date_str.count("-")==2 else 0
        c.execute("INSERT OR IGNORE INTO gov_raw (site_name,source_url,page_url,title,publish_date,content,summary,date_rank) VALUES (?,?,?,?,?,?,?,?)",
            (SITE_NAME, SITE_SOURCE, url, item["title"], date_str, text, summary, dr))
        if c.rowcount:
            inserted += 1
            try:
                # 2026-09-22: 先提交 gov_raw —— 下面手动写 FTS 会因触发器已写过同一
                #   rowid 而 IntegrityError，若不先 commit，这条记录会被一并回滚（静默丢数据）
                conn.commit()
                c.execute("INSERT INTO gov_search(rowid,site_name,title,summary) VALUES (?,?,?,?)", (c.lastrowid, SITE_NAME, item["title"], summary))
            except:
                pass
            print("  + {}... ({})".format(item["title"][:50], date_str))
        else:
            skipped += 1
    conn.commit()
    conn.close()
    print("\nDone! Inserted: {}, Skipped: {}".format(inserted, skipped))

if __name__ == "__main__":
    requests.packages.urllib3.disable_warnings()
    crawl("--incremental" in sys.argv)