#!/usr/bin/env python3
"""
纳溪区 — 环境保护
https://www.naxi.gov.cn/zw/fdzdgknr/zdmsxx/hjbh
CMS: 自定义
分页: hjbh (page1), hjbh_2 ... hjbh_48
列表: ul.newsList > li > a + span
详情: /zw/fdzdgknr/zdmsxx/hjbh/content_XXXXXX
正文: div.conTxt
"""
import requests
import sqlite3
import re
import time
import os
from datetime import datetime
from bs4 import BeautifulSoup, NavigableString

DB_PATH = "/root/search.db"
SITE_NAME = "纳溪区-环境保护"
LIST_URL = "https://www.naxi.gov.cn/zw/fdzdgknr/zdmsxx/hjbh"
BASE_DOMAIN = "https://www.naxi.gov.cn"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
}

# ─── 正文取文本（2026-09-11）：行内节点直接拼接，只在块级边界 / <br> 处换行 ───
# ⚠️ 不要用 el.get_text("\n") 取正文 —— 它是「每个**文本节点**之间插 \n」，Word 粘贴的
#    公文把一行拆成 <span>提取码：</span>pwaj<span>。查阅…</span>，这些行内节点于是各自
#    成行（福泉 id=2095080103703914437 实例：`提取码：`/`pwaj`/`。查阅…` 各占一行）。
_BLOCK_TAGS = {'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd', 'div',
               'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
               'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'nav', 'ol',
               'p', 'pre', 'section', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th',
               'ul', 'center', 'caption'}


def body_text(el):
    """块级边界出换行、行内节点直接拼接、<br> 出换行（≈ 浏览器看到的换行结构）。"""
    if el is None:
        return ''
    import re as _re
    from bs4 import NavigableString
    out = []

    def walk(node):
        for ch in node.children:
            if isinstance(ch, NavigableString):
                out.append(str(ch))
            elif getattr(ch, 'name', None) == 'br':
                out.append('\n')
            elif getattr(ch, 'name', None) in _BLOCK_TAGS:
                out.append('\n')
                walk(ch)
                out.append('\n')
            else:
                walk(ch)
    walk(el)
    t = ''.join(out)
    t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
    t = _re.sub(r'\n{3,}', '\n\n', t)
    return t.strip()


def fetch_list(page=1):
    if page == 1:
        url = LIST_URL
    else:
        url = f"{LIST_URL}_{page}"
    r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
    r.encoding = "utf-8"
    return r.text

def extract_items(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    ul = soup.find("ul", class_="newsList")
    if not ul:
        return items
    for li in ul.find_all("li", recursive=False):
        a = li.find("a", href=True)
        if not a:
            continue
        href = a["href"]
        title = a.get_text(strip=True)
        if not title or len(title) < 5:
            continue
        if href.startswith("http"):
            full_url = href
        elif href.startswith("/"):
            full_url = BASE_DOMAIN + href
        else:
            full_url = LIST_URL + "/" + href
        date_str = ""
        span = li.find("span")
        if span:
            date_str = span.get_text(strip=True)
        items.append({"title": title, "url": full_url, "date": date_str})
    return items

def extract_content_with_tables(element):
    """Extract text from element, preserving <table> HTML and proper paragraph breaks."""
    parts = []
    # Handle tables first - keep their HTML
    for table in element.find_all("table"):
        table_html = str(table)
        # Clean up excessive whitespace in table HTML
        table_html = re.sub(r">\s+<", "><", table_html)
        table_html = re.sub(r"\s{2,}", " ", table_html)
        parts.append(table_html)
        # Remove the table from the soup so we don't double-process it
        table.decompose()

    # Unwrap inline spans
    for span in element.find_all("span"):
        span.unwrap()

    # Extract p/li text
    seen = set()
    for el in element.find_all(["p", "li"]):
        text = el.get_text(strip=True)
        if text and text not in seen:
            parts.append(text)
            seen.add(text)

    return "\n\n".join(parts) if parts else body_text(element)

def fetch_detail(url):
    try:
        time.sleep(1.2)  # Rate limit: max 1 req/sec
        r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "html.parser")
        for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):
            tag.decompose()

        detail_date = ""
        prop = soup.find("div", class_="property")
        if prop:
            m = re.search(r"(\d{4}-\d{1,2}-\d{1,2})", prop.get_text())
            if m:
                detail_date = m.group(1)

        content_div = soup.find("div", class_="conTxt")
        if not content_div:
            content_div = soup.find("div", class_=lambda c: c and ("printArea" in (c or "")))
        if content_div:
            content = extract_content_with_tables(content_div)
        else:
            content = body_text(soup)

        content = re.sub(r"\n{3,}", "\n\n", content)
        content = re.sub(r" {2,}", " ", content)
        return content.strip(), detail_date
    except Exception as e:
        print(f"  [ERROR] detail: {e}", flush=True)
        return "", ""

def main():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    existing = set()
    for row in c.execute("SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)):
        existing.add(row[0])
    latest_db_date = "1900-01-01"
    row = c.execute("SELECT MAX(publish_date) FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchone()
    if row and row[0]:
        latest_db_date = row[0]
    conn.close()
    print(f"Latest DB date: {latest_db_date}", flush=True)
    print(f"Existing URLs: {len(existing)}", flush=True)

    all_items = []
    empty_pages = 0
    for page in range(1, 49):
        try:
            html = fetch_list(page)
        except Exception as e:
            print(f"  [ERROR] Page {page}: {e}", flush=True)
            time.sleep(5)
            try:
                html = fetch_list(page)
            except:
                break
        items = extract_items(html)
        print(f"  Page {page}: {len(items)} items", flush=True)
        if len(items) == 0:
            empty_pages += 1
            if empty_pages >= 3:
                break
        else:
            empty_pages = 0
        all_items.extend(items)

    print(f"Total list items: {len(all_items)}", flush=True)

    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    new_count = skip_count = error_count = 0

    for item in all_items:
        url = item["url"]
        title = item["title"]
        date_str = item["date"]
        if url in existing:
            skip_count += 1
            continue

        content, detail_date = fetch_detail(url)
        if not detail_date:
            detail_date = date_str
        if not content or len(content) < 50:
            print(f"  [SHORT] {title[:30]}... ({len(content)})", flush=True)
            if not content:
                error_count += 1
                continue

        if not detail_date:
            detail_date = datetime.now().strftime("%Y-%m-%d")
        detail_date = re.sub(r"[^\d-]", "", detail_date)[:10]
        dr = int(detail_date.replace("-", "")) if detail_date.count("-") == 2 else 0

        c.execute("""
            INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, date_rank)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """, (SITE_NAME, LIST_URL, url, title, detail_date, content, content[:500], dr))
        new_count += 1
        if new_count % 10 == 0:
            conn.commit()
            print(f"  Progress: {new_count} new / {skip_count} skip", flush=True)

    conn.commit()
    conn.close()

    # Rebuild FTS
    print("Rebuilding FTS...", flush=True)
    conn2 = sqlite3.connect(DB_PATH, timeout=60)
    c2 = conn2.cursor()
    # QC20260926 去掉手写 gov_search 整站删除(抢锁源; FTS 由 gov_raw 触发器维护) 
    # c2.execute("DELETE FROM gov_search WHERE site_name=?", (SITE_NAME,))
    conn2.commit()
    conn2.close()

    print(f"\n{'='*50}", flush=True)
    print(f"Site: {SITE_NAME}", flush=True)
    print(f"Total list items: {len(all_items)}", flush=True)
    print(f"New: {new_count}", flush=True)
    print(f"Skipped (existing): {skip_count}", flush=True)
    print(f"Errors: {error_count}", flush=True)
    print(f"{'='*50}", flush=True)

if __name__ == "__main__":
    main()