#!/usr/bin/env python3
"""徐圩新区 - 公示公告"""
import json, re, sys, requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = "/root/search.db"
SITE_NAME = "徐圩新区-公示公告"
CATEGORY = "公示公告"
GROUP = "徐圩新区"
BASE_URL = "http://www.xwxq.gov.cn/xxxq/gsgg/gsgg.html"
MAX_PAGES = 3

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept-Language": "zh-CN,zh;q=0.9",
}

SQL = """INSERT OR IGNORE INTO gov_raw
    (site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""


def init_db():
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=5000")
    return conn


def table_to_md(table, *args, **kwargs):
    """保留 HTML 表格结构（不转 md）"""
    return str(table)

def extract_content(content_div, detail_url):
    parts = []
    processed = set()
    for el in content_div.find_all(["p", "table", "img"], recursive=True):
        if id(el) in processed:
            continue
        processed.add(id(el))
        if el.name == "table":
            md = table_to_md(el)
            if md:
                parts.append(md)
        elif el.name == "img":
            src = el.get("src", "")
            if src:
                full_src = urljoin(detail_url, src) if not src.startswith("http") else src
                parts.append(f"![]({full_src})")
        elif el.name == "p":
            txt = el.get_text(" ", strip=True)
            if txt:
                txt = re.sub(r'[^\S\n]+', ' ', txt)
                parts.append(txt)
    return "\n\n".join(parts)


def extract_attachments(soup, detail_url):
    attachments = []
    for a_tag in soup.find_all("a", href=True):
        href = a_tag["href"].strip().lower()
        if any(href.endswith(ext) for ext in [".pdf", ".doc", ".docx", ".xls", ".xlsx", ".rar", ".zip"]):
            name = a_tag.get_text(strip=True) or href.split("/")[-1]
            full_url = urljoin(detail_url, a_tag["href"]) if not a_tag["href"].startswith("http") else a_tag["href"]
            if not any(a["url"] == full_url for a in attachments):
                attachments.append({"name": name, "url": full_url})
    return attachments


def extract_list_page():
    """Page 1 has all items inline. AJAX pages blocked by WAF."""
    try:
        r = requests.get(BASE_URL, headers=HEADERS, timeout=30)
        r.raise_for_status()
    except Exception as e:
        print(f"  [ERR] list page: {e}", file=sys.stderr)
        return []

    r.encoding = "utf-8"
    soup = BeautifulSoup(r.text, "html.parser")
    items = []

    # Each item is wrapped in its own ul.right_news_list
    for ul in soup.find_all("ul", class_="right_news_list"):
        li = ul.find("li")
        if not li:
            continue
        a_tag = li.find("a")
        if not a_tag or not a_tag.get("href"):
            continue
        href = a_tag["href"].strip()
        title = a_tag.get_text(strip=True)
        if not title:
            continue

        # Date from span.r: [YYYY-MM-DD]
        date_str = ""
        r_span = li.find("span", class_="r")
        if r_span:
            txt = r_span.get_text(strip=True)
            m = re.search(r'(\d{4}-\d{2}-\d{2})', txt)
            if m:
                date_str = m.group(1)

        detail_url = urljoin(BASE_URL, href)
        items.append({"title": title, "url": detail_url, "date": date_str})

    print(f"  Page 1: {len(items)} items")
    return items


def extract_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.raise_for_status()
    except Exception:
        return None, None, None, []

    r.encoding = "utf-8"
    soup = BeautifulSoup(r.text, "html.parser")

    # Title: div.art-title
    title_div = soup.find("div", class_="art-title")
    title = title_div.get_text(strip=True) if title_div else ""

    # Date: div.art-menu > span "发布日期：YYYY-MM-DD"
    date_str = ""
    art_menu = soup.find("div", class_="art-menu")
    if art_menu:
        for span in art_menu.find_all("span"):
            txt = span.get_text(strip=True)
            m = re.search(r'发布日期[：:]\s*(\d{4}-\d{2}-\d{2})', txt)
            if m:
                date_str = m.group(1)
                break

    # Content: div.art-main#menu
    content = ""
    art_main = soup.find("div", class_="art-main")
    if art_main:
        for tag in art_main.find_all(["script", "style"]):
            tag.decompose()
        content = extract_content(art_main, url)

    attachments = extract_attachments(soup, url)

    if len(content.strip()) < 20:
        fallback = f"[{title or url.split('/')[-1]}]({url})"
        if attachments:
            for a in attachments:
                fallback += f"\n\n附件：[{a['name']}]({a['url']})"
        content = fallback

    return title, date_str, content, attachments


def crawl(test_mode=False):
    conn = init_db()
    cur = conn.cursor()
    total = 0
    errors = 0

    items = extract_list_page()
    total_listed = len(items)

    for item in items:
        url = item["url"]
        cur.execute("SELECT id FROM gov_raw WHERE page_url = ?", (url,))
        if cur.fetchone():
            continue

        title, date_str, content, attachments = extract_detail(url)
        if not title:
            errors += 1
            continue

        final_title = title or item["title"]
        final_date = date_str or item["date"]
        att_json = json.dumps(attachments, ensure_ascii=False) if attachments else ""
        summary = (content[:200] if content else final_title).strip()
        if not summary:
            summary = final_title

        try:
            cur.execute(SQL, (
                SITE_NAME, url, url, final_title.strip(), final_date,
                summary, content, CATEGORY, att_json, GROUP,
            ))
            conn.commit()
            if cur.rowcount > 0:
                total += 1
        except Exception as e:
            print(f"  DB error: {e}", file=sys.stderr)
            errors += 1

        if test_mode and total >= 5:
            break

    conn.close()
    print(f"\n[DONE] {SITE_NAME}")
    print(f"  New: {total}, Errors: {errors}, Listed total: {total_listed}")
    return total


if __name__ == "__main__":
    test_mode = "--test" in sys.argv
    crawl(test_mode=test_mode)