#!/usr/bin/env python3
"""芦山县人民政府 - 公示公告"""
import json, re, sys, time, requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = "/root/search.db"
SITE_NAME = "芦山县-公示公告"
CATEGORY = "公示公告"
BASE_URL = "http://www.yals.gov.cn/gongkai/jichu/40016.html"
SITE_ID = "301144949"
MAX_PAGES = 100

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}

SQL = """INSERT OR IGNORE INTO gov_raw
    (site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""


def init_db():
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=5000")
    return conn


def decode_resp(r):
    raw = r.content
    ct = r.headers.get("Content-Type", "")
    if "gb2312" in ct.lower() or "gbk" in ct.lower():
        return raw.decode("gbk", errors="replace")
    raw_lower = raw[:2000].lower()
    if b"gb2312" in raw_lower or b"gbk" in raw_lower or b"charset=gb" in raw_lower:
        return raw.decode("gbk", errors="replace")
    return raw.decode("utf-8", errors="replace")


def table_to_md(table, *args, **kwargs):
    """保留 HTML 表格结构（不转 md）"""
    return str(table)

def extract_content(xqing_div, detail_url):
    """Extract content from xqing-web-box div."""
    parts = []
    processed = set()

    for el in xqing_div.find_all(["p", "table", "img", "h1", "h2", "h3", "h4"], recursive=True):
        if id(el) in processed:
            continue
        processed.add(id(el))

        if el.name == "table":
            md = table_to_md(el)
            if md:
                parts.append(md)

        elif el.name == "img":
            src = el.get("src", "")
            if src:
                alt = el.get("alt", "")
                full_src = urljoin(detail_url, src) if not src.startswith("http") else src
                parts.append(f"![{alt}]({full_src})" if alt else f"![]({full_src})")
            # Mark any nested img in processed
            for sub_img in el.find_all("img", recursive=True):
                processed.add(id(sub_img))

        elif el.name == "p":
            txt = el.get_text(" ", strip=True)
            if txt:
                # Space cleanup
                txt = re.sub(r'[^\S\n]+', ' ', txt)
                parts.append(txt)

        else:
            # h1-h4 headings
            txt = el.get_text(" ", strip=True)
            if txt:
                parts.append(txt)

    return "\n\n".join(parts)


def extract_attachments(soup, detail_url):
    """Extract attachment links from xqing-web-file div."""
    file_div = soup.find("div", class_="xqing-web-file")
    if not file_div:
        return []
    attachments = []
    for a_tag in file_div.find_all("a", href=True):
        href = a_tag["href"].strip()
        name = a_tag.get_text(strip=True)
        if not name:
            name = href.split("/")[-1]
        full_url = urljoin(detail_url, href) if not href.startswith("http") else href
        attachments.append({"name": name, "url": full_url})
    return attachments


def extract_list_page(page_num):
    """Extract list items from a given page number."""
    if page_num == 1:
        url = BASE_URL
    else:
        url = f"{BASE_URL}?page={page_num}"

    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.raise_for_status()
    except Exception as e:
        print(f"  [ERR] page {page_num}: {e}", file=sys.stderr)
        return []

    html = decode_resp(r)
    soup = BeautifulSoup(html, "html.parser")
    items = []

    ul = soup.select_one("div.data-list > ul.content-list")
    if not ul:
        print(f"  [WARN] page {page_num}: no content-list found", file=sys.stderr)
        return []

    for li in ul.find_all("li", recursive=False):
        a_tag = li.find("a")
        if not a_tag or not a_tag.get("href"):
            continue
        href = a_tag["href"].strip()
        title = a_tag.get("title", "") or a_tag.get_text(strip=True)
        if not title:
            continue

        date_span = li.find("span")
        date_str = date_span.get_text(strip=True) if date_span else ""

        detail_url = urljoin(BASE_URL, href)
        items.append({"title": title, "url": detail_url, "date": date_str})

    print(f"  Page {page_num}: {len(items)} items")
    return items


def extract_detail(url):
    """Extract detail page content."""
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.raise_for_status()
    except Exception as e:
        return None, None, None, []

    html = decode_resp(r)
    soup = BeautifulSoup(html, "html.parser")

    # Title: <h1>
    h1 = soup.find("h1")
    title = h1.get_text(strip=True) if h1 else ""

    # Date from div.intro > span
    date_str = ""
    intro = soup.find("div", class_="intro")
    if intro:
        for span in intro.find_all("span"):
            txt = span.get_text(strip=True)
            m = re.search(r'发布时间：\s*(\d{4}-\d{2}-\d{2})', txt)
            if m:
                date_str = m.group(1)
                break

    # Content
    xqing = soup.find("div", class_="xqing-web-box")
    content = ""
    if xqing:
        content = extract_content(xqing, url)

    # Attachments
    attachments = extract_attachments(soup, url)

    # Empty content fallback: if too short and we have attachments
    if len(content.strip()) < 20:
        fallback = f"[{title or url.split('/')[-1]}]({url})"
        if attachments:
            attach_links = []
            for a in attachments:
                attach_links.append(f"附件：[{a['name']}]({a['url']})")
            fallback += "\n\n" + "\n\n".join(attach_links)
        content = fallback

    return title, date_str, content, attachments


def crawl(test_mode=False, max_pages=MAX_PAGES):
    conn = init_db()
    cur = conn.cursor()
    total = 0
    errors = 0
    total_listed = 0

    for pn in range(1, max_pages + 1):
        items = extract_list_page(pn)
        total_listed += len(items)

        if not items:
            break

        for item in items:
            url = item["url"]
            # Check if already in DB
            cur.execute("SELECT id FROM gov_raw WHERE page_url = ?", (url,))
            if cur.fetchone():
                continue

            title, date_str, content, attachments = extract_detail(url)
            if not title:
                errors += 1
                continue

            final_title = title or item["title"]
            final_date = date_str or item["date"]
            att_json = json.dumps(attachments, ensure_ascii=False) if attachments else ""
            summary = (content[:200] if content else final_title).strip()
            if not summary:
                summary = final_title

            try:
                cur.execute(SQL, (
                    SITE_NAME, url, url, final_title.strip(), final_date,
                    summary, content, CATEGORY, att_json, CATEGORY,
                ))
                conn.commit()
                if cur.rowcount > 0:
                    total += 1
            except Exception as e:
                print(f"  DB error: {e}", file=sys.stderr)
                errors += 1

            if test_mode:
                # In test mode, just process first 5 items
                if total >= 5:
                    break

        if test_mode and total >= 5:
            break

    conn.close()
    print(f"\n[DONE] {SITE_NAME}")
    print(f"  New: {total}, Errors: {errors}, Listed total: {total_listed}")
    print(f"  (列表共 {total_listed} 条, 其中 {errors} 条详情处理异常)")
    return total


if __name__ == "__main__":
    mode = "test" if "--test" in sys.argv else "full"
    max_p = 5 if mode == "test" else MAX_PAGES
    if len(sys.argv) > 1 and sys.argv[1].isdigit():
        max_p = int(sys.argv[1])
    crawl(test_mode=(mode == "test"), max_pages=max_p)