#!/usr/bin/env python3
"""
汉源县人民政府 - 环境保护
URL: http://www.hanyuan.gov.cn/gongkai/kuozhan/306.html
CMS: 自定义政府信息公开平台 (.NET, LayUI)
List: ?page=N 分页 (20条/页, 共30页)
Detail: div.msg-content (表格HTML保留, 段落文本)
"""
import sys, os, re, json, time, sqlite3, html as html_mod
from datetime import datetime, timedelta
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

# ── 配置 ──
BASE = "http://www.hanyuan.gov.cn"
LIST_URL = BASE + "/gongkai/kuozhan/306.html"
SITE = "汉源县人民政府"
COLUMN = "环境保护"
PROVINCE = "四川"
DELAY = 3        # 请求间隔秒数，避免WAF
TOTAL_PAGES = 5   # 最多5页

# DB
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
session = requests.Session()
session.headers.update(HEADERS)


def log(msg):
    print(msg, file=sys.stderr, flush=True)


def strip_html(text):
    """Convert HTML to clean plain text for FTS summary."""
    if not text:
        return ""
    text = html_mod.unescape(text)
    text = re.sub(r'</?(?:p|div|h[1-6]|li|tr|blockquote|section|article|table|br\s*/?)[^>]*>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'<[^>]+>', '', text)
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    return text.strip()


def fetch(url, timeout=(10, 30)):
    """请求政府站"""
    try:
        r = session.get(url, timeout=timeout)
        r.encoding = "utf-8"
        return r.text if r.status_code == 200 else None
    except Exception as e:
        log(f"  [ERR] fetch failed: {e}")
        return None


def get_list_items(html):
    """从列表页提取条目"""
    items = []
    soup = BeautifulSoup(html, "html.parser")
    ul = soup.find("ul", class_="content-list")
    if not ul:
        return items
    for li in ul.find_all("li", recursive=False):
        a = li.find("a")
        span = li.find("span")
        if a and span:
            href = a.get("href", "").strip()
            title = a.get_text(strip=True)
            date_str = span.get_text(strip=True)
            if href and title:
                full_url = urljoin(BASE, href)
                items.append({"url": full_url, "title": title, "date": date_str})
    return items


def extract_detail(detail_url, list_title):
    """提取详情页"""
    html = fetch(detail_url)
    if not html:
        return None, [], list_title

    soup = BeautifulSoup(html, "html.parser")

    # 标题：meta ArticleTitle
    meta_title = soup.find("meta", attrs={"name": "ArticleTitle"})
    if meta_title and meta_title.get("content"):
        full_title = meta_title["content"].strip()
    else:
        h1 = soup.find("h1")
        full_title = h1.get_text(strip=True) if h1 else list_title

    # 日期
    date_str = ""
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date and meta_date.get("content"):
        date_str = meta_date["content"].strip()[:10]
    if not date_str:
        m = re.search(r"发布时间[：:]\s*(\d{4}-\d{1,2}-\d{1,2})", html)
        if m:
            date_str = m.group(1)

    # 附件
    attachments = []
    for a_tag in soup.find_all("a", href=re.compile(r'\.(doc|docx|pdf|xls|xlsx|zip|rar)$', re.I)):
        href = a_tag.get("href", "").strip()
        if href:
            name = a_tag.get_text(strip=True) or "附件"
            attachments.append({"name": name, "url": urljoin(detail_url, href)})

    # 正文：div.msg-content
    content_div = soup.find("div", class_="msg-content")
    if not content_div:
        # 后备
        for cls in ["news-content", "article-content", "content", "nr", "main-content"]:
            content_div = soup.find("div", class_=cls)
            if content_div:
                break

    if not content_div:
        return None, attachments, full_title

    # 移除无用元素
    for tag in content_div.find_all(["script", "style", "meta"]):
        tag.decompose()

    def _extract_block(el, depth=0):
        """递归提取元素内容，保持段落分隔和表格HTML"""
        if depth > 5:
            return []
        parts = []
        for child in el.children:
            if child.name is None:
                continue
            tn = child.name.lower()
            if tn == "table":
                tbl_soup = BeautifulSoup(str(child), "html.parser")
                for sp in tbl_soup.find_all(["span", "font"]):
                    sp.unwrap()
                parts.append(str(tbl_soup))
            elif tn == "p":
                text = child.get_text(separator="", strip=True)
                if text:
                    inner_tables = child.find_all("table")
                    if inner_tables:
                        for t in inner_tables:
                            tbl_s = BeautifulSoup(str(t), "html.parser")
                            for sp in tbl_s.find_all(["span", "font"]):
                                sp.unwrap()
                            parts.append(str(tbl_s))
                    else:
                        parts.append(text)
            elif tn in ("h1", "h2", "h3", "h4", "h5"):
                text = child.get_text(separator="", strip=True)
                if text:
                    parts.append(text)
            elif tn == "div":
                sub = _extract_block(child, depth + 1)
                parts.extend(sub)
            elif tn == "ul":
                for li in child.find_all("li", recursive=False):
                    text = li.get_text(separator="", strip=True)
                    if text:
                        parts.append(text)
        return parts

    # 遍历 msg-content 的子元素
    parts = []
    for child in content_div.children:
        if child.name is None:
            continue
        tag_name = child.name.lower()
        if tag_name == "h1":
            continue
        if tag_name == "div":
            cls_list = child.get("class", [])
            if any(c in ["intro", "title", "time", "info", "bt"] for c in cls_list):
                continue
        sub = _extract_block(child)
        parts.extend(sub)

    content = "\n\n".join(parts).strip()

    if len(content) < 20 and attachments:
        content = f'<p><a href="{detail_url}">{full_title}</a></p>'
        for att in attachments:
            content += f"\n[{att['name']}]({att['url']})"
    elif content and attachments:
        content += "\n\n**附件：**"
        for att in attachments:
            content += f"\n[{att['name']}]({att['url']})"

    return content, attachments, full_title


def import_to_db(record):
    """写入 search.db + FTS"""
    try:
        db = sqlite3.connect(DB_PATH, timeout=10)
        title = (record.get("title") or "")[:500]
        page_url = (record.get("page_url") or "")[:1000]
        content = strip_html(record.get("content") or "")
        publish_date = (record.get("publish_date") or "")[:20]
        site_name = (record.get("site_name") or "unknown")[:100]
        attachments_str = json.dumps(record.get("attachments") or [], ensure_ascii=False)

        old_rowids = db.execute(
            "SELECT rowid FROM gov_raw WHERE page_url = ?", (page_url,)
        ).fetchall()
        for (rid,) in old_rowids:
            db.execute("DELETE FROM gov_search WHERE rowid = ?", (rid,))

        db.execute(
            "INSERT OR REPLACE INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?)",
            (title, page_url, content, publish_date, site_name, page_url, attachments_str)
        )

        new_rowid = db.execute("SELECT last_insert_rowid()").fetchone()[0]
        summary = content[:500] if content else title[:500]
        db.execute(
            "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)",
            (new_rowid, title, site_name, summary)
        )
        db.commit()
        db.close()
        return True
    except Exception as e:
        log(f"  [ERR] DB import failed: {e}")
        return False


def output_item(item, idx):
    record = {
        "title": item.get("title", ""),
        "page_url": item.get("url", ""),
        "publish_date": item.get("date", ""),
        "content": item.get("content", ""),
        "attachments": item.get("attachments", []),
        "site_name": f"{SITE}-{COLUMN}",
        "column": COLUMN,
        "province": PROVINCE,
    }
    print(json.dumps(record, ensure_ascii=False))
    ok = import_to_db(record)
    return ok


def crawl_all(label="incremental"):
    """爬取：增量(第1页) 或 全量(前5页)"""
    if label == "incremental":
        log(f"\n{'='*50}")
        log(f"🏠 {SITE} - {COLUMN}")
        log(f"📄 增量（第1页）")
        log(f"{'='*50}")
        cutoff = datetime.now() - timedelta(hours=48)
        urls = [LIST_URL]
    else:
        log(f"\n{'='*50}")
        log(f"🏠 {SITE} - {COLUMN}")
        log(f"📄 全量前{TOTAL_PAGES}页")
        log(f"{'='*50}")
        cutoff = datetime.now() - timedelta(days=365 * 3)
        urls = [LIST_URL]
        for p in range(2, TOTAL_PAGES + 1):
            urls.append(f"{LIST_URL}?page={p}")

    all_items = []
    seen_urls = set()
    for page_url in urls:
        html = fetch(page_url)
        if not html:
            log(f"  [WARN] 页面获取失败: {page_url}")
            continue
        items = get_list_items(html)
        log(f"  {page_url.split('?')[0].split('/')[-1] if 'page' in page_url else '第1页'}: {len(items)}条")
        for it in items:
            if it["url"] not in seen_urls:
                seen_urls.add(it["url"])
                all_items.append(it)

    log(f"合计: {len(all_items)}条")

    total = len(all_items)
    ok_count = 0
    for idx, item in enumerate(all_items, 1):
        try:
            item_date = datetime.strptime(item["date"], "%Y-%m-%d")
            if item_date < cutoff:
                continue
        except (ValueError, KeyError):
            pass

        content, attachments, full_title = extract_detail(item["url"], item["title"])
        if content is None:
            log(f"  [{idx}/{total}] ⏭️ 详情空: {item['title'][:40]}")
            continue
        item["title"] = full_title
        item["content"] = content
        item["attachments"] = attachments
        if output_item(item, idx):
            ok_count += 1
        if idx % 20 == 0:
            log(f"  [PROGRESS] {idx}/{total}")
        time.sleep(DELAY)

    log(f"\n[DONE] 共处理 {ok_count} 条")
    return ok_count


if __name__ == "__main__":
    mode = sys.argv[1] if len(sys.argv) > 1 else "incremental"
    crawl_all(mode)