#!/usr/bin/env python3
"""
威海市文登区人民政府 - 环境影响评价审批信息
URL: http://www.wendeng.gov.cn/col/col62890/index.html?number=WDA50150301
CMS: TRS WCM (jpage dataproxy POST 分页)
List: HTML第1页内嵌XML datastore (~6条/页)
      补充数据从 dataproxy.jsp POST 获取
Detail: div#zoom (表格HTML保留，段落提取文本)
"""
import sys, os, re, json, time, sqlite3, html
from datetime import datetime
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# ── 配置 ──
BASE = "http://www.wendeng.gov.cn"
LIST_URL = BASE + "/col/col62890/index.html?number=WDA50150301"
PROXY_URL = BASE + "/module/web/jpage/dataproxy.jsp"
SITE = "威海市文登区人民政府"
COLUMN = "环境影响评价审批信息"
PROVINCE = "山东"
TOTAL_PAGES = 5  # 最多5页

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

PROXY_PARAMS = {
    "page": 1, "appid": 1, "webid": 137, "path": "/",
    "columnid": 62890, "unitid": 148240,
}

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
session = requests.Session()
session.headers.update(HEADERS)
session.verify = False


def log(msg):
    print(msg, file=sys.stderr, flush=True)


def fetch(url, post_data=None, retries=3):
    for attempt in range(retries):
        try:
            if post_data:
                r = session.post(url, data=post_data, timeout=30)
            else:
                r = session.get(url, timeout=30)
            r.encoding = "utf-8"
            return r.text
        except Exception as e:
            log(f"  [WARN] 请求失败 (尝试 {attempt+1}/{retries}): {e}")
            time.sleep(2)
    return ""


def parse_list_items(cdata_text):
    """从CDATA片段解析列表项：url, title, date"""
    items = []
    records = re.findall(r"<record><!\[CDATA\[(.*?)\]\]></record>", cdata_text, re.DOTALL)
    for cdata in records:
        a_match = re.search(r"<a\s+href='([^']+)'\s*title='([^']*)'", cdata)
        span_match = re.search(r"<span>([^<]+)</span>", cdata)
        if a_match and span_match:
            href = a_match.group(1).strip()
            title = a_match.group(2).strip()
            date_str = span_match.group(1).strip()
            full_url = urljoin(BASE, href)
            items.append({"url": full_url, "title": title, "date": date_str})
    return items


def get_page1_items():
    """从HTML第1页内嵌datastore获取列表"""
    html = fetch(LIST_URL)
    if not html:
        return []
    m = re.search(r"<datastore>(.*?)</datastore>", html, re.DOTALL)
    if not m:
        return []
    return parse_list_items(m.group(1))


def get_proxy_items():
    """从dataproxy page=1获取补充数据"""
    xml_text = fetch(PROXY_URL, post_data=dict(PROXY_PARAMS))
    if not xml_text:
        return []
    return parse_list_items(xml_text)


def extract_detail(detail_url, list_title):
    """提取详情页：完整标题(meta ArticleTitle)、正文(表格HTML保留)、附件、日期"""
    html = fetch(detail_url)
    if not html:
        return None, [], list_title

    soup = BeautifulSoup(html, "html.parser")

    # 标题：优先 meta ArticleTitle
    meta_title = soup.find("meta", attrs={"name": "ArticleTitle"})
    if meta_title and meta_title.get("content"):
        full_title = meta_title["content"].strip()
    else:
        title_tag = soup.find("title")
        full_title = title_tag.get_text(strip=True) if title_tag else list_title
        # 去掉站点名前缀
        full_title = re.sub(r'^威海市文登区人民政府\s*\S*\s*', '', full_title)

    if not full_title or full_title == list_title:
        full_title = list_title

    # 日期：meta PubDate
    date_str = ""
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date and meta_date.get("content"):
        date_str = meta_date["content"].strip()[:10]

    # 附件（含doc/docx/pdf等）
    attachments = []
    for a_tag in soup.find_all("a", href=re.compile(r'\.(doc|docx|pdf|xls|xlsx|zip|rar)$', re.I)):
        href = a_tag.get("href", "").strip()
        if href:
            name = a_tag.get_text(strip=True) or "附件"
            attachments.append({"name": name, "url": urljoin(detail_url, href)})

    # 正文：div#zoom
    zoom = soup.find("div", id="zoom")
    if not zoom:
        return full_title, attachments, full_title

    # 移除无用元素
    for tag in zoom.find_all(["script", "style", "meta"]):
        tag.decompose()

    parts = []
    for child in zoom.children:
        if child.name is None:
            continue
        tag_name = child.name.lower()

        if tag_name == "table":
            # 表格HTML原样保留
            tbl_soup = BeautifulSoup(str(child), "html.parser")
            for sp in tbl_soup.find_all(["span", "font"]):
                sp.unwrap()
            parts.append(str(tbl_soup))
        elif tag_name == "p":
            inner_tables = child.find_all("table")
            if inner_tables:
                for t in inner_tables:
                    tbl_soup = BeautifulSoup(str(t), "html.parser")
                    for sp in tbl_soup.find_all(["span", "font"]):
                        sp.unwrap()
                    parts.append(str(tbl_soup))
            else:
                text = child.get_text(separator="", strip=True)
                if text:
                    parts.append(text)
        elif tag_name == "div":
            inner_tables = child.find_all("table", recursive=False)
            if inner_tables:
                for t in inner_tables:
                    tbl_soup = BeautifulSoup(str(t), "html.parser")
                    for sp in tbl_soup.find_all(["span", "font"]):
                        sp.unwrap()
                    parts.append(str(tbl_soup))
            else:
                text = child.get_text(separator="", strip=True)
                if text and len(text) > 10:
                    parts.append(text)
        elif tag_name in ("h1", "h2", "h3", "h4", "h5"):
            text = child.get_text(separator="", strip=True)
            if text:
                parts.append(text)

    content = "\n\n".join(parts).strip()

    # 空内容回退
    if len(content) < 20 and attachments:
        content = f'<p><a href="{detail_url}">{full_title}</a></p>'
        for att in attachments:
            content += f"\n[{att['name']}]({att['url']})"
    elif content and attachments:
        content += "\n\n**附件：**"
        for att in attachments:
            content += f"\n[{att['name']}]({att['url']})"

    return content, attachments, full_title


def import_to_db(record):
    """将单条记录写入 search.db (gov_raw + gov_search FTS)"""
    try:
        db = sqlite3.connect(DB_PATH, timeout=10)
        title = (record.get("title") or "")[:500]
        page_url = (record.get("page_url") or "")[:1000]
        content = record.get("content") or ""
        publish_date = (record.get("publish_date") or "")[:20]
        site_name = (record.get("site_name") or "unknown")[:100]
        attachments_str = json.dumps(record.get("attachments") or [], ensure_ascii=False)

        # 删除旧数据（如果page_url已存在）
        old_rowids = db.execute(
            "SELECT rowid FROM gov_raw WHERE page_url = ?", (page_url,)
        ).fetchall()
        for (rid,) in old_rowids:
            db.execute("DELETE FROM gov_search WHERE rowid = ?", (rid,))

        # INSERT OR REPLACE
        db.execute(
            "INSERT OR REPLACE INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments, script_name) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?, 'crawl_wendeng.py')",
            (title, page_url, content, publish_date, site_name, page_url, attachments_str)
        )

        # FTS增量同步
        new_rowid = db.execute("SELECT last_insert_rowid()").fetchone()[0]
        summary = content[:500] if content else title[:500]
        # 2026-09-22: 先提交 gov_raw —— 库上触发器已维护 FTS，下面这条手动写入会因
        #   rowid 重复而 IntegrityError；不先 commit 会把 gov_raw 那条一并回滚（静默丢数据）
        db.commit()
        db.execute(
            "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)",
            (new_rowid, title, site_name, summary)
        )
        db.commit()
        db.close()
        log(f"  \u2705 {title[:30]}")
        return True
    except Exception as e:
        log(f"  [ERR] DB import failed: {e}")
        return False


def output_item(item, idx):
    record = {
        "title": item.get("title", ""),
        "page_url": item.get("url", ""),
        "publish_date": item.get("date", ""),
        "content": item.get("content", ""),
        "attachments": item.get("attachments", []),
        "site_name": SITE,
        "column": COLUMN,
        "province": PROVINCE,
    }
    ok = import_to_db(record)
    return ok


def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--pages", type=int, default=TOTAL_PAGES)
    args = parser.parse_args()

    total_pages = min(args.pages, TOTAL_PAGES)
    count = 0

    # 第1页：从HTML内嵌datastore获取
    items = get_page1_items()
    seen = {item["url"] for item in items}
    log(f"\u2460 HTML第1页: {len(items)}条")

    # 补充数据：从dataproxy page=1获取，去重
    extra = get_proxy_items()
    extra_deduped = [e for e in extra if e["url"] not in seen]
    # 如果已有第1页数据，补充的dataproxy数据中多翻页以覆盖total_pages
    # 按顺序取够 (total_pages * 每页条数 - 已有的第1页条数)
    page_size = max(len(items), 6)  # 每页约6条
    need = total_pages * page_size - len(items)
    if need > 0 and extra_deduped:
        items += extra_deduped[:need]
    log(f"\u2461 dataproxy补充: {len(extra_deduped)}条 (需{need}条, 共{len(items)}条)")

    if not items:
        log("没有找到任何列表项，退出")
        return

    for idx, item in enumerate(items, 1):
        url = item["url"]
        list_title = item["title"]
        log(f"  ({idx}/{len(items)}) 详情: {url.split('/')[-1]}")
        content, attachments, full_title = extract_detail(url, list_title)
        if content is None:
            log(f"    \u2757 详情提取失败: {url}")
            continue
        item["title"] = full_title
        item["content"] = content
        item["attachments"] = attachments
        ok = output_item(item, idx)
        if ok:
            count += 1
        time.sleep(0.5)  # 礼貌延迟

    log(f"\n\u2705 {SITE}-{COLUMN} 爬取完成，共入库 {count} 条")


if __name__ == "__main__":
    main()