#!/usr/bin/env python3
"""
嘉兴市桐乡市 - 建设项目环境影响评价信息公示 (浙江政务服务网)
https://jxtx.zjzwfw.gov.cn/col/col1460343/index.html
CMS: 浙江政务网JCMS, 单页无分页, 9条
列表: <a class="bt_link" href="/art/...">标题</a>
详情: div#zoom 正文
"""

import sys, os, re, json, requests, time
from bs4 import BeautifulSoup

BASE_URL = "https://jxtx.zjzwfw.gov.cn"
LIST_URL = "https://jxtx.zjzwfw.gov.cn/col/col1460343/index.html"
SITE_NAME = "嘉兴桐乡市-建设项目环评公示"
IMPORT_SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "import_jsonl.py")

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

TBL_MARKER_PREFIX = "___TBL_"
TBL_MARKER_SUFFIX = "___"


def extract_table_tags(html_str):
    """从HTML字符串中提取所有<table>标签，用唯一标记替换，返回(modified_html, [(marker, table_html)])"""
    markers = []
    
    def replace_table(m):
        idx = len(markers)
        key = f"{TBL_MARKER_PREFIX}{idx}{TBL_MARKER_SUFFIX}"
        markers.append((key, m.group(0)))
        return key
    
    # 用非贪婪匹配提取table（从<table到</table>），注意嵌套table的情况
    modified = re.sub(
        r"<table[^>]*>.*?</table>",
        replace_table,
        html_str,
        flags=re.DOTALL | re.IGNORECASE
    )
    return modified, markers


def parse_list(html):
    """解析列表页"""
    items = []
    for m in re.finditer(r"<a[^>]+href='(/(?:art|col)/[^']+?)'[^>]+title='([^']+)'", html):
        href = m.group(1).strip()
        title = m.group(2).strip()
        full_url = f"{BASE_URL}{href}" if href.startswith("/") else href
        date_str = ""
        dm = re.search(r"(\d{4}-\d{1,2}-\d{1,2})", html[m.end():m.end()+300])
        if dm: date_str = dm.group(1)
        items.append({"title": title, "url": full_url, "date": date_str})
    return items


def get_content(url):
    """获取详情页正文，表格HTML原样保留"""
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        r.raise_for_status()
    except:
        return "", "", ""

    soup = BeautifulSoup(r.text, "html.parser")
    full_title = ""
    pub_date = ""

    title_tag = soup.find("title")
    if title_tag:
        t = title_tag.get_text(strip=True)
        for sep in ["_浙江", "_", " - "]:
            idx = t.find(sep)
            if idx > 0: t = t[:idx]; break
        full_title = t
    if not full_title:
        h1 = soup.find("h1")
        if h1: full_title = h1.get_text(strip=True)

    for m in re.finditer(r'<meta[^>]+PubDate[^>]+content="(\d{4}-\d{1,2}-\d{1,2})"', r.text):
        pub_date = m.group(1)
        break
    if not pub_date:
        dm = re.search(r"(\d{4}-\d{1,2}-\d{1,2})", r.text[:2000])
        if dm: pub_date = dm.group(1)

    content_html = ""
    zoom = soup.find(id="zoom") or soup.select_one(".art_con, .content, .article-content, .TRS_Editor")
    if zoom:
        # 1) 提取zoom的HTML字符串，把table替换为标记
        zoom_html = str(zoom)
        zoom_html, table_markers = extract_table_tags(zoom_html)

        # 2) 重新解析替换后的HTML
        zoom_soup = BeautifulSoup(zoom_html, "html.parser")

        # 3) 移除script/style
        for tag in zoom_soup.find_all(["script", "style"]):
            tag.decompose()

        # 4) 附件/图片相对路径绝对化
        for a in zoom_soup.find_all("a", href=True):
            h = a["href"]
            if h.startswith("/"):
                a["href"] = BASE_URL + h
        for img in zoom_soup.find_all("img", src=True):
            h = img["src"]
            if h.startswith("/"):
                img["src"] = BASE_URL + h

        # 5) 保留 p 段落 HTML；内联标签 unwrap 但保留 <p>/<table>/<img>/<a>
        for tag in zoom_soup.find_all(["span", "font", "b", "strong", "i", "em", "u"]):
            tag.unwrap()

        # 6) 序列化保留 <p> 结构 (search_app 渲染要求 content 含闭合 <p>)
        content_html = zoom_soup.decode_contents()
        content_html = re.sub(r"<p[^>]*>", "<p>", content_html)
        content_html = re.sub(r"\s*\n\s*", "", content_html)
        content_html = content_html.strip()

        # 7) 恢复表格HTML
        for key, tbl_html in table_markers:
            content_html = content_html.replace(key, tbl_html)

    if not content_html or len(content_html) < 20:
        content_html = f'<p><a href="{url}">{full_title or "无正文"}</a></p>'

    return content_html, pub_date, full_title


def main():
    r = requests.get(LIST_URL, headers=HEADERS, timeout=20)
    r.encoding = "utf-8"
    items = parse_list(r.text)
    print(f"列表: {len(items)} 条", file=sys.stderr)

    all_items = []
    for i, item in enumerate(items):
        print(f"  [{i+1}/{len(items)}] {item['title'][:40]}...", end=" ", file=sys.stderr)
        content, pub_date, full_title = get_content(item["url"])
        date = item["date"] or pub_date
        use_title = full_title or item["title"]
        all_items.append({
            "title": use_title,
            "page_url": item["url"],
            "content": content,
            "publish_date": date,
            "site_name": SITE_NAME,
            "attachments": [],
        })
        print("OK", file=sys.stderr)
        time.sleep(0.3)

    if not all_items:
        print("no data", file=sys.stderr)
        return

    # 直接写入 search.db（避免 import_jsonl.py FTS重建超时）
    import sqlite3
    DB_PATH = "/root/search.db"
    db = sqlite3.connect(DB_PATH, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=OFF")

    added = 0
    for item in all_items:
        try:
            db.execute(
                "INSERT OR IGNORE INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments) VALUES (?,?,?,?,?,?,?,?)",
                (item["title"][:500], item["page_url"][:1000], item["content"],
                 item["publish_date"][:20], item["site_name"], item["page_url"],
                 "synced", json.dumps(item.get("attachments", []), ensure_ascii=False) if item.get("attachments") else "")
            )
            if db.total_changes > 0:
                added += 1
        except:
            pass
    db.commit()

    # 增量FTS更新
    db.execute("""
        INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary)
        SELECT r.rowid, r.title, r.site_name, substr(r.content, 1, 500)
        FROM gov_raw r
        WHERE r.site_name = ?
        AND r.content IS NOT NULL AND r.content != ''
        AND r.rowid NOT IN (SELECT rowid FROM gov_search)
    """, (SITE_NAME,))
    db.commit()
    db.close()

    print(f"DB: +{added} / {len(all_items)} items", file=sys.stderr)

    for item in all_items:
        print(json.dumps(item, ensure_ascii=False))

if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"time: {time.time()-t0:.1f}s", file=sys.stderr)