#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
隆阳区人民政府 - 环境影响评价
https://www.longyang.gov.cn/xxgk/bm/bsssthjjlyfj/fdzdgknr/sthj/hjyxpj.htm

大汉版通 CMS：列表相对路径 info/2918/{id}.htm + 倒序分页 hjyxpj/{k}.htm（第N页=86-N）
详情：meta ArticleTitle/PubDate/ContentSource + 正文 div.m-4.content + 附件 div.attach
"""
import sys, os, re, time, json
sys.path.insert(0, '/root/gov_crawler')
from crawler_lib import fetch_page, clean_html
import sqlite3
from urllib.parse import urljoin

SITE_NAME = "隆阳区-环境影响评价"
BASE = "https://www.longyang.gov.cn"
LIST_BASE = f"{BASE}/xxgk/bm/bsssthjjlyfj/fdzdgknr/sthj"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
THREE_YEARS_AGO = "2023-06-01"
TOTAL_PAGES = 85
PAGE_SIZE = 15

def fetch_list_page(page_no):
    """第1页: hjyxpj.htm；第N页(N>=2): hjyxpj/{86-N}.htm"""
    if page_no == 1:
        url = f"{LIST_BASE}/hjyxpj.htm"
    else:
        url = f"{LIST_BASE}/hjyxpj/{86 - page_no}.htm"
    html = fetch_page(url, encoding='utf-8')
    return html

def parse_list(html):
    """<li>【隆阳区生态环境局】<a href="../../../../../info/2918/14126999.htm" target="_blank" title="完整标题">截断标题</a><span style="float: right;">2026-07-31</span></li>"""
    items = []
    pattern = re.compile(
        r'<li>\s*【[^】]*】\s*<a\s+href="([^"]+)"[^>]*title="([^"]+)"[^>]*>.*?</a>\s*<span[^>]*>\s*(\d{4}-\d{2}-\d{2})\s*</span>',
        re.DOTALL
    )
    for m in pattern.finditer(html):
        href, title, date_str = m.group(1).strip(), m.group(2).strip(), m.group(3)
        if href and title:
            items.append((title, href, date_str))
    return items

def fetch_detail(url):
    """详情页: meta ArticleTitle/PubDate + 正文 div.m-4.content"""
    html = fetch_page(url, encoding='utf-8')
    if not html:
        return None, None, None

    pub_date = ""
    m = re.search(r'PubDate"\s+Content="(\d{4}-\d{2}-\d{2})', html)
    if m:
        pub_date = m.group(1)

    content = ""
    m = re.search(r'<div\s+class="m-4\s+content"[^>]*>(.*?)</div>\s*<div\s+class="attach"', html, re.DOTALL)
    if m:
        content = m.group(1).strip()

    if content:
        # 移除脚本/样式/二维码区
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL | re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL | re.I)
        content = re.sub(r'<div\s+class="ewm"[^>]*>.*?(?:</div>\s*){2}', '', content, flags=re.DOTALL | re.I)
        # 移除 vsbcontent 装饰段落标记（保留内容）
        content = re.sub(r'<p\s+class="vsbcontent_(?:start|end)"[^>]*>', '<p>', content)
        # 保留结构化 HTML，清理 style/class
        content = clean_html(content)

    title = ""
    m = re.search(r'ArticleTitle"\s+Content="([^"]+)"', html)
    if m:
        title = m.group(1).strip()

    return title or None, content or None, pub_date or None

def insert_to_db(items):
    if not items:
        print("  ⏭ 无数据")
        return
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in items:
        try:
            cur = db.execute("INSERT OR IGNORE INTO gov_raw "
                "(site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
                item.get("site_name", "")[:200],
                item.get("url", ""),
                item.get("url", ""),
                (item.get("title") or "")[:500],
                (item.get("pub_date") or "")[:10],
                (item.get("summary") or "")[:500],
                item.get("content", ""),
                "active", "", (item.get("tags") or "")[:100],
            ))
            if cur.rowcount > 0:
                ok += 1
            else:
                skip += 1
        except Exception as e:
            print(f"    ⚠ 入库异常: {e}")
            skip += 1
    db.commit()
    db.execute("INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?", (SITE_NAME,))
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}, FTS已同步")

def main():
    print(f"\n{'='*50}")
    print(f"🏠 {SITE_NAME}")
    print(f"{'='*50}")

    all_items = []
    seen_urls = set()

    MAX_PAGES = 1  # 增量用，每天只跑第1页
    if len(sys.argv) > 1 and sys.argv[1] == "full":
        MAX_PAGES = TOTAL_PAGES
        print(f"📋 全量模式：{TOTAL_PAGES}页")
    elif len(sys.argv) > 1 and sys.argv[1].isdigit():
        MAX_PAGES = min(int(sys.argv[1]), TOTAL_PAGES)

    for page_no in range(1, MAX_PAGES + 1):
        print(f"  📄 第 {page_no}/{MAX_PAGES} 页...", end=" ", flush=True)
        html = fetch_list_page(page_no)
        if not html:
            print("❌ 无响应")
            break
        items = parse_list(html)
        if not items:
            print("0 条（可能已到底）")
            break
        print(f"✅ {len(items)} 条")

        for title, href, date_str in items:
            full_url = urljoin(f"{LIST_BASE}/hjyxpj.htm", href)
            if full_url in seen_urls:
                continue
            seen_urls.add(full_url)

            if date_str and date_str < THREE_YEARS_AGO:
                continue

            print(f"    {title[:50]}...", end=" ", flush=True)
            dt, content, pub_date = fetch_detail(full_url)
            if dt:
                title = dt
            if pub_date:
                date_str = pub_date

            summary = re.sub(r'<[^>]+>', ' ', content or '').strip()[:300]
            summary = re.sub(r'\s+', ' ', summary)

            all_items.append({
                "site_name": SITE_NAME,
                "title": title,
                "url": full_url,
                "content": content or "",
                "pub_date": date_str or "",
                "summary": summary,
                "tags": "环评公示",
            })
            print(f"✅ {date_str}")
            time.sleep(0.3)

        if len(items) < PAGE_SIZE:
            print("    (末页)")
            break
        time.sleep(0.5)

    if all_items:
        insert_to_db(all_items)
    print(f"\n✅ 完成! 共 {len(all_items)} 条")


if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")