#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
河北省发展和改革委员会 - 行政执法公示/事后公开/其他执法行为
https://hbdrc.hebei.gov.cn/ztzl/xzzfgs/shgk/qtzfxw/

TRS CMS：静态分页 createPageHTML(98, 0, "index", "html")
列表：tab_zfgs 表格（序号/项目名称/审批事项/批复文号/办结日期）
详情：同款表格，含审批事项/批准文号/项目名称/项目代码/行政相对人/批准内容/批准日期/批准机关 等
"""
import sys, os, re, time, json
sys.path.insert(0, '/root/gov_crawler')
from crawler_lib import fetch_page, clean_html
import sqlite3
from urllib.parse import urljoin

SITE_NAME = "河北省发改委-其他执法行为"
BASE = "https://hbdrc.hebei.gov.cn"
LIST_BASE = f"{BASE}/ztzl/xzzfgs/shgk/qtzfxw"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
THREE_YEARS_AGO = "2023-06-01"
TOTAL_PAGES = 98
PAGE_SIZE = 10


def fetch_list_page(page_no):
    """第1页: index.html；第N页: index_{N}.html"""
    if page_no == 1:
        url = f"{LIST_BASE}/index.html"
    else:
        url = f"{LIST_BASE}/index_{page_no}.html"
    html = fetch_page(url, encoding='utf-8')
    return html


def parse_list(html):
    """tab_zfgs 表格：<tr><td><a href="./202607/t20260730_137618.html">1</a></td>
    <td><a href="..." title="完整标题">截断标题</a></td><td>审批事项</td><td>批复文号</td><td>办结日期</td></tr>"""
    items = []
    idx = html.find('tab_zfgs')
    if idx < 0:
        return items
    end = html.find('</table>', idx)
    if end < 0:
        end = len(html)
    seg = html[idx:end]
    rows = re.findall(r'<tr[^>]*>(.*?)</tr>', seg, re.DOTALL)
    for r in rows:
        cells = re.findall(r'<t[dh][^>]*>(.*?)</t[dh]>', r, re.DOTALL)
        if len(cells) < 5:
            continue
        # 链接 + 标题
        href_m = re.search(r'<a[^>]*href="([^"]+)"', cells[1])
        title_m = re.search(r'title="([^"]+)"', cells[1])
        if not href_m:
            continue
        href = href_m.group(1).strip()
        title = title_m.group(1).strip() if title_m else re.sub(r'<[^>]+>', '', cells[1]).strip()
        if not title or '首页' in title or '关于本站' in title or '联系我们' in title or '法律声明' in title:
            continue
        # 审批事项/批复文号/办结日期
        approve_item = re.sub(r'<[^>]+>', '', cells[2]).strip() if len(cells) > 2 else ''
        doc_no = re.sub(r'<[^>]+>', '', cells[3]).strip() if len(cells) > 3 else ''
        date_str = re.sub(r'<[^>]+>', '', cells[4]).strip() if len(cells) > 4 else ''
        # 跳过导航行/非数据行（标题须含项目特征或链接须为 ./202XXX/ 或 /ztzl/ 路径）
        if not re.match(r'^\d{4}\d{2}/', href) and '/202' not in href:
            continue
        if approve_item == '审批事项' and doc_no == '批复文号':
            continue
        items.append((title, href, date_str, approve_item, doc_no))
    return items


def fetch_detail(url):
    """详情页 tab_zfgs 表格 → 字段字典 + HTML 内容"""
    html = fetch_page(url, encoding='utf-8')
    if not html:
        return None, None, None

    fields = {}
    idx = html.find('tab_zfgs')
    if idx >= 0:
        end = html.find('</table>', idx)
        if end < 0:
            end = len(html)
        seg = html[idx:end]
        rows = re.findall(r'<tr[^>]*>(.*?)</tr>', seg, re.DOTALL)
        for r in rows:
            cells = re.findall(r'<t[dh][^>]*>(.*?)</t[dh]>', r, re.DOTALL)
            clean = [re.sub(r'\s+', ' ', re.sub(r'<[^>]+>', '', c)).strip() for c in cells]
            clean = [c for c in clean if c]
            if len(clean) >= 2:
                key = clean[0].rstrip('：:')
                val = clean[1]
                fields[key] = val

    title = fields.get('项目名称', '')
    pub_date = fields.get('批准日期', '')[:10]
    content = build_content_html(fields)
    return title or None, content or None, pub_date or None


def build_content_html(fields):
    """字段 → 结构化 HTML 表格"""
    order = ['审批事项', '批准文号', '项目名称', '项目代码', '行政相对人名称',
             '行政相对人代码类型', '统一社会信用代码', '行政相对人代码', '批准内容',
             '批准日期', '批准机关', '当前状态', '地方编码', '招标方案核准意见']
    parts = ['<table border="1" cellspacing="0" cellpadding="6" style="border-collapse:collapse;width:100%">']
    for k in order:
        if k in fields and fields[k]:
            parts.append(f'<tr><td style="width:160px;font-weight:bold;background:#f5f5f5">{k}</td><td>{fields[k]}</td></tr>')
    # 未在 order 中的额外字段
    for k, v in fields.items():
        if k not in order and v:
            parts.append(f'<tr><td style="width:160px;font-weight:bold;background:#f5f5f5">{k}</td><td>{v}</td></tr>')
    parts.append('</table>')
    return ''.join(parts)


def insert_to_db(items):
    if not items:
        print("  ⏭ 无数据")
        return
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in items:
        try:
            cur = db.execute("INSERT OR IGNORE INTO gov_raw "
                "(site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
                item.get("site_name", "")[:200],
                item.get("url", ""),
                item.get("url", ""),
                (item.get("title") or "")[:500],
                (item.get("pub_date") or "")[:10],
                (item.get("summary") or "")[:500],
                item.get("content", ""),
                "active", "", (item.get("tags") or "")[:100],
            ))
            if cur.rowcount > 0:
                ok += 1
            else:
                skip += 1
        except Exception as e:
            print(f"    ⚠ 入库异常: {e}")
            skip += 1
    db.commit()
    db.execute("INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?", (SITE_NAME,))
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}, FTS已同步")


def main():
    print(f"\n{'='*50}")
    print(f"🏠 {SITE_NAME}")
    print(f"{'='*50}")

    all_items = []
    seen_urls = set()

    MAX_PAGES = 1  # 增量用，每天只跑第1页
    if len(sys.argv) > 1 and sys.argv[1] == "full":
        MAX_PAGES = TOTAL_PAGES
        print(f"📋 全量模式：{TOTAL_PAGES}页")
    elif len(sys.argv) > 1 and sys.argv[1].isdigit():
        MAX_PAGES = min(int(sys.argv[1]), TOTAL_PAGES)

    for page_no in range(1, MAX_PAGES + 1):
        print(f"  📄 第 {page_no}/{MAX_PAGES} 页...", end=" ", flush=True)
        html = fetch_list_page(page_no)
        if not html:
            print("❌ 无响应")
            break
        items = parse_list(html)
        if not items:
            print("0 条（可能已到底）")
            break
        print(f"✅ {len(items)} 条")

        for title, href, date_str, approve_item, doc_no in items:
            full_url = urljoin(f"{LIST_BASE}/index.html", href)
            if full_url in seen_urls:
                continue
            seen_urls.add(full_url)

            if date_str and date_str < THREE_YEARS_AGO:
                continue

            print(f"    {title[:45]}...", end=" ", flush=True)
            dt, content, pub_date = fetch_detail(full_url)
            if dt:
                title = dt
            if pub_date:
                date_str = pub_date

            summary = re.sub(r'<[^>]+>', ' ', content or '').strip()[:300]
            summary = re.sub(r'\s+', ' ', summary)

            all_items.append({
                "site_name": SITE_NAME,
                "title": title,
                "url": full_url,
                "content": content or "",
                "pub_date": date_str or "",
                "summary": summary,
                "tags": "行政执法公示",
            })
            print(f"✅ {date_str}")
            time.sleep(0.3)

        if len(items) < PAGE_SIZE:
            print("    (末页)")
            break
        time.sleep(0.5)

    if all_items:
        insert_to_db(all_items)
    print(f"\n✅ 完成! 共 {len(all_items)} 条")


if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")