#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
突泉县人民政府 - 措施及实施情况（农牧补贴/公示）
http://www.tq.gov.cn/tq/zwgk/zfxxgk89/fdzdgknr92/4617194/snbt11/csjssqk9951/index.html

大汉版通 zkhl Aisite CMS：
列表: 每页15条, 分页 df12b020-{N}.html（5页≈40条）
详情: meta ArticleTitle/PubDate/ContentSource + div.detail_txt#mainText 正文 + 附件 .doc/.pdf
"""
import sys, os, re, time, json
sys.path.insert(0, '/root/gov_crawler')
from crawler_lib import fetch_page, clean_html
import sqlite3
from urllib.parse import urljoin

SITE_NAME = "突泉县-措施及实施情况"
BASE = "http://www.tq.gov.cn"
LIST_BASE = f"{BASE}/tq/zwgk/zfxxgk89/fdzdgknr92/4617194/snbt11/csjssqk9951"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
THREE_YEARS_AGO = "2023-06-01"
TOTAL_PAGES = 5
PAGE_SIZE = 15


def fetch_list_page(page_no):
    if page_no == 1:
        url = f"{LIST_BASE}/index.html"
    else:
        url = f"{LIST_BASE}/df12b020-{page_no}.html"
    html = fetch_page(url, encoding='utf-8')
    return html


def parse_list(html):
    """<a href="/tq/2026-06/23/article_xxx.html" title="完整标题">标题</a>（含日期）"""
    items = []
    pattern = re.compile(
        r'href="(/tq/\d{4}-\d{2}/\d{2}/article_[^"]+\.html)"[^>]*>([^<]{4,200})</a>',
        re.DOTALL
    )
    for m in pattern.finditer(html):
        href, title = m.group(1).strip(), m.group(2).strip()
        # 标题实体清理（&middot; &nbsp; 前缀等）
        title = title.replace('&middot;', '').replace('&nbsp;', '').replace('·', '').strip()
        title = re.sub(r'^\s*[·•]\s*', '', title).strip()
        if href and title and '首页' not in title and '关于本站' not in title:
            items.append((title, href, ''))
    # 去重
    seen = set()
    uniq = []
    for t, h, d in items:
        if h not in seen:
            seen.add(h)
            uniq.append((t, h, d))
    return uniq


def fetch_detail(url):
    """详情页: meta + div.detail_txt#mainText + 附件"""
    html = fetch_page(url, encoding='utf-8')
    if not html:
        return None, None, None

    pub_date = ""
    m = re.search(r'<meta name="PubDate" content="(\d{4}-\d{2}-\d{2})', html)
    if m:
        pub_date = m.group(1)

    title = ""
    m = re.search(r'<meta name="ArticleTitle" content="([^"]*)"', html)
    if m:
        title = m.group(1).strip()

    content = ""
    m = re.search(r'<div class="detail_txt"[^>]*id="mainText"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if not m:
        m = re.search(r'<div class="detail_txt"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()

    if content:
        # 清理脚本/样式
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL | re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL | re.I)
        # 标题实体清理
        content = content.replace('&middot;', '').replace('&nbsp;', ' ').replace('&ldquo;', '“').replace('&rdquo;', '”')
        # 清理无意义 span/font 包裹（保留文本）
        content = re.sub(r'<span[^>]*>', '', content)
        content = re.sub(r'</span>', '', content)
        content = re.sub(r'<font[^>]*>', '', content)
        content = re.sub(r'</font>', '', content)
        # 段落规范化：p 之间 \n\n，块级元素换行
        content = re.sub(r'</p>', '</p>\n\n', content)
        content = re.sub(r'<br\s*/?>', '\n', content)
        content = re.sub(r'</(div|tr)>', '</\g<1>>\n', content)
        # 附件嵌入（正文后追加附件链接）
        atts = re.findall(
            r'<a[^>]*href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|zip|rar|wps|et|wpt))"[^>]*>([^<]{1,100})</a>',
            html, re.IGNORECASE
        )
        att_parts = []
        for att_href, att_text in atts:
            att_full = urljoin(BASE, att_href)
            att_text = att_text.strip().replace('&middot;', '').replace('&nbsp;', ' ').strip()
            att_parts.append(f'<p><a href="{att_full}" target="_blank">{att_text}</a></p>')
        if att_parts:
            content += '\n\n<p><strong>附件：</strong></p>\n' + '\n'.join(att_parts)
        # 最终清理：空行合并
        content = re.sub(r'\n{3,}', '\n\n', content)
        content = clean_html(content)

    return title or None, content or None, pub_date or None


def insert_to_db(items):
    if not items:
        print("  ⏭ 无数据")
        return
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in items:
        try:
            cur = db.execute("INSERT OR IGNORE INTO gov_raw "
                "(site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
                item.get("site_name", "")[:200],
                item.get("url", ""),
                item.get("url", ""),
                (item.get("title") or "")[:500],
                (item.get("pub_date") or "")[:10],
                (item.get("summary") or "")[:500],
                item.get("content", ""),
                "active", "", (item.get("tags") or "")[:100],
            ))
            if cur.rowcount > 0:
                ok += 1
            else:
                skip += 1
        except Exception as e:
            print(f"    ⚠ 入库异常: {e}")
            skip += 1
    db.commit()
    db.execute("INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?", (SITE_NAME,))
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}, FTS已同步")


def main():
    print(f"\n{'='*50}")
    print(f"🏠 {SITE_NAME}")
    print(f"{'='*50}")

    all_items = []
    seen_urls = set()

    MAX_PAGES = 1  # 增量用，每天只跑第1页
    if len(sys.argv) > 1 and sys.argv[1] == "full":
        MAX_PAGES = TOTAL_PAGES
        print(f"📋 全量模式：{TOTAL_PAGES}页")
    elif len(sys.argv) > 1 and sys.argv[1].isdigit():
        MAX_PAGES = min(int(sys.argv[1]), TOTAL_PAGES)

    for page_no in range(1, MAX_PAGES + 1):
        print(f"  📄 第 {page_no}/{MAX_PAGES} 页...", end=" ", flush=True)
        html = fetch_list_page(page_no)
        if not html:
            print("❌ 无响应")
            break
        items = parse_list(html)
        if not items:
            print("0 条（可能已到底）")
            break
        print(f"✅ {len(items)} 条")

        for title, href, date_str in items:
            full_url = urljoin(BASE, href)
            if full_url in seen_urls:
                continue
            seen_urls.add(full_url)

            print(f"    {title[:45]}...", end=" ", flush=True)
            dt, content, pub_date = fetch_detail(full_url)
            if dt:
                title = dt
            if pub_date:
                date_str = pub_date

            summary = re.sub(r'<[^>]+>', ' ', content or '').strip()[:300]
            summary = re.sub(r'\s+', ' ', summary)

            all_items.append({
                "site_name": SITE_NAME,
                "title": title,
                "url": full_url,
                "content": content or "",
                "pub_date": date_str or "",
                "summary": summary,
                "tags": "措施及实施情况",
            })
            print(f"✅ {date_str}")
            time.sleep(0.3)

        if len(items) < PAGE_SIZE:
            print("    (末页)")
            break
        time.sleep(0.5)

    if all_items:
        insert_to_db(all_items)
    print(f"\n✅ 完成! 共 {len(all_items)} 条")


if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")