#!/usr/bin/env python3
"""
和县人民政府-生态环境分局 (hx.gov.cn)
建设项目环评文件（报告书、报告表）审批(非政府投资项目)
列表: /xxgk/opennessTarget/?branch_id=...&column_code=350101&page=N (AJAX)
详情: /xxgk/openness/detail/content/{id}.html
"""
import sys, os, re, time, json
sys.path.insert(0, '/root/gov_crawler')
from crawler_lib import fetch_page, clean_html
import sqlite3
import os

SITE_NAME = "和县生态环境分局"
BASE = "https://www.hx.gov.cn"
THREE_YEARS_AGO = "2023-06-01"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

# 栏目配置
BRANCH_ID = "57a3df762c262ea9a00aad33"
COLUMN_CODE = "350101"
MAX_PAGES = 5  # pagecount from pagination

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Cookie": "token_verified=true",
    "Referer": f"{BASE}/xxgk/opennessContent/?branch_id={BRANCH_ID}&column_code={COLUMN_CODE}",
}

def insert_to_db(items):
    if not items:
        print("  ⏭ 无数据")
        return
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in items:
        try:
            db.execute("""INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date,
                 summary, content, status, category, tags)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (
                item.get("site_name","")[:200],
                item.get("url",""),
                item.get("url",""),
                (item.get("title") or "")[:500],
                (item.get("pub_date") or "")[:10],
                (item.get("summary") or "")[:500],
                item.get("content",""),
                "active", "", (item.get("tags") or "")[:100],
            ))
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except:
            skip += 1
    db.commit()
    db.execute("""INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
        SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r
        WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?""",
        (SITE_NAME,))
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}, FTS已同步")

def fetch_with_cookie(url):
    """带 token_verified cookie 的请求"""
    import urllib.request
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=30)
        html = resp.read().decode('utf-8', errors='replace')
        return html
    except Exception as e:
        print(f"    ❌ 请求失败: {e}")
        return None

def get_list_url(page):
    return f"{BASE}/xxgk/opennessTarget/?branch_id={BRANCH_ID}&column_code={COLUMN_CODE}&page={page}"

def parse_list(html):
    """解析列表页，提取 (title, url, date_str)"""
    items = []
    # <tr> <td class="xh">1</td> <td class="bt"><a href="...">title</a></td> <td class="cwrq">2026-06-04</td> </tr>
    pattern = re.compile(
        r'<td\s+class="xh"[^>]*>\d+</td>\s*<td\s+class="bt"[^>]*><a\s+href="(/xxgk/openness/detail/content/([^"]+))"[^>]*>(.*?)</a>.*?<td\s+class="cwrq"[^>]*>\s*(\d{4}-\d{2}-\d{2})\s*</td>',
        re.DOTALL
    )
    for m in pattern.finditer(html):
        href = m.group(1).strip()
        content_id = m.group(2).strip()
        title = m.group(3).strip()
        date_str = m.group(4).strip()
        if title:
            items.append((title, href, content_id, date_str))
    return items

def fetch_detail(url):
    """获取详情页的标题、正文、日期"""
    html = fetch_with_cookie(url)
    if not html:
        return None, None, None, None, None, None
    
    # 标题: <h1 id="downloads_title">
    title = ""
    m = re.search(r'<h1[^>]*id="downloads_title"[^>]*>(.*?)</h1>', html, re.DOTALL)
    if m:
        title = m.group(1).strip()
    if not title:
        # fallback: meta ArticleTitle
        m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', html)
        if m:
            title = m.group(1).strip()
    if not title:
        # fallback: page <title>
        m = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
        if m:
            title = m.group(1).strip().replace('-和县人民政府', '').strip()
    
    # 发布时间: from meta PubDate or 成文日期 td
    pub_date = ""
    m = re.search(r'<meta\s+name="PubDate"\s+content="(\d{4}-\d{2}-\d{2})', html)
    if m:
        pub_date = m.group(1)
    if not pub_date:
        m = re.search(r'成文日期[：:].*?<td[^>]*>(\d{4}-\d{2}-\d{2})', html)
        if m:
            pub_date = m.group(1)
    
    # 发布机构
    publisher = ""
    m = re.search(r'发布机构[：:].*?<td[^>]*>(.*?)</td>', html)
    if m:
        publisher = m.group(1).strip()
    
    # 文号
    doc_no = ""
    m = re.search(r'文[ 　]*号[：:].*?<td[^>]*>(.*?)</td>', html)
    if m:
        doc_no = m.group(1).strip()
    
    # 正文: <div id="downloads_body">...</div>
    content = ""
    m = re.search(r'<div\s+id="downloads_body"[^>]*>(.*?)</div>\s*</div>\s*</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
    if not content:
        m = re.search(r'<div\s+id="downloads_body"[^>]*>(.*?)</div>', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
    
    if content:
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'\s*style="[^"]*"', '', content)
        content = re.sub(r'\s*class="[^"]*"', '', content)
        # 保留 <p>, <br>, <table>, <a> 等结构化标签
        content = clean_html(content)
        # 清理嵌套<p>：如果<p>内直接以另一个<p>开头，外层的<p>是包裹层，去掉
        content = re.sub(r'<p>\s*(<p)', r'\1', content)
        # 清理空段落和空div
        content = re.sub(r'<p>\s*</p>', '', content)
        content = re.sub(r'<div>\s*</div>', '', content)
        content = re.sub(r'<div>&nbsp;</div>', '', content)
        # 清理段落末尾残留的</p>（嵌套解包后的多余闭标签）
        content = re.sub(r'</p>\s*</p>', '</p>', content)
        # 清理页面模板残留（扫码、PDF提示等）
        content = re.sub(r'<p>\s*扫一扫在手机打开当前页\s*</p>', '', content)
        content = re.sub(r'扫一扫在手机打开当前页', '', content)
        content = re.sub(r'如果内容不能正常显示.*?(?:</p>|$)', '', content, flags=re.DOTALL)
        content = re.sub(r'\[?\s*在线安装\s*\]?\s*[,，]?\s*(?:或下载本PDF文档。?)?', '', content)
        # 移除正文中已包含的附件HTML，避免和后面extra重复
        # 附件容器在div#qrcode内（可能无闭合标签）
        content = re.sub(r'<div\s+id="qrcode"[^>]*>', '', content)
        content = re.sub(r'class="attachments"[^>]*>.*?</div>', '', content, flags=re.DOTALL)
        # 去重：对纯文本去重（去掉同一文本段落的第二次出现）
        texts = re.findall(r'>([^<]+)<', content)
        seen_texts = {}
        for t in texts:
            t = t.strip()
            if len(t) > 20:
                if t in seen_texts:
                    # 第二次出现 → 删除该段落
                    content = content.replace(f'>{t}<', '><', 1)
                else:
                    seen_texts[t] = 1
    
    # 附件下载
    attachments = []
    m = re.search(r'<div\s+class="m-xgxx"[^>]*>.*?<ul>(.*?)</ul>', html, re.DOTALL)
    if m:
        attach_html = m.group(1)
        for am in re.finditer(r'<li[^>]*>(.*?)(?:<li\s+class="linkxz"[^>]*><a\s+href="(/xxgk/download/[^"]+)"[^>]*>文件下载</a>)?</li>', attach_html, re.DOTALL):
            name = am.group(1).strip() if am.group(1) else ""
            link = BASE + am.group(2) if am.group(2) else ""
            if name or link:
                attachments.append({"name": name, "url": link})
    
    return title or None, content or None, pub_date or None, publisher, doc_no, attachments


def main():
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")
    all_items, seen = [], set()
    
    for page in range(1, MAX_PAGES + 1):
        url = get_list_url(page)
        print(f"  📄 第 {page} 页...", end=" ", flush=True)
        html = fetch_with_cookie(url)
        if not html:
            print("❌ 请求失败")
            break
        
        items = parse_list(html)
        if not items:
            print("0 条（可能是末页）")
            break
        
        print(f"✅ {len(items)} 条")
        
        # 提取分页信息，检测是否最后一页
        pagination_match = re.search(r'<pagination\s+currentpage="(\d+)"[^>]*pagecount="(\d+)"', html)
        if pagination_match:
            current = int(pagination_match.group(1))
            total = int(pagination_match.group(2))
            print(f"    (第{current}/{total}页)")
            if current >= total:
                print("    (末页)")
        
        for i, (title, href, content_id, date_str) in enumerate(items, 1):
            full_url = BASE + href
            if full_url in seen:
                continue
            seen.add(full_url)
            
            if date_str and date_str < THREE_YEARS_AGO:
                continue
            
            print(f"    [{i}/{len(items)}] {title[:50]}...", end=" ", flush=True)
            dt, content, pub_date, publisher, doc_no, attachments = fetch_detail(full_url)
            if dt:
                title = dt
            if pub_date:
                date_str = pub_date
            
            # 将附件信息追加到正文末尾
            extra = ""
            if attachments:
                extra = '<div class="attachments"><h3>文件下载</h3><ul>'
                for att in attachments:
                    name = att.get("name", "")
                    link = att.get("url", "")
                    if link:
                        extra += f'<li><a href="{link}">{name}</a></li>'
                    else:
                        extra += f'<li>{name}</li>'
                extra += '</ul></div>'
            full_content = (content or "") + extra
            
            summary = re.sub(r'<[^>]+>', ' ', full_content).strip()[:300]
            summary = re.sub(r'\s+', ' ', summary)
            
            tags = f"环评审批-和县"
            if doc_no:
                tags += f" {doc_no}"
            
            all_items.append({
                "site_name": SITE_NAME, "title": title, "url": full_url,
                "content": full_content, "pub_date": date_str or "",
                "summary": summary, "tags": tags,
            })
            print(f"✅ {date_str}")
            time.sleep(0.3)
        
        # 检测是否没有分页控件（最后一页无更多链接）
        if len(items) < 20:
            print("    (末页, 不足20条)")
            break
    
    if all_items:
        insert_to_db(all_items)
    print(f"\n✅ 完成! 共 {len(all_items)} 条")


if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")