#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
旌德县人民政府-重大行政决策预公开 爬虫 (Playwright) - 修复版
站点: www.ahjd.gov.cn
"""
import sys, os, re, json, sqlite3, math, warnings
from bs4 import BeautifulSoup
warnings.filterwarnings('ignore')
from playwright.sync_api import sync_playwright
import urllib.parse

_HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, _HERE)
sys.path.insert(0, os.path.join(_HERE, '..', 'crawler'))

SITE_NAME = "旌德县-重大行政决策预公开"
DOMAIN = "www.ahjd.gov.cn"
BASE_URL = "https://www.ahjd.gov.cn"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")


def insert_db(items):
    if not items:
        return
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA busy_timeout=8000")
    db.execute("PRAGMA synchronous=NORMAL")
    db.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT, source_url TEXT UNIQUE, page_url TEXT,
        title TEXT, publish_date TEXT, summary TEXT,
        content TEXT, status TEXT, category TEXT, tags TEXT,
        attachments TEXT
    )""")
    db.execute("""CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
        title, site_name, summary, content='gov_raw', content_rowid='id'
    )""")
    db.commit()
    ok = skip = 0
    for item in items:
        try:
            title = (item.get("title") or "")[:500]
            summary = (item.get("summary") or title)[:500]
            content = item.get("content") or ""
            pub_date = (item.get("publish_date") or "")[:10]
            source_url = item.get("source_url") or ""
            page_url = item.get("url") or source_url
            attachments = item.get("attachments") or ""
            db.execute("""INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date,
                 summary, content, status, attachments)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, source_url, page_url, title, pub_date,
                 summary, content, "active", attachments))
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except:
            skip += 1
    db.commit()
    try:
        db.execute("""INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
            SELECT r.id, r.title, r.site_name, r.summary
            FROM gov_raw r
            WHERE r.site_name=? AND r.id NOT IN (SELECT rowid FROM gov_search)""",
            (SITE_NAME,))
        db.commit()
    except Exception as e:
        print(f"  [FTS ERROR] {e}")
    db.close()
    print(f"  [DB] 新增: {ok}, 跳过: {skip}")


def clean_html(html):
    html = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r'<!--.*?-->', '', html, flags=re.DOTALL)
    return html.strip()


def extract_text(html):
    if not html:
        return ""
    text = html
    # 表格保留为HTML
    tables = re.findall(r'<table[^>]*>.*?</table>', text, re.DOTALL | re.IGNORECASE)
    placeholders = []
    for i, tbl in enumerate(tables):
        ph = f'__TABLE_{i}__'
        placeholders.append((ph, tbl))
        text = text.replace(tbl, ph, 1)

    text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</p>', '\n\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</div>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</tr>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</th>', '<__THEND__>', text, flags=re.IGNORECASE)
    text = re.sub(r'</td>', '<__TDEND__>', text, flags=re.IGNORECASE)
    text = re.sub(r'<[^>]+>', '', text)
    text = re.sub(r'<__THEND__>', '\n', text)
    text = re.sub(r'<__TDEND__>', '\n', text)
    text = re.sub(r'&nbsp;', ' ', text)
    text = re.sub(r'&lt;', '<', text)
    text = re.sub(r'&gt;', '>', text)
    text = re.sub(r'&amp;', '&', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t]+(?=[\u4e00-\u9fff])', '', text)

    # 恢复表格为Markdown
    for ph, tbl in placeholders:
        md = html_table_to_html(tbl)
        text = text.replace(ph, '\n' + md + '\n')
    return text.strip()


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)


def extract_content_from_detail(html):
    """从详情页提取正文内容"""
    m = re.search(r'<div[^>]*class="[^"]*m-detailbox[^"]*"[^>]*>(.*?)</div>\s*(?:<div[^>]*class="[^"]*(?:is-footer|footer))', html, re.DOTALL | re.IGNORECASE)
    if not m:
        m = re.search(r'<div[^>]*class="[^"]*m-detailbox[^"]*"[^>]*>(.*?)</div>\s*</div>\s*</div>', html, re.DOTALL | re.IGNORECASE)

    if m:
        content_html = m.group(1)
        # 在原始HTML中找"文本下载"以后的区域，跳过元数据
        tdx = content_html.find('文本下载')
        if tdx > 0:
            # 从"文本下载"后面的第一个行/列容器开始找
            after_td = content_html[tdx + len('文本下载'):]
            # 查找下一个<div class="row"> 或 <div class="col-md-12">
            body_m = re.search(r'<(?:div|p|table)[^>]*>', after_td)
            if body_m:
                # 从匹配位置开始取到 m-detailbox 结束前
                body_start = tdx + len('文本下载') + body_m.start()
                # 取到下一个"用微信扫描二维码"或页面分享区域前
                end_m = re.search(r'(?:用微信扫描二维码|分享至好友和朋友圈)', content_html[body_start:])
                if end_m:
                    body_html = content_html[body_start:body_start + end_m.start()]
                else:
                    body_html = content_html[body_start:]
                text = extract_text(clean_html(body_html))
            else:
                text = extract_text(clean_html(content_html))
        else:
            # 没找到"文本下载"，用原有的 col-md-12 方案
            cleaned = clean_html(content_html)
            # 找 "字体：[" 后面的内容（可能被span拆分）
            font_pos = cleaned.find('小]')
            if font_pos == -1:
                font_pos = cleaned.find('字体')
            if font_pos != -1 and font_pos > 0:
                after_font = cleaned[font_pos + 2:]
                # 找下一个 col-md-12
                body_m = re.search(r'<div[^>]*class="[^"]*col-md-12[^"]*"[^>]*>', after_font)
                if body_m:
                    body_html = after_font[body_m.end():]
                    end_pos = body_html.find('</div>')
                    if end_pos > 0:
                        body_html = body_html[:end_pos]
                    text = extract_text(body_html)
                else:
                    text = extract_text(after_font)
            else:
                text = extract_text(cleaned)

        # 清理残留的导航类文字
        text = re.sub(r'(?:用微信扫描二维码|分享至好友和朋友圈|打印).*', '', text, flags=re.DOTALL)
        lines = [l.strip() for l in text.split('\n') if l.strip() and len(l.strip()) > 3]
        # 去掉开头几行如果是重复标题或元数据
        while lines and any(x in lines[0] for x in ['发布时间：', '来源：', '浏览次数', '字体']):
            lines = lines[1:]
        text = '\n\n'.join(lines)
        if len(text) > 5:
            return text.strip()

    # 2. 兜底：取 <body> 但只保留正文区
    bm = re.search(r'<body[^>]*>(.*?)</body>', html, re.DOTALL)
    if bm:
        body = bm.group(1)
        cleaned = clean_html(body)
        text = extract_text(cleaned)
        # 去掉菜单/版权
        text = re.sub(r'无障碍浏览.*?注册', '', text)
        text = re.sub(r'主办单位.*|联系电话.*|邮箱.*|备案号.*|政府网站.*', '', text)
        text = re.sub(r'\n{3,}', '\n\n', text).strip()
        # 取最后部分（正文通常在页面后半段）
        lines = [l.strip() for l in text.split('\n') if l.strip() and len(l.strip()) > 4]
        # 过滤掉导航类文字
        lines = [l for l in lines if not any(x in l for x in [
            '首页', '走进旌德', '新闻中心', '政府信息公开', '政务服务',
            '互动交流', '解读回应', '县长之窗', '无障碍', '长辈版', '登录', '注册',
            '网站地图', '联系我们', '版权声明', '页面纠错', '隐私保护',
        ])]
        if lines:
            return '\n\n'.join(lines)
    return ""


def fetch_list_with_pw(page, list_page_url):
    """用 Playwright 获取列表页并返回文章列表"""
    page.goto(list_page_url, timeout=20000)
    page.wait_for_load_state("networkidle", timeout=10000)
    html = page.content()

    items = []
    # 匹配所有 OpennessContent 链接
    for m in re.finditer(
        r'<a[^>]*href="(/OpennessContent/show/\d+\.html)"[^>]*title="([^"]*)"[^>]*>',
        html
    ):
        href = m.group(1)
        title = m.group(2).strip()

        # 跳过"| 文件"类冗余链接（inner text 很短）
        a_end = m.end()
        a_close = html.find('</a>', a_end)
        inner = ''
        if a_close != -1:
            inner = html[a_end:a_close].strip()

        # 如果inner text是"| 文件"或长度<3，说明是附件链接不是主体链接
        if inner in ('| 文件', '文件', '') or len(inner) < 3:
            continue

        # 找对应的 date span
        date_span = re.search(r'<span[^>]*>(\d{4}-\d{2}-\d{2})</span>', html[m.end():m.end()+300])
        date_str = date_span.group(1) if date_span else ""

        # 去重
        if not any(i['url'] == BASE_URL + href for i in items):
            items.append({
                "title": title,
                "url": BASE_URL + href,
                "date": date_str,
            })
    return items


def fetch_detail_with_pw(page, url):
    """用 Playwright 获取详情页内容"""
    page.goto(url, timeout=20000)
    page.wait_for_load_state("networkidle", timeout=10000)
    html = page.content()

    # 完整标题
    detail_title = None
    tt = re.search(r'<title>([^<]+)</title>', html)
    if tt:
        t = tt.group(1).strip()
        t = re.sub(r'[-–—]\s*旌德县人民政府$', '', t)
        t = t.strip()
        if t:
            detail_title = t

    content_text = extract_content_from_detail(html)

    return {
        "title": detail_title,
        "content": content_text,
    }


def run(max_pages=None):
    print(f"\n{'='*60}")
    print(f"🚀 {SITE_NAME}")
    print(f"{'='*60}")

    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        ctx = browser.new_context(
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
            locale="zh-CN"
        )
        page = ctx.new_page()

        # 先测第1页获取总页数
        print("\n📥 获取列表...")
        first_url = f"{BASE_URL}/XxgkContent/showList/836/112385/page_1.html"
        items1 = fetch_list_with_pw(page, first_url)
        total_pages = max_pages if max_pages else 2
        print(f"  页1: {len(items1)} 条")

        all_items = list(items1)
        for pn in range(2, total_pages + 1):
            url = f"{BASE_URL}/XxgkContent/showList/836/112385/page_{pn}.html"
            items = fetch_list_with_pw(page, url)
            seen_urls = {i["url"] for i in all_items}
            new_items = [i for i in items if i["url"] not in seen_urls]
            all_items.extend(new_items)
            print(f"  第 {pn}/{total_pages} 页: {len(items)} 条 (新增 {len(new_items)})")

        print(f"  共获取: {len(all_items)} 条")

        # 逐条详情
        success = fail = 0
        db_items = []

        for i, item in enumerate(all_items, 1):
            print(f"\n[{i}/{len(all_items)}] {item['title'][:60]}...")
            try:
                detail = fetch_detail_with_pw(page, item["url"])
                if not detail or not detail.get("content"):
                    print(f"   ❌ 正文为空")
                    fail += 1
                    continue

                db_title = detail.get("title") or item["title"]
                content_text = detail.get("content", "")

                db_items.append({
                    "title": db_title,
                    "source_url": item["url"],
                    "url": item["url"],
                    "publish_date": item["date"][:10],
                    "summary": db_title[:200],
                    "content": content_text,
                    "attachments": "",
                })
                success += 1
                print(f"   ✅ 正文: {len(content_text)} 字, 日期: {item['date']}")
            except Exception as e:
                print(f"   ❌ 错误: {e}")
                fail += 1

        browser.close()

    print(f"\n{'='*60}")
    print(f"📦 写入数据库... 成功: {success}, 失败: {fail}")
    if db_items:
        insert_db(db_items)
        # 先删旧数据再写新数据
        # 但因为INSERT OR IGNORE，自动去重
    print(f"{'='*60}")
    print(f"✅ 完成! 共处理 {len(all_items)} 条，成功 {success}，失败 {fail}")


if __name__ == "__main__":
    max_p = None
    args = sys.argv[1:]
    for i, arg in enumerate(args):
        if arg == "--pages" and i + 1 < len(args):
            try:
                max_p = int(args[i + 1])
            except:
                pass
    run(max_pages=max_p)