#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
双江县人民政府-通知通告 爬虫
站点: www.shuangjiang.gov.cn
TLS 兼容: --ciphers AES128-GCM-SHA256
"""
import sys, os, re, subprocess, sqlite3, warnings
from bs4 import BeautifulSoup
import urllib.parse
warnings.filterwarnings('ignore')

_HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, _HERE)
sys.path.insert(0, os.path.join(_HERE, '..', 'crawler'))

SITE_NAME = "双江县-通知通告"
DOMAIN = "www.shuangjiang.gov.cn"
BASE_URL = "https://www.shuangjiang.gov.cn"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
CURL_OPTS = ["curl", "-s", "-L", "--max-time", "20", "-k", "--ciphers", "AES128-GCM-SHA256"]


def curl(url):
    result = subprocess.run(CURL_OPTS + [url], capture_output=True, text=True, timeout=25)
    return result.stdout


def insert_db(items):
    if not items:
        return
    db = sqlite3.connect(SEARCH_DB)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA busy_timeout=8000")
    db.execute("PRAGMA synchronous=NORMAL")
    db.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT, source_url TEXT UNIQUE, page_url TEXT,
        title TEXT, publish_date TEXT, summary TEXT,
        content TEXT, status TEXT, category TEXT, tags TEXT,
        attachments TEXT
    )""")
    db.execute("""CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
        title, site_name, summary, content='gov_raw', content_rowid='id'
    )""")
    db.commit()
    ok = skip = 0
    for item in items:
        try:
            title = (item.get("title") or "")[:500]
            summary = (item.get("summary") or title)[:500]
            content = item.get("content") or ""
            pub_date = (item.get("publish_date") or "")[:10]
            source_url = item.get("source_url") or ""
            page_url = item.get("url") or source_url
            attachments = item.get("attachments") or ""
            db.execute("""INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date,
                 summary, content, status, attachments)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, source_url, page_url, title, pub_date,
                 summary, content, "active", attachments))
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except:
            skip += 1
    db.commit()
    try:
        db.execute("""INSERT INTO gov_search(rowid, title, site_name, summary)
            SELECT r.id, r.title, r.site_name, r.summary
            FROM gov_raw r
            WHERE r.site_name=? AND r.id NOT IN (SELECT rowid FROM gov_search)""",
            (SITE_NAME,))
        db.commit()
    except Exception as e:
        print(f"  [FTS ERROR] {e}")
    db.close()
    print(f"  [DB] 新增: {ok}, 跳过: {skip}")


def clean_html(html):
    html = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r'<!--.*?-->', '', html, flags=re.DOTALL)
    return html.strip()


def extract_text(html):
    if not html:
        return ""
    text = html
    # 表格保留
    tables = re.findall(r'<table[^>]*>.*?</table>', text, re.DOTALL | re.IGNORECASE)
    placeholders = []
    for i, tbl in enumerate(tables):
        ph = f'__TABLE_{i}__'
        placeholders.append((ph, tbl))
        text = text.replace(tbl, ph, 1)
    text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</p>', '\n\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</div>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</tr>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</th>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</td>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'<(?!/?(?:table|tr|td|th|thead|tbody|tfoot|img))[^>]*>', '', text)
    text = re.sub(r'&nbsp;', ' ', text)
    text = re.sub(r'&lt;', '<', text)
    text = re.sub(r'&gt;', '>', text)
    text = re.sub(r'&amp;', '&', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t]+(?=[\u4e00-\u9fff])', '', text)
    for ph, tbl in placeholders:
        md = html_table_to_html(tbl)
        text = text.replace(ph, '\n' + md + '\n')
    return text.strip()


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)


def fetch_list_page(page_url):
    """获取列表页"""
    html = curl(page_url)
    items = []

    dl_m = re.search(r'<dl[^>]*class="textList1"[^>]*>(.*?)</dl>', html, re.DOTALL)
    if not dl_m:
        return []

    area = dl_m.group(1)
    for m in re.finditer(r'<dd[^>]*>(.*?)</dd>', area, re.DOTALL):
        dd = m.group(1)
        # 跳过菜单项（href=/index.html 之类）
        # 注意: title= 可能在 href= 之前或之后
        title_m = re.search(r'title="([^"]*)"', dd)
        href_m = re.search(r'href="(/artview/\d+/\d+\.html)"', dd)
        if not title_m or not href_m:
            continue
        title = title_m.group(1).strip()
        href = href_m.group(1)

        # 找日期
        date_m = re.search(r'<span[^>]*>(\d{4}-\d{2}-\d{2})</span>', dd)
        date_str = date_m.group(1) if date_m else ""

        items.append({
            "title": title,
            "url": BASE_URL + href,
            "date": date_str,
        })
    return items


def extract_content_from_detail(html):
    """详情页正文提取"""
    m = re.search(r'<div[^>]*class="articleBox"[^>]*>(.*?)<div[^>]*class="text"[^>]*>', html, re.DOTALL)
    if m:
        inner = m.group(1)
        # 检查是否有 PDF 嵌入
        pdf_m = re.search(r'<iframe[^>]*src="([^"]*\.pdf)"', inner)
        if pdf_m:
            pdf_url = pdf_m.group(1)
            if not pdf_url.startswith('http'):
                pdf_url = BASE_URL + pdf_url
            text = f"[附件：PDF文件]({pdf_url})"
            return text
        text = extract_text(clean_html(inner))
        if len(text) > 5:
            return text.strip()
    # 兜底
    m = re.search(r'<div[^>]*class="articleBox"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        inner = m.group(1)
        pdf_m = re.search(r'<iframe[^>]*src="([^"]*\.pdf)"', inner)
        if pdf_m:
            pdf_url = pdf_m.group(1)
            if not pdf_url.startswith('http'):
                pdf_url = BASE_URL + pdf_url
            return f"[附件：PDF文件]({pdf_url})"
        text = extract_text(clean_html(inner))
        if len(text) > 5:
            return text.strip()
    return ""


def fetch_detail(url):
    """获取详情"""
    html = curl(url)
    if not html:
        return None

    # 标题
    title = None
    tt = re.search(r'<div[^>]*class="articleTitle"[^>]*>(.*?)</div>', html)
    if tt:
        title = tt.group(1).strip()
    if not title:
        tt = re.search(r'<title>([^<]+)</title>', html)
        if tt:
            t = tt.group(1).strip()
            t = re.sub(r'[-–—|]\s*双江县人民政府$', '', t).strip()
            if t:
                title = t

    # 日期
    date_str = ""
    dp = re.search(r'<span[^>]*>(\d{4}-\d{2}-\d{2})</span>', html)
    if dp:
        date_str = dp.group(1)
    if not date_str:
        dp = re.search(r'(\d{4}-\d{2}-\d{2})\s+来源', html)
        if dp:
            date_str = dp.group(1)

    content = extract_content_from_detail(html)
    return {"title": title, "content": content, "date": date_str}


def run(max_pages=None):
    print(f"\n{'='*60}")
    print(f"🚀 {SITE_NAME}")
    print(f"{'='*60}")

    # 先取第1页确定总页数
    print("\n📥 获取列表...")
    items1 = fetch_list_page(f"{BASE_URL}/tztg.html")
    print(f"  第1页: {len(items1)} 条")

    # 从页1找总页数
    html1 = curl(f"{BASE_URL}/tztg.html")
    total_pages = 24  # 已知
    mm = re.search(r'max\s*=\s*"(\d+)"', html1)
    if mm:
        total_pages = int(mm.group(1))
    mm2 = re.search(r'/24页', html1)
    if mm2:
        total_pages = 24
    if max_pages and max_pages < total_pages:
        total_pages = max_pages
    print(f"  总页数: {total_pages}")

    all_items = list(items1)
    for pn in range(2, total_pages + 1):
        url = f"{BASE_URL}/tztg/index_{pn}.html"
        items = fetch_list_page(url)
        if not items:
            print(f"  第 {pn}/{total_pages} 页: 空")
            continue
        seen_urls = {i["url"] for i in all_items}
        new_items = [i for i in items if i["url"] not in seen_urls]
        all_items.extend(new_items)
        print(f"  第 {pn}/{total_pages} 页: {len(items)} 条 (新增 {len(new_items)})")

    print(f"\n  共获取: {len(all_items)} 条")

    # 逐条详情
    success = fail = 0
    db_items = []

    for i, item in enumerate(all_items, 1):
        print(f"\n[{i}/{len(all_items)}] {item['title'][:60]}...")
        detail = fetch_detail(item["url"])
        if not detail:
            print(f"   ❌ 请求失败")
            fail += 1
            continue

        content_text = detail.get("content", "")
        if not content_text or len(content_text) < 3:
            print(f"   ❌ 正文为空")
            fail += 1
            continue

        db_title = detail.get("title") or item["title"]
        pub_date = detail.get("date") or item["date"]

        db_items.append({
            "title": db_title,
            "source_url": item["url"],
            "url": item["url"],
            "publish_date": pub_date[:10],
            "summary": db_title[:200],
            "content": content_text,
            "attachments": "",
        })
        success += 1
        print(f"   ✅ 正文: {len(content_text)} 字, 日期: {pub_date}")

    print(f"\n{'='*60}")
    print(f"📦 写入数据库... 成功: {success}, 失败: {fail}")
    if db_items:
        insert_db(db_items)
    print(f"{'='*60}")
    print(f"✅ 完成! 共处理 {len(all_items)} 条, 成功 {success}, 失败 {fail}")


if __name__ == "__main__":
    max_p = None
    args = sys.argv[1:]
    for i, arg in enumerate(args):
        if arg == "--pages" and i + 1 < len(args):
            try:
                max_p = int(args[i + 1])
            except:
                pass
    run(max_pages=max_p)