#!/usr/bin/env python3
"""
佛冈县政府 — 通知公告 爬虫
===================
政府CMS标准模式：
  python3 crawl_fogang.py              # 增量（默认：只爬第1页最新）
  python3 crawl_fogang.py --full       # 全量：20页
  python3 crawl_fogang.py --pages=5    # 指定页数
  python3 crawl_fogang.py --sync       # 同步已有数据到服务器

数据流：本地 quality_results.db → SSH → 服务器 search.db
"""

import sys, os, re, json, sqlite3, hashlib, time, base64, subprocess, html as html_mod
from datetime import datetime
from urllib.parse import urljoin

# ─── 配置 ───
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
QUALITY_DB = os.path.join(BASE_DIR, "quality_results.db")
SERVER_SSH = "root@1.94.217.116"
SERVER_DB = "/root/quality_results.db"
SERVER_SEARCH_DB = "/root/search.db"

BASE_URL = "http://www.fogang.gov.cn"
LIST_URL = "http://www.fogang.gov.cn/ywdt/gggs/tzgg/index.html"
TOTAL_PAGES = 20

SITE_NAME = "佛冈县人民政府"
DOMAIN = "www.fogang.gov.cn"

# ─── HTTP 工具 ───
import ssl, urllib.request
CTX = ssl._create_unverified_context()
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "Chrome/125.0.0.0 Safari/537.36",
}

def http_get(url, timeout=20):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=timeout, context=CTX)
        raw = resp.read()
        text = raw.decode("utf-8", errors="replace")
        return text
    except Exception as e:
        return None

# ─── 列表页解析 ───
def parse_list_page(html):
    """从佛冈列表页提取所有文章"""
    items = []
    seen_urls = set()

    # 找到列表区域 (方案A: pageList listContent infoList)
    area_m = re.search(
        r'<div[^>]*class="pageList\s+listContent\s+infoList"[^>]*>(.*?)'
        r'<div[^>]*class="pagediv',
        html, re.DOTALL
    )
    if not area_m:
        # 方案B: mainContent
        area_m = re.search(
            r'id="mainContent"[^>]*>(.*?)<!--\s*main\s+End',
            html, re.DOTALL
        )
    if not area_m:
        # 方案C: 直接在 body 里找 li
        area_m = re.search(
            r'<body[^>]*>(.*?)</body>',
            html, re.DOTALL
        )
    if not area_m:
        return items
    area = area_m.group(1)

    # 匹配 li 项
    for m in re.finditer(
        r'<li[^>]*>.*?<a[^>]*href="([^"]+)"[^>]*title="([^"]*)"[^>]*>(.*?)</a>'
        r'.*?<span[^>]*class="time"[^>]*>([^<]+)</span>',
        area, re.DOTALL
    ):
        href = m.group(1).strip()
        title = m.group(2).strip() or re.sub(r'<[^>]+>', '', m.group(3)).strip()
        date = m.group(4).strip()

        if not title or not href:
            continue
        # ⚠️ 域名过滤: 列表页可能混入外链 (xjhbcy 等), 非本域 URL 直接跳过
        if DOMAIN not in href:
            continue
        # 拼接完整URL
        if not href.startswith('http'):
            href = urljoin(BASE_URL, href)
        if href in seen_urls:
            continue
        seen_urls.add(href)

        items.append({
            'title': title,
            'url': href,
            'date': date,
        })
    
    return items

def get_detail(url):
    """获取详情页：标题、正文HTML、日期"""
    html = http_get(url)
    if not html:
        return None

    # 标题：<title> 或 h1.article-title
    title = ''
    tm = re.search(r'<title>([^<]+)</title>', html)
    if tm:
        title = tm.group(1)
        # 去掉站点后缀
        title = re.sub(r'\s*-\s*佛冈县政府门户网站\s*$', '', title).strip()
    if not title:
        tm = re.search(r'<h1[^>]*class="article-title"[^>]*>(.*?)</h1>', html, re.DOTALL)
        if tm:
            title = re.sub(r'<[^>]+>', '', tm.group(1)).strip()

    # 正文：<div id="zoomcon"> 或 <div class="article-content article-content-body">
    content = ''
    cm = re.search(
        r'<div[^>]*class="article-content\s+article-content-body"[^>]*id="zoomcon"[^>]*>'
        r'(.*?)</div>\s*<div[^>]*class="article-reldocuments"',
        html, re.DOTALL
    )
    if not cm:
        cm = re.search(
            r'id="zoomcon"[^>]*>(.*?)</div>\s*<div[^>]*class="article-reldocuments"',
            html, re.DOTALL
        )
    if not cm:
        cm = re.search(
            r'<div[^>]*class="article-content[^"]*"[^>]*>(.*?)</div>\s*<div[^>]*class="article-(?:reldocuments|auxiliary|extended)"',
            html, re.DOTALL
        )
    if cm:
        content = cm.group(1).strip()
        # 清理：去掉 script/style 但保留 table/p/br 等结构标签
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        # 去掉多余的空格和空标签属性
        content = re.sub(r'<p>\s*<br\s*/?>\s*</p>', '', content)
        # 2026-08-11 修复: 去掉 10000 截断 — 正文必须完整入库

    return {
        'title': title,
        'content': content,
    }

def extract_plain_text(html_content):
    """从HTML提取纯文本，保留段落结构"""
    if not html_content:
        return ''
    # 在块级标签后插入换行
    text = re.sub(r'</(p|div|li|h[1-6]|tr|td|th)>', r'</\1>\n\n', html_content)
    text = re.sub(r'<br\s*/?>', '\n', text)
    # 去掉所有HTML标签
    text = re.sub(r'<[^>]+>', '', text)
    # 解码实体
    text = html_mod.unescape(text)
    # 清理空格
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    return text.strip()

# ─── DB 操作 ───
def ensure_table():
    conn = sqlite3.connect(QUALITY_DB, timeout=60)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS crawl_results (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            site_id INTEGER DEFAULT 0,
            title TEXT,
            url TEXT UNIQUE,
            content TEXT,
            publish_date TEXT,
            summary TEXT,
            domain TEXT,
            category TEXT DEFAULT '',
            content_hash TEXT,
            crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    """)
    conn.commit()
    conn.close()

def is_url_crawled(url):
    conn = sqlite3.connect(QUALITY_DB, timeout=60)
    cur = conn.execute("SELECT id FROM crawl_results WHERE url=?", (url,))
    r = cur.fetchone()
    conn.close()
    return r is not None

def store_item(item):
    url = item['url']
    content = item.get('content', '')
    summary = item.get('summary', extract_plain_text(content)[:300])
    date = item['date']
    title = item['title']

    content_hash = hashlib.md5((content or '').encode()).hexdigest()
    
    conn = sqlite3.connect(QUALITY_DB, timeout=60)
    try:
        conn.execute(
            "INSERT OR IGNORE INTO crawl_results "
            "(title, url, content, publish_date, summary, domain, content_hash) "
            "VALUES (?,?,?,?,?,?,?)",
            (title, url, content, date, summary, DOMAIN, content_hash)
        )
        conn.commit()
        return conn.total_changes > 0
    finally:
        conn.close()

# ─── 同步到服务器 ───
def sync_to_server():
    """将本地数据直接写入 search.db（服务器本地模式）"""
    print("\n📤 同步到 search.db...")

    conn = sqlite3.connect(QUALITY_DB, timeout=60)
    rows = conn.execute(
        "SELECT title, url, content, publish_date, summary FROM crawl_results WHERE domain=? ORDER BY id",
        (DOMAIN,)
    ).fetchall()
    conn.close()

    if not rows:
        print("  本地没有数据")
        return

    dst = sqlite3.connect("/root/search.db", timeout=60)
    dst.execute("PRAGMA journal_mode=WAL")
    dst.execute("PRAGMA busy_timeout=300000")

    site_name = "佛冈县人民政府"
    new_count = 0
    for r in rows:
        title, url, content, pub_date, summary = r
        try:
            dst.execute(
                "INSERT OR IGNORE INTO gov_raw "
                "(title, page_url, content, publish_date, summary, site_name, tags) "
                "VALUES (?,?,?,?,?,?,?)",
                (title, url, (content or "")[:500000], pub_date or "",
                 (summary or "")[:300], site_name, "")
            )
            if dst.total_changes > 0:
                new_count += 1
        except Exception as e:
            print(f"  Error: {e}")

    if new_count > 0:
        dst.commit()
        # Update FTS
        dst.execute(
            "INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) "
            "SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r "
            "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
            (site_name,))
        dst.commit()

    total = dst.execute(
        "SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (site_name,)).fetchone()[0]
    dst.close()

    print(f"  OK {new_count}/{len(rows)} 条同步到 search.db (DB共{total}条)")


# ─── 主逻辑 ───
def main():
    args = sys.argv[1:]
    full_mode = '--full' in args
    sync_only = '--sync' in args
    
    # 确定页数
    if sync_only:
        sync_to_server()
        return
    
    if full_mode:
        pages_to_crawl = TOTAL_PAGES
        mode = "全量"
    else:
        pages_match = [a for a in args if a.startswith('--pages=')]
        pages_to_crawl = int(pages_match[0].split('=')[1]) if pages_match else 1
        mode = f"增量（{pages_to_crawl}页）"
    
    print(f"🔍 佛冈通知公告爬虫 — {mode}模式")
    print(f"  总页数: {TOTAL_PAGES}, 本次爬取: {pages_to_crawl}页")
    print()
    
    ensure_table()
    
    new_total = 0
    skip_total = 0
    for page in range(1, pages_to_crawl + 1):
        if page == 1:
            url = LIST_URL
        else:
            url = f"http://www.fogang.gov.cn/ywdt/gggs/tzgg/index_{page}.html"
        
        html = http_get(url)
        if not html:
            print(f"  [p{page:2d}/{pages_to_crawl}] ✗ 获取列表页失败")
            continue
        
        items = parse_list_page(html)
        if not items:
            print(f"  [p{page:2d}/{pages_to_crawl}] ✗ 未解析到条目")
            continue
        
        page_new = 0
        page_skip = 0
        for item in items:
            if is_url_crawled(item['url']):
                page_skip += 1
                skip_total += 1
                continue
            
            # 获取详情页
            detail = get_detail(item['url'])
            if detail:
                item['title'] = detail['title'] or item['title']
                item['content'] = detail['content'] or ''
                item['summary'] = extract_plain_text(detail.get('content', ''))[:300]
            
            store_item(item)
            page_new += 1
            new_total += 1
    
        print(f"  [p{page:2d}/{pages_to_crawl}] ✓ {len(items)}条 (新增{page_new} 跳过{page_skip})", end='\r')
        time.sleep(0.5)  # 礼貌间隔
    
    print()
    print(f"\n📊 完成！新增: {new_total} | 跳过: {skip_total}")
    
    # 同步到服务器
    if new_total > 0:
        sync_to_server()
    else:
        print("  无新数据，跳过同步")

if __name__ == '__main__':
    main()