#!/usr/bin/env python3
"""
政府公告爬虫 v2 - 独立运行版（不依赖 Scrapy 完整栈）
使用火车头对齐规则（ljobx_rules.py + ljobx_list_rules.py）
每30分钟增量爬取 → 入库 → 可推送到服务器
"""

import sys, os, re, json, sqlite3, hashlib, time
from datetime import datetime
from urllib.parse import urlparse, urljoin

# ─── 配置 ───
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
SOURCES_CSV = os.path.join(BASE_DIR, "data", "sources.csv")
DB_PATH = os.path.join(BASE_DIR, "crawler_results.db")
RULES_DIR = os.path.join(BASE_DIR, "utils")

# ─── 规则加载 ───
def load_detail_rules():
    """加载 ljobx_rules.py （1051条详情页规则）"""
    sys.path.insert(0, RULES_DIR)
    from ljobx_rules import LJOBX_RULES
    return LJOBX_RULES

def load_list_rules():
    """加载 ljobx_list_rules.py（913条列表页规则）"""
    sys.path.insert(0, RULES_DIR)
    from ljobx_list_rules import LJOBX_LIST_RULES
    return LJOBX_LIST_RULES

# ─── 数据库 ───
def init_db():
    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()
    cur.executescript("""
        CREATE TABLE IF NOT EXISTS crawl_sites (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            name TEXT NOT NULL,
            url TEXT NOT NULL,
            domain TEXT,
            source_type TEXT DEFAULT 'gov',
            enabled INTEGER DEFAULT 1,
            last_crawled_at TEXT,
            total_items INTEGER DEFAULT 0,
            error_count INTEGER DEFAULT 0
        );
        CREATE TABLE IF NOT EXISTS crawl_results (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            site_id INTEGER NOT NULL,
            title TEXT NOT NULL,
            url TEXT UNIQUE NOT NULL,
            content TEXT,
            publish_date TEXT,
            summary TEXT,
            domain TEXT,
            category TEXT DEFAULT 'gov',
            crawled_at TEXT DEFAULT (datetime('now','localtime')),
            content_hash TEXT
        );
        CREATE INDEX IF NOT EXISTS idx_results_url ON crawl_results(url);
        CREATE INDEX IF NOT EXISTS idx_results_domain ON crawl_results(domain);
        CREATE INDEX IF NOT EXISTS idx_results_date ON crawl_results(publish_date);
        CREATE INDEX IF NOT EXISTS idx_results_site ON crawl_results(site_id);
    """)
    conn.commit()
    return conn

# ─── 站点加载 ───
def load_sites_csv():
    """从 sources.csv 加载站点"""
    import csv
    sites = []
    with open(SOURCES_CSV, 'r', encoding='utf-8-sig') as f:
        reader = csv.reader(f)
        for row in reader:
            if len(row) >= 2:
                name = row[0].strip()
                url = row[1].strip()
                status = row[2].strip() if len(row) > 2 else ''
                note = row[3].strip() if len(row) > 3 else ''
                if url and url.startswith('http'):
                    domain = urlparse(url).netloc
                    sites.append({
                        'name': name, 'url': url, 'domain': domain,
                        'status': status, 'note': note
                    })
    return sites

# ─── 列表页解析 ───
def parse_list_page(html, url, list_rules, detail_rules):
    """解析列表页：先用ljobx规则过滤，再用通用选择器"""
    import requests
    from bs4 import BeautifulSoup
    
    domain = urlparse(url).netloc
    soup = BeautifulSoup(html, 'html.parser')
    items = []
    
    # 1. 获取ljobx列表规则
    list_rule = list_rules.get(domain, ("", "", "", "", (0, 0)))
    area_start, area_end, url_must, url_forbid, pagination = list_rule
    
    # 2. 如果有区域规则，缩小搜索范围
    search_soup = soup
    if area_start and area_end:
        html_text = str(soup)
        start_idx = html_text.find(area_start)
        end_idx = html_text.find(area_end, start_idx)
        if start_idx >= 0 and end_idx > start_idx:
            search_soup = BeautifulSoup(html_text[start_idx:end_idx], 'html.parser')
    
    # 3. 提取所有链接
    for a_tag in search_soup.find_all('a', href=True):
        href = a_tag['href']
        if href.startswith('#') or href.startswith('javascript') or href.startswith('mailto'):
            continue
        
        full_url = urljoin(url, href)
        title = (a_tag.get('title') or a_tag.get_text() or '').strip()
        if not title:
            continue
        
        # 4. ljobx URL规则过滤
        if url_must and url_must not in full_url:
            continue
        if url_forbid and url_forbid in full_url:
            continue
        
        # 5. 过滤导航链接
        nav_keywords = ['首页', '上一页', '下一页', '末页', '返回', '关于我们', '联系我们']
        if any(kw in title for kw in nav_keywords):
            continue
        
        # 6. 提取日期
        date = ''
        date_patterns = [
            r'(20\d{2})[-/](\d{1,2})[-/](\d{1,2})',
            r'(\d{4})年(\d{1,2})月(\d{1,2})日',
        ]
        # 从周围文本找日期
        parent = a_tag.parent
        if parent:
            parent_text = parent.get_text()
            for pat in date_patterns:
                m = re.search(pat, parent_text)
                if m:
                    date = m.group(0)
                    break
        
        # 7. 标题质量过滤
        if len(title) < 6:
            continue
            
        items.append({
            'title': title,
            'url': full_url,
            'date': date,
        })
    
    # 8. 去重
    seen = set()
    unique_items = []
    for item in items:
        if item['url'] not in seen:
            seen.add(item['url'])
            unique_items.append(item)
    
    return unique_items[:50]  # 最多取50条

# ─── 详情页解析 ───
def parse_detail_page(html, url, detail_rules):
    """提取正文：先用ljobx规则，再用通用选择器"""
    from bs4 import BeautifulSoup
    
    domain = urlparse(url).netloc
    soup = BeautifulSoup(html, 'html.parser')
    
    # 尝试用ljobx规则提取
    rule = detail_rules.get(domain)
    if rule:
        title_start, title_end, content_start, content_end = rule
        
        # 提取标题
        title = ''
        if title_start and title_end:
            m = re.search(re.escape(title_start) + r'(.*?)' + re.escape(title_end), html, re.DOTALL)
            if m:
                title = BeautifulSoup(m.group(1), 'html.parser').get_text().strip()
        
        # 提取正文
        content = ''
        if content_start and content_end:
            if content_start == '<(*)>' and content_end == '</html>':
                # 全文模式
                m = re.search(r'<body[^>]*>(.*?)</body>', html, re.DOTALL)
                if m:
                    content = BeautifulSoup(m.group(1), 'html.parser').get_text()
            else:
                m = re.search(re.escape(content_start) + r'(.*?)' + re.escape(content_end), html, re.DOTALL)
                if m:
                    content = BeautifulSoup(m.group(1), 'html.parser').get_text()
        
        if content:
            content = re.sub(r'\s+', ' ', content).strip()
            if len(content) > 100:
                return {'title': title or '', 'content': content}
    
    # 通用选择器兜底
    content_selectors = [
        'div.content', 'div.article', 'div.text', '#content',
        'div.TRS_Editor', 'div.news-content', 'div.article-content',
        'div.detail', 'div.info-content', 'div.maintext', '#zoom',
        'article', 'div.show_content', 'div.con_text', 'div.bt_content',
        'td.content', 'div.page-content', 'div.txtcontent-div',
    ]
    
    # 标题选择器
    title_selectors = ['h1', 'h2', 'title', 'meta[property="og:title"]']
    
    for sel in title_selectors:
        if sel.startswith('meta'):
            t = soup.find('meta', {'property': 'og:title'})
            if t and t.get('content'):
                title = t['content'].strip()
                break
        else:
            t = soup.select_one(sel)
            if t:
                title = t.get_text().strip()
                if len(title) > 5:
                    break
    
    for sel in content_selectors:
        node = soup.select_one(sel)
        if node:
            text = node.get_text(separator='\n').strip()
            text = re.sub(r'\s+', ' ', text).strip()
            if len(text) > 100:
                # 过滤页脚噪声
                noise_keywords = ['版权所有', 'ICP', '备案号', '技术支持', '网站地图', '关于我们']
                lines = text.split('\n')
                clean_lines = [l for l in lines if not any(k in l for k in noise_keywords)]
                text = '\n'.join(clean_lines).strip()
                if len(text) > 100:
                    return {'title': title or '', 'content': text}
    
    return {'title': title or '', 'content': ''}

# ─── 主流程 ───
def crawl_site(site, conn, list_rules, detail_rules, limit_items=10):
    """爬取单个站点"""
    import requests
    
    name = site['name']
    url = site['url']
    domain = site['domain']
    
    print(f"\n📋 {name} ({url})")
    
    # 1. 判断是否已爬过
    cur = conn.cursor()
    cur.execute("SELECT id FROM crawl_sites WHERE url=? AND domain=?", (url, domain))
    row = cur.fetchone()
    if row:
        site_id = row[0]
        cur.execute("UPDATE crawl_sites SET last_crawled_at=datetime('now') WHERE id=?", (site_id,))
    else:
        cur.execute("INSERT INTO crawl_sites (name, url, domain) VALUES (?,?,?)", (name, url, domain))
        site_id = cur.lastrowid
    conn.commit()
    
    # 2. 获取列表页
    try:
        r = requests.get(url, timeout=20, headers={
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)',
            'Accept': 'text/html,application/xhtml+xml',
            'Accept-Language': 'zh-CN,zh;q=0.9',
        })
        r.encoding = r.apparent_encoding  # auto-detect encoding
        if r.status_code != 200:
            print(f"  ❌ HTTP {r.status_code}")
            cur.execute("UPDATE crawl_sites SET error_count=error_count+1 WHERE id=?", (site_id,))
            conn.commit()
            return 0
    except Exception as e:
        print(f"  ❌ 请求失败: {e}")
        return 0
    
    # 3. 解析列表页
    items = parse_list_page(r.text, url, list_rules, detail_rules)
    print(f"  发现 {len(items)} 条")
    
    if not items:
        return 0
    
    # 4. 取前N条详情
    new_count = 0
    for item in items[:limit_items]:
        detail_url = item['url']
        
        # 去重
        cur.execute("SELECT id FROM crawl_results WHERE url=?", (detail_url,))
        if cur.fetchone():
            continue
        
        # 获取详情
        try:
            r2 = requests.get(detail_url, timeout=20, headers={
                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)',
            })
            r2.encoding = r2.apparent_encoding
            if r2.status_code != 200:
                continue
        except:
            continue
        
        result = parse_detail_page(r2.text, detail_url, detail_rules)
        content = result.get('content', '')
        title = item['title'] or result.get('title', '')
        
        if not content or len(content) < 100:
            continue
        
        content_hash = hashlib.md5(content.encode()).hexdigest()
        
        try:
            cur.execute("""
                INSERT OR IGNORE INTO crawl_results 
                (site_id, title, url, content, publish_date, domain, category, content_hash, summary)
                VALUES (?,?,?,?,?,?,?,?,?)
            """, (
                site_id, title[:500], detail_url, content, item.get('date',''),
                domain, 'gov', content_hash, content[:200]
            ))
            if cur.rowcount > 0:
                new_count += 1
                print(f"  ✅ {title[:50]}")
        except Exception as e:
            print(f"  ⚠ {e}")
    
    conn.commit()
    cur.execute("UPDATE crawl_sites SET last_crawled_at=datetime('now'), total_items=total_items+? WHERE id=?", (new_count, site_id))
    conn.commit()
    
    return new_count

def main():
    import argparse
    
    parser = argparse.ArgumentParser(description='政府公告爬虫 v2')
    parser.add_argument('--limit', type=int, default=5, help='爬取站点数')
    parser.add_argument('--items', type=int, default=10, help='每站详情数')
    parser.add_argument('--list', action='store_true', help='列出可用站点')
    args = parser.parse_args()
    
    # 加载
    detail_rules = load_detail_rules()
    list_rules = load_list_rules()
    all_sites = load_sites_csv()
    
    # 只取 Done 站点
    active_sites = [s for s in all_sites if s['status'] == 'Done']
    
    print(f"总站点: {len(all_sites)}, 可用: {len(active_sites)}")
    
    if args.list:
        for i, s in enumerate(active_sites[:30]):
            has_detail = '📄' if s['domain'] in detail_rules else '  '
            has_list = '📋' if s['domain'] in list_rules else '  '
            print(f"  {has_detail}{has_list} [{i+1}] {s['name']} ({s['domain']})")
        return
    
    # 初始化DB
    conn = init_db()
    print(f"数据库: {DB_PATH}")
    
    total_new = 0
    limit = min(args.limit, len(active_sites))
    
    for i, site in enumerate(active_sites[:limit]):
        n = crawl_site(site, conn, list_rules, detail_rules, args.items)
        total_new += n
        if i < limit - 1:
            time.sleep(2)  # 礼貌延迟
    
    conn.close()
    print(f"\n{'='*40}")
    print(f"完成！爬取 {limit} 个站点，新增 {total_new} 条")
    print(f"数据库: {DB_PATH}")
    
    # 输出摘要
    conn2 = sqlite3.connect(DB_PATH)
    total = conn2.execute("SELECT COUNT(*) FROM crawl_results").fetchone()[0]
    print(f"总记录: {total} 条")
    conn2.close()

if __name__ == "__main__":
    main()