#!/usr/bin/env python3
"""Batch import all A-class JSONL files into search.db.

Handles varying field names across scripts:
  - url/link/source_url → page_url
  - date/publish_date → publish_date
  - category/group → group_name
"""
import json, sqlite3, os, sys, re, html

DB_PATH = os.environ.get("SEARCH_DB", "/root/search.db")

# A-class JSONL files with their config name and description
BATCH = [
    ("/root/gov_crawler/output/hbnq_gsgg.jsonl",      "内丘县人民政府-公告公示"),
    ("/root/gov_crawler/output/panjin_notice.jsonl",    "盘锦辽滨沿海经济技术开发区-通知公告"),
    ("/root/gov_crawler/output/panjin_hjbh.jsonl",      "盘锦辽滨沿海经济技术开发区-环境保护"),
    ("/root/gov_crawler/output/guazhou.jsonl",          "瓜州县人民政府-公示公告"),
    ("/root/gov_crawler/output/gdhuanke_notice.jsonl",  "广东环科技术咨询有限公司-公示公告"),
    ("/root/gov_crawler/output/gdiecc.jsonl",           "广东省国际工程咨询有限公司-公示公告"),
    ("/root/gov_crawler/output/fxxq.jsonl",             "新邱区-通知公告"),
    ("/tmp/hscc_output.jsonl",                          "恒申集团-信息公示"),
]

def get_url(item):
    """Get page_url from record, handling different field names."""
    return item.get("page_url") or item.get("url") or item.get("link") or item.get("source_url") or ""

def get_date(item):
    """Get publish_date from record."""
    return item.get("publish_date") or item.get("date") or ""

def get_group(item):
    """Get group from record."""
    return item.get("group") or item.get("category") or ""

def strip_html(text):
    if not text:
        return ""
    text = html.unescape(text)
    text = re.sub(r'</?(?:p|div|h[1-6]|li|tr|blockquote|section|article|table|br\s*/?)[^>]*>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'<[^>]+>', '', text)
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    return text.strip()

def import_file(path, config_name):
    if not os.path.exists(path):
        return f"❌ 文件不存在"
    if os.path.getsize(path) == 0:
        return f"⚠️ 文件为空"
    
    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    added = 0
    skipped = 0
    errors = 0
    
    with open(path) as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                item = json.loads(line)
            except:
                errors += 1
                continue
            
            title = (item.get("title") or "")[:500]
            page_url = get_url(item)[:1000]
            if not page_url:
                errors += 1
                continue
            
            content = item.get("content") or ""
            publish_date = get_date(item)[:20]
            site_name = (item.get("site_name") or config_name)[:100]
            group_name = get_group(item)[:100]
            
            # Attachments as JSON string
            att = item.get("attachments") or []
            if isinstance(att, str):
                attachments_str = att
            else:
                attachments_str = json.dumps(att, ensure_ascii=False) if att else ""
            
            # Summary: plain text version
            if item.get("summary"):
                summary = item["summary"][:500]
            else:
                summary = strip_html(content)[:500]
            
            try:
                c.execute("""
                    INSERT OR IGNORE INTO gov_raw 
                        (title, page_url, content, publish_date, site_name, 
                         source_url, status, attachments, group_name, summary)
                    VALUES (?, ?, ?, ?, ?, ?, 'synced', ?, ?, ?)
                """, (title, page_url, content, publish_date, site_name,
                      page_url, attachments_str, group_name, summary))
                if c.rowcount > 0:
                    added += 1
                else:
                    skipped += 1
            except Exception as e:
                print(f"  DB error: {e}")
                errors += 1
    
    conn.commit()
    conn.close()
    return f"+{added} skip={skipped} err={errors}"

def main():
    print("=" * 60)
    print("A类脚本批量入库")
    print("=" * 60)
    print(f"DB: {DB_PATH}")
    print()
    
    total_added = 0
    for path, config_name in BATCH:
        print(f"[{config_name}]")
        print(f"  📄 {path}")
        result = import_file(path, config_name)
        print(f"  {result}")
        print()
        # Extract number from result
        m = re.search(r'\+(\d+)', result)
        if m:
            total_added += int(m.group(1))
    
    # Verify
    conn = sqlite3.connect(DB_PATH)
    total = conn.execute("SELECT COUNT(*) FROM gov_raw").fetchone()[0]
    conn.close()
    print(f"📊 总入库: {total} 条")
    print(f"本次新增: {total_added} 条")

if __name__ == "__main__":
    main()