#!/usr/bin/env python3
"""Daily backfill: classify industry for all rows that slipped through without industry.

2026-08-11 修复: 原 LIMIT 1000 每轮只处理 1000 条 (361K other 要 361 天) →
改为一次取出全部未分类行全量 classify, 分批 UPDATE。crontab 注册 18:00。
"""
import sqlite3
import sys
import os

DB = "/root/search.db"
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))

sys.path.insert(0, "/root/gov_crawler")
from crawler_lib import classify_industry

db = sqlite3.connect(DB, timeout=120)
db.execute("PRAGMA busy_timeout=120000")
db.execute("PRAGMA journal_mode=WAL")
db.execute("PRAGMA synchronous=OFF")

# 全部未分类行 (含项目/工程关键词, 可能可分类)
rows = db.execute(
    "SELECT id, title FROM gov_raw "
    "WHERE (title LIKE '%项目%' OR title LIKE '%工程%') "
    "AND (industry IS NULL OR industry = '' OR industry = 'other') "
    "ORDER BY id"
).fetchall()

if not rows:
    print("No unclassified rows found.")
    db.close()
    exit(0)

print(f"未分类行: {len(rows)}")
updates = []
hits = 0
for rid, title in rows:
    ind = classify_industry(title or "")
    if ind and ind != 'other':
        updates.append((ind, rid))
        hits += 1
    if len(updates) >= 5000:
        db.executemany("UPDATE gov_raw SET industry = ? WHERE id = ?", updates)
        db.commit()
        updates = []

if updates:
    db.executemany("UPDATE gov_raw SET industry = ? WHERE id = ?", updates)
    db.commit()

print(f"Processed {len(rows)} rows, {hits} classified.")
db.close()