#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""gov_entity 增量维护 (2026-09-03 上线)
每晚 cron 4:20 运行: 
  1) 清理孤儿 (gov_entity 引用的 doc 已在 gov_raw 删除的)
  2) 找缺失 doc (gov_raw 中无实体的新入库行) → 提取 完整公司名+电话 → 补插
用法: python3 rebuild_entity_inc.py [--full] [--limit N]
  --full   全量重建 (灾难恢复后或首次), 流式分批 ~650 篇/s (54.5万≈14min)
"""
import sqlite3, re, sys, time

DB = '/root/search.db'   # 软链 → /mnt/data/search.db
FULL = '--full' in sys.argv
LIMIT = None
for _a in sys.argv:
    if _a.startswith('--limit='):
        LIMIT = int(_a.split('=')[1])

# ── 提取规则 (与试点 /mnt/data/drill/entity_extract.py v3 一致) ──
COMPANY_RE = re.compile(
    r'[\u4e00-\u9fffA-Za-z0-9（）()]{2,28}?'
    r'(?:有限责任公司|股份有限公司|集团有限公司|有限公司)'
)
STOP_PREFIX = set('该本我贵此全各其这那另上共下前')
PHONE_RE = re.compile(r'1[3-9]\d{9}|0\d{2,3}-?\d{7,8}|400-?\d{7}')
FUNC_TOKENS = ['受理', '依法', '解除', '报批', '委托', '同意', '编制', '评价', '承接',
               '负责', '经过', '我局', '已受理', '予以', '进行', '组织', '开展', '受托',
               '经', '由', '拟', '受', '对', '向', '为', '在', '将', '请', '现', '与', '及',
               '关于', '结合', '联系', '可联系', '如需']
CLAUSE_PUNCT = '，。；：？！、\n\r\t ()（）【】「」『』""\'\''
ADDR_MARKERS = '号栋幢室巷日'
GENERIC_WORDS = {'科技', '环保', '环境', '工程', '建设', '建筑', '实业', '发展', '控股',
                 '投资', '材料', '机械', '电子', '信息', '能源', '劳务', '运输', '商贸',
                 '贸易', '农业', '生物', '制药', '化工', '置业', '咨询', '设计', '监理',
                 '检测', '评估', '评价', '技术', '服务', '管理', '网络', '传媒', '文化',
                 '清洁', '资源', '综合', '集团', '有限', '责任', '公司'}
DATE_RE = re.compile(r'^(?:\d{1,2}月\d{1,2}日|\d{1,2}月|月\d{1,2}日|\d{1,2}日|\d+年)')


def norm_phone(m):
    return re.sub(r'\D', '', m)


def clean_company(name):
    name = name.strip()
    cut = 0
    for i, ch in enumerate(name):
        if ch in CLAUSE_PUNCT:
            cut = i + 1
    name = name[cut:].strip()
    name = DATE_RE.sub('', name)
    best = -1
    for i, ch in enumerate(name[:25]):
        if ch in ADDR_MARKERS:
            best = i
    if best >= 0:
        name = name[best + 1:].strip()
    best = 0
    for tok in FUNC_TOKENS:
        idx = name.rfind(tok)
        if idx >= 0:
            best = max(best, idx + len(tok))
    name = name[best:].strip('（(）) ')
    while name and name[0] in STOP_PREFIX and len(name) > 2:
        name = name[1:]
    name = name.strip('。，;；、 ')
    if len(name) < 5 or not ('有限' in name or name.endswith('公司') or name.endswith('集团')):
        return ''
    core = re.sub(r'(?:有限责任公司|股份有限公司|集团有限公司|有限公司|公司|集团)$', '', name)
    core_clean = ''.join(ch for ch in core if ch not in GENERIC_WORDS and not ch.isdigit())
    if len(core_clean) < 2:
        return ''
    if name[0].isdigit() or name[0] in CLAUSE_PUNCT or name[0] in '，。、':
        return ''
    return name


def extract(doc_id, title, content, summary):
    text = (title or '') + '\n' + (content or '') + '\n' + (summary or '')
    comps = set()
    for m in COMPANY_RE.finditer(text):
        cn = clean_company(m.group(0))
        if cn:
            comps.add(cn)
    phones = {norm_phone(m) for m in PHONE_RE.findall(text)}
    return [(doc_id, 'company', v) for v in comps] + [(doc_id, 'phone', v) for v in phones]


def main():
    conn = sqlite3.connect(DB, timeout=120)
    conn.execute("PRAGMA busy_timeout=120000")
    conn.execute("PRAGMA journal_mode=WAL")
    c = conn.cursor()
    w = conn.cursor()

    # 1) 孤儿清理 (每晚, 防 gov_raw 去重删除后残留)
    t0 = time.time()
    conn.execute("CREATE TABLE IF NOT EXISTS gov_entity_proc(doc_id INTEGER PRIMARY KEY)")
    if FULL:
        # 全量重建 = 清空重来 (防重复)
        w.execute("DELETE FROM gov_entity")
        w.execute("DELETE FROM gov_entity_proc")
        conn.commit()
        print(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] 全量模式: 已清空 gov_entity/gov_entity_proc", flush=True)
        orphans = 0
    else:
        c.execute("SELECT COUNT(*) FROM gov_entity e WHERE NOT EXISTS (SELECT 1 FROM gov_raw r WHERE r.id = e.doc_id)")
        orphans = c.fetchone()[0]
    if orphans:
        w.execute("DELETE FROM gov_entity WHERE NOT EXISTS (SELECT 1 FROM gov_raw r WHERE r.id = gov_entity.doc_id)")
        w.execute("DELETE FROM gov_entity_proc WHERE NOT EXISTS (SELECT 1 FROM gov_raw r WHERE r.id = gov_entity_proc.doc_id)")
        conn.commit()
    print(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] 孤儿清理: {orphans} 条 ({time.time()-t0:.0f}s)", flush=True)

    # 2) 缺失 doc (按已处理登记表判, 无实体的文档也登记, 避免每晚空扫)
    if FULL:
        c.execute("SELECT COUNT(*) FROM gov_raw")
    else:
        c.execute("SELECT COUNT(*) FROM gov_raw r WHERE NOT EXISTS (SELECT 1 FROM gov_entity_proc p WHERE p.doc_id = r.id)")
    missing = c.fetchone()[0]
    if missing == 0:
        print(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] 无缺失, 完成", flush=True)
        return
    print(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] 待处理 {missing} 篇 {'(全量重建)' if FULL else '(增量)'}", flush=True)

    if FULL:
        c.execute("SELECT id, title, content, summary FROM gov_raw ORDER BY id")
    else:
        c.execute("""SELECT r.id, r.title, r.content, r.summary FROM gov_raw r
                     WHERE NOT EXISTS (SELECT 1 FROM gov_entity_proc p WHERE p.doc_id = r.id)
                     ORDER BY r.id""")
    t_start = time.time()
    done = 0
    buf = []
    buf_proc = []
    while True:
        if LIMIT and done >= LIMIT:
            break
        row = c.fetchone()
        if row is None:
            break
        buf.extend(extract(*row))
        buf_proc.append((row[0],))
        done += 1
        if len(buf) >= 50000:
            w.executemany('INSERT INTO gov_entity VALUES (?,?,?)', buf)
            w.executemany('INSERT OR IGNORE INTO gov_entity_proc VALUES (?)', buf_proc)
            conn.commit()
            buf = []
            buf_proc = []
        if done % 20000 == 0:
            el = time.time() - t_start
            print(f"  {done}/{missing} 篇, {el:.0f}s ({done/el:.0f} 篇/s)", flush=True)
    if buf:
        w.executemany('INSERT INTO gov_entity VALUES (?,?,?)', buf)
    if buf_proc:
        w.executemany('INSERT OR IGNORE INTO gov_entity_proc VALUES (?)', buf_proc)
    if buf or buf_proc:
        conn.commit()
    el = time.time() - t_start
    n = conn.execute("SELECT COUNT(*) FROM gov_entity").fetchone()[0]
    print(f"✅ 处理 {done} 篇 {el:.0f}s ({done/el:.0f} 篇/s), gov_entity 现有 {n} 行", flush=True)


if __name__ == '__main__':
    main()