#!/usr/bin/env python3
"""
Re-import IC13 TXT files with content comparison:
Only append if content after normalization actually differs.

标题提取 2026-09-10 起单点复用 /root/gov_crawler/originals_sync.py:
  - 删除本文件自身的简化 extract_title(07-08 v4), 改为 import originals_sync 的
    extract_title/_tidy_title(09-03 v3/v4+tidy 全套: R1前缀族/Method NL/孤儿括号/
    规模尾缀/marker cut/中英比60%/净度守护) —— 与 originals note 62K 引擎同一套逻辑,
    标题知识只维护 originals_sync.py 一处(三处同步 md5 校验)。
  - resolve_title() 复刻 originals_sync main() 的净度守护兜底(extract 结果含 tab/_____/
    长英文/超100字/重复拼接 → _tidy_title → 仍脏回退纯 id_code)。
  - id_code/source='pharma'/append-on-change(content normalize 对比) 语义不变。
"""
import os, sys, sqlite3, datetime, re

sys.path.insert(0, '/root/gov_crawler')          # originals_sync.py 单点 (与本地 md5 同步)
import originals_sync as osx                      # 仅 import 函数, main 有 __main__ guard 无副作用

DB = '/root/originals.db'
IC13_DIR = '/tmp/ic13_copy_new/'
TODAY = datetime.date.today().strftime('%Y-%m-%d')


def normalize(text):
    if not text:
        return ''
    t = text.strip()
    t = re.sub(r'\s+', ' ', t)
    return t


def read_txt(filepath):
    for enc in ('gbk', 'gb18030', 'utf-8'):
        try:
            with open(filepath, 'r', encoding=enc) as f:
                return f.read()
        except (UnicodeDecodeError, UnicodeError):
            continue
    with open(filepath, 'r', encoding='latin-1') as f:
        return f.read()


def resolve_title(fname, text, id_code):
    """复刻 originals_sync main() 的提取+净度守护兜底, 保证 IC13 与 originals note 标题标尺一致"""
    title = osx.extract_title(fname, text)
    if not title:
        return id_code
    _dup = re.search(r'(.{8,}?)\1', title)
    if re.search(r'[\t_]{2,}|[A-Za-z]{4,}', title) or len(title) > 100 or (_dup and len(_dup.group(1)) >= 8):
        _tt = osx._tidy_title(title)
        if _tt:
            title = _tt
        else:
            title = id_code
    return title


def main():
    if not os.path.isdir(IC13_DIR):
        print(f'Directory {IC13_DIR} not found. Nothing to do.')
        return

    conn = sqlite3.connect(DB)
    conn.execute('PRAGMA journal_mode=OFF')
    conn.execute('PRAGMA synchronous=OFF')
    c = conn.cursor()

    existing = {}
    for row in c.execute(
        "SELECT id_code, content FROM tsk_data WHERE source='pharma' AND id_code IS NOT NULL AND id_code != ''"
    ):
        existing[row[0].strip()] = row[1] if row[1] else ''

    max_id = c.execute("SELECT MAX(id) FROM tsk_data").fetchone()[0] or 0

    files = sorted([f for f in os.listdir(IC13_DIR) if f.endswith('.txt')])
    print(f'Existing pharma id_codes: {len(existing)}')
    print(f'IC13 files: {len(files)}')

    new_count = update_count = skip_count = error_count = 0
    updates = []
    inserts = []

    for fname in files:
        id_code = fname.replace('.txt', '')
        filepath = os.path.join(IC13_DIR, fname)

        try:
            text = read_txt(filepath)
        except Exception as e:
            print(f'  ERROR reading {fname}: {e}')
            error_count += 1
            continue

        # Split: first line is title candidate, rest is content
        parts = text.split('\n', 1)
        content_new = parts[1].strip() if len(parts) > 1 else ''
        title = resolve_title(fname, text, id_code)

        if id_code in existing:
            old_content = existing[id_code]
            old_norm = normalize(old_content)
            new_norm = normalize(content_new)

            if old_norm == new_norm:
                skip_count += 1
                continue

            SEPARATOR = f'\n\n---\n更新日期: {TODAY}\n---\n\n'
            if old_content:
                content_new = old_content + SEPARATOR + content_new
            updates.append((title, content_new, id_code))
            update_count += 1
        else:
            max_id += 1
            inserts.append((max_id, id_code, title, content_new))
            new_count += 1

    print(f'\nProcessing...')
    if updates:
        c.executemany(
            "UPDATE tsk_data SET title=?, content=? WHERE id_code=? AND source='pharma'",
            updates
        )
        print(f'  Updates: {update_count}')
    if inserts:
        c.executemany(
            "INSERT INTO tsk_data (id, source, id_code, title, content) VALUES (?, 'pharma', ?, ?, ?)",
            inserts
        )
        print(f'  New inserts: {new_count}')

    conn.commit()
    total = c.execute("SELECT COUNT(*) FROM tsk_data WHERE source='pharma'").fetchone()[0]
    conn.close()

    print(f'Skipped (no change): {skip_count}')
    print(f'  Errors: {error_count}')
    print(f'  Total pharma records: {total}')

if __name__ == '__main__':
    main()