#!/usr/bin/env python3
"""
Originals sync with mtime-based incremental detection.
Uses SQL temp-file approach for all server-side DB operations.
Usage: python3 originals_sync.py              # Full sync
       python3 originals_sync.py <offset>      # From offset
       python3 originals_sync.py <offset> <limit>
"""
import os, re, sys, json, subprocess, time

# 默认 CloudStorage 路径；可用环境变量 ORIGINALS_DIR 覆盖（File Provider EDEADLK 时
# 改用 Group Containers 路径：~/Library/Group Containers/UBF8T346G9.OneDriveSyncClientSuite/OneDrive.noindex/OneDrive/originals note）
ORIGINALS_DIR = os.environ.get("ORIGINALS_DIR", "/Users/chniir0000outlook.com/Library/CloudStorage/OneDrive-个人/originals note")
SSH_TARGET = "hw-backup"
DB = "/root/originals.db"

# ── helpers ────────────────────────────────────────────────

def run_sql_on_server(sql, timeout=60):
    """Write SQL to temp file, scp to server, execute."""
    local_tmp = "/tmp/originals_sql_temp.sql"
    with open(local_tmp, 'w', encoding='utf-8') as f:
        f.write(sql)
    subprocess.run(f"scp {local_tmp} {SSH_TARGET}:/tmp/osql.sql", shell=True, capture_output=True, timeout=15)
    r = subprocess.run(
        f"ssh {SSH_TARGET} \"sqlite3 {DB} < /tmp/osql.sql\"",
        shell=True, capture_output=True, text=True, timeout=timeout
    )
    os.remove(local_tmp)
    subprocess.run(f"ssh {SSH_TARGET} \"rm -f /tmp/osql.sql\"", shell=True, capture_output=True, timeout=10)
    return r

def query_server(sql, timeout=30):
    """Run SELECT query and return stdout (raw text)."""
    local_tmp = "/tmp/originals_query_temp.sql"
    with open(local_tmp, 'w', encoding='utf-8') as f:
        f.write(sql)
    subprocess.run(f"scp {local_tmp} {SSH_TARGET}:/tmp/oq.sql", shell=True, capture_output=True, timeout=15)
    r = subprocess.run(
        f"ssh {SSH_TARGET} \"sqlite3 -separator '|' {DB} < /tmp/oq.sql\"",
        shell=True, capture_output=True, text=True, timeout=timeout
    )
    os.remove(local_tmp)
    subprocess.run(f"ssh {SSH_TARGET} \"rm -f /tmp/oq.sql\"", shell=True, capture_output=True, timeout=10)
    return r

def classify(filename):
    name = filename.replace('.txt', '')
    m7 = re.match(r'^(\d{7})(?:\s+(.*))?$', name)
    if m7: return m7.group(1), 'plant'
    m9 = re.match(r'^(\d{9})(?:\s+(.*))?$', name)
    if m9: return m9.group(1), 'cpi'
    md = re.match(r'^(\d+)-', name)
    if md:
        code = md.group(1)
        return code, 'cpi' if len(code) == 9 else ('plant' if len(code) == 7 else 'pharma')
    return '', 'pharma'

def _good_end(t):
    """合格标题结尾: 项目词结尾, 可带（N期）/裸期次/（N期）尾"""
    return bool(t and re.search(r'(项目|工程|装置|技改|扩建|一体化|基地|生产线|厂)(?:（[^（）]{0,14}）|\([^()]{0,14}\)|[一二三四五六七八九十\d]+期)?$', t))

def _tidy_title(t):
    """净度后处理: 去重复拼接片段/尾残留/超长。返回清洗后标题, 太脏返回 None"""
    if not t:
        return None
    t = t.strip()
    # 连续重复片段截断: 'X项目X项目' → 'X项目';  '公司公司项目' → '公司项目'(切掉第一份)
    # (最短重复单元, 单元>=6字; 切片语义保留后续内容, 勿用 t=unit 丢尾巴)
    for unit_len in range(6, len(t) // 2 + 1):
        unit = t[:unit_len]
        if t.startswith(unit + unit):
            t = t[len(unit):].strip()
            break
    def _tail_clean(s):
        # 去尾部杂质/标签前缀/项目双写折叠
        s = re.sub(r'(项目概况|环境影响评价|环境影响报告|环评|现状环境影响评价|的公示|公示$|第一次|第二次|报批前|公众参与|征求意见稿)信息?$', '', s)
        s = re.sub(r'^(项目标题|标题|项目名称|项目名)\s*[:：]?\s*', '', s)
        s = re.sub(r'(项目){2,}$', '项目', s)
        # 行尾孤儿括号/行首孤儿闭括号(上游合并单元格残留: '...项目【' 无对应关闭 → 截掉)
        s = re.sub(r'[【\[（(]+$', '', s).strip()
        s = re.sub(r'^[】\]）)]+', '', s).strip()
        # 规模尾缀: '...建设项目中型项目' → '...建设项目'(上游把规模级别拼进名称末尾)
        s = re.sub(r'(项目|工程)([大中小])型项目$', r'\1', s)
        return s.strip()
    # 字段标记截断: 名称后拼接的字段文案/公告尾(备案/编号/地址/环评公示等)一律截掉
    for _marker in ['项目编号', '项目基本信息', '详细地址', '项目选址', '预算投资', '投资总额',
                    '投资额', '占地面积', '备案', '工业项目', '旗县产业类项目', '跟踪', '（VIP）', '(VIP)',
                    '版本', '一期建设', '二期建设', '三期建设', '四期建设', '当前阶段', '立项申报', '手续审批',
                    '项目名称', '项目标题', '项目名', '（VIP', '(VIP', '（跟踪',
                    '环境影响评价', '公众参与', '受理', '拟审批', '公示', '公告']:
        _i = t.find(_marker)
        if _i > 0:
            t = t[:_i].strip()
    # 截断后暴露的新尾部 → 再清一轮
    t = _tail_clean(t)
    # 尾部产能尾巴剪: '...项目52万吨尿素'/'...示范项目26万吨PC' → 截到首个项目词尾
    # (尾巴=数字+产品串; 以（开头的期次括号受保护; 仅当整体还不是合格结尾时触发)
    if not _good_end(t):
        _mm = re.search(r'(项目|工程|装置|技改|扩建|基地|生产线|厂)', t)
        if _mm:
            _pre = t[:_mm.end()].strip()
            _tail2 = t[_mm.end():].strip()
            if (_tail2 and not _tail2.startswith(('（', '(')) and re.search(r'\d', _tail2)
                    and len(_pre) >= 8 and re.search(r'(项目|工程|装置|技改|扩建|基地|生产线|厂)', _pre)):
                t = _tail_clean(_pre)
    # 若含 电话/邮箱/Sheet/www/______ → 截到其前
    for bad in ['电话', '邮箱', 'Sheet', 'www.', '______']:
        idx = t.find(bad)
        if idx > 0:
            t = t[:idx].strip()
    # 中英混杂且中文不足 60% → 不干净
    cjk = len(re.findall(r'[\u4e00-\u9fff]', t))
    if len(t) > 0 and cjk / len(t) < 0.5:
        return None
    if not t or len(t) < 8:
        return None
    return t[:100]

def extract_title(filename, content):
    name = filename.replace('.txt', '')
    
    if content:
        _all_lines = [l.strip().split('\r')[0].strip() for l in content.split('\n')]
        # R0: 首行 tab 内容带 项目名称/项目标题 标签 → 该标签值即权威标题(300627153类:
        # 首行=真名, 正文深处别家/他项目行勿覆盖)。值须 tidy 后 good_end。
        if _all_lines:
            _m0 = re.match(r'^\d+\t(?:项目名称|项目标题|项目名)\s*[:：]\s*(.+)$', _all_lines[0])
            if _m0:
                _v0 = re.sub(r'^\s*[\-—]+', '', _m0.group(1).strip().rstrip('。.；;，,、:：'))
                _t0 = _tidy_title(_v0)
                if _t0 and _good_end(_t0):
                    return _t0
        # R1 v2: 强信号前缀行 → 冒号后中文项目名（多候选择优, 2026-09-03）
        # 前缀族排序: 名称字段(项目信息/名称/标题/工程) > 公司字段(公司名称/企业名称) > 元数据列(TYPE/Researcher)
        # Kelly New Project 格式: 'Researcher: <项目名>' / 'TYPE: 1、<项目名>（VIP）（跟踪N）' / '项目标题：<项目名>…'
        # Vivian/Charles 的 'Researcher: Vivian' 会被 长度/中文/关键词 预检自然滤掉
        # 同一前缀族内按行序; 无 good_end 候选时退回首个通过候选(仍优于 [来源Sheet] 垃圾)
        _R1_PREF = {
            '项目信息': 0, '项目名称': 0, '工程名称': 0, '建设项目名称': 0, '项目标题': 0,
            '公司名称&项目名称': 0, '项目名称&公司名称': 0,
            '公司名称': 1, '企业名称': 1,
            'TYPE': 2, 'Researcher': 2,
        }
        _LABEL_JUNK = re.compile(r'^(项目编号|项目基本信息|项目名称及概要|项目概况|目录|项目类型|项目所在地)')
        _STAGE_PFX = re.compile(r'^(拟在建|报批立项|手续审批|施工图设计|工程设计|立项申报|已投产|正在落实|拟建|在建|竣工|暂停|已竣工)\s*')
        def _try_join(_base, _li):
            # 软换行续行拼接: 候选非合格结尾时, 顺延拼接后续行(无冒号/非字段行)直到 good_end
            _acc = _base
            for _j in range(_li + 1, min(_li + 4, len(_all_lines))):
                _nxt = _all_lines[_j].strip()
                if (not _nxt or re.search(r'[:：]', _nxt)
                        or re.match(r'^(公司名称|项目信息|项目名称|工程名称|企业名称|Researcher|TYPE|ID|Unnamed|项目跟踪版本|项目基础|更新时间|发布时间|跟踪版本号|预算投资|所属|建设等级|联系人|手机|邮箱|电话|项目类型|公司简介)', _nxt)):
                    break
                _acc = (_acc + _nxt).strip()
                _jt = _tidy_title(_acc)
                if _jt and _good_end(_jt):
                    return _jt
            return None
        _cands = []
        for _li, _l in enumerate(_all_lines):
            _pm = re.match(r'^([^:：]{0,12}?)\s*[:：]\s*(.+)$', _l)
            if not _pm:
                continue
            _rank = _R1_PREF.get(_pm.group(1).strip())
            if _rank is None:
                continue
            _v = _pm.group(2).strip().rstrip('。.；;，,、:：')
            # 只剥列表序号 '1、/1./1)' 等(带分隔符), 不剥真实产能数字 '14000吨/年…'
            _v = re.sub(r'^\s*[\-—]+', '', _v)
            _v = re.sub(r'^\d+\s*[、.．)）]\s*', '', _v)
            # 超长合并行: 切掉 ' 2090594025641992194 …' 整段ID尾巴(空格+6位以上数字起)
            _v = re.split(r'\s+\d{6,}', _v)[0].strip()
            _v = _STAGE_PFX.sub('', _v)
            if not (8 <= len(_v) <= 120 and re.search(r'[\u4e00-\u9fff]', _v)
                    and not re.fullmatch(r'[\d\s.．、，,。%％/／()（）-]+', _v)
                    and not re.search(r'(电话|邮箱|手机|传真|Sheet|www\.)', _v, re.I)
                    and not _LABEL_JUNK.match(_v)):
                continue
            _vv = _tidy_title(_v)
            if _vv:
                if _good_end(_vv):
                    _cands.append((_rank, _li, _vv))
                else:
                    _jv = _try_join(_vv, _li)
                    if _jv:
                        _cands.append((_rank, _li, _jv))
                    elif re.search(r'(项目|工程|装置|技改|扩建|年产|基地|一体化|生产线|联产)', _vv):
                        _cands.append((_rank, _li, _vv))
        if _cands:
            _good = [c for c in _cands if _good_end(c[2])]
            _pool = _good if _good else _cands
            # 公司名优先(有限公司/公司/集团/厂…): 项目标题通常带业主; 顺带避开无公司副本行的OCR错字
            _has_comp = lambda s: bool(re.search(r'(有限公司|有限责任公司|股份有限公司|公司|集团|厂)', s))
            _pool.sort(key=lambda c: (0 if _has_comp(c[2]) else 1, c[0], c[1]))
            _best = _pool[0][2]
            # 公司名前缀补全: Tai/Other 型文件 公司名称/企业名称 行是独立纯公司名,
            # 当选中的 good_end 缺公司名时拼上(守卫: 公司值4-40字/以公司后缀结尾/无项目关键词/防重叠)
            if not _has_comp(_best):
                for _l2 in _all_lines:
                    _m2 = re.match(r'^(公司名称|企业名称)\s*[:：]\s*(.+)$', _l2)
                    if not _m2:
                        continue
                    _cv = _m2.group(2).strip().rstrip('。.；;，,、:：')
                    _cv = re.sub(r'^\s*[\-—]+', '', _cv)
                    _cv2 = _tidy_title(_cv)
                    if (_cv2 and 4 <= len(_cv2) <= 40
                            and re.search(r'(有限公司|有限责任公司|股份有限公司|公司|集团|厂)$', _cv2)
                            and not re.search(r'(项目|工程|装置|技改|扩建|年产|基地|一体化|生产线|联产)', _cv2)
                            and not _best.startswith(_cv2[:6])):
                        _merged = _cv2 + _best
                        if len(_merged) <= 100 and _good_end(_merged):
                            _best = _merged
                        break
            return _best
        # R2: 首行 tab 多字段中 ______ 分隔的尾段中文项目名（3494431 类）
        # 样本: '3494431\t新疆三五九…______Aksu…______…PLANT\t300765632___Will___N___新疆三五九新材料有限公司___新疆三五九新材料有限公司年产3万吨氯化聚合物生产线建设项目'
        if _all_lines:
            _fl = _all_lines[0]
            for _seg in re.split(r'\t', _fl):
                _pu = re.split(r'_{3,}', _seg)
                if len(_pu) >= 2:
                    _tail = _pu[-1].strip()
                    if (10 <= len(_tail) <= 120 and ord(_tail[0]) > 0x7f
                            and re.search(r'(项目|工程|装置)$', _tail)
                            and not re.search(r'(电话|邮箱|手机|传真|Sheet|www\.|GRASSROOT|PLANT$)', _tail, re.I)):
                        _tt = _tidy_title(_tail)
                        if _tt:
                            return _tt

    # Common tab-based extraction for all files with 3+ tab parts
    # 格式: [ID]\t[混排标题]\t[中文项目名称] 或更多字段
    if content:
        first_line = content.split('\n')[0].split('\r')[0]
        parts = first_line.split('\t')
        if len(parts) >= 3:
            p2 = parts[2].split('  ')[0].strip()
            # Truncate at ______ markers to get clean project name
            p2_clean = re.split(r'_{3,}', p2)[0].strip() or p2
            
            # Method 1: Clean project name directly in part[2] (truncated at ______)
            # Company name must appear at start; skip if starts with article openers
            if not any(p2_clean.startswith(w) for w in ['者', '记', '我', '今', '近', '昨', '项', '本', '从', '位', '工', '随', '来']):
                starts_with_company = re.match(r'^[\u4e00-\u9fff]+?(?:有限公司?|有限责任公司|化工)', p2_clean[:25])
                starts_with_company = starts_with_company or re.match(r'^[\u4e00-\u9fff]+?(?:公司|化工)', p2_clean[:20])
                if p2_clean and len(p2_clean) > 10 and ord(p2_clean[0]) > 0x7f \
                   and (re.search(r'(项目|工程|装置|技改|一体化|基地)$', p2_clean) or starts_with_company) \
                   and any(kw in p2_clean for kw in ['项目', '装置', '扩建', '技改', '年产', '工程']):
                    _m1 = _tidy_title(p2_clean)
                    if _m1 and re.search(r'(项目|装置|扩建|技改|年产|工程|基地|一体化|生产线)', _m1):
                        return _m1
            
            # Method 2: Continuous text with ______ markers
            if len(p2) > 200 and '______________' in p2:
                between = re.findall(r'_{3,}(.*?)(?=_{3,}|\(推荐\))', p2)
                for b in between:
                    b = b.strip()
                    if b.endswith('项目') and len(b) > 10 \
                       and re.match(r'^[\u4e00-\u9fff]+?(?:有限公司?|有限责任公司|化工)', b[:30]):
                        _m2t = _tidy_title(b)
                        if _m2t and re.search(r'(项目|工程|装置|技改|扩建|基地|一体化|生产线)', _m2t):
                            return _m2t
            
            # Method 3: Extract project description from article body + company name
            if len(p2) > 100:
                chinese_company = parts[1].split('______')[0] if '______' in parts[1] else ''
                proj_desc = re.search(r'(年产\d+[万千][吨瓦].*?(?:工程|项目|装置))', p2[:500])
                if chinese_company and proj_desc:
                    combined = chinese_company + proj_desc.group(1)
                    if len(combined) < 200:
                        _m3t = _tidy_title(combined)
                        if _m3t and re.search(r'(项目|工程|装置|技改|扩建|基地|一体化|生产线)', _m3t):
                            return _m3t
    
    # Fallback for files where content is in part[1] (parts[2:] are empty)
    if content and len(parts) >= 2 and (len(parts) < 3 or not parts[2].strip()):
        p1 = parts[1].strip()
        # Try regex: company name + project description
        proj_match = re.match(r'^([\u4e00-\u9fff]+?(?:有限公司?|有限责任公司).*?(?:项目|工程|装置))', p1[:200])
        if proj_match:
            _p1t = _tidy_title(proj_match.group(1))
            if _p1t:
                return _p1t
    
    # Fallback for multi-tab files: try part[4] (pure Chinese company name) if part[2] starts with non-Chinese
    if content and len(parts) > 4 and parts[4].strip():
        cn_name = parts[4].strip()
        if cn_name and ord(cn_name[0]) > 0x7f and len(cn_name) < 100:
            # Try to get project description from part[7] (English title) or part[2]
            for src in [parts[7] if len(parts) > 7 else '', parts[2]]:
                desc = re.search(r'(?:GRASSROOT|UNIT ADDITION|PHASE)\s*(.*?)(?:\t|$)', src)
                if desc:
                    eng = desc.group(1).strip().lower()
                    # Remove common English words from title
                    short = cn_name
                    if len(short) > 5:
                        return short
            return cn_name
    
    # Filename-based extraction
    m9 = re.match(r'^(\d{9})(?:\s+(.*))?$', name)
    if m9 and m9.group(2):
        return m9.group(2).strip()
    m7 = re.match(r'^(\d{7})(?:\s+(.*))?$', name)
    if m7 and m7.group(2):
        return m7.group(2).strip()
    if not re.match(r'^\d+$', name):
        return name.strip()
    
    # Method NL: 多行非 tab 格式（首行纯ID → 日期行 → 标题行连续重复两次 → 正文）
    # 样本: "300646655\n1/20/2025 2:53:00 PM\n中韩石化武汉80万/年乙烯工程\n中韩石化武汉80万/年乙烯工程\n\n<正文>"
    # 标题关键词须含 工程/项目/装置 等（工程结尾同样有效，如 中韩石化武汉80万/年乙烯工程）
    # ⚠️ 只在前 8 行找：正文内也会有重复句（如 300752861 "目前2万吨/年磷酸铁锂项目已投产…"），必须限制位置
    if content:
        lines = [l.strip().split('\r')[0].strip() for l in content.split('\n')][:8]
        for i in range(len(lines) - 1):
            a, b = lines[i], lines[i + 1]
            if (a and a == b and 6 <= len(a) <= 80
                    and ord(a[0]) > 0x7f
                    and re.search(r'(工程|项目|装置|技改|扩建|年产|基地|一体化)', a)
                    and not re.search(r'(电话|邮箱|传真|手机|地址|Sheet)', a)):
                _nl = _tidy_title(a)
                if _nl:
                    return _nl

    # Final fallback: strip ID from first line
    if content:
        fl = content.split('\n')[0].strip().split('\r')[0].strip()
        fd = re.sub(r'^\d+\t', '', fl)
        if fd and fd != fl:
            _fb = _tidy_title(fd)
            if _fb:
                return _fb
            return fd.strip()[:100]
        return fl[:100].strip() or name
    return name

# ── main ───────────────────────────────────────────────────

def main():
    offset = int(sys.argv[1]) if len(sys.argv) > 1 else 0
    limit = int(sys.argv[2]) if len(sys.argv) > 2 else 0

    # 1. Ensure tracker table + UNIQUE index on id_code
    run_sql_on_server(
        "CREATE TABLE IF NOT EXISTS sync_tracker "
        "(filename TEXT PRIMARY KEY, mtime REAL, last_synced TEXT);"
    )
    run_sql_on_server(
        "CREATE UNIQUE INDEX IF NOT EXISTS idx_tsk_data_id_code "
        "ON tsk_data(id_code) WHERE id_code IS NOT NULL AND id_code != '';"
    )

    # 2. Load tracker
    r = query_server("SELECT filename, mtime FROM sync_tracker;")
    tracker = {}
    for line in r.stdout.strip().split('\n'):
        if '|' in line:
            fn, mt = line.split('|', 1)
            tracker[fn] = float(mt)
    print(f"[INFO] Tracker has {len(tracker)} entries")

    # 3. Scan files (mtime only, no reads)
    files = sorted([f for f in os.listdir(ORIGINALS_DIR)
                    if f.endswith('.txt') and f != '000alerts.txt'])
    if limit > 0:
        files = files[offset:offset+limit]
    elif offset > 0:
        files = files[offset:]

    print(f"[INFO] Scanning {len(files)} files (offset={offset}, limit={'all' if limit==0 else limit})")

    changed = []
    skipped = 0
    for filename in files:
        fp = os.path.join(ORIGINALS_DIR, filename)
        try:
            cmt = os.stat(fp).st_mtime
        except:
            skipped += 1
            continue
        pmt = tracker.get(filename)
        if pmt is not None and abs(cmt - pmt) < 0.01:
            skipped += 1
        else:
            changed.append((filename, cmt))

    print(f"[INFO] mtime: {len(changed)} changed, {skipped} skipped (unchanged)")

    if not changed:
        print("[DONE] No changes since last sync")
        return

    # 4. Process changed files (batch mode)
    imported = 0
    errors = 0
    sbreak = {}
    BATCH_SIZE = 100

    for batch_start in range(0, len(changed), BATCH_SIZE):
        batch = changed[batch_start:batch_start+BATCH_SIZE]
        data_sql_lines = []
        tracker_sql_lines = []

        for filename, cmt in batch:
            fp = os.path.join(ORIGINALS_DIR, filename)
            try:
                # Try UTF-8 first, fall back to GBK/GB18030 (most files are GBK encoded)
                try:
                    with open(fp, 'r', encoding='utf-8') as fh:
                        content = fh.read()
                except UnicodeDecodeError:
                    # Try strict GBK; if that fails (corrupt bytes), use errors='replace'
                    try:
                        with open(fp, 'r', encoding='gbk') as fh:
                            content = fh.read()
                    except UnicodeDecodeError:
                        try:
                            with open(fp, 'r', encoding='gb18030') as fh:
                                content = fh.read()
                        except UnicodeDecodeError:
                            raw = open(fp, 'rb').read()
                            content = raw.decode('gbk', errors='replace')
            except Exception as e:
                print(f"[ERR] Read {filename}: {e}")
                errors += 1
                continue

            id_code, source = classify(filename)
            title = extract_title(filename, content)
            # 统一净度守护: tab/______/英文串/超长/重复拼接 杂质 → tidy; 仍脏 → 退回文件名
            _dup = re.search(r'(.{8,}?)\1', title or '')
            if title and (re.search(r'[\t_]{2,}|[A-Za-z]{4,}', title) or len(title) > 100 or (_dup and len(_dup.group(1)) >= 8)):
                _tt = _tidy_title(title)
                if _tt:
                    title = _tt
                else:
                    title = (id_code if id_code else filename.replace('.txt', ''))
            sbreak[source] = sbreak.get(source, 0) + 1

            ic = f"'{id_code}'" if id_code else "NULL"
            tl = title.replace("'", "''")
            ct = content[:50000].replace("'", "''")
            data_sql_lines.append(f"INSERT OR REPLACE INTO tsk_data (source, id_code, title, content) VALUES ('{source}',{ic},'{tl}','{ct}');")

            ts = time.strftime('%Y-%m-%d %H:%M:%S')
            fn_s = filename.replace("'", "''")
            tracker_sql_lines.append(f"INSERT OR REPLACE INTO sync_tracker (filename, mtime, last_synced) VALUES ('{fn_s}', {cmt}, '{ts}');")
            imported += 1

        if data_sql_lines:
            sql = "BEGIN TRANSACTION;\n" + "\n".join(data_sql_lines + tracker_sql_lines) + "\nCOMMIT;"
            run_sql_on_server(sql, timeout=60)

        pct = min(100, (batch_start + len(batch)) * 100 // len(changed))
        print(f"  [{batch_start+len(batch)}/{len(changed)}] +{imported} err{errors} ({pct}%)")

    # Stats
    r1 = query_server("SELECT COUNT(*) FROM tsk_data;")
    r2 = query_server("SELECT source, COUNT(*) FROM tsk_data GROUP BY source ORDER BY source;")

    print(f"\n{'='*50}")
    print(f"[DONE]")
    print(f"  Skipped (mtime unchanged): {skipped}")
    print(f"  Imported (new/changed):    {imported}")
    print(f"  Read errors:               {errors}")
    print(f"  Breakdown: {json.dumps(sbreak, ensure_ascii=False)}")
    print(f"  DB total: {r1.stdout.strip()}")
    for line in r2.stdout.strip().split('\n'):
        if line.strip():
            parts = line.split('|')
            print(f"    {parts[0]}: {parts[1] if len(parts)>1 else ''}")

if __name__ == '__main__':
    main()