#!/usr/bin/env python3
"""清水河-部门文件 全量并发 runner
3 worker 并行抓取，每 worker 独立 playwright + JSONL，最后合并
用法: python3 qsh_bmwj_runner.py [起始页] [结束页] [worker数]
"""
import subprocess, sys, os, time, glob, json
SCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "crawl_qingshuihe_bmwj.py")
TOTAL_PAGES = 334
WORKERS = 3

def main():
    workers = WORKERS
    if len(sys.argv) > 1:
        workers = int(sys.argv[1])
    pages_per_worker = (TOTAL_PAGES + workers - 1) // workers
    procs = []
    ts = time.strftime("%Y%m%d_%H%M%S")
    for w in range(workers):
        start = w * pages_per_worker + 1
        end = min((w + 1) * pages_per_worker, TOTAL_PAGES)
        out = f"/tmp/qsh_bmwj_w{w}_{ts}.jsonl"
        print(f"worker{w}: pages {start}-{end} -> {out}", flush=True)
        p = subprocess.Popen(
            ["python3", SCRIPT, "--pages", str(end - start + 1), "--start", str(start), "--output", out],
            stdout=open(f"/tmp/qsh_bmwj_w{w}_{ts}.log", "w"),
            stderr=subprocess.STDOUT,
        )
        procs.append((p, out, f"/tmp/qsh_bmwj_w{w}_{ts}.log"))
    for p, out, log in procs:
        p.wait()
        print(f"worker done: {out} rc={p.returncode}")
        # 日志尾部
        try:
            lines = open(log).readlines()
            print("  " + "".join(lines[-3:]).strip())
        except Exception:
            pass
    # 合并
    merged = f"/tmp/qsh_bmwj_all_{ts}.jsonl"
    total = 0
    with open(merged, "w", encoding="utf-8") as mf:
        for p, out, log in procs:
            if os.path.exists(out):
                for line in open(out, encoding="utf-8"):
                    mf.write(line)
                    total += 1
    print(f"合并完成: {merged} 共{total}条")
    # 去重
    seen = set()
    dedup = f"/tmp/qsh_bmwj_dedup_{ts}.jsonl"
    with open(dedup, "w", encoding="utf-8") as df:
        for line in open(merged, encoding="utf-8"):
            try:
                r = json.loads(line)
                u = r["page_url"]
                if u not in seen:
                    seen.add(u)
                    df.write(line)
            except Exception:
                pass
    print(f"去重: {len(seen)} 条 -> {dedup}")

if __name__ == "__main__":
    main()