# -*- coding: utf-8 -*-
"""去重 ocr_raw.jsonl: 每个 idx 只保留第一条"""
import json, os
from pathlib import Path

WORK_DIR = Path(__file__).resolve().parent
src = WORK_DIR / "ocr_raw.jsonl"
dst = WORK_DIR / "ocr_raw_clean.jsonl"

sys_out_ok = False
try:
    import sys
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
    sys_out_ok = True
except:
    pass

lines = [l for l in src.read_text(encoding="utf-8").strip().split("\n") if l.strip()]
seen = set()
deduped = []
for l in lines:
    rec = json.loads(l)
    idx = rec.get("idx")
    if idx not in seen:
        seen.add(idx)
        deduped.append(l)

deduped.sort(key=lambda x: json.loads(x).get("idx", 0))

# write to clean file
dst.write_text("\n".join(deduped) + "\n", encoding="utf-8")
print(f"deduped: {len(lines)} -> {len(deduped)} lines")

# verify clean file
v = dst.read_text(encoding="utf-8").strip().split("\n")
print(f"verify clean: {len(v)} lines")

# replace original
os.replace(str(dst), str(src))

# verify original
v2 = src.read_text(encoding="utf-8").strip().split("\n")
print(f"verify original: {len(v2)} lines, {src.stat().st_size//1024}KB")