Buckets:
| """Structural audit of rustensai/russian-handwriting-ocr, text columns only. | |
| Questions: how many unique references; rows per reference; does source_folder | |
| identify one essay (pages of it); do references leak across train/validation? | |
| Uploads structure_report.md to the eval bucket. | |
| """ | |
| import hashlib | |
| import json | |
| from collections import Counter, defaultdict | |
| import pyarrow.parquet as pq | |
| from huggingface_hub import HfFileSystem, batch_bucket_files, list_repo_files | |
| REPO = "rustensai/russian-handwriting-ocr" | |
| BUCKET = "ktrapeznikov/russian-hw-ocr-eval" | |
| COLS = ["messages", "source_folder", "dataset_name", "image_type"] | |
| fs = HfFileSystem() | |
| files = [f for f in list_repo_files(REPO, repo_type="dataset") if f.endswith(".parquet")] | |
| def ref_of(messages): | |
| for m in messages: | |
| if m["role"] == "assistant": | |
| return "".join(c.get("text") or "" for c in m["content"] if c.get("type") == "text") | |
| return "" | |
| rows = [] | |
| for path in sorted(files): | |
| split = "train" if "train" in path else "validation" | |
| with fs.open(f"datasets/{REPO}/{path}") as f: | |
| t = pq.read_table(f, columns=COLS) | |
| d = t.to_pydict() | |
| for msgs, folder, dsname, itype in zip(d["messages"], d["source_folder"], d["dataset_name"], d["image_type"]): | |
| ref = ref_of(msgs) | |
| h = hashlib.md5(ref.encode()).hexdigest()[:12] | |
| rows.append({"split": split, "folder": folder, "dataset": dsname, | |
| "itype": itype, "ref_hash": h, "ref_len": len(ref)}) | |
| print(f"{path}: +{len(d['messages'])} rows (total {len(rows)})", flush=True) | |
| n = len(rows) | |
| by_hash = defaultdict(list) | |
| for r in rows: | |
| by_hash[r["ref_hash"]].append(r) | |
| rows_per_ref = Counter(len(v) for v in by_hash.values()) | |
| ref_lens = [r["ref_len"] for r in rows] | |
| # folder -> unique refs (is a folder one essay?) | |
| folder_refs = defaultdict(set) | |
| folder_rows = Counter() | |
| for r in rows: | |
| folder_refs[r["folder"]].add(r["ref_hash"]) | |
| folder_rows[r["folder"]] += 1 | |
| multi_ref_folders = {f: len(s) for f, s in folder_refs.items() if len(s) > 1} | |
| # train/val leakage: same reference text in both splits | |
| train_hashes = {r["ref_hash"] for r in rows if r["split"] == "train"} | |
| val_hashes = {r["ref_hash"] for r in rows if r["split"] == "validation"} | |
| leaked = train_hashes & val_hashes | |
| val_rows_leaked = sum(1 for r in rows if r["split"] == "validation" and r["ref_hash"] in leaked) | |
| train_rows_leaked = sum(1 for r in rows if r["split"] == "train" and r["ref_hash"] in leaked) | |
| by_ds = Counter(r["dataset"] for r in rows) | |
| by_it = Counter(r["itype"] for r in rows) | |
| report = { | |
| "total_rows": n, | |
| "unique_references": len(by_hash), | |
| "rows_per_reference_distribution": dict(sorted(rows_per_ref.items())), | |
| "ref_len_chars": {"mean": round(sum(ref_lens) / n), "min": min(ref_lens), "max": max(ref_lens)}, | |
| "folders": len(folder_refs), | |
| "folders_with_multiple_unique_refs": len(multi_ref_folders), | |
| "avg_rows_per_folder": round(n / len(folder_refs), 1), | |
| "train_val_ref_leakage": { | |
| "unique_refs_in_both": len(leaked), | |
| "val_rows_whose_ref_also_in_train": val_rows_leaked, | |
| "val_rows_total": sum(1 for r in rows if r["split"] == "validation"), | |
| "train_rows_whose_ref_also_in_val": train_rows_leaked, | |
| }, | |
| "by_dataset_name": dict(by_ds), | |
| "by_image_type": dict(by_it), | |
| } | |
| print(json.dumps(report, ensure_ascii=False, indent=2)) | |
| md = [ | |
| f"# Structural audit: {REPO}", | |
| "", | |
| f"- Rows: {n} (train {sum(1 for r in rows if r['split']=='train')}, " | |
| f"validation {sum(1 for r in rows if r['split']=='validation')})", | |
| f"- Unique reference texts: {len(by_hash)}", | |
| f"- Rows per unique reference: {dict(sorted(rows_per_ref.items()))}", | |
| f"- Reference length (chars): mean {report['ref_len_chars']['mean']}, " | |
| f"min {report['ref_len_chars']['min']}, max {report['ref_len_chars']['max']}", | |
| f"- source_folders: {len(folder_refs)}; folders containing >1 unique reference: {len(multi_ref_folders)}", | |
| f"- Avg rows per folder: {report['avg_rows_per_folder']}", | |
| "", | |
| "## Train/validation leakage (same full reference text in both splits)", | |
| "", | |
| f"- Unique references appearing in both splits: {len(leaked)}", | |
| f"- Validation rows whose reference also appears in train: " | |
| f"{val_rows_leaked}/{sum(1 for r in rows if r['split']=='validation')}", | |
| "", | |
| "## Composition", | |
| "", | |
| f"- dataset_name: {dict(by_ds)}", | |
| f"- image_type: {dict(by_it)}", | |
| ] | |
| with open("structure_report.md", "w", encoding="utf-8") as f: | |
| f.write("\n".join(md)) | |
| batch_bucket_files(BUCKET, add=[("structure_report.md", "audits/structure_report.md")]) | |
| print("Uploaded audits/structure_report.md") | |
Xet Storage Details
- Size:
- 4.66 kB
- Xet hash:
- b428bf2e9a06cf7c70577d2a6ec7a3c3c9b379cae6725a2a2adcb85f60274b9f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.