"""Dataset verification — recompute BLAKE3 over the local files referenced by a shard manifest, surface mismatches. Pure stdlib + `mindxtrain.provenance.hashing`. """ from __future__ import annotations import json from pathlib import Path from pydantic import BaseModel, ConfigDict, Field from mindxtrain.provenance.hashing import blake3_file class VerifyResult(BaseModel): model_config = ConfigDict(extra="forbid") manifest_path: Path matched: int = 0 mismatched: list[str] = Field(default_factory=list) missing: list[str] = Field(default_factory=list) def verify_dataset(manifest_path: Path, root: Path) -> VerifyResult: """Walk the manifest, recompute hashes, return mismatches. Manifest format expected: {"shards": [{"path": "shard-00000.tar", "blake3": "...."}, ...]} `path` is resolved relative to `root`. """ manifest_path = Path(manifest_path) root = Path(root) raw = json.loads(manifest_path.read_text()) shards = raw.get("shards") or [] result = VerifyResult(manifest_path=manifest_path) for shard in shards: rel = shard.get("path", "") expected = shard.get("blake3", "") if not rel or not expected: continue local = root / rel if not local.exists(): result.missing.append(rel) continue actual = blake3_file(local) if actual != expected: result.mismatched.append(rel) else: result.matched += 1 return result __all__ = ["VerifyResult", "verify_dataset"]