Spaces:
Sleeping
Sleeping
File size: 2,861 Bytes
16607b8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | """Verify train/val/test seed split JSON files (disjointness, counts, checksums)."""
from __future__ import annotations
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
import splits # noqa: E402
_EXPECTED_TOTAL = {"train": 2100, "val": 450, "test": 450}
_EXPECTED_PER_ARCH = {"train": 700, "val": 150, "test": 150}
def _fail(msg: str) -> None:
print(msg, file=sys.stderr)
sys.exit(1)
def main() -> None:
docs: dict[str, dict] = {}
sets_by_split: dict[str, set[int]] = {}
for name in splits._SPLIT_NAMES:
path = splits._split_path(name)
if not path.is_file():
_fail(f"missing split file: {path}")
data = json.loads(path.read_text(encoding="utf-8"))
docs[name] = data
if data.get("split") != name:
_fail(f"{path}: expected split={name!r}, got {data.get('split')!r}")
if data.get("version") != splits.SPLIT_VERSION:
_fail(
f"{path}: expected version={splits.SPLIT_VERSION!r}, got {data.get('version')!r}"
)
if data.get("master_seed") != splits.MASTER_SEED:
_fail(
f"{path}: expected master_seed={splits.MASTER_SEED!r}, got {data.get('master_seed')!r}"
)
sb = data["seeds_by_archetype"]
actual = splits._checksum_seeds_only(sb)
expected = data["checksum"]
if actual != expected:
_fail(
f"{path}: checksum mismatch: expected {expected!r}, got {actual!r}"
)
for arch in ("easy", "medium", "hard"):
n = len(sb[arch])
want = _EXPECTED_PER_ARCH[name]
if n != want:
_fail(
f"{path}: {arch}: expected {want} seeds, got {n}"
)
flat = [*sb["easy"], *sb["medium"], *sb["hard"]]
if len(flat) != _EXPECTED_TOTAL[name]:
_fail(
f"{path}: expected {_EXPECTED_TOTAL[name]} total seeds, got {len(flat)}"
)
if len(set(flat)) != len(flat):
_fail(f"{path}: duplicate seed within split")
sets_by_split[name] = set(flat)
if (
sets_by_split["train"] & sets_by_split["val"]
or sets_by_split["train"] & sets_by_split["test"]
or sets_by_split["val"] & sets_by_split["test"]
):
_fail("splits overlap: same seed appears in more than one split")
union_size = len(
sets_by_split["train"] | sets_by_split["val"] | sets_by_split["test"]
)
if union_size != 3000:
_fail(f"expected 3000 globally unique seeds, got {union_size}")
print(
"verify_splits: OK (checksums, per-archetype counts, global disjointness)"
)
if __name__ == "__main__":
main()
|