File size: 3,121 Bytes
20b4034 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | #!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import os
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def require(path: Path, min_bytes: int = 1) -> None:
if not path.is_file() or path.stat().st_size < min_bytes:
raise FileNotFoundError(f"missing or truncated: {path}")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--checksums", action="store_true")
parser.add_argument("--materialized", action="store_true")
args = parser.parse_args()
required = {
ROOT / "data/covt/data.json": 100_000_000,
ROOT / "checkpoints/stage1/checkpoint-32000.bin": 1_000_000_000,
ROOT / "models/base/Qwen3-VL-4B-Instruct/model-00001-of-00002.safetensors": 4_000_000_000,
ROOT / "models/base/Qwen3-VL-4B-Instruct/model-00002-of-00002.safetensors": 3_000_000_000,
ROOT / "models/teachers/sam_vit_h_4b8939.pth": 2_000_000_000,
ROOT / "models/teachers/VGGT-1B/model.pt": 4_000_000_000,
ROOT / "models/teachers/table5_baseline.pth": 1_000_000,
ROOT / "models/teachers/siglip2-large-patch16-256/model.safetensors": 3_000_000_000,
ROOT / "models/teachers/dinov2-large-processor/preprocessor_config.json": 100,
ROOT / "cache/torch/hub/checkpoints/dinov2_vitl14_pretrain.pth": 1_000_000_000,
ROOT / "cache/torch/hub/facebookresearch_dinov2_main/hubconf.py": 1_000,
ROOT / "environment/unvideo-conda.tar.gz": 1_000_000_000,
}
for path, size in required.items():
require(path, size)
stats = json.loads((ROOT / "data/covt/stats.json").read_text(encoding="utf-8"))
shards = sorted((ROOT / "data/covt/shards").glob("images-*.tar"))
if len(shards) != stats["shard_count"]:
raise AssertionError(f"shard count {len(shards)} != {stats['shard_count']}")
for shard in shards:
require(Path(f"{shard}.sha256"), 64)
if args.materialized:
image_count = sum(1 for path in (ROOT / "data/covt/images").iterdir() if path.is_file())
if image_count != stats["image_count"]:
raise AssertionError(f"image count {image_count} != {stats['image_count']}")
if args.checksums:
subprocess.run(["sha256sum", "-c", "MANIFEST.sha256"], cwd=ROOT, check=True)
config = ROOT / "configs/stage2_covt.yaml"
if not config.is_file():
subprocess.run([str(ROOT / ".env/unvideo/bin/python"), str(ROOT / "scripts/render_config.py")], check=True)
env = os.environ.copy()
env["PYTHONPATH"] = ":".join(
[
str(ROOT / "code/umm/runtime"),
str(ROOT / "code/umm"),
str(ROOT / "third_party/transformers/src"),
str(ROOT / "third_party/diffusers/src"),
]
)
env["UMM_STAGE2_CONFIG"] = str(config)
subprocess.run(
[str(ROOT / ".env/unvideo/bin/python"), str(ROOT / "code/umm/stage2_covt/validate_setup.py")],
cwd=ROOT,
env=env,
check=True,
)
print("UMM Stage-2 bundle validation passed")
if __name__ == "__main__":
main()
|