"""Read the pinned checkpoints and Apple Vision validation data on the server.""" import gzip import hashlib import json import os from pathlib import Path import re import streamlit as st from transformer_inference import Transformer CONTRACT_PATH = Path(__file__).with_name("data") / "transformer_contract.json" CONTRACT = json.loads(CONTRACT_PATH.read_text(encoding="utf-8")) def read_verified(path, expected_hash): payload = Path(path).read_bytes() if hashlib.sha256(payload).hexdigest() != expected_hash: raise ValueError("A frozen validation file failed its checksum") return payload def load_validation(download): """Return only the approved validation rows, ordered by native clip ID.""" files = CONTRACT["dataset"]["files"] vocab_path = "metadata/vocabulary.json" vocabulary_rows = json.loads(read_verified(download(vocab_path), files[vocab_path])) vectors = {row["name"]: row["embedding"] for row in vocabulary_rows if row["type"] == "word"} labels = CONTRACT["classNames"] vocabulary = [{"word": label, "vector": vectors[label]} for label in labels] allowed = set(labels) clips = {} seen = set() for name, expected in files.items(): if name == vocab_path: continue payload = read_verified(download(name), expected) for line in gzip.decompress(payload).splitlines(): row = json.loads(line) identity = row["video_id"] words = [label["name"] for label in row["labels"] if label["type"] == "word"] if (identity in seen or row["split"] != "validation" or row["dataset"] != "INCLUDE" or row["extractor"] != "AppleVision" or row["native_format_version"] != 1 or len(words) != 1): raise ValueError("Unexpected validation record") seen.add(identity) if words[0] not in allowed: continue frames = [] for index, frame in enumerate(row["frames"]): if frame["frame_index"] != index or frame["source_model"] != "AppleVision": raise ValueError("Unexpected frame schema") frames.append({ "timestamp": frame["timestamp_s"], "body": frame["body"], "leftHand": frame["left_hand"], "rightHand": frame["right_hand"], }) if not frames or len(frames) != row["frame_count"]: raise ValueError("Missing validation frames") clips[identity] = (words[0], row["duration_s"], frames) if len(seen) != 743 or len(clips) != 720: raise ValueError("Frozen validation cohort size differs") return vocabulary, [clips[key] for key in sorted(clips)] def settings(): if os.environ.get("ISL_PREVIEW_ASSETS"): return "local" revision = os.environ.get("ISL_MODEL_REVISION", "") if os.environ.get("ISL_ARTIFACT_READ_TOKEN") and re.fullmatch(r"[0-9a-f]{40}", revision): return revision return None @st.cache_resource def load_assets(revision): dataset = CONTRACT["dataset"] if revision == "local": # The local preview accepts a server-configured directory with symlinked # assets. It never accepts file paths from visitors. root = Path(os.environ["ISL_PREVIEW_ASSETS"]) vocabulary, clips = load_validation(lambda name: root / "dataset" / name) paths = {arm: root / filename for arm, filename in CONTRACT["modelFiles"].items()} else: from huggingface_hub import hf_hub_download token = os.environ["ISL_ARTIFACT_READ_TOKEN"] def download(name): return hf_hub_download(dataset["id"], name, repo_type="dataset", revision=dataset["revision"], token=token) vocabulary, clips = load_validation(download) paths = {arm: hf_hub_download(CONTRACT["modelRepo"], name, revision=revision, token=token) for arm, name in CONTRACT["modelFiles"].items()} models = {arm: Transformer(path, vocabulary, arm) for arm, path in paths.items()} return clips, models def replay_clip(clips, models, index, expected): label, duration, frames = clips[index] if (label, duration) != (expected["label"], expected["duration"]): raise ValueError("Live validation clip differs from the archived panel") output = {"id": expected["id"], "label": label, "duration": duration, "videoPath": expected["videoPath"]} for arm, model in models.items(): rows = list(model.replay(frames, duration)) ticks = [{key: row[key] for key in ("time", "label", "score")} for row in rows] last = rows[-1] early = next((row for row in rows if row["confirmationTime"] is not None), None) output[arm] = { "last": ticks[-1], "early": None if early is None else { "time": last["confirmationTime"], "label": last["confirmationLabel"], "score": early["score"], }, "ticks": ticks, } return output @st.cache_data(max_entries=1) def evaluate_validation(revision, expected_clips): clips, models = load_assets(revision) return [replay_clip(clips, models, index, expected) for index, expected in enumerate(expected_clips)]