File size: 5,336 Bytes
e35b538 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | """Read the pinned checkpoints and Apple Vision validation data on the server."""
import gzip
import hashlib
import json
import os
from pathlib import Path
import re
import streamlit as st
from transformer_inference import Transformer
CONTRACT_PATH = Path(__file__).with_name("data") / "transformer_contract.json"
CONTRACT = json.loads(CONTRACT_PATH.read_text(encoding="utf-8"))
def read_verified(path, expected_hash):
payload = Path(path).read_bytes()
if hashlib.sha256(payload).hexdigest() != expected_hash:
raise ValueError("A frozen validation file failed its checksum")
return payload
def load_validation(download):
"""Return only the approved validation rows, ordered by native clip ID."""
files = CONTRACT["dataset"]["files"]
vocab_path = "metadata/vocabulary.json"
vocabulary_rows = json.loads(read_verified(download(vocab_path), files[vocab_path]))
vectors = {row["name"]: row["embedding"] for row in vocabulary_rows if row["type"] == "word"}
labels = CONTRACT["classNames"]
vocabulary = [{"word": label, "vector": vectors[label]} for label in labels]
allowed = set(labels)
clips = {}
seen = set()
for name, expected in files.items():
if name == vocab_path:
continue
payload = read_verified(download(name), expected)
for line in gzip.decompress(payload).splitlines():
row = json.loads(line)
identity = row["video_id"]
words = [label["name"] for label in row["labels"] if label["type"] == "word"]
if (identity in seen or row["split"] != "validation" or row["dataset"] != "INCLUDE"
or row["extractor"] != "AppleVision" or row["native_format_version"] != 1
or len(words) != 1):
raise ValueError("Unexpected validation record")
seen.add(identity)
if words[0] not in allowed:
continue
frames = []
for index, frame in enumerate(row["frames"]):
if frame["frame_index"] != index or frame["source_model"] != "AppleVision":
raise ValueError("Unexpected frame schema")
frames.append({
"timestamp": frame["timestamp_s"],
"body": frame["body"], "leftHand": frame["left_hand"], "rightHand": frame["right_hand"],
})
if not frames or len(frames) != row["frame_count"]:
raise ValueError("Missing validation frames")
clips[identity] = (words[0], row["duration_s"], frames)
if len(seen) != 743 or len(clips) != 720:
raise ValueError("Frozen validation cohort size differs")
return vocabulary, [clips[key] for key in sorted(clips)]
def settings():
if os.environ.get("ISL_PREVIEW_ASSETS"):
return "local"
revision = os.environ.get("ISL_MODEL_REVISION", "")
if os.environ.get("ISL_ARTIFACT_READ_TOKEN") and re.fullmatch(r"[0-9a-f]{40}", revision):
return revision
return None
@st.cache_resource
def load_assets(revision):
dataset = CONTRACT["dataset"]
if revision == "local":
# The local preview accepts a server-configured directory with symlinked
# assets. It never accepts file paths from visitors.
root = Path(os.environ["ISL_PREVIEW_ASSETS"])
vocabulary, clips = load_validation(lambda name: root / "dataset" / name)
paths = {arm: root / filename for arm, filename in CONTRACT["modelFiles"].items()}
else:
from huggingface_hub import hf_hub_download
token = os.environ["ISL_ARTIFACT_READ_TOKEN"]
def download(name):
return hf_hub_download(dataset["id"], name, repo_type="dataset",
revision=dataset["revision"], token=token)
vocabulary, clips = load_validation(download)
paths = {arm: hf_hub_download(CONTRACT["modelRepo"], name, revision=revision, token=token)
for arm, name in CONTRACT["modelFiles"].items()}
models = {arm: Transformer(path, vocabulary, arm) for arm, path in paths.items()}
return clips, models
def replay_clip(clips, models, index, expected):
label, duration, frames = clips[index]
if (label, duration) != (expected["label"], expected["duration"]):
raise ValueError("Live validation clip differs from the archived panel")
output = {"id": expected["id"], "label": label, "duration": duration,
"videoPath": expected["videoPath"]}
for arm, model in models.items():
rows = list(model.replay(frames, duration))
ticks = [{key: row[key] for key in ("time", "label", "score")} for row in rows]
last = rows[-1]
early = next((row for row in rows if row["confirmationTime"] is not None), None)
output[arm] = {
"last": ticks[-1],
"early": None if early is None else {
"time": last["confirmationTime"], "label": last["confirmationLabel"], "score": early["score"],
},
"ticks": ticks,
}
return output
@st.cache_data(max_entries=1)
def evaluate_validation(revision, expected_clips):
clips, models = load_assets(revision)
return [replay_clip(clips, models, index, expected)
for index, expected in enumerate(expected_clips)]
|