File size: 5,336 Bytes
e35b538
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
"""Read the pinned checkpoints and Apple Vision validation data on the server."""

import gzip
import hashlib
import json
import os
from pathlib import Path
import re

import streamlit as st

from transformer_inference import Transformer

CONTRACT_PATH = Path(__file__).with_name("data") / "transformer_contract.json"
CONTRACT = json.loads(CONTRACT_PATH.read_text(encoding="utf-8"))


def read_verified(path, expected_hash):
    payload = Path(path).read_bytes()
    if hashlib.sha256(payload).hexdigest() != expected_hash:
        raise ValueError("A frozen validation file failed its checksum")
    return payload


def load_validation(download):
    """Return only the approved validation rows, ordered by native clip ID."""
    files = CONTRACT["dataset"]["files"]
    vocab_path = "metadata/vocabulary.json"
    vocabulary_rows = json.loads(read_verified(download(vocab_path), files[vocab_path]))
    vectors = {row["name"]: row["embedding"] for row in vocabulary_rows if row["type"] == "word"}
    labels = CONTRACT["classNames"]
    vocabulary = [{"word": label, "vector": vectors[label]} for label in labels]
    allowed = set(labels)
    clips = {}
    seen = set()
    for name, expected in files.items():
        if name == vocab_path:
            continue
        payload = read_verified(download(name), expected)
        for line in gzip.decompress(payload).splitlines():
            row = json.loads(line)
            identity = row["video_id"]
            words = [label["name"] for label in row["labels"] if label["type"] == "word"]
            if (identity in seen or row["split"] != "validation" or row["dataset"] != "INCLUDE"
                    or row["extractor"] != "AppleVision" or row["native_format_version"] != 1
                    or len(words) != 1):
                raise ValueError("Unexpected validation record")
            seen.add(identity)
            if words[0] not in allowed:
                continue
            frames = []
            for index, frame in enumerate(row["frames"]):
                if frame["frame_index"] != index or frame["source_model"] != "AppleVision":
                    raise ValueError("Unexpected frame schema")
                frames.append({
                    "timestamp": frame["timestamp_s"],
                    "body": frame["body"], "leftHand": frame["left_hand"], "rightHand": frame["right_hand"],
                })
            if not frames or len(frames) != row["frame_count"]:
                raise ValueError("Missing validation frames")
            clips[identity] = (words[0], row["duration_s"], frames)
    if len(seen) != 743 or len(clips) != 720:
        raise ValueError("Frozen validation cohort size differs")
    return vocabulary, [clips[key] for key in sorted(clips)]


def settings():
    if os.environ.get("ISL_PREVIEW_ASSETS"):
        return "local"
    revision = os.environ.get("ISL_MODEL_REVISION", "")
    if os.environ.get("ISL_ARTIFACT_READ_TOKEN") and re.fullmatch(r"[0-9a-f]{40}", revision):
        return revision
    return None


@st.cache_resource
def load_assets(revision):
    dataset = CONTRACT["dataset"]
    if revision == "local":
        # The local preview accepts a server-configured directory with symlinked
        # assets. It never accepts file paths from visitors.
        root = Path(os.environ["ISL_PREVIEW_ASSETS"])
        vocabulary, clips = load_validation(lambda name: root / "dataset" / name)
        paths = {arm: root / filename for arm, filename in CONTRACT["modelFiles"].items()}
    else:
        from huggingface_hub import hf_hub_download
        token = os.environ["ISL_ARTIFACT_READ_TOKEN"]
        def download(name):
            return hf_hub_download(dataset["id"], name, repo_type="dataset",
                                   revision=dataset["revision"], token=token)
        vocabulary, clips = load_validation(download)
        paths = {arm: hf_hub_download(CONTRACT["modelRepo"], name, revision=revision, token=token)
                 for arm, name in CONTRACT["modelFiles"].items()}
    models = {arm: Transformer(path, vocabulary, arm) for arm, path in paths.items()}
    return clips, models


def replay_clip(clips, models, index, expected):
    label, duration, frames = clips[index]
    if (label, duration) != (expected["label"], expected["duration"]):
        raise ValueError("Live validation clip differs from the archived panel")
    output = {"id": expected["id"], "label": label, "duration": duration,
              "videoPath": expected["videoPath"]}
    for arm, model in models.items():
        rows = list(model.replay(frames, duration))
        ticks = [{key: row[key] for key in ("time", "label", "score")} for row in rows]
        last = rows[-1]
        early = next((row for row in rows if row["confirmationTime"] is not None), None)
        output[arm] = {
            "last": ticks[-1],
            "early": None if early is None else {
                "time": last["confirmationTime"], "label": last["confirmationLabel"], "score": early["score"],
            },
            "ticks": ticks,
        }
    return output


@st.cache_data(max_entries=1)
def evaluate_validation(revision, expected_clips):
    clips, models = load_assets(revision)
    return [replay_clip(clips, models, index, expected)
            for index, expected in enumerate(expected_clips)]