SignLanguageTranslate / transformer_runtime.py
spsarolkar's picture
Deploy SignLanguageTranslate with Transformer evaluation
e35b538 verified
Raw History Blame Contribute Delete
5.34 kB
"""Read the pinned checkpoints and Apple Vision validation data on the server."""
import gzip
import hashlib
import json
import os
from pathlib import Path
import re
import streamlit as st
from transformer_inference import Transformer
CONTRACT_PATH = Path(__file__).with_name("data") / "transformer_contract.json"
CONTRACT = json.loads(CONTRACT_PATH.read_text(encoding="utf-8"))
def read_verified(path, expected_hash):
payload = Path(path).read_bytes()
if hashlib.sha256(payload).hexdigest() != expected_hash:
raise ValueError("A frozen validation file failed its checksum")
return payload
def load_validation(download):
"""Return only the approved validation rows, ordered by native clip ID."""
files = CONTRACT["dataset"]["files"]
vocab_path = "metadata/vocabulary.json"
vocabulary_rows = json.loads(read_verified(download(vocab_path), files[vocab_path]))
vectors = {row["name"]: row["embedding"] for row in vocabulary_rows if row["type"] == "word"}
labels = CONTRACT["classNames"]
vocabulary = [{"word": label, "vector": vectors[label]} for label in labels]
allowed = set(labels)
clips = {}
seen = set()
for name, expected in files.items():
if name == vocab_path:
continue
payload = read_verified(download(name), expected)
for line in gzip.decompress(payload).splitlines():
row = json.loads(line)
identity = row["video_id"]
words = [label["name"] for label in row["labels"] if label["type"] == "word"]
if (identity in seen or row["split"] != "validation" or row["dataset"] != "INCLUDE"
or row["extractor"] != "AppleVision" or row["native_format_version"] != 1
or len(words) != 1):
raise ValueError("Unexpected validation record")
seen.add(identity)
if words[0] not in allowed:
continue
frames = []
for index, frame in enumerate(row["frames"]):
if frame["frame_index"] != index or frame["source_model"] != "AppleVision":
raise ValueError("Unexpected frame schema")
frames.append({
"timestamp": frame["timestamp_s"],
"body": frame["body"], "leftHand": frame["left_hand"], "rightHand": frame["right_hand"],
})
if not frames or len(frames) != row["frame_count"]:
raise ValueError("Missing validation frames")
clips[identity] = (words[0], row["duration_s"], frames)
if len(seen) != 743 or len(clips) != 720:
raise ValueError("Frozen validation cohort size differs")
return vocabulary, [clips[key] for key in sorted(clips)]
def settings():
if os.environ.get("ISL_PREVIEW_ASSETS"):
return "local"
revision = os.environ.get("ISL_MODEL_REVISION", "")
if os.environ.get("ISL_ARTIFACT_READ_TOKEN") and re.fullmatch(r"[0-9a-f]{40}", revision):
return revision
return None
@st.cache_resource
def load_assets(revision):
dataset = CONTRACT["dataset"]
if revision == "local":
# The local preview accepts a server-configured directory with symlinked
# assets. It never accepts file paths from visitors.
root = Path(os.environ["ISL_PREVIEW_ASSETS"])
vocabulary, clips = load_validation(lambda name: root / "dataset" / name)
paths = {arm: root / filename for arm, filename in CONTRACT["modelFiles"].items()}
else:
from huggingface_hub import hf_hub_download
token = os.environ["ISL_ARTIFACT_READ_TOKEN"]
def download(name):
return hf_hub_download(dataset["id"], name, repo_type="dataset",
revision=dataset["revision"], token=token)
vocabulary, clips = load_validation(download)
paths = {arm: hf_hub_download(CONTRACT["modelRepo"], name, revision=revision, token=token)
for arm, name in CONTRACT["modelFiles"].items()}
models = {arm: Transformer(path, vocabulary, arm) for arm, path in paths.items()}
return clips, models
def replay_clip(clips, models, index, expected):
label, duration, frames = clips[index]
if (label, duration) != (expected["label"], expected["duration"]):
raise ValueError("Live validation clip differs from the archived panel")
output = {"id": expected["id"], "label": label, "duration": duration,
"videoPath": expected["videoPath"]}
for arm, model in models.items():
rows = list(model.replay(frames, duration))
ticks = [{key: row[key] for key in ("time", "label", "score")} for row in rows]
last = rows[-1]
early = next((row for row in rows if row["confirmationTime"] is not None), None)
output[arm] = {
"last": ticks[-1],
"early": None if early is None else {
"time": last["confirmationTime"], "label": last["confirmationLabel"], "score": early["score"],
},
"ticks": ticks,
}
return output
@st.cache_data(max_entries=1)
def evaluate_validation(revision, expected_clips):
clips, models = load_assets(revision)
return [replay_clip(clips, models, index, expected)
for index, expected in enumerate(expected_clips)]