SignLanguageTranslate / transformer_comparison.py
spsarolkar's picture
Deploy SignLanguageTranslate with Transformer evaluation
e35b538 verified
Raw History Blame Contribute Delete
9.19 kB
"""Streamlit comparison of the archived Transformer predictions on one INCLUDE panel."""
import csv
import gzip
import io
import json
from pathlib import Path
from urllib.parse import quote
import streamlit as st
import streamlit.components.v1 as components
ARMS = {
"rolling": "Rolling Transformer · epoch 31",
"causal": "Causal Transformer · epoch 102",
}
DATA = Path(__file__).with_name("data") / "include_validation.json.gz"
VIDEO_REPO = "spsarolkar/AI4Bharat-INCLUDE-dataset"
VIDEO_REVISION = "d12293ab05cb5dc2cb6a342c364132a18adc49c9"
@st.cache_data
def load_panel():
with gzip.open(DATA, "rt", encoding="utf-8") as source:
return json.load(source)
def outcome_counts(clips, arm):
correct = wrong_early = correct_early = unconfirmed = 0
for clip in clips:
prediction = clip[arm]
correct += (prediction["last"] or {}).get("label") == clip["label"]
early = prediction["early"]
if early is None:
unconfirmed += 1
elif early["label"] == clip["label"]:
correct_early += 1
else:
wrong_early += 1
return correct, wrong_early, correct_early, unconfirmed
def score(value, total):
return f"{value / total:.2%} ({value}/{total})"
def export_scores(clips):
buffer = io.StringIO()
writer = csv.writer(buffer)
writer.writerow(("case", "expected", "model", "last_guess", "last_guess_seconds",
"first_confirmation", "confirmation_seconds", "outcome"))
for clip in clips:
for arm, model in ARMS.items():
prediction = clip[arm]
last, early = prediction["last"] or {}, prediction["early"] or {}
outcome = ("Unconfirmed" if not early else
"Correct" if early["label"] == clip["label"] else "Wrong")
writer.writerow((clip["id"], clip["label"], model, last.get("label"), last.get("time"),
early.get("label"), early.get("time"), outcome))
return buffer.getvalue()
def public_video(path):
if not path:
return None
parts = path.split("/")
if len(parts) != 5 or parts[0] != "videos" or parts[1] not in {"train", "val", "test"}:
raise ValueError("Unexpected public video path")
if any(part in {"", ".", ".."} for part in parts):
raise ValueError("Invalid public video path")
return f"https://huggingface.co/datasets/{VIDEO_REPO}/resolve/{VIDEO_REVISION}/{quote(path)}"
def render_playback(video_url, clip):
# Inline script data is escaped before interpolation into the iframe document.
payload = json.dumps({
"video": video_url, "arms": ARMS, "duration": clip["duration"],
"predictions": {arm: clip[arm] for arm in ARMS},
}, ensure_ascii=True).replace("<", "\\u003c")
template = Path(__file__).with_name("validation_playback.html").read_text(encoding="utf-8")
components.html(template.replace("/* PAYLOAD */", payload), height=480, scrolling=True)
def render_transformer_comparison():
st.title("SignLanguageTranslate")
st.header("Indian Sign Language model comparison")
try:
panel = load_panel()
except (OSError, ValueError):
st.error("The archived evaluation could not be loaded.")
return
if panel.get("videoSource") != {"repoId": VIDEO_REPO, "revision": VIDEO_REVISION}:
st.error("The archived evaluation points to a different video dataset revision.")
return
clips = panel["clips"]
st.caption("720 isolated INCLUDE clips from the saved iPad validation split. "
"Both models were evaluated on the same Apple Vision keypoints.")
from transformer_runtime import evaluate_validation, load_assets, replay_clip, settings
revision = settings()
live = False
if revision:
if st.button("Run both models on all 720 validation clips"):
try:
with st.spinner("Running both models on the frozen validation clips…"):
st.session_state["transformer_evaluation"] = evaluate_validation(revision, clips)
except (OSError, ValueError, RuntimeError):
st.error("Live evaluation could not load the pinned artifacts or finish inference.")
results = st.session_state.get("transformer_evaluation")
if results is not None:
clips = results
live = True
else:
st.info("The server is showing the archived validation results. Live inference requires the Space's private model and dataset configuration.")
st.subheader("Validation accuracy")
st.caption("Computed by the Space" if live else "Archived evaluation on the Mac")
table = []
for arm, name in ARMS.items():
correct, wrong_early, correct_early, unconfirmed = outcome_counts(clips, arm)
table.append({
"Model": name,
"Final accuracy": score(correct, len(clips)),
"Wrong early confirmations": score(wrong_early, len(clips)),
"Correct early confirmations": correct_early,
"Unconfirmed clips": unconfirmed,
})
st.dataframe(table, use_container_width=True, hide_index=True)
st.caption("Final accuracy uses the last scheduled prediction before the clip ends. "
"An early confirmation is the first label repeated across three consecutive updates.")
st.download_button("Download validation results (CSV)", export_scores(clips),
file_name="include-validation-results.csv", mime="text/csv")
st.subheader("Replay predictions")
labels = sorted({clip["label"] for clip in clips}, key=str.casefold)
selected_label = st.selectbox("Expected sign", labels, index=labels.index("Woman") if "Woman" in labels else 0)
matching = [clip for clip in clips if clip["label"] == selected_label]
default_clip = next((i for i, item in enumerate(matching) if item["id"] == "include-004"), 0)
clip = st.selectbox("Validation clip", matching, index=default_clip,
format_func=lambda item: f"{item['id']} · {item['duration']:.2f} s"
+ (" · source video available" if item["videoPath"] else ""))
if revision and not live:
if st.button("Run both models on this clip", key=f"predict-{clip['id']}"):
try:
with st.spinner("Running both models on the saved poses…"):
inputs, models = load_assets(revision)
index = next(i for i, item in enumerate(clips) if item["id"] == clip["id"])
st.session_state["transformer_clip"] = replay_clip(inputs, models, index, clip)
except (OSError, ValueError, RuntimeError):
st.error("Live inference could not load the pinned artifacts or predict this clip.")
result = st.session_state.get("transformer_clip")
if result is not None and result["id"] == clip["id"]:
clip = result
live = True
video_url = public_video(clip["videoPath"])
if video_url:
split = {"val": "validation", "train": "train", "test": "test"}[clip["videoPath"].split("/")[1]]
st.caption(f"Public video repository split: {split}.")
st.markdown(f"[Open source video]({video_url})")
else:
st.caption("Use the timeline slider to review this clip’s archived predictions.")
st.caption(("Predictions computed by the Space" if live else "Archived model predictions") + " at 250 ms intervals.")
render_playback(video_url, clip)
with st.expander("All prediction updates"):
updates = {}
for arm, name in ARMS.items():
for tick in clip[arm]["ticks"]:
updates.setdefault(tick["time"], {"Time (s)": tick["time"]})[name] = tick["label"]
st.dataframe(list(updates.values()), use_container_width=True, hide_index=True)
with st.expander("Dataset splits and model sources"):
st.markdown(
"- This evaluation uses 720 of 743 clips in the saved iPad validation split. "
"The models share 256 output labels. This panel covers 254 labels, and 23 clips "
"were excluded because their labels are outside the model vocabulary. "
"Epoch 102 was selected using this validation panel. Signer independence is unverified.\n"
"- The legacy LSTM uses 167 output labels and a custom test manifest. Of its 412 clips, "
"245 match the public INCLUDE train split, 25 validation, 66 test, and 76 have no exact match. "
"Training membership for the published legacy weights remains unverified.\n"
"- [INCLUDE source and attribution](https://huggingface.co/datasets/ai4bharat/INCLUDE) · "
"[Public video dataset](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-dataset) · "
"[Apple Vision keypoint dataset card](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-keypoints) · "
"[Native Transformer checkpoint card](https://huggingface.co/spsarolkar/isl-transformer-mlx) · "
"[Legacy LSTM model card](https://huggingface.co/sunilsarolkar/isl-translation-model)"
)