"""Streamlit comparison of the archived Transformer predictions on one INCLUDE panel.""" import csv import gzip import io import json from pathlib import Path from urllib.parse import quote import streamlit as st import streamlit.components.v1 as components ARMS = { "rolling": "Rolling Transformer · epoch 31", "causal": "Causal Transformer · epoch 102", } DATA = Path(__file__).with_name("data") / "include_validation.json.gz" VIDEO_REPO = "spsarolkar/AI4Bharat-INCLUDE-dataset" VIDEO_REVISION = "d12293ab05cb5dc2cb6a342c364132a18adc49c9" @st.cache_data def load_panel(): with gzip.open(DATA, "rt", encoding="utf-8") as source: return json.load(source) def outcome_counts(clips, arm): correct = wrong_early = correct_early = unconfirmed = 0 for clip in clips: prediction = clip[arm] correct += (prediction["last"] or {}).get("label") == clip["label"] early = prediction["early"] if early is None: unconfirmed += 1 elif early["label"] == clip["label"]: correct_early += 1 else: wrong_early += 1 return correct, wrong_early, correct_early, unconfirmed def score(value, total): return f"{value / total:.2%} ({value}/{total})" def export_scores(clips): buffer = io.StringIO() writer = csv.writer(buffer) writer.writerow(("case", "expected", "model", "last_guess", "last_guess_seconds", "first_confirmation", "confirmation_seconds", "outcome")) for clip in clips: for arm, model in ARMS.items(): prediction = clip[arm] last, early = prediction["last"] or {}, prediction["early"] or {} outcome = ("Unconfirmed" if not early else "Correct" if early["label"] == clip["label"] else "Wrong") writer.writerow((clip["id"], clip["label"], model, last.get("label"), last.get("time"), early.get("label"), early.get("time"), outcome)) return buffer.getvalue() def public_video(path): if not path: return None parts = path.split("/") if len(parts) != 5 or parts[0] != "videos" or parts[1] not in {"train", "val", "test"}: raise ValueError("Unexpected public video path") if any(part in {"", ".", ".."} for part in parts): raise ValueError("Invalid public video path") return f"https://huggingface.co/datasets/{VIDEO_REPO}/resolve/{VIDEO_REVISION}/{quote(path)}" def render_playback(video_url, clip): # Inline script data is escaped before interpolation into the iframe document. payload = json.dumps({ "video": video_url, "arms": ARMS, "duration": clip["duration"], "predictions": {arm: clip[arm] for arm in ARMS}, }, ensure_ascii=True).replace("<", "\\u003c") template = Path(__file__).with_name("validation_playback.html").read_text(encoding="utf-8") components.html(template.replace("/* PAYLOAD */", payload), height=480, scrolling=True) def render_transformer_comparison(): st.title("SignLanguageTranslate") st.header("Indian Sign Language model comparison") try: panel = load_panel() except (OSError, ValueError): st.error("The archived evaluation could not be loaded.") return if panel.get("videoSource") != {"repoId": VIDEO_REPO, "revision": VIDEO_REVISION}: st.error("The archived evaluation points to a different video dataset revision.") return clips = panel["clips"] st.caption("720 isolated INCLUDE clips from the saved iPad validation split. " "Both models were evaluated on the same Apple Vision keypoints.") from transformer_runtime import evaluate_validation, load_assets, replay_clip, settings revision = settings() live = False if revision: if st.button("Run both models on all 720 validation clips"): try: with st.spinner("Running both models on the frozen validation clips…"): st.session_state["transformer_evaluation"] = evaluate_validation(revision, clips) except (OSError, ValueError, RuntimeError): st.error("Live evaluation could not load the pinned artifacts or finish inference.") results = st.session_state.get("transformer_evaluation") if results is not None: clips = results live = True else: st.info("The server is showing the archived validation results. Live inference requires the Space's private model and dataset configuration.") st.subheader("Validation accuracy") st.caption("Computed by the Space" if live else "Archived evaluation on the Mac") table = [] for arm, name in ARMS.items(): correct, wrong_early, correct_early, unconfirmed = outcome_counts(clips, arm) table.append({ "Model": name, "Final accuracy": score(correct, len(clips)), "Wrong early confirmations": score(wrong_early, len(clips)), "Correct early confirmations": correct_early, "Unconfirmed clips": unconfirmed, }) st.dataframe(table, use_container_width=True, hide_index=True) st.caption("Final accuracy uses the last scheduled prediction before the clip ends. " "An early confirmation is the first label repeated across three consecutive updates.") st.download_button("Download validation results (CSV)", export_scores(clips), file_name="include-validation-results.csv", mime="text/csv") st.subheader("Replay predictions") labels = sorted({clip["label"] for clip in clips}, key=str.casefold) selected_label = st.selectbox("Expected sign", labels, index=labels.index("Woman") if "Woman" in labels else 0) matching = [clip for clip in clips if clip["label"] == selected_label] default_clip = next((i for i, item in enumerate(matching) if item["id"] == "include-004"), 0) clip = st.selectbox("Validation clip", matching, index=default_clip, format_func=lambda item: f"{item['id']} · {item['duration']:.2f} s" + (" · source video available" if item["videoPath"] else "")) if revision and not live: if st.button("Run both models on this clip", key=f"predict-{clip['id']}"): try: with st.spinner("Running both models on the saved poses…"): inputs, models = load_assets(revision) index = next(i for i, item in enumerate(clips) if item["id"] == clip["id"]) st.session_state["transformer_clip"] = replay_clip(inputs, models, index, clip) except (OSError, ValueError, RuntimeError): st.error("Live inference could not load the pinned artifacts or predict this clip.") result = st.session_state.get("transformer_clip") if result is not None and result["id"] == clip["id"]: clip = result live = True video_url = public_video(clip["videoPath"]) if video_url: split = {"val": "validation", "train": "train", "test": "test"}[clip["videoPath"].split("/")[1]] st.caption(f"Public video repository split: {split}.") st.markdown(f"[Open source video]({video_url})") else: st.caption("Use the timeline slider to review this clip’s archived predictions.") st.caption(("Predictions computed by the Space" if live else "Archived model predictions") + " at 250 ms intervals.") render_playback(video_url, clip) with st.expander("All prediction updates"): updates = {} for arm, name in ARMS.items(): for tick in clip[arm]["ticks"]: updates.setdefault(tick["time"], {"Time (s)": tick["time"]})[name] = tick["label"] st.dataframe(list(updates.values()), use_container_width=True, hide_index=True) with st.expander("Dataset splits and model sources"): st.markdown( "- This evaluation uses 720 of 743 clips in the saved iPad validation split. " "The models share 256 output labels. This panel covers 254 labels, and 23 clips " "were excluded because their labels are outside the model vocabulary. " "Epoch 102 was selected using this validation panel. Signer independence is unverified.\n" "- The legacy LSTM uses 167 output labels and a custom test manifest. Of its 412 clips, " "245 match the public INCLUDE train split, 25 validation, 66 test, and 76 have no exact match. " "Training membership for the published legacy weights remains unverified.\n" "- [INCLUDE source and attribution](https://huggingface.co/datasets/ai4bharat/INCLUDE) · " "[Public video dataset](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-dataset) · " "[Apple Vision keypoint dataset card](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-keypoints) · " "[Native Transformer checkpoint card](https://huggingface.co/spsarolkar/isl-transformer-mlx) · " "[Legacy LSTM model card](https://huggingface.co/sunilsarolkar/isl-translation-model)" )