Download transformer_comparison.py from spsarolkar/SignLanguageTranslate: direct link, hf CLI and curl.
- Browser
- Download file 9.19 kB
-
https://huggingface.co/spaces/spsarolkar/SignLanguageTranslate/resolve/main/transformer_comparison.py
- Command line
-
hf download hf://spaces/spsarolkar/SignLanguageTranslate/transformer_comparison.py
-
curl -L -o transformer_comparison.py https://huggingface.co/spaces/spsarolkar/SignLanguageTranslate/resolve/main/transformer_comparison.py
9.19 kB
| """Streamlit comparison of the archived Transformer predictions on one INCLUDE panel.""" | |
| import csv | |
| import gzip | |
| import io | |
| import json | |
| from pathlib import Path | |
| from urllib.parse import quote | |
| import streamlit as st | |
| import streamlit.components.v1 as components | |
| ARMS = { | |
| "rolling": "Rolling Transformer · epoch 31", | |
| "causal": "Causal Transformer · epoch 102", | |
| } | |
| DATA = Path(__file__).with_name("data") / "include_validation.json.gz" | |
| VIDEO_REPO = "spsarolkar/AI4Bharat-INCLUDE-dataset" | |
| VIDEO_REVISION = "d12293ab05cb5dc2cb6a342c364132a18adc49c9" | |
| def load_panel(): | |
| with gzip.open(DATA, "rt", encoding="utf-8") as source: | |
| return json.load(source) | |
| def outcome_counts(clips, arm): | |
| correct = wrong_early = correct_early = unconfirmed = 0 | |
| for clip in clips: | |
| prediction = clip[arm] | |
| correct += (prediction["last"] or {}).get("label") == clip["label"] | |
| early = prediction["early"] | |
| if early is None: | |
| unconfirmed += 1 | |
| elif early["label"] == clip["label"]: | |
| correct_early += 1 | |
| else: | |
| wrong_early += 1 | |
| return correct, wrong_early, correct_early, unconfirmed | |
| def score(value, total): | |
| return f"{value / total:.2%} ({value}/{total})" | |
| def export_scores(clips): | |
| buffer = io.StringIO() | |
| writer = csv.writer(buffer) | |
| writer.writerow(("case", "expected", "model", "last_guess", "last_guess_seconds", | |
| "first_confirmation", "confirmation_seconds", "outcome")) | |
| for clip in clips: | |
| for arm, model in ARMS.items(): | |
| prediction = clip[arm] | |
| last, early = prediction["last"] or {}, prediction["early"] or {} | |
| outcome = ("Unconfirmed" if not early else | |
| "Correct" if early["label"] == clip["label"] else "Wrong") | |
| writer.writerow((clip["id"], clip["label"], model, last.get("label"), last.get("time"), | |
| early.get("label"), early.get("time"), outcome)) | |
| return buffer.getvalue() | |
| def public_video(path): | |
| if not path: | |
| return None | |
| parts = path.split("/") | |
| if len(parts) != 5 or parts[0] != "videos" or parts[1] not in {"train", "val", "test"}: | |
| raise ValueError("Unexpected public video path") | |
| if any(part in {"", ".", ".."} for part in parts): | |
| raise ValueError("Invalid public video path") | |
| return f"https://huggingface.co/datasets/{VIDEO_REPO}/resolve/{VIDEO_REVISION}/{quote(path)}" | |
| def render_playback(video_url, clip): | |
| # Inline script data is escaped before interpolation into the iframe document. | |
| payload = json.dumps({ | |
| "video": video_url, "arms": ARMS, "duration": clip["duration"], | |
| "predictions": {arm: clip[arm] for arm in ARMS}, | |
| }, ensure_ascii=True).replace("<", "\\u003c") | |
| template = Path(__file__).with_name("validation_playback.html").read_text(encoding="utf-8") | |
| components.html(template.replace("/* PAYLOAD */", payload), height=480, scrolling=True) | |
| def render_transformer_comparison(): | |
| st.title("SignLanguageTranslate") | |
| st.header("Indian Sign Language model comparison") | |
| try: | |
| panel = load_panel() | |
| except (OSError, ValueError): | |
| st.error("The archived evaluation could not be loaded.") | |
| return | |
| if panel.get("videoSource") != {"repoId": VIDEO_REPO, "revision": VIDEO_REVISION}: | |
| st.error("The archived evaluation points to a different video dataset revision.") | |
| return | |
| clips = panel["clips"] | |
| st.caption("720 isolated INCLUDE clips from the saved iPad validation split. " | |
| "Both models were evaluated on the same Apple Vision keypoints.") | |
| from transformer_runtime import evaluate_validation, load_assets, replay_clip, settings | |
| revision = settings() | |
| live = False | |
| if revision: | |
| if st.button("Run both models on all 720 validation clips"): | |
| try: | |
| with st.spinner("Running both models on the frozen validation clips…"): | |
| st.session_state["transformer_evaluation"] = evaluate_validation(revision, clips) | |
| except (OSError, ValueError, RuntimeError): | |
| st.error("Live evaluation could not load the pinned artifacts or finish inference.") | |
| results = st.session_state.get("transformer_evaluation") | |
| if results is not None: | |
| clips = results | |
| live = True | |
| else: | |
| st.info("The server is showing the archived validation results. Live inference requires the Space's private model and dataset configuration.") | |
| st.subheader("Validation accuracy") | |
| st.caption("Computed by the Space" if live else "Archived evaluation on the Mac") | |
| table = [] | |
| for arm, name in ARMS.items(): | |
| correct, wrong_early, correct_early, unconfirmed = outcome_counts(clips, arm) | |
| table.append({ | |
| "Model": name, | |
| "Final accuracy": score(correct, len(clips)), | |
| "Wrong early confirmations": score(wrong_early, len(clips)), | |
| "Correct early confirmations": correct_early, | |
| "Unconfirmed clips": unconfirmed, | |
| }) | |
| st.dataframe(table, use_container_width=True, hide_index=True) | |
| st.caption("Final accuracy uses the last scheduled prediction before the clip ends. " | |
| "An early confirmation is the first label repeated across three consecutive updates.") | |
| st.download_button("Download validation results (CSV)", export_scores(clips), | |
| file_name="include-validation-results.csv", mime="text/csv") | |
| st.subheader("Replay predictions") | |
| labels = sorted({clip["label"] for clip in clips}, key=str.casefold) | |
| selected_label = st.selectbox("Expected sign", labels, index=labels.index("Woman") if "Woman" in labels else 0) | |
| matching = [clip for clip in clips if clip["label"] == selected_label] | |
| default_clip = next((i for i, item in enumerate(matching) if item["id"] == "include-004"), 0) | |
| clip = st.selectbox("Validation clip", matching, index=default_clip, | |
| format_func=lambda item: f"{item['id']} · {item['duration']:.2f} s" | |
| + (" · source video available" if item["videoPath"] else "")) | |
| if revision and not live: | |
| if st.button("Run both models on this clip", key=f"predict-{clip['id']}"): | |
| try: | |
| with st.spinner("Running both models on the saved poses…"): | |
| inputs, models = load_assets(revision) | |
| index = next(i for i, item in enumerate(clips) if item["id"] == clip["id"]) | |
| st.session_state["transformer_clip"] = replay_clip(inputs, models, index, clip) | |
| except (OSError, ValueError, RuntimeError): | |
| st.error("Live inference could not load the pinned artifacts or predict this clip.") | |
| result = st.session_state.get("transformer_clip") | |
| if result is not None and result["id"] == clip["id"]: | |
| clip = result | |
| live = True | |
| video_url = public_video(clip["videoPath"]) | |
| if video_url: | |
| split = {"val": "validation", "train": "train", "test": "test"}[clip["videoPath"].split("/")[1]] | |
| st.caption(f"Public video repository split: {split}.") | |
| st.markdown(f"[Open source video]({video_url})") | |
| else: | |
| st.caption("Use the timeline slider to review this clip’s archived predictions.") | |
| st.caption(("Predictions computed by the Space" if live else "Archived model predictions") + " at 250 ms intervals.") | |
| render_playback(video_url, clip) | |
| with st.expander("All prediction updates"): | |
| updates = {} | |
| for arm, name in ARMS.items(): | |
| for tick in clip[arm]["ticks"]: | |
| updates.setdefault(tick["time"], {"Time (s)": tick["time"]})[name] = tick["label"] | |
| st.dataframe(list(updates.values()), use_container_width=True, hide_index=True) | |
| with st.expander("Dataset splits and model sources"): | |
| st.markdown( | |
| "- This evaluation uses 720 of 743 clips in the saved iPad validation split. " | |
| "The models share 256 output labels. This panel covers 254 labels, and 23 clips " | |
| "were excluded because their labels are outside the model vocabulary. " | |
| "Epoch 102 was selected using this validation panel. Signer independence is unverified.\n" | |
| "- The legacy LSTM uses 167 output labels and a custom test manifest. Of its 412 clips, " | |
| "245 match the public INCLUDE train split, 25 validation, 66 test, and 76 have no exact match. " | |
| "Training membership for the published legacy weights remains unverified.\n" | |
| "- [INCLUDE source and attribution](https://huggingface.co/datasets/ai4bharat/INCLUDE) · " | |
| "[Public video dataset](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-dataset) · " | |
| "[Apple Vision keypoint dataset card](https://huggingface.co/datasets/spsarolkar/AI4Bharat-INCLUDE-keypoints) · " | |
| "[Native Transformer checkpoint card](https://huggingface.co/spsarolkar/isl-transformer-mlx) · " | |
| "[Legacy LSTM model card](https://huggingface.co/sunilsarolkar/isl-translation-model)" | |
| ) | |