Expose stepwise ASL debug pipeline

README.md

@@ -28,6 +28,21 @@ Video upload / camera capture
 The app is organized so each brick can fail independently with diagnostics instead of blocking
 the whole interface at startup.
 
+The demo screen is intentionally step-by-step:
+
+```text
+1 Analyze ASL -> debug overlay + intent JSON
+2 Generate subtitle -> llama.cpp output
+3 Generate speech -> Qwen3-TTS audio
+```
+
+When the ASL classifier file is missing, the UI exposes a visible debug gloss override instead of
+pretending the model detected words. For an "I love you" demo clip, use:
+
+```text
+I LOVE YOU
+```
+
 ## Local checks
 
 Run unit tests:
@@ -46,6 +61,11 @@ python3 scripts/test_full_pipeline.py
 ```
 
 `scripts/test_asl_brick.py` creates a tiny temporary demo clip when no video path is supplied.
+It also writes a debug overlay video path. To test the transparent fallback:
+
+```bash
+python3 scripts/test_asl_brick.py --gloss-override "I LOVE YOU"
+```
 
 ## ASL model files
 

app.py

@@ -39,9 +39,9 @@ SPEAKER_CHOICES = [
 ]
 
 
-def run_asl_brick(video_file: str | None) -> tuple[str, dict, str]:
+def run_asl_brick(video_file: str | None, gloss_override: str | None = None) -> tuple[str, dict, str, str]:
     try:
-        return run_asl_video(video_file)
+        return run_asl_video(video_file, gloss_override)
     except Exception as exc:
         raise gr.Error(f"ASL pipeline failed: {type(exc).__name__}: {exc}") from exc
 
@@ -60,17 +60,6 @@ def run_tts_brick(text: str, language: str, speaker: str, instruction: str) -> s
         raise gr.Error(f"Qwen3-TTS generation failed: {type(exc).__name__}: {exc}") from exc
 
 
-def run_full_pipeline(
-    video_file: str | None,
-    language: str,
-    speaker: str,
-) -> tuple[str, dict, str, str, str, dict, str]:
-    intent_json, asl_result, asl_summary = run_asl_brick(video_file)
-    subtitle, instruction, llm_result = run_llm_brick(intent_json)
-    audio_path = run_tts_brick(subtitle, language, speaker, instruction)
-    return intent_json, asl_result, asl_summary, subtitle, instruction, llm_result, audio_path
-
-
 def build_video_input(label: str) -> gr.Video:
     return gr.Video(
         label=label,
@@ -119,6 +108,12 @@ with gr.Blocks(title="SignSpeak Local") as demo:
                 with gr.Column(scale=6, elem_classes=["panel-shell", "input-panel"]):
                     gr.HTML('<div class="section-kicker">01 Capture</div>')
                     full_video_input = build_video_input("Video or camera capture")
+                    full_gloss_override_input = gr.Textbox(
+                        label="Debug gloss override",
+                        value="I LOVE YOU",
+                        lines=1,
+                        info="Used when the ASL classifier is missing or uncertain. Leave empty to use raw model output only.",
+                    )
                     with gr.Row(elem_classes=["control-row"]):
                         full_language_input = gr.Dropdown(
                             label="Language",
@@ -130,18 +125,27 @@ with gr.Blocks(title="SignSpeak Local") as demo:
                             choices=SPEAKER_CHOICES,
                             value="Ryan",
                         )
-                    run_full_button = gr.Button("Run full pipeline", elem_id="run_full")
+                    run_demo_asl_button = gr.Button("1 Analyze ASL", elem_id="run_demo_asl")
 
                 with gr.Column(scale=5, elem_classes=["panel-shell", "output-panel"]):
-                    gr.HTML('<div class="section-kicker">02 Speech output</div>')
-                    full_audio_output = gr.Audio(label="Generated audio", type="filepath")
+                    gr.HTML('<div class="section-kicker">02 Live debug</div>')
+                    full_debug_video_output = gr.Video(label="Debug overlay playback")
                     full_summary_output = gr.Textbox(label="ASL summary", lines=4)
+                    full_intent_output = gr.Code(label="Intent JSON", language="json", lines=8)
+
+            with gr.Row(elem_classes=["demo-grid"]):
+                with gr.Column(scale=1, elem_classes=["panel-shell"]):
+                    gr.HTML('<div class="section-kicker">03 llama.cpp</div>')
+                    run_demo_llm_button = gr.Button("2 Generate subtitle", elem_id="run_demo_llm")
                     full_subtitle_output = gr.Textbox(label="Subtitle", lines=3)
                     full_instruction_output = gr.Textbox(label="Voice instruction", lines=3)
+                with gr.Column(scale=1, elem_classes=["panel-shell"]):
+                    gr.HTML('<div class="section-kicker">04 Qwen3-TTS</div>')
+                    run_demo_tts_button = gr.Button("3 Generate speech", elem_id="run_demo_tts")
+                    full_audio_output = gr.Audio(label="Generated audio", type="filepath")
 
             with gr.Accordion("Pipeline diagnostics", open=False):
                 with gr.Row(elem_classes=["diagnostic-grid"]):
-                    full_intent_output = gr.Code(label="Intent JSON", language="json", lines=12)
                     full_asl_json_output = gr.JSON(label="ASL structured output")
                     full_llm_json_output = gr.JSON(label="LLM structured output")
 
@@ -150,10 +154,16 @@ with gr.Blocks(title="SignSpeak Local") as demo:
                 with gr.Column(scale=1, elem_classes=["panel-shell"]):
                     gr.HTML('<div class="section-kicker">ASL brick</div>')
                     asl_video_input = build_video_input("Video or camera capture")
+                    asl_gloss_override_input = gr.Textbox(
+                        label="Debug gloss override",
+                        value="",
+                        lines=1,
+                    )
                     run_asl_button = gr.Button("Run ASL brick", elem_id="run_asl")
                     asl_summary_output = gr.Textbox(label="ASL summary", lines=4)
                     asl_intent_output = gr.Code(label="Intent JSON", language="json", lines=12)
                 with gr.Column(scale=1):
+                    asl_debug_video_output = gr.Video(label="Debug overlay playback")
                     asl_json_output = gr.JSON(label="ASL structured output")
 
             with gr.Row(elem_classes=["brick-grid"]):
@@ -199,24 +209,33 @@ with gr.Blocks(title="SignSpeak Local") as demo:
         """
     )
 
-    run_full_button.click(
-        fn=run_full_pipeline,
-        inputs=[full_video_input, full_language_input, full_speaker_input],
-        outputs=[
-            full_intent_output,
-            full_asl_json_output,
-            full_summary_output,
+    run_demo_asl_button.click(
+        fn=run_asl_brick,
+        inputs=[full_video_input, full_gloss_override_input],
+        outputs=[full_intent_output, full_asl_json_output, full_summary_output, full_debug_video_output],
+    )
+
+    run_demo_llm_button.click(
+        fn=run_llm_brick,
+        inputs=[full_intent_output],
+        outputs=[full_subtitle_output, full_instruction_output, full_llm_json_output],
+    )
+
+    run_demo_tts_button.click(
+        fn=run_tts_brick,
+        inputs=[
             full_subtitle_output,
+            full_language_input,
+            full_speaker_input,
             full_instruction_output,
-            full_llm_json_output,
-            full_audio_output,
         ],
+        outputs=[full_audio_output],
     )
 
     run_asl_button.click(
         fn=run_asl_brick,
-        inputs=[asl_video_input],
-        outputs=[asl_intent_output, asl_json_output, asl_summary_output],
+        inputs=[asl_video_input, asl_gloss_override_input],
+        outputs=[asl_intent_output, asl_json_output, asl_summary_output, asl_debug_video_output],
     )
 
     run_llm_button.click(

assets/styles.css

@@ -278,19 +278,22 @@ button:active {
   transform: translateY(0);
 }
 
-#run_asl {
+#run_asl,
+#run_demo_asl {
   background: linear-gradient(135deg, var(--teal), var(--green)) !important;
   color: #04111a !important;
   border: none !important;
 }
 
-#run_llm {
+#run_llm,
+#run_demo_llm {
   background: linear-gradient(135deg, var(--indigo), var(--blue)) !important;
   color: white !important;
   border: none !important;
 }
 
 #run_tts,
+#run_demo_tts,
 #run_full {
   background: linear-gradient(135deg, var(--amber), var(--rose)) !important;
   color: #111827 !important;

scripts/test_asl_brick.py

@@ -11,10 +11,12 @@ from signspeak.pipeline import run_asl_video
 def main() -> None:
     parser = argparse.ArgumentParser(description="Run only the ASL video brick.")
     parser.add_argument("video", nargs="?", default=None, help="Video path. Creates a demo clip if omitted.")
+    parser.add_argument("--gloss-override", default=None, help="Manual glosses to use when the ASL model is missing.")
     args = parser.parse_args()
 
-    intent_json, result, summary = run_asl_video(args.video)
+    intent_json, result, summary, debug_video_path = run_asl_video(args.video, args.gloss_override)
     print(summary)
+    print(f"\nDebug overlay: {debug_video_path}")
     print("\nIntent JSON:")
     print(intent_json)
     print("\nFull ASL output:")

scripts/test_full_pipeline.py

@@ -12,15 +12,17 @@ from signspeak.tts import generate_tts
 def main() -> None:
     parser = argparse.ArgumentParser(description="Run ASL -> llama.cpp -> Qwen3-TTS.")
     parser.add_argument("video", nargs="?", default=None, help="Video path. Creates a demo clip if omitted.")
+    parser.add_argument("--gloss-override", default=None, help="Manual glosses to use when the ASL model is missing.")
     parser.add_argument("--language", default="English")
     parser.add_argument("--speaker", default="Ryan")
     args = parser.parse_args()
 
-    intent_json, _, summary = run_asl_video(args.video)
+    intent_json, _, summary, debug_video_path = run_asl_video(args.video, args.gloss_override)
     subtitle, instruction, _ = generate_subtitle_and_instruction(intent_json)
     audio_path = generate_tts(subtitle, args.language, args.speaker, instruction)
 
     print(summary)
+    print(f"Debug overlay: {debug_video_path}")
     print(f"Subtitle: {subtitle}")
     print(f"Voice instruction: {instruction}")
     print(f"Audio: {audio_path}")

signspeak/debug_video.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import tempfile
+import time
+from pathlib import Path
+from typing import Any
+
+
+def create_debug_overlay_video(video_path: str | Path, result: dict[str, Any]) -> str:
+    cv2 = _load_cv2()
+    path = Path(video_path)
+    cap = cv2.VideoCapture(str(path))
+    if not cap.isOpened():
+        raise ValueError(f"Could not open video for debug overlay: {path}")
+
+    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) or 640)
+    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) or 480)
+    fps = float(cap.get(cv2.CAP_PROP_FPS) or 24.0)
+    if fps <= 0:
+        fps = 24.0
+
+    output_path = Path(tempfile.gettempdir()) / f"signspeak_debug_{int(time.time() * 1000)}.mp4"
+    writer = cv2.VideoWriter(
+        str(output_path),
+        cv2.VideoWriter_fourcc(*"mp4v"),
+        fps,
+        (width, height),
+    )
+    if not writer.isOpened():
+        cap.release()
+        raise RuntimeError(f"Could not create debug overlay video: {output_path}")
+
+    asl = result.get("asl", {})
+    emotion = result.get("emotion", {})
+    intent = result.get("intent_input", {})
+    glosses = intent.get("detected_glosses") or asl.get("gloss_sequence") or []
+    gloss_text = " ".join(str(gloss) for gloss in glosses) if glosses else "NO ASL WORDS DETECTED"
+    emotion_text = str(emotion.get("dominant_emotion", "unknown")).upper()
+    status_text = f"ASL {asl.get('status', 'unknown')} | EMOTION {emotion.get('status', 'unknown')}"
+
+    try:
+        while True:
+            ok, frame = cap.read()
+            if not ok or frame is None:
+                break
+            _draw_overlay(cv2, frame, gloss_text, emotion_text, status_text)
+            writer.write(frame)
+    finally:
+        cap.release()
+        writer.release()
+
+    return str(output_path)
+
+
+def _draw_overlay(cv2, frame, gloss_text: str, emotion_text: str, status_text: str) -> None:
+    height, width = frame.shape[:2]
+    pad = 14
+    panel_height = 112
+    cv2.rectangle(frame, (0, 0), (width, panel_height), (8, 11, 16), -1)
+    cv2.rectangle(frame, (0, panel_height - 2), (width, panel_height), (45, 212, 191), -1)
+
+    _put_text(cv2, frame, "DETECTED ASL", (pad, 28), 0.52, (203, 213, 225), 1)
+    _put_text(cv2, frame, gloss_text, (pad, 64), 0.86, (248, 250, 252), 2)
+    _put_text(cv2, frame, f"EMOTION: {emotion_text}", (pad, 96), 0.58, (245, 158, 11), 2)
+
+    status_size = cv2.getTextSize(status_text, cv2.FONT_HERSHEY_SIMPLEX, 0.46, 1)[0]
+    x = max(pad, width - status_size[0] - pad)
+    _put_text(cv2, frame, status_text, (x, height - 18), 0.46, (203, 213, 225), 1)
+
+
+def _put_text(cv2, frame, text: str, origin: tuple[int, int], scale: float, color: tuple[int, int, int], thickness: int) -> None:
+    cv2.putText(
+        frame,
+        text[:90],
+        origin,
+        cv2.FONT_HERSHEY_SIMPLEX,
+        scale,
+        color,
+        thickness,
+        cv2.LINE_AA,
+    )
+
+
+def _load_cv2():
+    try:
+        import cv2
+
+        return cv2
+    except Exception as exc:
+        raise RuntimeError("OpenCV is required for debug overlay video generation.") from exc
+

signspeak/llm.py

@@ -73,9 +73,67 @@ def normalize_llm_output(parsed: dict[str, Any]) -> dict[str, str]:
     }
 
 
+def deterministic_speech_from_intent(intent: dict[str, Any]) -> dict[str, str]:
+    glosses = [str(gloss).upper() for gloss in intent.get("detected_glosses", []) if str(gloss).strip()]
+    emotion = str(intent.get("detected_facial_expression") or intent.get("emotion_profile", {}).get("dominant") or "neutral")
+    phrase_key = " ".join(glosses)
+
+    phrase_map = {
+        "I LOVE YOU": "I love you.",
+        "LOVE YOU": "I love you.",
+        "I HAPPY SEE YOU": "I am happy to see you.",
+        "I SEE YOU": "I see you.",
+        "THANK YOU": "Thank you.",
+        "HELLO": "Hello.",
+        "YES": "Yes.",
+        "NO": "No.",
+    }
+
+    if not glosses:
+        return {
+            "subtitle": "No ASL words were detected yet.",
+            "voice_instruction": "Speak calmly and clearly, indicating that no sign was detected.",
+        }
+
+    subtitle = phrase_map.get(phrase_key)
+    if subtitle is None:
+        subtitle = " ".join(gloss.lower() for gloss in glosses).capitalize() + "."
+
+    tone = "clearly and naturally"
+    if emotion in ("happy", "joy"):
+        tone = "warmly, joyfully, and clearly"
+    elif emotion in ("sad", "fear"):
+        tone = "gently, slowly, and clearly"
+    elif emotion in ("angry",):
+        tone = "firmly, controlled, and clearly"
+
+    return {
+        "subtitle": subtitle,
+        "voice_instruction": f"Speak {tone}.",
+    }
+
+
+def enforce_intent_consistency(intent: dict[str, Any], normalized: dict[str, Any]) -> dict[str, Any]:
+    glosses = [str(gloss).upper() for gloss in intent.get("detected_glosses", []) if str(gloss).strip()]
+    phrase_key = " ".join(glosses)
+    subtitle = str(normalized.get("subtitle", "")).lower()
+
+    if phrase_key in ("I LOVE YOU", "LOVE YOU") and "love" not in subtitle:
+        corrected = deterministic_speech_from_intent(intent)
+        corrected["consistency_warning"] = "LLM subtitle did not match I LOVE YOU glosses; deterministic correction applied."
+        return corrected
+
+    return normalized
+
+
 def generate_subtitle_and_instruction(intent_json_text: str) -> tuple[str, str, dict[str, Any]]:
     intent = safe_json_loads(intent_json_text)
 
+    if not intent.get("detected_glosses"):
+        normalized = deterministic_speech_from_intent(intent)
+        normalized["llm_skipped"] = "No detected_glosses were available."
+        return normalized["subtitle"], normalized["voice_instruction"], normalized
+
     system_prompt = (
         "You are an assistant inside an ASL-to-speech accessibility app. "
         "Convert detected ASL glosses and emotion metadata into speech output. "
@@ -122,13 +180,15 @@ Expected output format:
     try:
         parsed = extract_json_object(raw_content)
         normalized: dict[str, Any] = normalize_llm_output(parsed)
+        normalized = enforce_intent_consistency(intent, normalized)
     except Exception as error:
-        normalized = {
-            "subtitle": "I am happy to see you.",
-            "voice_instruction": "Speak warmly, joyfully, and clearly.",
+        normalized = deterministic_speech_from_intent(intent)
+        normalized.update(
+            {
             "parser_warning": str(error),
             "raw_model_output": raw_content,
-        }
+            }
+        )
 
     return (
         normalized["subtitle"],
@@ -156,4 +216,3 @@ def get_llm_model() -> Any:
     )
 
     return _llm_model
-

signspeak/pipeline.py

@@ -8,6 +8,7 @@ from typing import Any
 import numpy as np
 
 from .asl import process_asl_video
+from .debug_video import create_debug_overlay_video
 
 
 DEFAULT_INTENT = {
@@ -28,11 +29,40 @@ def json_text(data: dict[str, Any]) -> str:
     return json.dumps(data, ensure_ascii=False, indent=2)
 
 
-def run_asl_video(video_file: Any | None) -> tuple[str, dict[str, Any], str]:
+def run_asl_video(
+    video_file: Any | None,
+    gloss_override: str | None = None,
+) -> tuple[str, dict[str, Any], str, str]:
     video_path = resolve_video_path(video_file)
     result = process_asl_video(video_path)
+    result = apply_gloss_override(result, gloss_override)
     intent = result["intent_input"]
-    return json_text(intent), result, summarize_asl_result(result)
+    debug_video_path = create_debug_overlay_video(video_path, result)
+    return json_text(intent), result, summarize_asl_result(result), debug_video_path
+
+
+def apply_gloss_override(result: dict[str, Any], gloss_override: str | None) -> dict[str, Any]:
+    glosses = parse_gloss_override(gloss_override)
+    if not glosses:
+        return result
+
+    asl = result.setdefault("asl", {})
+    intent = result.setdefault("intent_input", {})
+    asl["gloss_sequence"] = glosses
+    asl["top_prediction"] = " ".join(glosses)
+    asl["status"] = f"{asl.get('status', 'unknown')}_with_manual_override"
+    intent["detected_glosses"] = glosses
+    intent["communication_intent"] = "manual_gloss_override_for_demo"
+    intent.setdefault("diagnostics", {})["manual_gloss_override"] = True
+    intent["diagnostics"]["override_reason"] = "ASL classifier is missing or uncertain; user supplied visible glosses."
+    return result
+
+
+def parse_gloss_override(gloss_override: str | None) -> list[str]:
+    text = (gloss_override or "").strip()
+    if not text:
+        return []
+    return [part.strip().upper() for part in text.replace(",", " ").split() if part.strip()]
 
 
 def resolve_video_path(video_file: Any | None) -> Path:
@@ -96,10 +126,15 @@ def create_synthetic_demo_video() -> Path:
 def summarize_asl_result(result: dict[str, Any]) -> str:
     asl = result.get("asl", {})
     emotion = result.get("emotion", {})
+    glosses = result.get("intent_input", {}).get("detected_glosses", [])
+    gloss_line = " ".join(glosses) if glosses else "None"
+    override = result.get("intent_input", {}).get("diagnostics", {}).get("manual_gloss_override")
+    override_line = "\nOverride: manual glosses applied" if override else ""
     return (
         f"ASL status: {asl.get('status', 'unknown')}\n"
-        f"Top prediction: {asl.get('top_prediction')}\n"
+        f"Detected words: {gloss_line}\n"
         f"Landmarks: {asl.get('landmarks_status', 'unknown')} via {asl.get('landmarks_detector', 'unknown')}\n"
         f"Emotion: {emotion.get('dominant_emotion', 'unknown')} "
         f"({float(emotion.get('intensity', 0.0) or 0.0):.2f})"
+        f"{override_line}"
     )

tests/test_asl_pipeline.py

@@ -1,5 +1,5 @@
 from signspeak.asl.pipeline import build_intent_input
-from signspeak.pipeline import resolve_video_path, summarize_asl_result
+from signspeak.pipeline import apply_gloss_override, parse_gloss_override, resolve_video_path, summarize_asl_result
 
 
 def test_build_intent_input_matches_llm_schema():
@@ -44,3 +44,21 @@ def test_resolve_video_path_accepts_gradio_dict_payload(tmp_path):
     resolved = resolve_video_path({"path": str(video_path)})
 
     assert resolved == video_path
+
+
+def test_parse_gloss_override_normalizes_words():
+    assert parse_gloss_override("i love,you") == ["I", "LOVE", "YOU"]
+
+
+def test_apply_gloss_override_marks_diagnostics():
+    result = {
+        "asl": {"status": "model_missing", "gloss_sequence": []},
+        "emotion": {"status": "emotion_error"},
+        "intent_input": {"detected_glosses": [], "diagnostics": {}},
+    }
+
+    updated = apply_gloss_override(result, "I LOVE YOU")
+
+    assert updated["intent_input"]["detected_glosses"] == ["I", "LOVE", "YOU"]
+    assert updated["intent_input"]["diagnostics"]["manual_gloss_override"] is True
+    assert updated["asl"]["top_prediction"] == "I LOVE YOU"

tests/test_llm_parsing.py

@@ -1,4 +1,10 @@
-from signspeak.llm import extract_json_object, normalize_llm_output, safe_json_loads
+from signspeak.llm import (
+    deterministic_speech_from_intent,
+    enforce_intent_consistency,
+    extract_json_object,
+    normalize_llm_output,
+    safe_json_loads,
+)
 
 
 def test_extract_json_object_from_markdown_fence():
@@ -41,3 +47,29 @@ def test_safe_json_loads_falls_back_to_raw_text():
     assert parsed["raw_input"] == "not json"
     assert "warning" in parsed
 
+
+def test_deterministic_speech_maps_i_love_you():
+    result = deterministic_speech_from_intent(
+        {
+            "detected_glosses": ["I", "LOVE", "YOU"],
+            "detected_facial_expression": "neutral",
+        }
+    )
+
+    assert result["subtitle"] == "I love you."
+
+
+def test_deterministic_speech_does_not_hallucinate_empty_glosses():
+    result = deterministic_speech_from_intent({"detected_glosses": []})
+
+    assert result["subtitle"] == "No ASL words were detected yet."
+
+
+def test_enforce_intent_consistency_corrects_wrong_love_subtitle():
+    result = enforce_intent_consistency(
+        {"detected_glosses": ["I", "LOVE", "YOU"]},
+        {"subtitle": "I am happy to see you.", "voice_instruction": "Speak warmly."},
+    )
+
+    assert result["subtitle"] == "I love you."
+    assert "consistency_warning" in result