import numpy as np import signspeak.asl.pipeline as asl_pipeline from signspeak.asl.pipeline import build_intent_input, process_asl_frames from signspeak.pipeline import apply_gloss_override, parse_gloss_override, resolve_video_path, summarize_asl_result def test_build_intent_input_matches_llm_schema(): asl = { "status": "ok", "gloss_sequence": ["I", "HAPPY", "SEE", "YOU"], "confidence": 0.91, "frames_used": 30, } emotion = { "status": "ok", "dominant_emotion": "happy", "intensity": 0.82, "emotion_scores": {"happy": 0.82, "neutral": 0.18}, } intent = build_intent_input(asl, emotion) assert intent["detected_glosses"] == ["I", "HAPPY", "SEE", "YOU"] assert intent["detected_facial_expression"] == "happy" assert intent["emotion_profile"]["confidence"] == 0.82 assert intent["sign_confidence"] == 0.91 assert intent["diagnostics"]["asl_status"] == "ok" assert intent["sign_detection"]["accepted"] is True def test_build_intent_input_exposes_rejected_top_prediction(): asl = { "status": "low_confidence", "gloss_sequence": [], "top_prediction": "talk", "confidence": 0.138, "confidence_threshold": 0.70, "top_predictions": [{"label": "talk", "confidence": 0.138}], "frames_used": 30, "landmarks_status": "ok", "landmarks_detector": "holistic", } emotion = { "status": "ok", "dominant_emotion": "neutral", "intensity": 0.29, "emotion_scores": {"neutral": 0.29}, } intent = build_intent_input(asl, emotion) assert intent["detected_glosses"] == [] assert intent["sign_detection"]["accepted"] is False assert intent["sign_detection"]["top_prediction"] == "talk" assert intent["sign_detection"]["confidence_threshold"] == 0.70 assert intent["diagnostics"]["top_prediction"] == "talk" def test_process_asl_frames_preserves_detector_diagnostics(monkeypatch): class FakeASLDetector: def predict_sequence_from_frames(self, frames): return { "status": "ok", "gloss_sequence": ["talk"], "top_prediction": "talk", "confidence": 0.96, "confidence_threshold": 0.70, "top_predictions": [{"label": "talk", "confidence": 0.96}], "segment_predictions": [ { "window_index": 0, "start_frame": 0, "end_frame": 8, "label": "talk", "confidence": 0.96, "accepted": True, } ], "frames_used": len(frames), "windows_used": 1, "sequence_window": 30, "sequence_stride": 15, "recognition_mode": "sliding_window_sequence", "keypoints_shape": [8, 543, 3], "landmarks_status": "ok", "landmarks_detector": "holistic", } monkeypatch.setattr(asl_pipeline, "ASLDetector", FakeASLDetector) monkeypatch.setenv("ASL_DETECTOR_BACKEND", "tflite") monkeypatch.setattr( asl_pipeline, "detect_emotion_on_frames", lambda frames: { "status": "ok", "dominant_emotion": "neutral", "intensity": 0.29, "emotion_scores": {"neutral": 0.29}, }, ) result = process_asl_frames([np.zeros((2, 2, 3), dtype=np.uint8)] * 8) assert result["asl"]["confidence_threshold"] == 0.70 assert result["asl"]["top_predictions"] == [{"label": "talk", "confidence": 0.96}] assert result["intent_input"]["detected_glosses"] == ["talk"] assert result["intent_input"]["sign_detection"]["top_prediction"] == "talk" assert result["intent_input"]["candidate_gloss_sequence"][0]["gloss"] == "talk" def test_predict_asl_sequence_auto_falls_back_to_tflite(monkeypatch): class FakeWLASLDetector: def predict_sequence_from_frames(self, frames): return { "status": "wlasl_i3d_error", "error": "missing torch", "gloss_sequence": [], } class FakeASLDetector: def predict_sequence_from_frames(self, frames): return { "status": "ok", "gloss_sequence": ["hello"], "top_prediction": "hello", "confidence": 0.91, "confidence_threshold": 0.70, "top_predictions": [{"label": "hello", "confidence": 0.91}], "segment_predictions": [], "frames_used": len(frames), "windows_used": 1, "sequence_window": 30, "sequence_stride": 15, "recognition_mode": "sliding_window_sequence", "model_backend": "tflite_250", } monkeypatch.setenv("ASL_DETECTOR_BACKEND", "auto") monkeypatch.setattr(asl_pipeline, "WLASLI3DDetector", FakeWLASLDetector) monkeypatch.setattr(asl_pipeline, "ASLDetector", FakeASLDetector) result = asl_pipeline.predict_asl_sequence([np.zeros((2, 2, 3), dtype=np.uint8)] * 30) assert result["gloss_sequence"] == ["hello"] assert result["fallback_from"]["backend"] == "wlasl_i3d" assert result["fallback_from"]["error"] == "missing torch" def test_predict_asl_sequence_defaults_to_auto(monkeypatch): class FakeWLASLDetector: def predict_sequence_from_frames(self, frames): return { "status": "ok", "gloss_sequence": ["thankyou"], "top_prediction": "thankyou", "confidence": 0.44, "confidence_threshold": 0.20, "top_predictions": [{"label": "thankyou", "confidence": 0.44}], "segment_predictions": [], "frames_used": len(frames), "windows_used": 1, "sequence_window": 64, "sequence_stride": 32, "recognition_mode": "wlasl_i3d_sliding_window", "model_backend": "wlasl_i3d_2000", } monkeypatch.delenv("ASL_DETECTOR_BACKEND", raising=False) monkeypatch.setattr(asl_pipeline, "WLASLI3DDetector", FakeWLASLDetector) result = asl_pipeline.predict_asl_sequence([np.zeros((2, 2, 3), dtype=np.uint8)] * 64) assert result["gloss_sequence"] == ["thankyou"] assert result["model_backend"] == "wlasl_i3d_2000" def test_summarize_asl_result_is_stable_for_missing_fields(): summary = summarize_asl_result( { "asl": {"status": "model_missing"}, "emotion": {"dominant_emotion": "unknown", "intensity": 0.0}, } ) assert "ASL status: model_missing" in summary assert "Emotion: unknown (0.00)" in summary def test_resolve_video_path_accepts_gradio_dict_payload(tmp_path): video_path = tmp_path / "capture.mp4" video_path.write_bytes(b"demo") resolved = resolve_video_path({"path": str(video_path)}) assert resolved == video_path def test_resolve_video_path_accepts_gradio_tuple_payload(tmp_path): video_path = tmp_path / "capture.mp4" video_path.write_bytes(b"demo") resolved = resolve_video_path((str(video_path),)) assert resolved == video_path def test_parse_gloss_override_normalizes_words(): assert parse_gloss_override("i love,you") == ["I", "LOVE", "YOU"] def test_apply_gloss_override_marks_diagnostics(): result = { "asl": {"status": "model_missing", "gloss_sequence": []}, "emotion": {"status": "emotion_error"}, "intent_input": {"detected_glosses": [], "diagnostics": {}}, } updated = apply_gloss_override(result, "I LOVE YOU") assert updated["intent_input"]["detected_glosses"] == ["I", "LOVE", "YOU"] assert updated["intent_input"]["diagnostics"]["manual_gloss_override"] is True assert updated["asl"]["top_prediction"] == "I LOVE YOU"