Integrate ASL TFLite model assets

.gitignore

@@ -2,3 +2,4 @@ __pycache__/
 *.py[cod]
 .pytest_cache/
 data/examples/*.mp4
+external/

README.md

@@ -66,11 +66,18 @@ python3 scripts/test_asl_brick.py --gloss-override "I LOVE YOU"
 
 ## ASL model files
 
-Place the ASL classifier assets here when available:
+The ASL classifier assets are stored under:
 
 ```text
 data/models/asl/model.tflite
 data/models/asl/train.csv
+data/models/asl/sign_to_prediction_index_map.json
+```
+
+They come from:
+
+```text
+https://github.com/jamesjbustos/sign-language-recognition
 ```
 
 Without these files, the ASL brick still samples frames and emits `model_missing` diagnostics.

data/models/asl/model.tflite

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af631244b8a7595a1925c8c7cd2b6b20b3086f6783061041d85f5bddbcb94da1
+size 3383908

data/models/asl/sign_to_prediction_index_map.json

@@ -0,0 +1 @@
+{"TV": 0, "after": 1, "airplane": 2, "all": 3, "alligator": 4, "animal": 5, "another": 6, "any": 7, "apple": 8, "arm": 9, "aunt": 10, "awake": 11, "backyard": 12, "bad": 13, "balloon": 14, "bath": 15, "because": 16, "bed": 17, "bedroom": 18, "bee": 19, "before": 20, "beside": 21, "better": 22, "bird": 23, "black": 24, "blow": 25, "blue": 26, "boat": 27, "book": 28, "boy": 29, "brother": 30, "brown": 31, "bug": 32, "bye": 33, "callonphone": 34, "can": 35, "car": 36, "carrot": 37, "cat": 38, "cereal": 39, "chair": 40, "cheek": 41, "child": 42, "chin": 43, "chocolate": 44, "clean": 45, "close": 46, "closet": 47, "cloud": 48, "clown": 49, "cow": 50, "cowboy": 51, "cry": 52, "cut": 53, "cute": 54, "dad": 55, "dance": 56, "dirty": 57, "dog": 58, "doll": 59, "donkey": 60, "down": 61, "drawer": 62, "drink": 63, "drop": 64, "dry": 65, "dryer": 66, "duck": 67, "ear": 68, "elephant": 69, "empty": 70, "every": 71, "eye": 72, "face": 73, "fall": 74, "farm": 75, "fast": 76, "feet": 77, "find": 78, "fine": 79, "finger": 80, "finish": 81, "fireman": 82, "first": 83, "fish": 84, "flag": 85, "flower": 86, "food": 87, "for": 88, "frenchfries": 89, "frog": 90, "garbage": 91, "gift": 92, "giraffe": 93, "girl": 94, "give": 95, "glasswindow": 96, "go": 97, "goose": 98, "grandma": 99, "grandpa": 100, "grass": 101, "green": 102, "gum": 103, "hair": 104, "happy": 105, "hat": 106, "hate": 107, "have": 108, "haveto": 109, "head": 110, "hear": 111, "helicopter": 112, "hello": 113, "hen": 114, "hesheit": 115, "hide": 116, "high": 117, "home": 118, "horse": 119, "hot": 120, "hungry": 121, "icecream": 122, "if": 123, "into": 124, "jacket": 125, "jeans": 126, "jump": 127, "kiss": 128, "kitty": 129, "lamp": 130, "later": 131, "like": 132, "lion": 133, "lips": 134, "listen": 135, "look": 136, "loud": 137, "mad": 138, "make": 139, "man": 140, "many": 141, "milk": 142, "minemy": 143, "mitten": 144, "mom": 145, "moon": 146, "morning": 147, "mouse": 148, "mouth": 149, "nap": 150, "napkin": 151, "night": 152, "no": 153, "noisy": 154, "nose": 155, "not": 156, "now": 157, "nuts": 158, "old": 159, "on": 160, "open": 161, "orange": 162, "outside": 163, "owie": 164, "owl": 165, "pajamas": 166, "pen": 167, "pencil": 168, "penny": 169, "person": 170, "pig": 171, "pizza": 172, "please": 173, "police": 174, "pool": 175, "potty": 176, "pretend": 177, "pretty": 178, "puppy": 179, "puzzle": 180, "quiet": 181, "radio": 182, "rain": 183, "read": 184, "red": 185, "refrigerator": 186, "ride": 187, "room": 188, "sad": 189, "same": 190, "say": 191, "scissors": 192, "see": 193, "shhh": 194, "shirt": 195, "shoe": 196, "shower": 197, "sick": 198, "sleep": 199, "sleepy": 200, "smile": 201, "snack": 202, "snow": 203, "stairs": 204, "stay": 205, "sticky": 206, "store": 207, "story": 208, "stuck": 209, "sun": 210, "table": 211, "talk": 212, "taste": 213, "thankyou": 214, "that": 215, "there": 216, "think": 217, "thirsty": 218, "tiger": 219, "time": 220, "tomorrow": 221, "tongue": 222, "tooth": 223, "toothbrush": 224, "touch": 225, "toy": 226, "tree": 227, "uncle": 228, "underwear": 229, "up": 230, "vacuum": 231, "wait": 232, "wake": 233, "water": 234, "wet": 235, "weus": 236, "where": 237, "white": 238, "who": 239, "why": 240, "will": 241, "wolf": 242, "yellow": 243, "yes": 244, "yesterday": 245, "yourself": 246, "yucky": 247, "zebra": 248, "zipper": 249}

requirements-asl-full.txt

@@ -1,7 +1 @@
 -r requirements.txt
-
-protobuf>=4.25,<5.0
-mediapipe==0.10.14
-tensorflow-cpu==2.17.1
-deepface==0.0.100
-tf-keras==2.17.0

requirements.txt

@@ -1,13 +1,35 @@
 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
 --prefer-binary
 
-gradio
+gradio>=5.50,<6.0
 qwen-tts
 soundfile
 torch
 huggingface-hub
 llama-cpp-python
-numpy>=1.26,<2.0
+numpy==1.26.4
 pandas>=2.1,<3.0
-opencv-python-headless==4.11.0.86
+protobuf==4.25.9
+opencv-contrib-python==4.11.0.86
+attrs>=26.1,<27
+mediapipe==0.10.14
+tensorflow-cpu==2.17.1
+tf-keras==2.17.0
+Flask
+flask-cors
+gdown
+mtcnn
+fire
+gunicorn
+lightphe
+lightdsa
+python-dotenv
+beautifulsoup4
+PySocks
+joblib
+lz4
+sympy
+deepface==0.0.100
+retina-face==0.0.18
+pillow==11.3.0
 pytest>=8.0,<9.0

signspeak/asl/asl_detector.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 from pathlib import Path
 from typing import Any
 
@@ -16,6 +17,7 @@ class ASLDetector:
         self.model_dir = Path(configured_model_dir) if configured_model_dir else default_model_dir
         self.model_path = self.model_dir / "model.tflite"
         self.train_csv_path = self.model_dir / "train.csv"
+        self.sign_map_path = self.model_dir / "sign_to_prediction_index_map.json"
         self.labels = self._load_labels()
 
     def predict_from_frames(self, frames: list[np.ndarray]) -> dict[str, Any]:
@@ -44,14 +46,7 @@ class ASLDetector:
 
         try:
             interpreter = self._load_interpreter()
-            interpreter.allocate_tensors()
-            input_details = interpreter.get_input_details()
-            output_details = interpreter.get_output_details()
-
-            input_data = self._prepare_input(keypoints, input_details[0])
-            interpreter.set_tensor(input_details[0]["index"], input_data)
-            interpreter.invoke()
-            output = interpreter.get_tensor(output_details[0]["index"])
+            output = self._predict(interpreter, keypoints)
 
             probs = self._softmax_if_needed(np.asarray(output).reshape(-1))
             top_idx = int(np.argmax(probs))
@@ -86,6 +81,34 @@ class ASLDetector:
 
             return Interpreter(model_path=str(self.model_path))
 
+    def _predict(self, interpreter: Any, keypoints: np.ndarray) -> np.ndarray:
+        signatures = interpreter.get_signature_list() if hasattr(interpreter, "get_signature_list") else {}
+        if "serving_default" in signatures:
+            prediction_fn = interpreter.get_signature_runner("serving_default")
+            input_name = self._signature_input_name(signatures["serving_default"])
+            input_data = np.nan_to_num(keypoints, nan=0.0, posinf=0.0, neginf=0.0).astype(np.float32)
+            prediction = prediction_fn(**{input_name: input_data})
+            output_name = self._signature_output_name(prediction)
+            return np.asarray(prediction[output_name])
+
+        interpreter.allocate_tensors()
+        input_details = interpreter.get_input_details()
+        output_details = interpreter.get_output_details()
+
+        input_data = self._prepare_input(keypoints, input_details[0])
+        interpreter.set_tensor(input_details[0]["index"], input_data)
+        interpreter.invoke()
+        return np.asarray(interpreter.get_tensor(output_details[0]["index"]))
+
+    def _signature_input_name(self, signature: dict[str, Any]) -> str:
+        inputs = signature.get("inputs") or ["inputs"]
+        return inputs[0]
+
+    def _signature_output_name(self, prediction: dict[str, Any]) -> str:
+        if "outputs" in prediction:
+            return "outputs"
+        return next(iter(prediction))
+
     def _prepare_input(self, keypoints: np.ndarray, input_detail: dict[str, Any]) -> np.ndarray:
         shape = input_detail.get("shape")
         dtype = input_detail.get("dtype", np.float32)
@@ -117,6 +140,16 @@ class ASLDetector:
         return fitted.reshape(target_shape)
 
     def _load_labels(self) -> list[str]:
+        if self.sign_map_path.exists():
+            try:
+                sign_to_index = json.loads(self.sign_map_path.read_text(encoding="utf-8"))
+                labels = [""] * len(sign_to_index)
+                for sign, index in sign_to_index.items():
+                    labels[int(index)] = str(sign)
+                return labels
+            except Exception:
+                pass
+
         if not self.train_csv_path.exists():
             return []
         try:

tests/test_asl_detector.py

@@ -0,0 +1,42 @@
+import json
+
+import numpy as np
+
+from signspeak.asl.asl_detector import ASLDetector
+
+
+def test_load_labels_prefers_prediction_index_map(tmp_path):
+    model_dir = tmp_path / "asl"
+    model_dir.mkdir()
+    (model_dir / "sign_to_prediction_index_map.json").write_text(
+        json.dumps({"love": 2, "hello": 0, "thanks": 1}),
+        encoding="utf-8",
+    )
+
+    detector = ASLDetector(model_dir=model_dir)
+
+    assert detector.labels == ["hello", "thanks", "love"]
+
+
+class FakeSignatureInterpreter:
+    def get_signature_list(self):
+        return {"serving_default": {"inputs": ["inputs"], "outputs": ["outputs"]}}
+
+    def get_signature_runner(self, signature_name):
+        assert signature_name == "serving_default"
+
+        def predict(**kwargs):
+            assert kwargs["inputs"].shape == (543, 3)
+            return {"outputs": np.asarray([[0.1, 0.8, 0.1]], dtype=np.float32)}
+
+        return predict
+
+
+def test_predict_uses_tflite_signature_runner(tmp_path):
+    detector = ASLDetector(model_dir=tmp_path)
+    keypoints = np.zeros((543, 3), dtype=np.float32)
+
+    output = detector._predict(FakeSignatureInterpreter(), keypoints)
+
+    assert output.shape == (1, 3)
+    assert float(output[0][1]) == np.float32(0.8)