Commit ·
2290cb9
1
Parent(s): 8f01235
Use single 30-frame ASL upload inference
Browse files- signspeak/asl/asl_detector.py +4 -38
- signspeak/asl/pipeline.py +1 -1
- tests/test_asl_detector.py +0 -10
signspeak/asl/asl_detector.py
CHANGED
|
@@ -48,25 +48,11 @@ class ASLDetector:
|
|
| 48 |
|
| 49 |
try:
|
| 50 |
interpreter = self._load_interpreter()
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
probs = self._softmax_if_needed(np.asarray(output).reshape(-1))
|
| 55 |
-
top_idx = int(np.argmax(probs))
|
| 56 |
-
window_results.append(
|
| 57 |
-
{
|
| 58 |
-
"window_index": window_index,
|
| 59 |
-
"probs": probs,
|
| 60 |
-
"top_idx": top_idx,
|
| 61 |
-
"confidence": float(probs[top_idx]),
|
| 62 |
-
}
|
| 63 |
-
)
|
| 64 |
-
|
| 65 |
-
best = max(window_results, key=lambda item: item["confidence"])
|
| 66 |
-
probs = best["probs"]
|
| 67 |
-
top_idx = int(best["top_idx"])
|
| 68 |
top_prediction = self._label_for_index(top_idx)
|
| 69 |
-
confidence = float(
|
| 70 |
accepted = confidence >= self.confidence_threshold
|
| 71 |
|
| 72 |
base.update(
|
|
@@ -77,8 +63,6 @@ class ASLDetector:
|
|
| 77 |
"confidence": confidence,
|
| 78 |
"confidence_threshold": self.confidence_threshold,
|
| 79 |
"top_predictions": self._top_predictions(probs),
|
| 80 |
-
"windows_analyzed": len(window_results),
|
| 81 |
-
"best_window_index": int(best["window_index"]),
|
| 82 |
}
|
| 83 |
)
|
| 84 |
return base
|
|
@@ -129,24 +113,6 @@ class ASLDetector:
|
|
| 129 |
return "outputs"
|
| 130 |
return next(iter(prediction))
|
| 131 |
|
| 132 |
-
def _keypoint_windows(self, keypoints: np.ndarray, window_size: int = 30, max_windows: int = 5) -> list[np.ndarray]:
|
| 133 |
-
frame_count = int(keypoints.shape[0])
|
| 134 |
-
if frame_count <= window_size:
|
| 135 |
-
return [keypoints.astype(np.float32)]
|
| 136 |
-
|
| 137 |
-
max_start = frame_count - window_size
|
| 138 |
-
if max_windows <= 1:
|
| 139 |
-
starts = [max_start // 2]
|
| 140 |
-
else:
|
| 141 |
-
starts = np.linspace(0, max_start, num=min(max_windows, max_start + 1), dtype=int).tolist()
|
| 142 |
-
|
| 143 |
-
unique_starts = []
|
| 144 |
-
for start in starts:
|
| 145 |
-
if start not in unique_starts:
|
| 146 |
-
unique_starts.append(int(start))
|
| 147 |
-
|
| 148 |
-
return [keypoints[start : start + window_size].astype(np.float32) for start in unique_starts]
|
| 149 |
-
|
| 150 |
def _prepare_input(self, keypoints: np.ndarray, input_detail: dict[str, Any]) -> np.ndarray:
|
| 151 |
shape = input_detail.get("shape")
|
| 152 |
dtype = input_detail.get("dtype", np.float32)
|
|
|
|
| 48 |
|
| 49 |
try:
|
| 50 |
interpreter = self._load_interpreter()
|
| 51 |
+
output = self._predict(interpreter, keypoints.astype(np.float32))
|
| 52 |
+
probs = self._softmax_if_needed(np.asarray(output).reshape(-1))
|
| 53 |
+
top_idx = int(np.argmax(probs))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
top_prediction = self._label_for_index(top_idx)
|
| 55 |
+
confidence = float(probs[top_idx])
|
| 56 |
accepted = confidence >= self.confidence_threshold
|
| 57 |
|
| 58 |
base.update(
|
|
|
|
| 63 |
"confidence": confidence,
|
| 64 |
"confidence_threshold": self.confidence_threshold,
|
| 65 |
"top_predictions": self._top_predictions(probs),
|
|
|
|
|
|
|
| 66 |
}
|
| 67 |
)
|
| 68 |
return base
|
|
|
|
| 113 |
return "outputs"
|
| 114 |
return next(iter(prediction))
|
| 115 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
def _prepare_input(self, keypoints: np.ndarray, input_detail: dict[str, Any]) -> np.ndarray:
|
| 117 |
shape = input_detail.get("shape")
|
| 118 |
dtype = input_detail.get("dtype", np.float32)
|
signspeak/asl/pipeline.py
CHANGED
|
@@ -12,7 +12,7 @@ from .video_utils import sample_video_frames, sample_video_frames_for_emotion
|
|
| 12 |
|
| 13 |
def process_asl_video(video_path: str | Path) -> dict[str, Any]:
|
| 14 |
path = Path(video_path)
|
| 15 |
-
asl_frames = sample_video_frames(path, target_frames=
|
| 16 |
emotion_frames = sample_video_frames_for_emotion(path, target_frames=12)
|
| 17 |
|
| 18 |
return process_asl_frames(asl_frames, emotion_frames, source=str(path))
|
|
|
|
| 12 |
|
| 13 |
def process_asl_video(video_path: str | Path) -> dict[str, Any]:
|
| 14 |
path = Path(video_path)
|
| 15 |
+
asl_frames = sample_video_frames(path, target_frames=30)
|
| 16 |
emotion_frames = sample_video_frames_for_emotion(path, target_frames=12)
|
| 17 |
|
| 18 |
return process_asl_frames(asl_frames, emotion_frames, source=str(path))
|
tests/test_asl_detector.py
CHANGED
|
@@ -85,13 +85,3 @@ def test_low_confidence_prediction_is_not_accepted(monkeypatch, tmp_path):
|
|
| 85 |
assert result["status"] == "low_confidence"
|
| 86 |
assert result["top_prediction"] == "where"
|
| 87 |
assert result["gloss_sequence"] == []
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
def test_keypoint_windows_cover_long_sequences(tmp_path):
|
| 91 |
-
detector = ASLDetector(model_dir=tmp_path)
|
| 92 |
-
keypoints = np.zeros((90, 543, 3), dtype=np.float32)
|
| 93 |
-
|
| 94 |
-
windows = detector._keypoint_windows(keypoints, window_size=30, max_windows=5)
|
| 95 |
-
|
| 96 |
-
assert len(windows) == 5
|
| 97 |
-
assert all(window.shape == (30, 543, 3) for window in windows)
|
|
|
|
| 85 |
assert result["status"] == "low_confidence"
|
| 86 |
assert result["top_prediction"] == "where"
|
| 87 |
assert result["gloss_sequence"] == []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|