fix: pin dropdown label backgrounds light (no dark FMS Test header)
#8
by BladeSzaSza - opened
- .hfignore +37 -37
- CLAUDE.md +199 -192
- README.md +118 -118
- app.py +11 -11
- docs/superpowers/plans/2026-06-09-pose-model-selector.md +734 -734
- docs/superpowers/plans/2026-06-09-pose-visualizer.md +914 -914
- docs/superpowers/plans/2026-06-13-full-fms-session-pdf.md +1209 -1209
- docs/superpowers/specs/2026-06-09-pose-model-selector-design.md +171 -171
- docs/superpowers/specs/2026-06-09-pose-visualizer-design.md +197 -197
- docs/superpowers/specs/2026-06-13-full-fms-session-pdf-design.md +154 -154
- formscout/agents/classifier.py +102 -102
- formscout/agents/ingest.py +7 -28
- formscout/agents/judge.py +125 -136
- formscout/agents/pdf_report.py +175 -115
- formscout/agents/pose2d.py +232 -232
- formscout/agents/report.py +139 -139
- formscout/agents/visualizer.py +418 -435
- formscout/analysis/__init__.py +1 -0
- formscout/analysis/charts.py +171 -0
- formscout/analysis/laban.py +127 -0
- formscout/analysis/relevant_joints.py +122 -0
- formscout/analysis/timeseries.py +49 -0
- formscout/config.py +15 -3
- formscout/pipeline.py +111 -111
- formscout/rubric/__init__.py +32 -32
- formscout/rubric/active_slr.py +51 -51
- formscout/rubric/hurdle_step.py +60 -60
- formscout/rubric/inline_lunge.py +58 -58
- formscout/rubric/rotary_stability.py +56 -56
- formscout/rubric/shoulder_mobility.py +46 -46
- formscout/rubric/trunk_stability_pushup.py +55 -55
- formscout/serving/__init__.py +20 -0
- formscout/serving/llama_cpp.py +148 -174
- formscout/serving/transformers_vlm.py +116 -0
- formscout/session.py +283 -194
- formscout/startup.py +47 -47
- formscout/types.py +3 -0
- formscout/ui/theme.py +272 -250
- requirements.txt +3 -1
- scripts/hf_upload.sh +97 -97
- scripts/serve_judge.sh +35 -35
- tests/test_analysis.py +145 -0
- tests/test_judge_backend.py +75 -0
- tests/test_keyframe.py +37 -37
- tests/test_pdf_report.py +51 -51
- tests/test_phase2.py +354 -354
- tests/test_session.py +94 -94
- tests/test_visualizer.py +176 -176
.hfignore
CHANGED
|
@@ -1,37 +1,37 @@
|
|
| 1 |
-
# Python
|
| 2 |
-
__pycache__/
|
| 3 |
-
*.py[cod]
|
| 4 |
-
*.egg-info/
|
| 5 |
-
dist/
|
| 6 |
-
build/
|
| 7 |
-
.eggs/
|
| 8 |
-
*.egg
|
| 9 |
-
|
| 10 |
-
# Virtual environments
|
| 11 |
-
.venv/
|
| 12 |
-
venv/
|
| 13 |
-
env/
|
| 14 |
-
|
| 15 |
-
# Secrets / local config
|
| 16 |
-
.env
|
| 17 |
-
.env.*
|
| 18 |
-
|
| 19 |
-
# Model weights (managed separately)
|
| 20 |
-
checkpoints/
|
| 21 |
-
*.pt
|
| 22 |
-
*.pth
|
| 23 |
-
*.gguf
|
| 24 |
-
*.bin
|
| 25 |
-
|
| 26 |
-
# Run artifacts
|
| 27 |
-
traces/
|
| 28 |
-
*.mp4
|
| 29 |
-
|
| 30 |
-
# Dev tooling
|
| 31 |
-
.pytest_cache/
|
| 32 |
-
.ruff_cache/
|
| 33 |
-
.DS_Store
|
| 34 |
-
.claude/
|
| 35 |
-
|
| 36 |
-
# Git
|
| 37 |
-
.git/
|
|
|
|
| 1 |
+
# Python
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
*.egg-info/
|
| 5 |
+
dist/
|
| 6 |
+
build/
|
| 7 |
+
.eggs/
|
| 8 |
+
*.egg
|
| 9 |
+
|
| 10 |
+
# Virtual environments
|
| 11 |
+
.venv/
|
| 12 |
+
venv/
|
| 13 |
+
env/
|
| 14 |
+
|
| 15 |
+
# Secrets / local config
|
| 16 |
+
.env
|
| 17 |
+
.env.*
|
| 18 |
+
|
| 19 |
+
# Model weights (managed separately)
|
| 20 |
+
checkpoints/
|
| 21 |
+
*.pt
|
| 22 |
+
*.pth
|
| 23 |
+
*.gguf
|
| 24 |
+
*.bin
|
| 25 |
+
|
| 26 |
+
# Run artifacts
|
| 27 |
+
traces/
|
| 28 |
+
*.mp4
|
| 29 |
+
|
| 30 |
+
# Dev tooling
|
| 31 |
+
.pytest_cache/
|
| 32 |
+
.ruff_cache/
|
| 33 |
+
.DS_Store
|
| 34 |
+
.claude/
|
| 35 |
+
|
| 36 |
+
# Git
|
| 37 |
+
.git/
|
CLAUDE.md
CHANGED
|
@@ -1,192 +1,199 @@
|
|
| 1 |
-
# CLAUDE.md
|
| 2 |
-
|
| 3 |
-
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
| 4 |
-
|
| 5 |
-
## Project overview
|
| 6 |
-
|
| 7 |
-
FormScout is a Gradio app (Hugging Face Space) that scores Functional Movement Screen (FMS) videos 0–3 per test with a written rationale and an annotated overlay. It is a **screening aid** — not a diagnosis, not an injury predictor. Built for the Build Small Hackathon (Backyard AI track). Full product spec is in `docs/FormScout-FMS-Spec.md`; the engineering contract is in `docs/plans/FormScout-Build-Prompt.md`.
|
| 8 |
-
|
| 9 |
-
**Current status:** Phase 2 complete. All 7 FMS test rubric scorers, JudgeAgent, MovementClassifierAgent, ReportAgent, PoseVisualizer (overlay video), and a user-selectable pose-model registry are implemented and tested (86/87 passing). Phase 3 is next (ST-GCN fine-tune + RAG retrieval).
|
| 10 |
-
|
| 11 |
-
## Common commands
|
| 12 |
-
|
| 13 |
-
```bash
|
| 14 |
-
# Run the Gradio app locally
|
| 15 |
-
python3 app.py
|
| 16 |
-
|
| 17 |
-
# Headless pipeline test (no Gradio)
|
| 18 |
-
python3 -m formscout.run sample.mp4
|
| 19 |
-
|
| 20 |
-
# Run all tests
|
| 21 |
-
pytest tests/
|
| 22 |
-
|
| 23 |
-
# Run a single test file or test
|
| 24 |
-
pytest tests/test_phase2.py
|
| 25 |
-
pytest tests/test_biomechanics.py::TestBiomechanicsAgent::test_deep_squat_score
|
| 26 |
-
|
| 27 |
-
# Lint / format
|
| 28 |
-
ruff check . && ruff format .
|
| 29 |
-
|
| 30 |
-
# Start the local VLM judge server (llama.cpp, port 8080)
|
| 31 |
-
./scripts/serve_judge.sh
|
| 32 |
-
|
| 33 |
-
# Push source tree to the HF model repo + Space (PRs; message from last commit)
|
| 34 |
-
./scripts/hf_upload.sh
|
| 35 |
-
|
| 36 |
-
# Run Svelte component tests (when frontend work is added)
|
| 37 |
-
npx vitest run
|
| 38 |
-
```
|
| 39 |
-
|
| 40 |
-
## Architecture
|
| 41 |
-
|
| 42 |
-
The pipeline is a sequence of **typed specialist agents**. Each agent accepts and returns a frozen dataclass from `formscout/types.py`. The Director in `formscout/pipeline.py` orchestrates them as a deterministic state machine (not an LLM).
|
| 43 |
-
|
| 44 |
-
### Agent pipeline
|
| 45 |
-
|
| 46 |
-
```
|
| 47 |
-
IngestAgent → Pose2DAgent → [Body3DAgent — optional]
|
| 48 |
-
→ MovementClassifierAgent → BiomechanicsAgent
|
| 49 |
-
→ rubric/score_test() → JudgeAgent → ReportAgent
|
| 50 |
-
```
|
| 51 |
-
|
| 52 |
-
The **Director** (`pipeline.py`) owns the flow. `app.py` creates one `Director()` instance and calls `director.run(video_path, test_name, side, model_key)` per submission. The Gradio UI passes `test_name` directly (from dropdown), bypassing the classifier; `model_key` selects the pose backend from `config.POSE_MODELS`.
|
| 53 |
-
|
| 54 |
-
`PoseVisualizer` (`formscout/agents/visualizer.py`) renders the annotated overlay video (skeleton, trails, velocity arrows) from `IngestResult` + `Pose2DResult`. It is called from `app.py` after the pipeline run — it is a UI-layer component, not a Director stage. It returns `None` on failure, never raises.
|
| 55 |
-
|
| 56 |
-
### The tiering rule (most important invariant)
|
| 57 |
-
|
| 58 |
-
**The 2D path is the default and must stand alone as a complete, functional pipeline.** `Body3DAgent` is only activated when `config.ENABLE_3D == True` AND the checkpoint loads successfully. If 3D is off or fails, `Body3DResult(used=False, ...)` is returned — this is a normal success path, not an error. `BiomechFeatures.view` is `"2d"` or `"3d"` so the `JudgeAgent` can caveat its rationale appropriately. Never put `Body3DAgent` on the critical path.
|
| 59 |
-
|
| 60 |
-
### Feature flags in `config.py` and their current state
|
| 61 |
-
|
| 62 |
-
| Flag | Default | Meaning |
|
| 63 |
-
|------|---------|---------|
|
| 64 |
-
| `ENABLE_JUDGE` | `True` | Judge/Classifier call Qwen3-VL via llama-server; graceful rubric fallback when the server is down |
|
| 65 |
-
| `ENABLE_3D` | `False` | When False, Body3DAgent returns `used=False` immediately |
|
| 66 |
-
| `ENABLE_STGCN` | `False` | Phase 3 — ST-GCN learned scoring head |
|
| 67 |
-
| `ENABLE_RAG` | `False` | Phase 3 — RetrievalAgent exemplar lookup |
|
| 68 |
-
|
| 69 |
-
All model IDs, thresholds, k-values, and feature flags live in `config.py` — never scattered literals.
|
| 70 |
-
|
| 71 |
-
###
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
- `
|
| 109 |
-
|
| 110 |
-
`
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
- **
|
| 136 |
-
- **
|
| 137 |
-
- **
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
-
|
| 142 |
-
-
|
| 143 |
-
-
|
| 144 |
-
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
|
| 157 |
-
|
|
| 158 |
-
|
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
##
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
- [ ]
|
| 191 |
-
- [ ]
|
| 192 |
-
- [ ]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CLAUDE.md
|
| 2 |
+
|
| 3 |
+
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
| 4 |
+
|
| 5 |
+
## Project overview
|
| 6 |
+
|
| 7 |
+
FormScout is a Gradio app (Hugging Face Space) that scores Functional Movement Screen (FMS) videos 0–3 per test with a written rationale and an annotated overlay. It is a **screening aid** — not a diagnosis, not an injury predictor. Built for the Build Small Hackathon (Backyard AI track). Full product spec is in `docs/FormScout-FMS-Spec.md`; the engineering contract is in `docs/plans/FormScout-Build-Prompt.md`.
|
| 8 |
+
|
| 9 |
+
**Current status:** Phase 2 complete. All 7 FMS test rubric scorers, JudgeAgent, MovementClassifierAgent, ReportAgent, PoseVisualizer (overlay video), and a user-selectable pose-model registry are implemented and tested (86/87 passing). Phase 3 is next (ST-GCN fine-tune + RAG retrieval).
|
| 10 |
+
|
| 11 |
+
## Common commands
|
| 12 |
+
|
| 13 |
+
```bash
|
| 14 |
+
# Run the Gradio app locally
|
| 15 |
+
python3 app.py
|
| 16 |
+
|
| 17 |
+
# Headless pipeline test (no Gradio)
|
| 18 |
+
python3 -m formscout.run sample.mp4
|
| 19 |
+
|
| 20 |
+
# Run all tests
|
| 21 |
+
pytest tests/
|
| 22 |
+
|
| 23 |
+
# Run a single test file or test
|
| 24 |
+
pytest tests/test_phase2.py
|
| 25 |
+
pytest tests/test_biomechanics.py::TestBiomechanicsAgent::test_deep_squat_score
|
| 26 |
+
|
| 27 |
+
# Lint / format
|
| 28 |
+
ruff check . && ruff format .
|
| 29 |
+
|
| 30 |
+
# Start the local VLM judge server (llama.cpp, port 8080)
|
| 31 |
+
./scripts/serve_judge.sh
|
| 32 |
+
|
| 33 |
+
# Push source tree to the HF model repo + Space (PRs; message from last commit)
|
| 34 |
+
./scripts/hf_upload.sh
|
| 35 |
+
|
| 36 |
+
# Run Svelte component tests (when frontend work is added)
|
| 37 |
+
npx vitest run
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
## Architecture
|
| 41 |
+
|
| 42 |
+
The pipeline is a sequence of **typed specialist agents**. Each agent accepts and returns a frozen dataclass from `formscout/types.py`. The Director in `formscout/pipeline.py` orchestrates them as a deterministic state machine (not an LLM).
|
| 43 |
+
|
| 44 |
+
### Agent pipeline
|
| 45 |
+
|
| 46 |
+
```
|
| 47 |
+
IngestAgent → Pose2DAgent → [Body3DAgent — optional]
|
| 48 |
+
→ MovementClassifierAgent → BiomechanicsAgent
|
| 49 |
+
→ rubric/score_test() → JudgeAgent → ReportAgent
|
| 50 |
+
```
|
| 51 |
+
|
| 52 |
+
The **Director** (`pipeline.py`) owns the flow. `app.py` creates one `Director()` instance and calls `director.run(video_path, test_name, side, model_key)` per submission. The Gradio UI passes `test_name` directly (from dropdown), bypassing the classifier; `model_key` selects the pose backend from `config.POSE_MODELS`.
|
| 53 |
+
|
| 54 |
+
`PoseVisualizer` (`formscout/agents/visualizer.py`) renders the annotated overlay video (skeleton, trails, velocity arrows) from `IngestResult` + `Pose2DResult`. It is called from `app.py` after the pipeline run — it is a UI-layer component, not a Director stage. It returns `None` on failure, never raises.
|
| 55 |
+
|
| 56 |
+
### The tiering rule (most important invariant)
|
| 57 |
+
|
| 58 |
+
**The 2D path is the default and must stand alone as a complete, functional pipeline.** `Body3DAgent` is only activated when `config.ENABLE_3D == True` AND the checkpoint loads successfully. If 3D is off or fails, `Body3DResult(used=False, ...)` is returned — this is a normal success path, not an error. `BiomechFeatures.view` is `"2d"` or `"3d"` so the `JudgeAgent` can caveat its rationale appropriately. Never put `Body3DAgent` on the critical path.
|
| 59 |
+
|
| 60 |
+
### Feature flags in `config.py` and their current state
|
| 61 |
+
|
| 62 |
+
| Flag | Default | Meaning |
|
| 63 |
+
|------|---------|---------|
|
| 64 |
+
| `ENABLE_JUDGE` | `True` | Judge/Classifier call Qwen3-VL via llama-server; graceful rubric fallback when the server is down |
|
| 65 |
+
| `ENABLE_3D` | `False` | When False, Body3DAgent returns `used=False` immediately |
|
| 66 |
+
| `ENABLE_STGCN` | `False` | Phase 3 — ST-GCN learned scoring head |
|
| 67 |
+
| `ENABLE_RAG` | `False` | Phase 3 — RetrievalAgent exemplar lookup |
|
| 68 |
+
|
| 69 |
+
All model IDs, thresholds, k-values, and feature flags live in `config.py` — never scattered literals.
|
| 70 |
+
|
| 71 |
+
### Judge backend selection (local vs Space)
|
| 72 |
+
|
| 73 |
+
`config.resolve_judge_backend()` picks the VLM backend via `FORMSCOUT_JUDGE_BACKEND` (`llama_cpp` | `transformers` | `auto`). `auto` (default) uses **llama-server locally** and the **in-process transformers backend on a Space** (detected via `SPACE_ID`). `JudgeAgent` gets its client from `serving.get_vlm_client()`.
|
| 74 |
+
|
| 75 |
+
- **`llama_cpp`** — `LlamaCppClient` → llama-server at `127.0.0.1:8080` (start with `scripts/serve_judge.sh`). The local path; works perfectly.
|
| 76 |
+
- **`transformers`** — `TransformersVLMClient` loads Qwen3-VL-8B via transformers, GPU-wrapped with `spaces.GPU` (ZeroGPU). Lazy model load, cached per process. On any load/inference failure it returns `{"fallback": True}` and the Judge falls back to the rubric. **Needs validation on real ZeroGPU hardware** — not exercised in CPU tests.
|
| 77 |
+
|
| 78 |
+
### Fallback chain (important for local dev and Spaces)
|
| 79 |
+
|
| 80 |
+
1. `ENABLE_JUDGE=False` → JudgeAgent returns rubric score wrapped as JudgeResult (no VLM needed)
|
| 81 |
+
2. `ENABLE_JUDGE=True` + selected backend unavailable / transformers load fails → same rubric fallback, logs a warning
|
| 82 |
+
3. `ENABLE_JUDGE=True` + backend available → calls Qwen3-VL-8B-Instruct (llama-server locally, transformers/ZeroGPU on a Space)
|
| 83 |
+
|
| 84 |
+
Start the VLM server with `scripts/serve_judge.sh` (downloads live in `checkpoints/qwen3-vl/`, gitignored). To use a fine-tuned GGUF, set `FORMSCOUT_JUDGE_GGUF` (and `FORMSCOUT_JUDGE_MMPROJ` if it ships its own projector) — no code change needed. Multimodal requests go through the OpenAI-compatible `/v1/chat/completions` endpoint (the legacy `/completion` + `image_data` path does not work with modern llama-server).
|
| 85 |
+
|
| 86 |
+
This means the app is **fully functional without any GPU or llama.cpp** — rubric scoring is pure Python.
|
| 87 |
+
|
| 88 |
+
### Rubric scorers
|
| 89 |
+
|
| 90 |
+
Each FMS test has a pure-function scorer in `formscout/rubric/`:
|
| 91 |
+
|
| 92 |
+
```
|
| 93 |
+
score_deep_squat / score_hurdle_step / score_inline_lunge /
|
| 94 |
+
score_shoulder_mobility / score_active_slr /
|
| 95 |
+
score_trunk_stability_pushup / score_rotary_stability
|
| 96 |
+
```
|
| 97 |
+
|
| 98 |
+
All accept `BiomechFeatures` and return `ScoreResult`. Dispatch via `rubric.score_test(features)`. **Rubric functions must remain pure** — no model calls, no I/O.
|
| 99 |
+
|
| 100 |
+
### Bilateral tests
|
| 101 |
+
|
| 102 |
+
`hurdle_step`, `inline_lunge`, `shoulder_mobility`, `active_slr` are bilateral. `ReportAgent` groups them by test name, takes the **lower** score, and always emits the asymmetry delta even when scores are equal. `composite` is `None` when any test is unscored.
|
| 103 |
+
|
| 104 |
+
### Types contract
|
| 105 |
+
|
| 106 |
+
Every agent I/O is a frozen dataclass from `formscout/types.py`. Key types:
|
| 107 |
+
|
| 108 |
+
- `IngestResult` — decoded frames (np.ndarray list), fps, duration, dimensions
|
| 109 |
+
- `Pose2DResult` — per-frame keypoints as `dict[int, {x, y, conf}]` (COCO 17 joints)
|
| 110 |
+
- `Body3DResult` — optional 3D joints, always has `used: bool`
|
| 111 |
+
- `MovementResult` — `test_name` (validated enum), `side` ("left"|"right"|"na")
|
| 112 |
+
- `BiomechFeatures` — `angles: dict`, `alignments: dict`, `view: "2d"|"3d"`, `symmetry_delta`
|
| 113 |
+
- `ScoreResult` — `score: int` (0–3), `rationale`, `needs_human`
|
| 114 |
+
- `JudgeResult` — same as ScoreResult + `compensation_tags`, `corrective_hint`; `score=None` when `needs_human=True`
|
| 115 |
+
- `PipelineState` — mutable accumulator threaded through the Director
|
| 116 |
+
|
| 117 |
+
`MovementResult` and `JudgeResult` validate their fields in `__post_init__` — passing invalid values raises immediately.
|
| 118 |
+
|
| 119 |
+
### Pose model selection and checkpoints
|
| 120 |
+
|
| 121 |
+
`config.POSE_MODELS` is a registry of pose backends: MediaPipe (CPU-friendly), five YOLO26 sizes (n/s/m/l/x), and Sapiens2 variants (Phase 3, need the custom `sapiens` repo installed). `config.DEFAULT_POSE_MODEL` is YOLO26n. The Gradio UI exposes a dropdown built from `config.available_pose_models()` (filters to checkpoints actually present) and passes the chosen `model_key` through `Director.run` to `Pose2DAgent`. `config.YOLO_POSE_MODEL` is a backward-compat alias only.
|
| 122 |
+
|
| 123 |
+
Checkpoints are **not** committed (`checkpoints/` is gitignored). `formscout/startup.py:ensure_checkpoints()` downloads missing YOLO26/MediaPipe files from the `silas-therapy/formscout-checkpoints` HF repo once at app startup. Models load once per process and are cached — never inside the inference hot path.
|
| 124 |
+
|
| 125 |
+
### llama.cpp serving
|
| 126 |
+
|
| 127 |
+
`formscout/serving/llama_cpp.py` provides `LlamaCppClient` (VLM, port 8080) and `EmbeddingClient` (embeddings, port 8081). Both check `/health` before use and return safe error dicts when unavailable. Only active when the corresponding `ENABLE_*` flag is True.
|
| 128 |
+
|
| 129 |
+
### Deploying to Hugging Face
|
| 130 |
+
|
| 131 |
+
The repo deploys to both `silas-therapy/small-functional-movement-screening` (model repo) and the Space of the same name (README frontmatter is the Space config). Use `./scripts/hf_upload.sh` — never raw `hf upload .`: the `hf` CLI does **not** read `.hfignore`, so a raw upload hashes the entire `.venv` (~44k files) and pushes torch binaries. The script parses `.hfignore` into `--exclude` globs, preflights the file count, creates PRs on both repos, and auto-switches to `hf upload-large-folder` (resumable, but no PR / no commit message) above 500 files.
|
| 132 |
+
|
| 133 |
+
## Key constraints and invariants
|
| 134 |
+
|
| 135 |
+
- **No cloud model APIs.** All inference runs on-Space (ZeroGPU). No OpenAI/Anthropic/Gemini calls.
|
| 136 |
+
- **Pain is never auto-scored.** Any clearing test or visible distress sets `needs_human=True` — enforced in rubric functions and JudgeAgent. `JudgeResult.score` must be `None` when `needs_human=True`.
|
| 137 |
+
- **Quality gates (Director, never silently skip):**
|
| 138 |
+
- Any agent `confidence < config.MIN_CONFIDENCE` (0.6) → warn or stop
|
| 139 |
+
- `|rubric.score - judge.score| >= 1` → flag disagreement
|
| 140 |
+
- `MovementResult.test_name == "unknown"` → stop pipeline, surface manual override
|
| 141 |
+
- `JudgeAgent.needs_human == True` → no numeric score emitted
|
| 142 |
+
- **Composite is null** when any test is unscored. Never show a partial 0–21 as complete.
|
| 143 |
+
- **Pipeline runs headless.** No Gradio imports in any agent file.
|
| 144 |
+
- **Safety banner** ("Screening aid — not a diagnosis…") must always be visible in the UI — appears at top and bottom of `app.py`.
|
| 145 |
+
|
| 146 |
+
## Engineering standards
|
| 147 |
+
|
| 148 |
+
- Every agent: one public entrypoint, typed dataclass I/O from `types.py`, `confidence: float` and `notes: str` on every result.
|
| 149 |
+
- Models load once at module/instance init — never inside the inference hot path.
|
| 150 |
+
- Every agent module docstring states: purpose, inputs, outputs, failure behavior, model param count, license, and gated status.
|
| 151 |
+
- `tracing.py` records structured per-agent I/O for any run; one full run gets exported to the Hub.
|
| 152 |
+
- Every agent ships with a pytest in `tests/` that runs without model downloads and asserts the typed contract.
|
| 153 |
+
|
| 154 |
+
## Model stack (~17.6B total — stay under 32B)
|
| 155 |
+
|
| 156 |
+
| Component | Model | Params | Status |
|
| 157 |
+
|---|---|---|---|
|
| 158 |
+
| 2D pose (primary) | YOLO26-Pose n/s/m/l/x (default: n) | 0.0007–0.058B | Ready (auto-downloaded at startup) |
|
| 159 |
+
| 2D pose (CPU alt) | MediaPipe Pose Landmarker (full) | ~0.004B | Ready (auto-downloaded at startup) |
|
| 160 |
+
| 2D pose (HQ alt) | `facebook/sapiens2-pose-0.4b/0.8b/1b/5b` | 0.4–5B | Phase 3 — needs custom `sapiens` repo |
|
| 161 |
+
| Segmentation | SAM 3.1 base | ~0.85B | Access accepted |
|
| 162 |
+
| 3D biomechanics | `facebook/sam-3d-body-dinov3` | ~0.84B | **Access ACCEPTED Jun 4 2026** |
|
| 163 |
+
| Learned scoring | ST-GCN (pyskl) | ~0.03B | Phase 3 |
|
| 164 |
+
| Judge + Classifier | Qwen3-VL-8B-Instruct (llama.cpp) | 8B | **Online** — `scripts/serve_judge.sh`, ENABLE_JUDGE=True |
|
| 165 |
+
| Retrieval | Qwen3-VL-Embedding-8B (llama.cpp) | 8B | Phase 3 |
|
| 166 |
+
|
| 167 |
+
Track the running sum in `MODEL_BUDGET.md`. The two Qwen3-VL-8B models share a backbone.
|
| 168 |
+
|
| 169 |
+
## Gradio + Svelte UI guidance
|
| 170 |
+
|
| 171 |
+
The UI uses **Gradio `gr.Blocks`** with custom CSS/theme (`formscout/ui/theme.py`). Custom Svelte components for score dial, asymmetry bars, rubric drawer are planned for Phase 4. Use `gradio-svelte-expert` agent for Svelte component work.
|
| 172 |
+
|
| 173 |
+
- ZeroGPU: wrap heavy inference (`Pose2DAgent.run`, `Body3DAgent.run`) in `@spaces.GPU` before deploying to Spaces.
|
| 174 |
+
- Verify Gradio APIs against current docs before use — pin exact versions in `requirements.txt`.
|
| 175 |
+
|
| 176 |
+
## Build phases
|
| 177 |
+
|
| 178 |
+
1. **Phase 0 — Recon:** ✅ Complete. See `RECON.md`.
|
| 179 |
+
2. **Phase 1 — Spine:** ✅ Complete. Deep Squat end-to-end.
|
| 180 |
+
3. **Phase 2 — All 7 tests:** ✅ Complete. Classifier, Judge, Report agents; all rubric scorers; Gradio UI.
|
| 181 |
+
4. **Phase 3 — Learned scoring + retrieval:** ST-GCN fine-tune on physio clips, publish to Hub. RetrievalAgent with embedding index.
|
| 182 |
+
5. **Phase 4 — Polish + ship:** Custom Svelte UI components, agent trace to Hub, blog post. (Overlay video done via `PoseVisualizer`; full 7-test session + PDF export done via `formscout/session.py` + `PdfReportAgent`.)
|
| 183 |
+
|
| 184 |
+
## Known issues
|
| 185 |
+
|
| 186 |
+
- `tests/test_biomechanics.py::TestBiomechanicsAgent::test_unimplemented_test_returns_low_confidence` fails: expects `"not yet implemented"` in `result.notes` but biomechanics returns empty string. Minor �� low priority.
|
| 187 |
+
|
| 188 |
+
## Badge checklist (definition of done)
|
| 189 |
+
|
| 190 |
+
- [ ] Space runs green; upload → scorecard works on real clips
|
| 191 |
+
- [ ] Param sum verified ≤ 32B in `MODEL_BUDGET.md`
|
| 192 |
+
- [ ] 🔌 **Off the Grid** — no cloud model APIs anywhere in the pipeline
|
| 193 |
+
- [ ] 🎯 **Well-Tuned** — fine-tuned ST-GCN head published to Hub with honest model card
|
| 194 |
+
- [ ] 🎨 **Off-Brand** — custom, non-default Gradio UI (scout/trail theme)
|
| 195 |
+
- [ ] 🦙 **Llama Champion** — VLM + embedder served via llama.cpp (GGUF)
|
| 196 |
+
- [ ] 📡 **Sharing is Caring** — one full agent trace (all I/O) published to Hub
|
| 197 |
+
- [ ] 📓 **Field Notes** — blog post written, honesty section (FMS limitations) front-and-center
|
| 198 |
+
- [ ] Demo video + social post recorded
|
| 199 |
+
- [ ] Safety banner present; pain/clearing never auto-scored; low-confidence flagged
|
README.md
CHANGED
|
@@ -1,118 +1,118 @@
|
|
| 1 |
-
---
|
| 2 |
-
title: FormScout
|
| 3 |
-
emoji: 🏔️
|
| 4 |
-
colorFrom: green
|
| 5 |
-
colorTo:
|
| 6 |
-
sdk: gradio
|
| 7 |
-
app_file: app.py
|
| 8 |
-
pinned: false
|
| 9 |
-
license: apache-2.0
|
| 10 |
-
short_description: FMS video scoring — movement screen aid
|
| 11 |
-
---
|
| 12 |
-
|
| 13 |
-
# FormScout
|
| 14 |
-
|
| 15 |
-
FMS (Functional Movement Screen) scoring pipeline — a screening aid that scores movement videos 0–3 per test with a written rationale and annotated overlay.
|
| 16 |
-
|
| 17 |
-
**⚠️ Screening aid — not a diagnosis. Pain or clearing tests require a clinician.**
|
| 18 |
-
|
| 19 |
-
## Running locally
|
| 20 |
-
|
| 21 |
-
### 1. Clone and install
|
| 22 |
-
|
| 23 |
-
```bash
|
| 24 |
-
git clone https://huggingface.co/silas-therapy/small-functional-movement-screening
|
| 25 |
-
cd small-functional-movement-screening
|
| 26 |
-
python3 -m venv .venv && source .venv/bin/activate
|
| 27 |
-
pip install -r requirements.txt
|
| 28 |
-
```
|
| 29 |
-
|
| 30 |
-
### 2. Start the VLM judge (optional but recommended)
|
| 31 |
-
|
| 32 |
-
The judge uses Qwen3-VL-8B-Instruct via llama.cpp. Without it the app falls back to the deterministic rubric score — fully functional, no GPU needed.
|
| 33 |
-
|
| 34 |
-
```bash
|
| 35 |
-
# Install llama.cpp once
|
| 36 |
-
brew install llama.cpp
|
| 37 |
-
|
| 38 |
-
# Download the model (one-time, ~6 GB)
|
| 39 |
-
python3 -c "
|
| 40 |
-
from huggingface_hub import hf_hub_download
|
| 41 |
-
for f in ['Qwen3VL-8B-Instruct-Q4_K_M.gguf', 'mmproj-Qwen3VL-8B-Instruct-F16.gguf']:
|
| 42 |
-
hf_hub_download('Qwen/Qwen3-VL-8B-Instruct-GGUF', f, local_dir='checkpoints/qwen3-vl')
|
| 43 |
-
"
|
| 44 |
-
|
| 45 |
-
# Start the server (keep this terminal open)
|
| 46 |
-
./scripts/serve_judge.sh
|
| 47 |
-
```
|
| 48 |
-
|
| 49 |
-
To use a fine-tuned GGUF instead of the default:
|
| 50 |
-
```bash
|
| 51 |
-
FORMSCOUT_JUDGE_GGUF=/path/to/finetuned.gguf ./scripts/serve_judge.sh
|
| 52 |
-
```
|
| 53 |
-
|
| 54 |
-
### 3. Launch the Gradio app
|
| 55 |
-
|
| 56 |
-
```bash
|
| 57 |
-
python3 app.py
|
| 58 |
-
# → http://127.0.0.1:7860
|
| 59 |
-
```
|
| 60 |
-
|
| 61 |
-
Upload a video, select the FMS test from the dropdown, and click **Analyze**.
|
| 62 |
-
|
| 63 |
-
### 4. Headless pipeline (no Gradio)
|
| 64 |
-
|
| 65 |
-
```bash
|
| 66 |
-
python3 -m formscout.run sample.mp4
|
| 67 |
-
```
|
| 68 |
-
|
| 69 |
-
### 5. Tests
|
| 70 |
-
|
| 71 |
-
```bash
|
| 72 |
-
pytest tests/ -v
|
| 73 |
-
```
|
| 74 |
-
|
| 75 |
-
### 6. Upload to Hugging Face
|
| 76 |
-
|
| 77 |
-
```bash
|
| 78 |
-
# Pushes source to both model repo and Space, opens a PR on each
|
| 79 |
-
./scripts/hf_upload.sh
|
| 80 |
-
|
| 81 |
-
# Or with a custom commit message
|
| 82 |
-
./scripts/hf_upload.sh "feat: my change"
|
| 83 |
-
```
|
| 84 |
-
|
| 85 |
-
## Architecture
|
| 86 |
-
|
| 87 |
-
Typed specialist agents orchestrated by a deterministic Director:
|
| 88 |
-
|
| 89 |
-
```
|
| 90 |
-
Ingest → Pose2D → [Body3D optional] → Biomechanics → Rubric Score → [Judge] → Report
|
| 91 |
-
```
|
| 92 |
-
|
| 93 |
-
| Agent | Model | Status |
|
| 94 |
-
|---|---|---|
|
| 95 |
-
| Pose2D | YOLO26l-Pose (0.026B) + MediaPipe fallback | ✅ |
|
| 96 |
-
| Body3D | SAM 3D Body DINOv3 (0.84B) | gated, off by default |
|
| 97 |
-
| Judge + Classifier | Qwen3-VL-8B-Instruct via llama.cpp (8B) | ✅ |
|
| 98 |
-
| Scoring Head | ST-GCN (0.03B) | Phase 3 |
|
| 99 |
-
| Retrieval | Qwen3-VL-Embedding-8B (8B) | Phase 3 |
|
| 100 |
-
|
| 101 |
-
See [CLAUDE.md](CLAUDE.md) for full architecture and invariants.
|
| 102 |
-
|
| 103 |
-
## Feature flags (`formscout/config.py`)
|
| 104 |
-
|
| 105 |
-
| Flag | Default | Meaning |
|
| 106 |
-
|---|---|---|
|
| 107 |
-
| `ENABLE_JUDGE` | `True` | VLM judge via llama-server; rubric fallback when server is down |
|
| 108 |
-
| `ENABLE_3D` | `False` | SAM 3D Body — off until integrated |
|
| 109 |
-
| `ENABLE_STGCN` | `False` | Phase 3 |
|
| 110 |
-
| `ENABLE_RAG` | `False` | Phase 3 |
|
| 111 |
-
|
| 112 |
-
## Model budget
|
| 113 |
-
|
| 114 |
-
~18B params total (under 32B cap). See [MODEL_BUDGET.md](MODEL_BUDGET.md).
|
| 115 |
-
|
| 116 |
-
## License
|
| 117 |
-
|
| 118 |
-
Apache-2.0. Built for the Build Small Hackathon (Backyard AI track).
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: FormScout
|
| 3 |
+
emoji: 🏔️
|
| 4 |
+
colorFrom: green
|
| 5 |
+
colorTo: yellow
|
| 6 |
+
sdk: gradio
|
| 7 |
+
app_file: app.py
|
| 8 |
+
pinned: false
|
| 9 |
+
license: apache-2.0
|
| 10 |
+
short_description: FMS video scoring — movement screen aid
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
# FormScout
|
| 14 |
+
|
| 15 |
+
FMS (Functional Movement Screen) scoring pipeline — a screening aid that scores movement videos 0–3 per test with a written rationale and annotated overlay.
|
| 16 |
+
|
| 17 |
+
**⚠️ Screening aid — not a diagnosis. Pain or clearing tests require a clinician.**
|
| 18 |
+
|
| 19 |
+
## Running locally
|
| 20 |
+
|
| 21 |
+
### 1. Clone and install
|
| 22 |
+
|
| 23 |
+
```bash
|
| 24 |
+
git clone https://huggingface.co/silas-therapy/small-functional-movement-screening
|
| 25 |
+
cd small-functional-movement-screening
|
| 26 |
+
python3 -m venv .venv && source .venv/bin/activate
|
| 27 |
+
pip install -r requirements.txt
|
| 28 |
+
```
|
| 29 |
+
|
| 30 |
+
### 2. Start the VLM judge (optional but recommended)
|
| 31 |
+
|
| 32 |
+
The judge uses Qwen3-VL-8B-Instruct via llama.cpp. Without it the app falls back to the deterministic rubric score — fully functional, no GPU needed.
|
| 33 |
+
|
| 34 |
+
```bash
|
| 35 |
+
# Install llama.cpp once
|
| 36 |
+
brew install llama.cpp
|
| 37 |
+
|
| 38 |
+
# Download the model (one-time, ~6 GB)
|
| 39 |
+
python3 -c "
|
| 40 |
+
from huggingface_hub import hf_hub_download
|
| 41 |
+
for f in ['Qwen3VL-8B-Instruct-Q4_K_M.gguf', 'mmproj-Qwen3VL-8B-Instruct-F16.gguf']:
|
| 42 |
+
hf_hub_download('Qwen/Qwen3-VL-8B-Instruct-GGUF', f, local_dir='checkpoints/qwen3-vl')
|
| 43 |
+
"
|
| 44 |
+
|
| 45 |
+
# Start the server (keep this terminal open)
|
| 46 |
+
./scripts/serve_judge.sh
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
To use a fine-tuned GGUF instead of the default:
|
| 50 |
+
```bash
|
| 51 |
+
FORMSCOUT_JUDGE_GGUF=/path/to/finetuned.gguf ./scripts/serve_judge.sh
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
### 3. Launch the Gradio app
|
| 55 |
+
|
| 56 |
+
```bash
|
| 57 |
+
python3 app.py
|
| 58 |
+
# → http://127.0.0.1:7860
|
| 59 |
+
```
|
| 60 |
+
|
| 61 |
+
Upload a video, select the FMS test from the dropdown, and click **Analyze**.
|
| 62 |
+
|
| 63 |
+
### 4. Headless pipeline (no Gradio)
|
| 64 |
+
|
| 65 |
+
```bash
|
| 66 |
+
python3 -m formscout.run sample.mp4
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
### 5. Tests
|
| 70 |
+
|
| 71 |
+
```bash
|
| 72 |
+
pytest tests/ -v
|
| 73 |
+
```
|
| 74 |
+
|
| 75 |
+
### 6. Upload to Hugging Face
|
| 76 |
+
|
| 77 |
+
```bash
|
| 78 |
+
# Pushes source to both model repo and Space, opens a PR on each
|
| 79 |
+
./scripts/hf_upload.sh
|
| 80 |
+
|
| 81 |
+
# Or with a custom commit message
|
| 82 |
+
./scripts/hf_upload.sh "feat: my change"
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
## Architecture
|
| 86 |
+
|
| 87 |
+
Typed specialist agents orchestrated by a deterministic Director:
|
| 88 |
+
|
| 89 |
+
```
|
| 90 |
+
Ingest → Pose2D → [Body3D optional] → Biomechanics → Rubric Score → [Judge] → Report
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
| Agent | Model | Status |
|
| 94 |
+
|---|---|---|
|
| 95 |
+
| Pose2D | YOLO26l-Pose (0.026B) + MediaPipe fallback | ✅ |
|
| 96 |
+
| Body3D | SAM 3D Body DINOv3 (0.84B) | gated, off by default |
|
| 97 |
+
| Judge + Classifier | Qwen3-VL-8B-Instruct via llama.cpp (8B) | ✅ |
|
| 98 |
+
| Scoring Head | ST-GCN (0.03B) | Phase 3 |
|
| 99 |
+
| Retrieval | Qwen3-VL-Embedding-8B (8B) | Phase 3 |
|
| 100 |
+
|
| 101 |
+
See [CLAUDE.md](CLAUDE.md) for full architecture and invariants.
|
| 102 |
+
|
| 103 |
+
## Feature flags (`formscout/config.py`)
|
| 104 |
+
|
| 105 |
+
| Flag | Default | Meaning |
|
| 106 |
+
|---|---|---|
|
| 107 |
+
| `ENABLE_JUDGE` | `True` | VLM judge via llama-server; rubric fallback when server is down |
|
| 108 |
+
| `ENABLE_3D` | `False` | SAM 3D Body — off until integrated |
|
| 109 |
+
| `ENABLE_STGCN` | `False` | Phase 3 |
|
| 110 |
+
| `ENABLE_RAG` | `False` | Phase 3 |
|
| 111 |
+
|
| 112 |
+
## Model budget
|
| 113 |
+
|
| 114 |
+
~18B params total (under 32B cap). See [MODEL_BUDGET.md](MODEL_BUDGET.md).
|
| 115 |
+
|
| 116 |
+
## License
|
| 117 |
+
|
| 118 |
+
Apache-2.0. Built for the Build Small Hackathon (Backyard AI track).
|
app.py
CHANGED
|
@@ -125,22 +125,22 @@ def _render_score_card(score: int, confidence: float, needs_human: bool) -> str:
|
|
| 125 |
if needs_human:
|
| 126 |
return """
|
| 127 |
<div class="score-card needs-review">
|
| 128 |
-
<div style="font-size: 1.2em; color: #
|
| 129 |
-
<div style="font-size: 0.9em; color: #
|
| 130 |
</div>
|
| 131 |
"""
|
| 132 |
|
| 133 |
conf_pct = int(confidence * 100)
|
| 134 |
-
conf_color = "#
|
| 135 |
|
| 136 |
return f"""
|
| 137 |
<div class="score-card">
|
| 138 |
<div class="score-value">{score}/3</div>
|
| 139 |
-
<div style="font-size: 0.95em; color: #
|
| 140 |
{SCORE_DESCRIPTIONS.get(score, '')}
|
| 141 |
</div>
|
| 142 |
<div style="margin-top: 12px;">
|
| 143 |
-
<div style="display: flex; justify-content: space-between; font-size: 0.8em; color: #
|
| 144 |
<span>Confidence</span>
|
| 145 |
<span style="color: {conf_color};">{conf_pct}%</span>
|
| 146 |
</div>
|
|
@@ -155,9 +155,9 @@ def _render_score_card(score: int, confidence: float, needs_human: bool) -> str:
|
|
| 155 |
def _render_empty_state() -> str:
|
| 156 |
"""Render placeholder when no video processed yet."""
|
| 157 |
return """
|
| 158 |
-
<div class="score-card" style="opacity: 0.
|
| 159 |
<div style="font-size: 2em; margin-bottom: 8px;">🏔️</div>
|
| 160 |
-
<div style="color: #
|
| 161 |
</div>
|
| 162 |
"""
|
| 163 |
|
|
@@ -319,7 +319,7 @@ def build_app() -> gr.Blocks:
|
|
| 319 |
gr.HTML("""
|
| 320 |
<div class="formscout-header">
|
| 321 |
<h1>🏔️ FormScout</h1>
|
| 322 |
-
<p style="color: #
|
| 323 |
Functional Movement Screen · Automated Scoring Aid
|
| 324 |
</p>
|
| 325 |
</div>
|
|
@@ -362,7 +362,7 @@ def build_app() -> gr.Blocks:
|
|
| 362 |
|
| 363 |
overlay_layers = gr.CheckboxGroup(
|
| 364 |
choices=["Skeleton", "Trails", "Velocity arrows"],
|
| 365 |
-
value=["Skeleton", "Trails"],
|
| 366 |
label="Overlay Layers",
|
| 367 |
)
|
| 368 |
|
|
@@ -414,9 +414,9 @@ def build_app() -> gr.Blocks:
|
|
| 414 |
gr.HTML(f'<div class="safety-banner" style="margin-top: 20px;">{DISCLAIMER}</div>')
|
| 415 |
|
| 416 |
gr.Markdown(
|
| 417 |
-
"<center style='color: #
|
| 418 |
"FormScout · ~18B params · Off the Grid · "
|
| 419 |
-
"<a href='https://
|
| 420 |
"</center>"
|
| 421 |
)
|
| 422 |
|
|
|
|
| 125 |
if needs_human:
|
| 126 |
return """
|
| 127 |
<div class="score-card needs-review">
|
| 128 |
+
<div style="font-size: 1.2em; color: #cf922a; margin-bottom: 8px;">⚠️ Needs Clinician Review</div>
|
| 129 |
+
<div style="font-size: 0.9em; color: #4a5f57;">Pain or clearing test detected — cannot auto-score</div>
|
| 130 |
</div>
|
| 131 |
"""
|
| 132 |
|
| 133 |
conf_pct = int(confidence * 100)
|
| 134 |
+
conf_color = "#2b8a8a" if confidence >= 0.7 else "#cf922a" if confidence >= 0.4 else "#d9534f"
|
| 135 |
|
| 136 |
return f"""
|
| 137 |
<div class="score-card">
|
| 138 |
<div class="score-value">{score}/3</div>
|
| 139 |
+
<div style="font-size: 0.95em; color: #4a5f57; margin-top: 4px;">
|
| 140 |
{SCORE_DESCRIPTIONS.get(score, '')}
|
| 141 |
</div>
|
| 142 |
<div style="margin-top: 12px;">
|
| 143 |
+
<div style="display: flex; justify-content: space-between; font-size: 0.8em; color: #6b7d75;">
|
| 144 |
<span>Confidence</span>
|
| 145 |
<span style="color: {conf_color};">{conf_pct}%</span>
|
| 146 |
</div>
|
|
|
|
| 155 |
def _render_empty_state() -> str:
|
| 156 |
"""Render placeholder when no video processed yet."""
|
| 157 |
return """
|
| 158 |
+
<div class="score-card" style="opacity: 0.6;">
|
| 159 |
<div style="font-size: 2em; margin-bottom: 8px;">🏔️</div>
|
| 160 |
+
<div style="color: #6b7d75;">Upload a video to begin</div>
|
| 161 |
</div>
|
| 162 |
"""
|
| 163 |
|
|
|
|
| 319 |
gr.HTML("""
|
| 320 |
<div class="formscout-header">
|
| 321 |
<h1>🏔️ FormScout</h1>
|
| 322 |
+
<p style="color: #4a5f57; font-size: 0.95em;">
|
| 323 |
Functional Movement Screen · Automated Scoring Aid
|
| 324 |
</p>
|
| 325 |
</div>
|
|
|
|
| 362 |
|
| 363 |
overlay_layers = gr.CheckboxGroup(
|
| 364 |
choices=["Skeleton", "Trails", "Velocity arrows"],
|
| 365 |
+
value=["Skeleton", "Trails", "Velocity arrows"],
|
| 366 |
label="Overlay Layers",
|
| 367 |
)
|
| 368 |
|
|
|
|
| 414 |
gr.HTML(f'<div class="safety-banner" style="margin-top: 20px;">{DISCLAIMER}</div>')
|
| 415 |
|
| 416 |
gr.Markdown(
|
| 417 |
+
"<center style='color: #6b7d75; font-size: 0.8em; margin-top: 12px;'>"
|
| 418 |
"FormScout · ~18B params · Off the Grid · "
|
| 419 |
+
"<a href='https://silastherapy.sk' style='color: #1f6e6e;'>Silas Therapy · Build Small Hackathon</a>"
|
| 420 |
"</center>"
|
| 421 |
)
|
| 422 |
|
docs/superpowers/plans/2026-06-09-pose-model-selector.md
CHANGED
|
@@ -1,734 +1,734 @@
|
|
| 1 |
-
# Pose Model Selector Implementation Plan
|
| 2 |
-
|
| 3 |
-
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
| 4 |
-
|
| 5 |
-
**Goal:** Replace the hard-coded YOLO26l default with a 10-model dropdown (MediaPipe, YOLO26 n→x, Sapiens2 0.4B→5B) wired end-to-end from UI through the Director to `Pose2DAgent`.
|
| 6 |
-
|
| 7 |
-
**Architecture:** Unified `POSE_MODELS` registry in `config.py` drives a `gr.Dropdown` in `app.py`; the selected key flows through `Director.run()` into `Pose2DAgent.run(model_key)`, which dispatches to one of three private sub-runners (`_run_yolo`, `_run_mediapipe`, `_run_sapiens2`), all producing the same COCO-17 `list[dict]` contract.
|
| 8 |
-
|
| 9 |
-
**Tech Stack:** `ultralytics` (YOLO), `onnxruntime` + `huggingface_hub` (MediaPipe), `transformers` (Sapiens2), `gradio` (UI).
|
| 10 |
-
|
| 11 |
-
---
|
| 12 |
-
|
| 13 |
-
## File map
|
| 14 |
-
|
| 15 |
-
| File | Change |
|
| 16 |
-
|---|---|
|
| 17 |
-
| `formscout/config.py` | Replace `YOLO_POSE_MODELS` with `POSE_MODELS` dict + `DEFAULT_POSE_MODEL` |
|
| 18 |
-
| `formscout/agents/pose2d.py` | Add `_run_yolo`, `_run_mediapipe`, `_run_sapiens2`; update `run()` signature |
|
| 19 |
-
| `formscout/pipeline.py` | Change `pose_model_path` param to `model_key` |
|
| 20 |
-
| `app.py` | Add `pose_model_dropdown`, fix `_map_inputs` + `process_video` |
|
| 21 |
-
| `requirements.txt` | Add `onnxruntime>=1.18` |
|
| 22 |
-
| `tests/test_pose2d.py` | Add mocked tests for each backend |
|
| 23 |
-
|
| 24 |
-
---
|
| 25 |
-
|
| 26 |
-
## Task 1: Add unified `POSE_MODELS` registry to `config.py`
|
| 27 |
-
|
| 28 |
-
**Files:**
|
| 29 |
-
- Modify: `formscout/config.py`
|
| 30 |
-
|
| 31 |
-
- [ ] **Step 1: Open `formscout/config.py` and replace the `YOLO_POSE_MODELS` block**
|
| 32 |
-
|
| 33 |
-
Replace lines 12–20 (the `YOLO_POSE_MODELS` dict and `YOLO_POSE_MODEL` / `YOLO_POSE_MODEL_HQ` lines) with:
|
| 34 |
-
|
| 35 |
-
```python
|
| 36 |
-
_YOLO_DIR = ROOT / "checkpoints" / "yolo26"
|
| 37 |
-
|
| 38 |
-
POSE_MODELS: dict[str, dict] = {
|
| 39 |
-
# ── MediaPipe (Qualcomm HF, ONNX Runtime) ──────────────────────────────
|
| 40 |
-
"MediaPipe-Pose ⬇ ~16 MB, CPU-friendly": {
|
| 41 |
-
"backend": "mediapipe",
|
| 42 |
-
"hf_id": "qualcomm/MediaPipe-Pose-Estimation",
|
| 43 |
-
"params_m": 4.2,
|
| 44 |
-
},
|
| 45 |
-
# ── YOLO26 (local checkpoints) ─────────────────────────────────────────
|
| 46 |
-
"YOLO26n — nano (0.7M, fastest)": {
|
| 47 |
-
"backend": "yolo",
|
| 48 |
-
"path": str(_YOLO_DIR / "yolo26n-pose.pt"),
|
| 49 |
-
"params_m": 0.7,
|
| 50 |
-
},
|
| 51 |
-
"YOLO26s — small (3.5M)": {
|
| 52 |
-
"backend": "yolo",
|
| 53 |
-
"path": str(_YOLO_DIR / "yolo26s-pose.pt"),
|
| 54 |
-
"params_m": 3.5,
|
| 55 |
-
},
|
| 56 |
-
"YOLO26m — medium (9M)": {
|
| 57 |
-
"backend": "yolo",
|
| 58 |
-
"path": str(_YOLO_DIR / "yolo26m-pose.pt"),
|
| 59 |
-
"params_m": 9.0,
|
| 60 |
-
},
|
| 61 |
-
"YOLO26l — large (25.9M)": {
|
| 62 |
-
"backend": "yolo",
|
| 63 |
-
"path": str(_YOLO_DIR / "yolo26l-pose.pt"),
|
| 64 |
-
"params_m": 25.9,
|
| 65 |
-
},
|
| 66 |
-
"YOLO26x — extra-large (57.6M)": {
|
| 67 |
-
"backend": "yolo",
|
| 68 |
-
"path": str(_YOLO_DIR / "yolo26x-pose.pt"),
|
| 69 |
-
"params_m": 57.6,
|
| 70 |
-
},
|
| 71 |
-
# ── Sapiens2 (HF download, transformers) ──────────────────────────────
|
| 72 |
-
"Sapiens2-0.4B ⬇ ~1.6 GB": {
|
| 73 |
-
"backend": "sapiens2",
|
| 74 |
-
"hf_id": "facebook/sapiens2-pose-0.4b",
|
| 75 |
-
"params_m": 400,
|
| 76 |
-
},
|
| 77 |
-
"Sapiens2-0.8B ⬇ ~3.2 GB": {
|
| 78 |
-
"backend": "sapiens2",
|
| 79 |
-
"hf_id": "facebook/sapiens2-pose-0.8b",
|
| 80 |
-
"params_m": 800,
|
| 81 |
-
},
|
| 82 |
-
"Sapiens2-1B ⬇ ~4 GB": {
|
| 83 |
-
"backend": "sapiens2",
|
| 84 |
-
"hf_id": "facebook/sapiens2-pose-1b",
|
| 85 |
-
"params_m": 1000,
|
| 86 |
-
},
|
| 87 |
-
"Sapiens2-5B ⬇ ~20 GB, large GPU": {
|
| 88 |
-
"backend": "sapiens2",
|
| 89 |
-
"hf_id": "facebook/sapiens2-pose-5b",
|
| 90 |
-
"params_m": 5000,
|
| 91 |
-
},
|
| 92 |
-
}
|
| 93 |
-
|
| 94 |
-
DEFAULT_POSE_MODEL = "YOLO26n — nano (0.7M, fastest)"
|
| 95 |
-
|
| 96 |
-
# Backward-compat aliases — kept for any direct references outside the agent
|
| 97 |
-
YOLO_POSE_MODEL = str(_YOLO_DIR / "yolo26l-pose.pt")
|
| 98 |
-
YOLO_POSE_MODEL_HQ = str(_YOLO_DIR / "yolo26x-pose.pt")
|
| 99 |
-
```
|
| 100 |
-
|
| 101 |
-
- [ ] **Step 2: Verify import is clean**
|
| 102 |
-
|
| 103 |
-
```bash
|
| 104 |
-
python3 -c "from formscout import config; print(list(config.POSE_MODELS.keys()))"
|
| 105 |
-
```
|
| 106 |
-
|
| 107 |
-
Expected: list of 10 model labels, starting with `MediaPipe-Pose...`
|
| 108 |
-
|
| 109 |
-
- [ ] **Step 3: Commit**
|
| 110 |
-
|
| 111 |
-
```bash
|
| 112 |
-
git add formscout/config.py
|
| 113 |
-
git commit -m "feat: unified POSE_MODELS registry with MediaPipe, YOLO26 n-x, Sapiens2 0.4-5B"
|
| 114 |
-
git push
|
| 115 |
-
```
|
| 116 |
-
|
| 117 |
-
---
|
| 118 |
-
|
| 119 |
-
## Task 2: Refactor `Pose2DAgent` — YOLO sub-runner + new `run()` signature
|
| 120 |
-
|
| 121 |
-
**Files:**
|
| 122 |
-
- Modify: `formscout/agents/pose2d.py`
|
| 123 |
-
- Modify: `tests/test_pose2d.py`
|
| 124 |
-
|
| 125 |
-
- [ ] **Step 1: Write failing test for the new `model_key` signature**
|
| 126 |
-
|
| 127 |
-
Add to `tests/test_pose2d.py`:
|
| 128 |
-
|
| 129 |
-
```python
|
| 130 |
-
def test_run_accepts_model_key(pose2d_agent):
|
| 131 |
-
"""run() must accept model_key kwarg, not model_path."""
|
| 132 |
-
import inspect
|
| 133 |
-
sig = inspect.signature(pose2d_agent.run)
|
| 134 |
-
assert "model_key" in sig.parameters
|
| 135 |
-
assert "model_path" not in sig.parameters
|
| 136 |
-
```
|
| 137 |
-
|
| 138 |
-
- [ ] **Step 2: Run to confirm it fails**
|
| 139 |
-
|
| 140 |
-
```bash
|
| 141 |
-
pytest tests/test_pose2d.py::TestPose2DAgent::test_run_accepts_model_key -v
|
| 142 |
-
```
|
| 143 |
-
|
| 144 |
-
Expected: FAIL — `model_path` still present in signature.
|
| 145 |
-
|
| 146 |
-
- [ ] **Step 3: Rewrite `formscout/agents/pose2d.py`**
|
| 147 |
-
|
| 148 |
-
Replace the entire file with:
|
| 149 |
-
|
| 150 |
-
```python
|
| 151 |
-
"""
|
| 152 |
-
Pose2DAgent — 2D per-frame keypoint extraction.
|
| 153 |
-
|
| 154 |
-
Backends: yolo (local ONNX), mediapipe (Qualcomm HF/ONNX Runtime),
|
| 155 |
-
sapiens2 (Meta HF/transformers).
|
| 156 |
-
All backends output COCO-17 keypoints: dict[int, {x, y, conf}] per frame.
|
| 157 |
-
|
| 158 |
-
Input: IngestResult
|
| 159 |
-
Output: Pose2DResult(keypoints per frame, fps, confidence)
|
| 160 |
-
Failure: Pose2DResult(confidence=0.0, notes=<reason>) — never raises.
|
| 161 |
-
"""
|
| 162 |
-
from __future__ import annotations
|
| 163 |
-
|
| 164 |
-
import logging
|
| 165 |
-
import numpy as np
|
| 166 |
-
|
| 167 |
-
from formscout import config
|
| 168 |
-
from formscout.types import IngestResult, Pose2DResult
|
| 169 |
-
|
| 170 |
-
logger = logging.getLogger(__name__)
|
| 171 |
-
|
| 172 |
-
COCO_KEYPOINTS = [
|
| 173 |
-
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 174 |
-
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 175 |
-
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 176 |
-
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 177 |
-
]
|
| 178 |
-
|
| 179 |
-
# BlazePose-33 → COCO-17 index mapping
|
| 180 |
-
_BLAZEPOSE_TO_COCO: dict[int, int] = {
|
| 181 |
-
0: 0, # nose
|
| 182 |
-
1: 2, # left_eye (inner → left_eye)
|
| 183 |
-
2: 1, # right_eye (inner → right_eye) — swapped: BlazePose 1=left_eye_inner
|
| 184 |
-
3: 3, # left_ear
|
| 185 |
-
4: 4, # right_ear
|
| 186 |
-
5: 5, # left_shoulder → COCO left_shoulder... wait
|
| 187 |
-
# Correct BlazePose-33 COCO mapping (canonical):
|
| 188 |
-
# BlazePose idx : COCO idx
|
| 189 |
-
# 0 nose → COCO 0
|
| 190 |
-
# 2 left_eye → COCO 1
|
| 191 |
-
# 5 right_eye → COCO 2
|
| 192 |
-
# 7 left_ear → COCO 3
|
| 193 |
-
# 8 right_ear → COCO 4
|
| 194 |
-
# 11 left_shoulder → COCO 5
|
| 195 |
-
# 12 right_shoulder → COCO 6
|
| 196 |
-
# 13 left_elbow → COCO 7
|
| 197 |
-
# 14 right_elbow → COCO 8
|
| 198 |
-
# 15 left_wrist → COCO 9
|
| 199 |
-
# 16 right_wrist → COCO 10
|
| 200 |
-
# 23 left_hip → COCO 11
|
| 201 |
-
# 24 right_hip → COCO 12
|
| 202 |
-
# 25 left_knee → COCO 13
|
| 203 |
-
# 26 right_knee → COCO 14
|
| 204 |
-
# 27 left_ankle → COCO 15
|
| 205 |
-
# 28 right_ankle → COCO 16
|
| 206 |
-
}
|
| 207 |
-
|
| 208 |
-
# BlazePose source index → COCO target index (correct mapping, no duplicates)
|
| 209 |
-
_BP_SRC = [0, 2, 5, 7, 8, 11, 12, 13, 14, 15, 16, 23, 24, 25, 26, 27, 28]
|
| 210 |
-
_BP_DST = list(range(17)) # COCO 0..16
|
| 211 |
-
|
| 212 |
-
_model_cache: dict[str, object] = {}
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
# ── YOLO backend ─────────────────────────────────────────────────────────────
|
| 216 |
-
|
| 217 |
-
def _get_yolo(path: str) -> object:
|
| 218 |
-
if path not in _model_cache:
|
| 219 |
-
from ultralytics import YOLO
|
| 220 |
-
_model_cache[path] = YOLO(path)
|
| 221 |
-
return _model_cache[path]
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
def _run_yolo(frames: list, path: str) -> list[dict]:
|
| 225 |
-
model = _get_yolo(path)
|
| 226 |
-
out = []
|
| 227 |
-
for frame in frames:
|
| 228 |
-
try:
|
| 229 |
-
results = model(frame, verbose=False)
|
| 230 |
-
kps: dict[int, dict] = {}
|
| 231 |
-
if results and results[0].keypoints is not None:
|
| 232 |
-
kp = results[0].keypoints
|
| 233 |
-
if kp.xy is not None and len(kp.xy) > 0:
|
| 234 |
-
xy = kp.xy[0].cpu().numpy()
|
| 235 |
-
conf = kp.conf[0].cpu().numpy()
|
| 236 |
-
for j in range(min(len(xy), 17)):
|
| 237 |
-
kps[j] = {"x": float(xy[j, 0]), "y": float(xy[j, 1]), "conf": float(conf[j])}
|
| 238 |
-
out.append(kps)
|
| 239 |
-
except Exception:
|
| 240 |
-
out.append({})
|
| 241 |
-
return out
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
# ── MediaPipe backend ────────────────────────────────────────────────────────
|
| 245 |
-
|
| 246 |
-
def _get_mediapipe_sessions(hf_id: str):
|
| 247 |
-
"""Return (detector_session, landmark_session) cached by hf_id."""
|
| 248 |
-
cache_key = f"mp:{hf_id}"
|
| 249 |
-
if cache_key not in _model_cache:
|
| 250 |
-
from huggingface_hub import snapshot_download
|
| 251 |
-
import onnxruntime as ort
|
| 252 |
-
from pathlib import Path
|
| 253 |
-
|
| 254 |
-
snap = Path(snapshot_download(hf_id))
|
| 255 |
-
onnx_files = sorted(snap.glob("**/*.onnx"), key=lambda p: p.stat().st_size)
|
| 256 |
-
if len(onnx_files) < 2:
|
| 257 |
-
raise RuntimeError(f"Expected 2 ONNX files in {snap}, found {len(onnx_files)}")
|
| 258 |
-
# Smaller file = pose detector; larger = pose landmark detector
|
| 259 |
-
det_sess = ort.InferenceSession(str(onnx_files[0]))
|
| 260 |
-
lmk_sess = ort.InferenceSession(str(onnx_files[-1]))
|
| 261 |
-
_model_cache[cache_key] = (det_sess, lmk_sess)
|
| 262 |
-
return _model_cache[cache_key]
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
def _preprocess_mediapipe(frame: np.ndarray, size: int = 256) -> np.ndarray:
|
| 266 |
-
"""Resize to size×size, normalize to [0,1], add batch dim → (1,3,H,W)."""
|
| 267 |
-
import cv2
|
| 268 |
-
img = cv2.resize(frame, (size, size)).astype(np.float32) / 255.0
|
| 269 |
-
return img.transpose(2, 0, 1)[None] # (1, 3, 256, 256)
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
def _run_mediapipe(frames: list, hf_id: str) -> list[dict]:
|
| 273 |
-
try:
|
| 274 |
-
det_sess, lmk_sess = _get_mediapipe_sessions(hf_id)
|
| 275 |
-
except Exception as e:
|
| 276 |
-
logger.warning("mediapipe load failed: %s", e)
|
| 277 |
-
return [{} for _ in frames]
|
| 278 |
-
|
| 279 |
-
import cv2
|
| 280 |
-
h_orig, w_orig = frames[0].shape[:2] if frames else (480, 640)
|
| 281 |
-
out = []
|
| 282 |
-
|
| 283 |
-
for frame in frames:
|
| 284 |
-
try:
|
| 285 |
-
h, w = frame.shape[:2]
|
| 286 |
-
inp = _preprocess_mediapipe(frame)
|
| 287 |
-
|
| 288 |
-
# Run landmark detector directly on full frame (single-person FMS use-case)
|
| 289 |
-
lmk_input_name = lmk_sess.get_inputs()[0].name
|
| 290 |
-
lmk_out = lmk_sess.run(None, {lmk_input_name: inp})
|
| 291 |
-
|
| 292 |
-
# lmk_out[0] shape: (1, 33, 3) — [x, y, visibility] normalized 0..1
|
| 293 |
-
landmarks = lmk_out[0][0] # (33, 3)
|
| 294 |
-
|
| 295 |
-
kps: dict[int, dict] = {}
|
| 296 |
-
for coco_idx, bp_idx in zip(_BP_DST, _BP_SRC):
|
| 297 |
-
if bp_idx < len(landmarks):
|
| 298 |
-
lm = landmarks[bp_idx]
|
| 299 |
-
kps[coco_idx] = {
|
| 300 |
-
"x": float(lm[0] * w),
|
| 301 |
-
"y": float(lm[1] * h),
|
| 302 |
-
"conf": float(lm[2]), # visibility score
|
| 303 |
-
}
|
| 304 |
-
out.append(kps)
|
| 305 |
-
except Exception:
|
| 306 |
-
out.append({})
|
| 307 |
-
|
| 308 |
-
return out
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
# ── Sapiens2 backend ─────────────────────────────────────────────────────────
|
| 312 |
-
|
| 313 |
-
# COCO-17 keypoint names in order (used to map Sapiens2 named output → COCO index)
|
| 314 |
-
_COCO_NAMES = [
|
| 315 |
-
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 316 |
-
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 317 |
-
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 318 |
-
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 319 |
-
]
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
def _get_sapiens2(hf_id: str) -> object:
|
| 323 |
-
if hf_id not in _model_cache:
|
| 324 |
-
from transformers import pipeline as hf_pipeline
|
| 325 |
-
_model_cache[hf_id] = hf_pipeline("pose-estimation", model=hf_id)
|
| 326 |
-
return _model_cache[hf_id]
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
def _run_sapiens2(frames: list, hf_id: str) -> list[dict]:
|
| 330 |
-
try:
|
| 331 |
-
pipe = _get_sapiens2(hf_id)
|
| 332 |
-
except Exception as e:
|
| 333 |
-
logger.warning("sapiens2 load failed: %s", e)
|
| 334 |
-
return [{} for _ in frames]
|
| 335 |
-
|
| 336 |
-
from PIL import Image
|
| 337 |
-
out = []
|
| 338 |
-
|
| 339 |
-
for frame in frames:
|
| 340 |
-
try:
|
| 341 |
-
pil_img = Image.fromarray(frame)
|
| 342 |
-
result = pipe(pil_img)
|
| 343 |
-
|
| 344 |
-
# result is a list of person dicts; take the first (highest confidence)
|
| 345 |
-
if not result:
|
| 346 |
-
out.append({})
|
| 347 |
-
continue
|
| 348 |
-
|
| 349 |
-
person = result[0]
|
| 350 |
-
keypoints = person.get("keypoints", [])
|
| 351 |
-
scores = person.get("keypoint_scores", [])
|
| 352 |
-
|
| 353 |
-
# Build name→(x,y,score) lookup from pipeline output
|
| 354 |
-
kp_lookup: dict[str, tuple] = {}
|
| 355 |
-
for i, kp in enumerate(keypoints):
|
| 356 |
-
name = kp.get("label", "") if isinstance(kp, dict) else ""
|
| 357 |
-
x = kp.get("x", 0.0) if isinstance(kp, dict) else float(kp[0])
|
| 358 |
-
y = kp.get("y", 0.0) if isinstance(kp, dict) else float(kp[1])
|
| 359 |
-
score = scores[i] if i < len(scores) else 0.0
|
| 360 |
-
if name:
|
| 361 |
-
kp_lookup[name] = (x, y, float(score))
|
| 362 |
-
|
| 363 |
-
kps: dict[int, dict] = {}
|
| 364 |
-
for coco_idx, name in enumerate(_COCO_NAMES):
|
| 365 |
-
if name in kp_lookup:
|
| 366 |
-
x, y, s = kp_lookup[name]
|
| 367 |
-
kps[coco_idx] = {"x": x, "y": y, "conf": s}
|
| 368 |
-
|
| 369 |
-
out.append(kps)
|
| 370 |
-
except Exception:
|
| 371 |
-
out.append({})
|
| 372 |
-
|
| 373 |
-
return out
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
# ── Agent ────────────────────────────────────────────────────────────────────
|
| 377 |
-
|
| 378 |
-
class Pose2DAgent:
|
| 379 |
-
"""Extracts COCO-17 keypoints per frame; dispatches to YOLO, MediaPipe, or Sapiens2."""
|
| 380 |
-
|
| 381 |
-
def run(self, ingest: IngestResult, model_key: str | None = None) -> Pose2DResult:
|
| 382 |
-
if not ingest.frames:
|
| 383 |
-
return Pose2DResult(keypoints=[], fps=ingest.fps, confidence=0.0, notes="no frames in ingest")
|
| 384 |
-
|
| 385 |
-
key = model_key or config.DEFAULT_POSE_MODEL
|
| 386 |
-
spec = config.POSE_MODELS.get(key)
|
| 387 |
-
if spec is None:
|
| 388 |
-
logger.warning("Unknown model_key %r — falling back to %s", key, config.DEFAULT_POSE_MODEL)
|
| 389 |
-
spec = config.POSE_MODELS[config.DEFAULT_POSE_MODEL]
|
| 390 |
-
|
| 391 |
-
backend = spec["backend"]
|
| 392 |
-
try:
|
| 393 |
-
if backend == "yolo":
|
| 394 |
-
kps_per_frame = _run_yolo(ingest.frames, spec["path"])
|
| 395 |
-
elif backend == "mediapipe":
|
| 396 |
-
kps_per_frame = _run_mediapipe(ingest.frames, spec["hf_id"])
|
| 397 |
-
elif backend == "sapiens2":
|
| 398 |
-
kps_per_frame = _run_sapiens2(ingest.frames, spec["hf_id"])
|
| 399 |
-
else:
|
| 400 |
-
return Pose2DResult(
|
| 401 |
-
keypoints=[{} for _ in ingest.frames],
|
| 402 |
-
fps=ingest.fps, confidence=0.0,
|
| 403 |
-
notes=f"unknown backend: {backend}",
|
| 404 |
-
)
|
| 405 |
-
except Exception as e:
|
| 406 |
-
return Pose2DResult(
|
| 407 |
-
keypoints=[{} for _ in ingest.frames],
|
| 408 |
-
fps=ingest.fps, confidence=0.0,
|
| 409 |
-
notes=str(e),
|
| 410 |
-
)
|
| 411 |
-
|
| 412 |
-
n_detected = sum(1 for f in kps_per_frame if f)
|
| 413 |
-
total_conf = sum(
|
| 414 |
-
sum(kp["conf"] for kp in f.values()) / len(f)
|
| 415 |
-
for f in kps_per_frame if f
|
| 416 |
-
)
|
| 417 |
-
overall_conf = (total_conf / n_detected) if n_detected > 0 else 0.0
|
| 418 |
-
notes = "" if n_detected > 0 else "no person detected in any frame"
|
| 419 |
-
|
| 420 |
-
return Pose2DResult(
|
| 421 |
-
keypoints=kps_per_frame,
|
| 422 |
-
fps=ingest.fps,
|
| 423 |
-
confidence=overall_conf,
|
| 424 |
-
notes=notes,
|
| 425 |
-
)
|
| 426 |
-
```
|
| 427 |
-
|
| 428 |
-
- [ ] **Step 4: Run the new signature test**
|
| 429 |
-
|
| 430 |
-
```bash
|
| 431 |
-
pytest tests/test_pose2d.py::TestPose2DAgent::test_run_accepts_model_key -v
|
| 432 |
-
```
|
| 433 |
-
|
| 434 |
-
Expected: PASS
|
| 435 |
-
|
| 436 |
-
- [ ] **Step 5: Run full existing pose2d test suite**
|
| 437 |
-
|
| 438 |
-
```bash
|
| 439 |
-
pytest tests/test_pose2d.py -v
|
| 440 |
-
```
|
| 441 |
-
|
| 442 |
-
Expected: all existing tests pass (they will skip if YOLO model unavailable in env — that's OK).
|
| 443 |
-
|
| 444 |
-
- [ ] **Step 6: Commit and push**
|
| 445 |
-
|
| 446 |
-
```bash
|
| 447 |
-
git add formscout/agents/pose2d.py tests/test_pose2d.py
|
| 448 |
-
git commit -m "feat: Pose2DAgent — three backends (yolo/mediapipe/sapiens2), model_key dispatch"
|
| 449 |
-
git push
|
| 450 |
-
```
|
| 451 |
-
|
| 452 |
-
---
|
| 453 |
-
|
| 454 |
-
## Task 3: Add `onnxruntime` to requirements
|
| 455 |
-
|
| 456 |
-
**Files:**
|
| 457 |
-
- Modify: `requirements.txt`
|
| 458 |
-
|
| 459 |
-
- [ ] **Step 1: Add onnxruntime**
|
| 460 |
-
|
| 461 |
-
Open `requirements.txt` and add after the existing `transformers` line:
|
| 462 |
-
|
| 463 |
-
```
|
| 464 |
-
onnxruntime>=1.18
|
| 465 |
-
```
|
| 466 |
-
|
| 467 |
-
- [ ] **Step 2: Verify it installs**
|
| 468 |
-
|
| 469 |
-
```bash
|
| 470 |
-
pip install onnxruntime --quiet && python3 -c "import onnxruntime; print(onnxruntime.__version__)"
|
| 471 |
-
```
|
| 472 |
-
|
| 473 |
-
Expected: version string printed, no errors.
|
| 474 |
-
|
| 475 |
-
- [ ] **Step 3: Commit and push**
|
| 476 |
-
|
| 477 |
-
```bash
|
| 478 |
-
git add requirements.txt
|
| 479 |
-
git commit -m "chore: add onnxruntime for MediaPipe ONNX backend"
|
| 480 |
-
git push
|
| 481 |
-
```
|
| 482 |
-
|
| 483 |
-
---
|
| 484 |
-
|
| 485 |
-
## Task 4: Update `Director.run()` — `pose_model_path` → `model_key`
|
| 486 |
-
|
| 487 |
-
**Files:**
|
| 488 |
-
- Modify: `formscout/pipeline.py`
|
| 489 |
-
|
| 490 |
-
- [ ] **Step 1: Update the signature and the `pose2d` call**
|
| 491 |
-
|
| 492 |
-
In `formscout/pipeline.py`, change `Director.run()`:
|
| 493 |
-
|
| 494 |
-
```python
|
| 495 |
-
def run(self, video_path: str, test_name: str = "deep_squat", side: str = "na", model_key: str | None = None) -> PipelineState:
|
| 496 |
-
"""
|
| 497 |
-
Run the full pipeline on a single video.
|
| 498 |
-
test_name/side serve as manual override when provided (skips classifier).
|
| 499 |
-
model_key selects the pose backend (see config.POSE_MODELS).
|
| 500 |
-
"""
|
| 501 |
-
state = PipelineState(video_path=video_path)
|
| 502 |
-
|
| 503 |
-
# ─── Ingest ───
|
| 504 |
-
state.ingest = self._ingest.run(video_path)
|
| 505 |
-
if state.ingest.confidence < config.MIN_CONFIDENCE:
|
| 506 |
-
state.errors.append("ingest: low confidence — video may be corrupt")
|
| 507 |
-
return state
|
| 508 |
-
|
| 509 |
-
# ─── Pose 2D ───
|
| 510 |
-
state.pose2d = self._pose2d.run(state.ingest, model_key=model_key)
|
| 511 |
-
# ... rest of method unchanged
|
| 512 |
-
```
|
| 513 |
-
|
| 514 |
-
(Only the signature line and the `self._pose2d.run(...)` call change — everything else stays the same.)
|
| 515 |
-
|
| 516 |
-
- [ ] **Step 2: Verify import is clean**
|
| 517 |
-
|
| 518 |
-
```bash
|
| 519 |
-
python3 -c "from formscout.pipeline import Director; d = Director(); print('ok')"
|
| 520 |
-
```
|
| 521 |
-
|
| 522 |
-
Expected: `ok` (models load lazily so no crash here).
|
| 523 |
-
|
| 524 |
-
- [ ] **Step 3: Commit and push**
|
| 525 |
-
|
| 526 |
-
```bash
|
| 527 |
-
git add formscout/pipeline.py
|
| 528 |
-
git commit -m "feat: Director.run() accepts model_key, threads to Pose2DAgent"
|
| 529 |
-
git push
|
| 530 |
-
```
|
| 531 |
-
|
| 532 |
-
---
|
| 533 |
-
|
| 534 |
-
## Task 5: Wire the UI — pose model dropdown in `app.py`
|
| 535 |
-
|
| 536 |
-
**Files:**
|
| 537 |
-
- Modify: `app.py`
|
| 538 |
-
|
| 539 |
-
- [ ] **Step 1: Update `process_video` to use `model_key` and the unified registry**
|
| 540 |
-
|
| 541 |
-
Replace the existing `process_video` function signature and the old `YOLO_POSE_MODELS.get()` lookup:
|
| 542 |
-
|
| 543 |
-
```python
|
| 544 |
-
def process_video(video_path: str, test_name: str, side: str, model_key: str):
|
| 545 |
-
"""Process an uploaded video through the FormScout pipeline."""
|
| 546 |
-
if not video_path:
|
| 547 |
-
return (
|
| 548 |
-
_render_empty_state(),
|
| 549 |
-
"Upload a video to begin analysis.",
|
| 550 |
-
"",
|
| 551 |
-
"",
|
| 552 |
-
)
|
| 553 |
-
|
| 554 |
-
director = Director()
|
| 555 |
-
state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
|
| 556 |
-
```
|
| 557 |
-
|
| 558 |
-
(Remove the `pose_model_path = config.YOLO_POSE_MODELS.get(...)` line entirely.)
|
| 559 |
-
|
| 560 |
-
- [ ] **Step 2: Add the `pose_model_dropdown` in `build_app()`**
|
| 561 |
-
|
| 562 |
-
Inside `build_app()`, after the `side_dropdown` block (around line 265) and before `submit_btn`, add:
|
| 563 |
-
|
| 564 |
-
```python
|
| 565 |
-
pose_model_dropdown = gr.Dropdown(
|
| 566 |
-
choices=list(config.POSE_MODELS.keys()),
|
| 567 |
-
value=config.DEFAULT_POSE_MODEL,
|
| 568 |
-
label="Pose Model",
|
| 569 |
-
)
|
| 570 |
-
```
|
| 571 |
-
|
| 572 |
-
- [ ] **Step 3: Update `_map_inputs` to pass the model key**
|
| 573 |
-
|
| 574 |
-
Replace the existing `_map_inputs` closure:
|
| 575 |
-
|
| 576 |
-
```python
|
| 577 |
-
def _map_inputs(video, test_display_name, side_display, pose_model_key):
|
| 578 |
-
"""Map UI display values to internal values."""
|
| 579 |
-
test_map = {name: val for name, val in FMS_TESTS}
|
| 580 |
-
test_name = test_map.get(test_display_name, "deep_squat")
|
| 581 |
-
side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
|
| 582 |
-
return process_video(video, test_name, side, pose_model_key)
|
| 583 |
-
```
|
| 584 |
-
|
| 585 |
-
- [ ] **Step 4: Update `submit_btn.click` to include `pose_model_dropdown`**
|
| 586 |
-
|
| 587 |
-
Replace the existing `.click(...)` call:
|
| 588 |
-
|
| 589 |
-
```python
|
| 590 |
-
submit_btn.click(
|
| 591 |
-
fn=_map_inputs,
|
| 592 |
-
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown],
|
| 593 |
-
outputs=[score_html, pipeline_md, score_details, alerts_md],
|
| 594 |
-
)
|
| 595 |
-
```
|
| 596 |
-
|
| 597 |
-
- [ ] **Step 5: Smoke-test the app starts**
|
| 598 |
-
|
| 599 |
-
```bash
|
| 600 |
-
python3 -c "from app import build_app; app = build_app(); print('app built ok')"
|
| 601 |
-
```
|
| 602 |
-
|
| 603 |
-
Expected: `app built ok` — no import or config errors.
|
| 604 |
-
|
| 605 |
-
- [ ] **Step 6: Commit and push**
|
| 606 |
-
|
| 607 |
-
```bash
|
| 608 |
-
git add app.py
|
| 609 |
-
git commit -m "feat: pose model dropdown in UI, wired through process_video → Director"
|
| 610 |
-
git push
|
| 611 |
-
```
|
| 612 |
-
|
| 613 |
-
---
|
| 614 |
-
|
| 615 |
-
## Task 6: Add mocked backend tests
|
| 616 |
-
|
| 617 |
-
**Files:**
|
| 618 |
-
- Modify: `tests/test_pose2d.py`
|
| 619 |
-
|
| 620 |
-
- [ ] **Step 1: Add mocked YOLO test**
|
| 621 |
-
|
| 622 |
-
Append to `tests/test_pose2d.py`:
|
| 623 |
-
|
| 624 |
-
```python
|
| 625 |
-
import unittest.mock as mock
|
| 626 |
-
import numpy as np
|
| 627 |
-
from formscout.types import IngestResult, Pose2DResult
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
def _blank_ingest_3():
|
| 631 |
-
frames = [np.zeros((480, 640, 3), dtype=np.uint8) for _ in range(3)]
|
| 632 |
-
return IngestResult(frames=frames, fps=30.0, duration=0.1, n_people=1, width=640, height=480)
|
| 633 |
-
|
| 634 |
-
|
| 635 |
-
class TestPose2DBackendsMocked:
|
| 636 |
-
"""Backend dispatch tests — no real model downloads."""
|
| 637 |
-
|
| 638 |
-
def test_yolo_backend_dispatches(self):
|
| 639 |
-
from formscout.agents.pose2d import Pose2DAgent, _run_yolo
|
| 640 |
-
fake_kps = [{0: {"x": 10.0, "y": 20.0, "conf": 0.9}} for _ in range(3)]
|
| 641 |
-
with mock.patch("formscout.agents.pose2d._run_yolo", return_value=fake_kps) as m:
|
| 642 |
-
agent = Pose2DAgent()
|
| 643 |
-
result = agent.run(_blank_ingest_3(), model_key="YOLO26n — nano (0.7M, fastest)")
|
| 644 |
-
m.assert_called_once()
|
| 645 |
-
assert isinstance(result, Pose2DResult)
|
| 646 |
-
assert len(result.keypoints) == 3
|
| 647 |
-
assert result.confidence > 0.0
|
| 648 |
-
|
| 649 |
-
def test_mediapipe_backend_dispatches(self):
|
| 650 |
-
from formscout.agents.pose2d import Pose2DAgent
|
| 651 |
-
fake_kps = [{i: {"x": float(i), "y": float(i), "conf": 0.8} for i in range(17)} for _ in range(3)]
|
| 652 |
-
with mock.patch("formscout.agents.pose2d._run_mediapipe", return_value=fake_kps) as m:
|
| 653 |
-
agent = Pose2DAgent()
|
| 654 |
-
result = agent.run(_blank_ingest_3(), model_key="MediaPipe-Pose ⬇ ~16 MB, CPU-friendly")
|
| 655 |
-
m.assert_called_once()
|
| 656 |
-
assert isinstance(result, Pose2DResult)
|
| 657 |
-
assert len(result.keypoints) == 3
|
| 658 |
-
assert all(len(f) == 17 for f in result.keypoints)
|
| 659 |
-
|
| 660 |
-
def test_sapiens2_backend_dispatches(self):
|
| 661 |
-
from formscout.agents.pose2d import Pose2DAgent
|
| 662 |
-
fake_kps = [{i: {"x": float(i), "y": float(i), "conf": 0.85} for i in range(17)} for _ in range(3)]
|
| 663 |
-
with mock.patch("formscout.agents.pose2d._run_sapiens2", return_value=fake_kps) as m:
|
| 664 |
-
agent = Pose2DAgent()
|
| 665 |
-
result = agent.run(_blank_ingest_3(), model_key="Sapiens2-0.4B ⬇ ~1.6 GB")
|
| 666 |
-
m.assert_called_once()
|
| 667 |
-
assert isinstance(result, Pose2DResult)
|
| 668 |
-
assert len(result.keypoints) == 3
|
| 669 |
-
|
| 670 |
-
def test_unknown_model_key_falls_back(self):
|
| 671 |
-
from formscout.agents.pose2d import Pose2DAgent
|
| 672 |
-
fake_kps = [{0: {"x": 1.0, "y": 2.0, "conf": 0.7}} for _ in range(3)]
|
| 673 |
-
with mock.patch("formscout.agents.pose2d._run_yolo", return_value=fake_kps):
|
| 674 |
-
agent = Pose2DAgent()
|
| 675 |
-
result = agent.run(_blank_ingest_3(), model_key="nonexistent-model-xyz")
|
| 676 |
-
assert isinstance(result, Pose2DResult) # graceful fallback, no crash
|
| 677 |
-
|
| 678 |
-
def test_confidence_zero_on_empty_keypoints(self):
|
| 679 |
-
from formscout.agents.pose2d import Pose2DAgent
|
| 680 |
-
with mock.patch("formscout.agents.pose2d._run_yolo", return_value=[{}, {}, {}]):
|
| 681 |
-
agent = Pose2DAgent()
|
| 682 |
-
result = agent.run(_blank_ingest_3(), model_key="YOLO26n — nano (0.7M, fastest)")
|
| 683 |
-
assert result.confidence == 0.0
|
| 684 |
-
assert "no person" in result.notes.lower()
|
| 685 |
-
```
|
| 686 |
-
|
| 687 |
-
- [ ] **Step 2: Run the new tests**
|
| 688 |
-
|
| 689 |
-
```bash
|
| 690 |
-
pytest tests/test_pose2d.py::TestPose2DBackendsMocked -v
|
| 691 |
-
```
|
| 692 |
-
|
| 693 |
-
Expected: all 5 tests PASS.
|
| 694 |
-
|
| 695 |
-
- [ ] **Step 3: Run the full test suite to check for regressions**
|
| 696 |
-
|
| 697 |
-
```bash
|
| 698 |
-
pytest tests/ -v --tb=short 2>&1 | tail -30
|
| 699 |
-
```
|
| 700 |
-
|
| 701 |
-
Expected: same pass/fail ratio as before (45/46 known passing). The one known failure (`test_unimplemented_test_returns_low_confidence`) is pre-existing — ignore it.
|
| 702 |
-
|
| 703 |
-
- [ ] **Step 4: Commit and push**
|
| 704 |
-
|
| 705 |
-
```bash
|
| 706 |
-
git add tests/test_pose2d.py
|
| 707 |
-
git commit -m "test: mocked backend dispatch tests for YOLO, MediaPipe, Sapiens2"
|
| 708 |
-
git push
|
| 709 |
-
```
|
| 710 |
-
|
| 711 |
-
---
|
| 712 |
-
|
| 713 |
-
## Self-review
|
| 714 |
-
|
| 715 |
-
**Spec coverage:**
|
| 716 |
-
- ✅ Unified `POSE_MODELS` registry (Task 1)
|
| 717 |
-
- ✅ `DEFAULT_POSE_MODEL = YOLO26n` (Task 1)
|
| 718 |
-
- ✅ Backward-compat `YOLO_POSE_MODEL` / `YOLO_POSE_MODEL_HQ` aliases (Task 1)
|
| 719 |
-
- ✅ `_run_yolo` sub-runner (Task 2)
|
| 720 |
-
- ✅ `_run_mediapipe` with ONNX Runtime + BlazePose→COCO-17 mapping (Task 2)
|
| 721 |
-
- ✅ `_run_sapiens2` with transformers pipeline + named-keypoint→COCO-17 mapping (Task 2)
|
| 722 |
-
- ✅ `Pose2DAgent.run(model_key)` dispatch + fallback on unknown key (Task 2)
|
| 723 |
-
- ✅ `onnxruntime` added to requirements (Task 3)
|
| 724 |
-
- ✅ `Director.run(model_key)` threads key to agent (Task 4)
|
| 725 |
-
- ✅ `pose_model_dropdown` in UI (Task 5)
|
| 726 |
-
- ✅ `_map_inputs` + `submit_btn.click` wired (Task 5)
|
| 727 |
-
- ✅ Error handling: unknown key → warning + fallback; download failure → confidence=0 (Task 2)
|
| 728 |
-
- ✅ Mocked tests for all three backends (Task 6)
|
| 729 |
-
|
| 730 |
-
**Placeholder scan:** None found.
|
| 731 |
-
|
| 732 |
-
**Type consistency:** `model_key: str | None` used consistently across `Pose2DAgent.run`, `Director.run`, `process_video`. `config.POSE_MODELS` and `config.DEFAULT_POSE_MODEL` referenced consistently.
|
| 733 |
-
|
| 734 |
-
**Note on Sapiens2 keypoint format:** The `_run_sapiens2` implementation uses **named keypoint lookup** (by label string) rather than assuming fixed indices 0–16 = COCO. This is the safe approach — the transformers pipeline returns labeled keypoints and the code maps by name. If the pipeline returns unnamed keypoints (index-only), the `kp_lookup` will be empty and the frame will gracefully return `{}`.
|
|
|
|
| 1 |
+
# Pose Model Selector Implementation Plan
|
| 2 |
+
|
| 3 |
+
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
| 4 |
+
|
| 5 |
+
**Goal:** Replace the hard-coded YOLO26l default with a 10-model dropdown (MediaPipe, YOLO26 n→x, Sapiens2 0.4B→5B) wired end-to-end from UI through the Director to `Pose2DAgent`.
|
| 6 |
+
|
| 7 |
+
**Architecture:** Unified `POSE_MODELS` registry in `config.py` drives a `gr.Dropdown` in `app.py`; the selected key flows through `Director.run()` into `Pose2DAgent.run(model_key)`, which dispatches to one of three private sub-runners (`_run_yolo`, `_run_mediapipe`, `_run_sapiens2`), all producing the same COCO-17 `list[dict]` contract.
|
| 8 |
+
|
| 9 |
+
**Tech Stack:** `ultralytics` (YOLO), `onnxruntime` + `huggingface_hub` (MediaPipe), `transformers` (Sapiens2), `gradio` (UI).
|
| 10 |
+
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
## File map
|
| 14 |
+
|
| 15 |
+
| File | Change |
|
| 16 |
+
|---|---|
|
| 17 |
+
| `formscout/config.py` | Replace `YOLO_POSE_MODELS` with `POSE_MODELS` dict + `DEFAULT_POSE_MODEL` |
|
| 18 |
+
| `formscout/agents/pose2d.py` | Add `_run_yolo`, `_run_mediapipe`, `_run_sapiens2`; update `run()` signature |
|
| 19 |
+
| `formscout/pipeline.py` | Change `pose_model_path` param to `model_key` |
|
| 20 |
+
| `app.py` | Add `pose_model_dropdown`, fix `_map_inputs` + `process_video` |
|
| 21 |
+
| `requirements.txt` | Add `onnxruntime>=1.18` |
|
| 22 |
+
| `tests/test_pose2d.py` | Add mocked tests for each backend |
|
| 23 |
+
|
| 24 |
+
---
|
| 25 |
+
|
| 26 |
+
## Task 1: Add unified `POSE_MODELS` registry to `config.py`
|
| 27 |
+
|
| 28 |
+
**Files:**
|
| 29 |
+
- Modify: `formscout/config.py`
|
| 30 |
+
|
| 31 |
+
- [ ] **Step 1: Open `formscout/config.py` and replace the `YOLO_POSE_MODELS` block**
|
| 32 |
+
|
| 33 |
+
Replace lines 12–20 (the `YOLO_POSE_MODELS` dict and `YOLO_POSE_MODEL` / `YOLO_POSE_MODEL_HQ` lines) with:
|
| 34 |
+
|
| 35 |
+
```python
|
| 36 |
+
_YOLO_DIR = ROOT / "checkpoints" / "yolo26"
|
| 37 |
+
|
| 38 |
+
POSE_MODELS: dict[str, dict] = {
|
| 39 |
+
# ── MediaPipe (Qualcomm HF, ONNX Runtime) ──────────────────────────────
|
| 40 |
+
"MediaPipe-Pose ⬇ ~16 MB, CPU-friendly": {
|
| 41 |
+
"backend": "mediapipe",
|
| 42 |
+
"hf_id": "qualcomm/MediaPipe-Pose-Estimation",
|
| 43 |
+
"params_m": 4.2,
|
| 44 |
+
},
|
| 45 |
+
# ── YOLO26 (local checkpoints) ─────────────────────────────────────────
|
| 46 |
+
"YOLO26n — nano (0.7M, fastest)": {
|
| 47 |
+
"backend": "yolo",
|
| 48 |
+
"path": str(_YOLO_DIR / "yolo26n-pose.pt"),
|
| 49 |
+
"params_m": 0.7,
|
| 50 |
+
},
|
| 51 |
+
"YOLO26s — small (3.5M)": {
|
| 52 |
+
"backend": "yolo",
|
| 53 |
+
"path": str(_YOLO_DIR / "yolo26s-pose.pt"),
|
| 54 |
+
"params_m": 3.5,
|
| 55 |
+
},
|
| 56 |
+
"YOLO26m — medium (9M)": {
|
| 57 |
+
"backend": "yolo",
|
| 58 |
+
"path": str(_YOLO_DIR / "yolo26m-pose.pt"),
|
| 59 |
+
"params_m": 9.0,
|
| 60 |
+
},
|
| 61 |
+
"YOLO26l — large (25.9M)": {
|
| 62 |
+
"backend": "yolo",
|
| 63 |
+
"path": str(_YOLO_DIR / "yolo26l-pose.pt"),
|
| 64 |
+
"params_m": 25.9,
|
| 65 |
+
},
|
| 66 |
+
"YOLO26x — extra-large (57.6M)": {
|
| 67 |
+
"backend": "yolo",
|
| 68 |
+
"path": str(_YOLO_DIR / "yolo26x-pose.pt"),
|
| 69 |
+
"params_m": 57.6,
|
| 70 |
+
},
|
| 71 |
+
# ── Sapiens2 (HF download, transformers) ─────────────────────────���─────
|
| 72 |
+
"Sapiens2-0.4B ⬇ ~1.6 GB": {
|
| 73 |
+
"backend": "sapiens2",
|
| 74 |
+
"hf_id": "facebook/sapiens2-pose-0.4b",
|
| 75 |
+
"params_m": 400,
|
| 76 |
+
},
|
| 77 |
+
"Sapiens2-0.8B ⬇ ~3.2 GB": {
|
| 78 |
+
"backend": "sapiens2",
|
| 79 |
+
"hf_id": "facebook/sapiens2-pose-0.8b",
|
| 80 |
+
"params_m": 800,
|
| 81 |
+
},
|
| 82 |
+
"Sapiens2-1B ⬇ ~4 GB": {
|
| 83 |
+
"backend": "sapiens2",
|
| 84 |
+
"hf_id": "facebook/sapiens2-pose-1b",
|
| 85 |
+
"params_m": 1000,
|
| 86 |
+
},
|
| 87 |
+
"Sapiens2-5B ⬇ ~20 GB, large GPU": {
|
| 88 |
+
"backend": "sapiens2",
|
| 89 |
+
"hf_id": "facebook/sapiens2-pose-5b",
|
| 90 |
+
"params_m": 5000,
|
| 91 |
+
},
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
DEFAULT_POSE_MODEL = "YOLO26n — nano (0.7M, fastest)"
|
| 95 |
+
|
| 96 |
+
# Backward-compat aliases — kept for any direct references outside the agent
|
| 97 |
+
YOLO_POSE_MODEL = str(_YOLO_DIR / "yolo26l-pose.pt")
|
| 98 |
+
YOLO_POSE_MODEL_HQ = str(_YOLO_DIR / "yolo26x-pose.pt")
|
| 99 |
+
```
|
| 100 |
+
|
| 101 |
+
- [ ] **Step 2: Verify import is clean**
|
| 102 |
+
|
| 103 |
+
```bash
|
| 104 |
+
python3 -c "from formscout import config; print(list(config.POSE_MODELS.keys()))"
|
| 105 |
+
```
|
| 106 |
+
|
| 107 |
+
Expected: list of 10 model labels, starting with `MediaPipe-Pose...`
|
| 108 |
+
|
| 109 |
+
- [ ] **Step 3: Commit**
|
| 110 |
+
|
| 111 |
+
```bash
|
| 112 |
+
git add formscout/config.py
|
| 113 |
+
git commit -m "feat: unified POSE_MODELS registry with MediaPipe, YOLO26 n-x, Sapiens2 0.4-5B"
|
| 114 |
+
git push
|
| 115 |
+
```
|
| 116 |
+
|
| 117 |
+
---
|
| 118 |
+
|
| 119 |
+
## Task 2: Refactor `Pose2DAgent` — YOLO sub-runner + new `run()` signature
|
| 120 |
+
|
| 121 |
+
**Files:**
|
| 122 |
+
- Modify: `formscout/agents/pose2d.py`
|
| 123 |
+
- Modify: `tests/test_pose2d.py`
|
| 124 |
+
|
| 125 |
+
- [ ] **Step 1: Write failing test for the new `model_key` signature**
|
| 126 |
+
|
| 127 |
+
Add to `tests/test_pose2d.py`:
|
| 128 |
+
|
| 129 |
+
```python
|
| 130 |
+
def test_run_accepts_model_key(pose2d_agent):
|
| 131 |
+
"""run() must accept model_key kwarg, not model_path."""
|
| 132 |
+
import inspect
|
| 133 |
+
sig = inspect.signature(pose2d_agent.run)
|
| 134 |
+
assert "model_key" in sig.parameters
|
| 135 |
+
assert "model_path" not in sig.parameters
|
| 136 |
+
```
|
| 137 |
+
|
| 138 |
+
- [ ] **Step 2: Run to confirm it fails**
|
| 139 |
+
|
| 140 |
+
```bash
|
| 141 |
+
pytest tests/test_pose2d.py::TestPose2DAgent::test_run_accepts_model_key -v
|
| 142 |
+
```
|
| 143 |
+
|
| 144 |
+
Expected: FAIL — `model_path` still present in signature.
|
| 145 |
+
|
| 146 |
+
- [ ] **Step 3: Rewrite `formscout/agents/pose2d.py`**
|
| 147 |
+
|
| 148 |
+
Replace the entire file with:
|
| 149 |
+
|
| 150 |
+
```python
|
| 151 |
+
"""
|
| 152 |
+
Pose2DAgent — 2D per-frame keypoint extraction.
|
| 153 |
+
|
| 154 |
+
Backends: yolo (local ONNX), mediapipe (Qualcomm HF/ONNX Runtime),
|
| 155 |
+
sapiens2 (Meta HF/transformers).
|
| 156 |
+
All backends output COCO-17 keypoints: dict[int, {x, y, conf}] per frame.
|
| 157 |
+
|
| 158 |
+
Input: IngestResult
|
| 159 |
+
Output: Pose2DResult(keypoints per frame, fps, confidence)
|
| 160 |
+
Failure: Pose2DResult(confidence=0.0, notes=<reason>) — never raises.
|
| 161 |
+
"""
|
| 162 |
+
from __future__ import annotations
|
| 163 |
+
|
| 164 |
+
import logging
|
| 165 |
+
import numpy as np
|
| 166 |
+
|
| 167 |
+
from formscout import config
|
| 168 |
+
from formscout.types import IngestResult, Pose2DResult
|
| 169 |
+
|
| 170 |
+
logger = logging.getLogger(__name__)
|
| 171 |
+
|
| 172 |
+
COCO_KEYPOINTS = [
|
| 173 |
+
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 174 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 175 |
+
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 176 |
+
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 177 |
+
]
|
| 178 |
+
|
| 179 |
+
# BlazePose-33 → COCO-17 index mapping
|
| 180 |
+
_BLAZEPOSE_TO_COCO: dict[int, int] = {
|
| 181 |
+
0: 0, # nose
|
| 182 |
+
1: 2, # left_eye (inner → left_eye)
|
| 183 |
+
2: 1, # right_eye (inner → right_eye) — swapped: BlazePose 1=left_eye_inner
|
| 184 |
+
3: 3, # left_ear
|
| 185 |
+
4: 4, # right_ear
|
| 186 |
+
5: 5, # left_shoulder → COCO left_shoulder... wait
|
| 187 |
+
# Correct BlazePose-33 COCO mapping (canonical):
|
| 188 |
+
# BlazePose idx : COCO idx
|
| 189 |
+
# 0 nose → COCO 0
|
| 190 |
+
# 2 left_eye → COCO 1
|
| 191 |
+
# 5 right_eye → COCO 2
|
| 192 |
+
# 7 left_ear → COCO 3
|
| 193 |
+
# 8 right_ear → COCO 4
|
| 194 |
+
# 11 left_shoulder → COCO 5
|
| 195 |
+
# 12 right_shoulder → COCO 6
|
| 196 |
+
# 13 left_elbow → COCO 7
|
| 197 |
+
# 14 right_elbow → COCO 8
|
| 198 |
+
# 15 left_wrist → COCO 9
|
| 199 |
+
# 16 right_wrist → COCO 10
|
| 200 |
+
# 23 left_hip → COCO 11
|
| 201 |
+
# 24 right_hip → COCO 12
|
| 202 |
+
# 25 left_knee → COCO 13
|
| 203 |
+
# 26 right_knee → COCO 14
|
| 204 |
+
# 27 left_ankle → COCO 15
|
| 205 |
+
# 28 right_ankle → COCO 16
|
| 206 |
+
}
|
| 207 |
+
|
| 208 |
+
# BlazePose source index → COCO target index (correct mapping, no duplicates)
|
| 209 |
+
_BP_SRC = [0, 2, 5, 7, 8, 11, 12, 13, 14, 15, 16, 23, 24, 25, 26, 27, 28]
|
| 210 |
+
_BP_DST = list(range(17)) # COCO 0..16
|
| 211 |
+
|
| 212 |
+
_model_cache: dict[str, object] = {}
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
# ── YOLO backend ─────────────────────────────────────────────────────────────
|
| 216 |
+
|
| 217 |
+
def _get_yolo(path: str) -> object:
|
| 218 |
+
if path not in _model_cache:
|
| 219 |
+
from ultralytics import YOLO
|
| 220 |
+
_model_cache[path] = YOLO(path)
|
| 221 |
+
return _model_cache[path]
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def _run_yolo(frames: list, path: str) -> list[dict]:
|
| 225 |
+
model = _get_yolo(path)
|
| 226 |
+
out = []
|
| 227 |
+
for frame in frames:
|
| 228 |
+
try:
|
| 229 |
+
results = model(frame, verbose=False)
|
| 230 |
+
kps: dict[int, dict] = {}
|
| 231 |
+
if results and results[0].keypoints is not None:
|
| 232 |
+
kp = results[0].keypoints
|
| 233 |
+
if kp.xy is not None and len(kp.xy) > 0:
|
| 234 |
+
xy = kp.xy[0].cpu().numpy()
|
| 235 |
+
conf = kp.conf[0].cpu().numpy()
|
| 236 |
+
for j in range(min(len(xy), 17)):
|
| 237 |
+
kps[j] = {"x": float(xy[j, 0]), "y": float(xy[j, 1]), "conf": float(conf[j])}
|
| 238 |
+
out.append(kps)
|
| 239 |
+
except Exception:
|
| 240 |
+
out.append({})
|
| 241 |
+
return out
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
# ── MediaPipe backend ────────────────────────────────────────────────────────
|
| 245 |
+
|
| 246 |
+
def _get_mediapipe_sessions(hf_id: str):
|
| 247 |
+
"""Return (detector_session, landmark_session) cached by hf_id."""
|
| 248 |
+
cache_key = f"mp:{hf_id}"
|
| 249 |
+
if cache_key not in _model_cache:
|
| 250 |
+
from huggingface_hub import snapshot_download
|
| 251 |
+
import onnxruntime as ort
|
| 252 |
+
from pathlib import Path
|
| 253 |
+
|
| 254 |
+
snap = Path(snapshot_download(hf_id))
|
| 255 |
+
onnx_files = sorted(snap.glob("**/*.onnx"), key=lambda p: p.stat().st_size)
|
| 256 |
+
if len(onnx_files) < 2:
|
| 257 |
+
raise RuntimeError(f"Expected 2 ONNX files in {snap}, found {len(onnx_files)}")
|
| 258 |
+
# Smaller file = pose detector; larger = pose landmark detector
|
| 259 |
+
det_sess = ort.InferenceSession(str(onnx_files[0]))
|
| 260 |
+
lmk_sess = ort.InferenceSession(str(onnx_files[-1]))
|
| 261 |
+
_model_cache[cache_key] = (det_sess, lmk_sess)
|
| 262 |
+
return _model_cache[cache_key]
|
| 263 |
+
|
| 264 |
+
|
| 265 |
+
def _preprocess_mediapipe(frame: np.ndarray, size: int = 256) -> np.ndarray:
|
| 266 |
+
"""Resize to size×size, normalize to [0,1], add batch dim → (1,3,H,W)."""
|
| 267 |
+
import cv2
|
| 268 |
+
img = cv2.resize(frame, (size, size)).astype(np.float32) / 255.0
|
| 269 |
+
return img.transpose(2, 0, 1)[None] # (1, 3, 256, 256)
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def _run_mediapipe(frames: list, hf_id: str) -> list[dict]:
|
| 273 |
+
try:
|
| 274 |
+
det_sess, lmk_sess = _get_mediapipe_sessions(hf_id)
|
| 275 |
+
except Exception as e:
|
| 276 |
+
logger.warning("mediapipe load failed: %s", e)
|
| 277 |
+
return [{} for _ in frames]
|
| 278 |
+
|
| 279 |
+
import cv2
|
| 280 |
+
h_orig, w_orig = frames[0].shape[:2] if frames else (480, 640)
|
| 281 |
+
out = []
|
| 282 |
+
|
| 283 |
+
for frame in frames:
|
| 284 |
+
try:
|
| 285 |
+
h, w = frame.shape[:2]
|
| 286 |
+
inp = _preprocess_mediapipe(frame)
|
| 287 |
+
|
| 288 |
+
# Run landmark detector directly on full frame (single-person FMS use-case)
|
| 289 |
+
lmk_input_name = lmk_sess.get_inputs()[0].name
|
| 290 |
+
lmk_out = lmk_sess.run(None, {lmk_input_name: inp})
|
| 291 |
+
|
| 292 |
+
# lmk_out[0] shape: (1, 33, 3) — [x, y, visibility] normalized 0..1
|
| 293 |
+
landmarks = lmk_out[0][0] # (33, 3)
|
| 294 |
+
|
| 295 |
+
kps: dict[int, dict] = {}
|
| 296 |
+
for coco_idx, bp_idx in zip(_BP_DST, _BP_SRC):
|
| 297 |
+
if bp_idx < len(landmarks):
|
| 298 |
+
lm = landmarks[bp_idx]
|
| 299 |
+
kps[coco_idx] = {
|
| 300 |
+
"x": float(lm[0] * w),
|
| 301 |
+
"y": float(lm[1] * h),
|
| 302 |
+
"conf": float(lm[2]), # visibility score
|
| 303 |
+
}
|
| 304 |
+
out.append(kps)
|
| 305 |
+
except Exception:
|
| 306 |
+
out.append({})
|
| 307 |
+
|
| 308 |
+
return out
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
# ── Sapiens2 backend ─────────────────────────────────────────────────────────
|
| 312 |
+
|
| 313 |
+
# COCO-17 keypoint names in order (used to map Sapiens2 named output → COCO index)
|
| 314 |
+
_COCO_NAMES = [
|
| 315 |
+
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 316 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 317 |
+
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 318 |
+
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 319 |
+
]
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def _get_sapiens2(hf_id: str) -> object:
|
| 323 |
+
if hf_id not in _model_cache:
|
| 324 |
+
from transformers import pipeline as hf_pipeline
|
| 325 |
+
_model_cache[hf_id] = hf_pipeline("pose-estimation", model=hf_id)
|
| 326 |
+
return _model_cache[hf_id]
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def _run_sapiens2(frames: list, hf_id: str) -> list[dict]:
|
| 330 |
+
try:
|
| 331 |
+
pipe = _get_sapiens2(hf_id)
|
| 332 |
+
except Exception as e:
|
| 333 |
+
logger.warning("sapiens2 load failed: %s", e)
|
| 334 |
+
return [{} for _ in frames]
|
| 335 |
+
|
| 336 |
+
from PIL import Image
|
| 337 |
+
out = []
|
| 338 |
+
|
| 339 |
+
for frame in frames:
|
| 340 |
+
try:
|
| 341 |
+
pil_img = Image.fromarray(frame)
|
| 342 |
+
result = pipe(pil_img)
|
| 343 |
+
|
| 344 |
+
# result is a list of person dicts; take the first (highest confidence)
|
| 345 |
+
if not result:
|
| 346 |
+
out.append({})
|
| 347 |
+
continue
|
| 348 |
+
|
| 349 |
+
person = result[0]
|
| 350 |
+
keypoints = person.get("keypoints", [])
|
| 351 |
+
scores = person.get("keypoint_scores", [])
|
| 352 |
+
|
| 353 |
+
# Build name→(x,y,score) lookup from pipeline output
|
| 354 |
+
kp_lookup: dict[str, tuple] = {}
|
| 355 |
+
for i, kp in enumerate(keypoints):
|
| 356 |
+
name = kp.get("label", "") if isinstance(kp, dict) else ""
|
| 357 |
+
x = kp.get("x", 0.0) if isinstance(kp, dict) else float(kp[0])
|
| 358 |
+
y = kp.get("y", 0.0) if isinstance(kp, dict) else float(kp[1])
|
| 359 |
+
score = scores[i] if i < len(scores) else 0.0
|
| 360 |
+
if name:
|
| 361 |
+
kp_lookup[name] = (x, y, float(score))
|
| 362 |
+
|
| 363 |
+
kps: dict[int, dict] = {}
|
| 364 |
+
for coco_idx, name in enumerate(_COCO_NAMES):
|
| 365 |
+
if name in kp_lookup:
|
| 366 |
+
x, y, s = kp_lookup[name]
|
| 367 |
+
kps[coco_idx] = {"x": x, "y": y, "conf": s}
|
| 368 |
+
|
| 369 |
+
out.append(kps)
|
| 370 |
+
except Exception:
|
| 371 |
+
out.append({})
|
| 372 |
+
|
| 373 |
+
return out
|
| 374 |
+
|
| 375 |
+
|
| 376 |
+
# ── Agent ────────────────────────────────────────────────────────────────────
|
| 377 |
+
|
| 378 |
+
class Pose2DAgent:
|
| 379 |
+
"""Extracts COCO-17 keypoints per frame; dispatches to YOLO, MediaPipe, or Sapiens2."""
|
| 380 |
+
|
| 381 |
+
def run(self, ingest: IngestResult, model_key: str | None = None) -> Pose2DResult:
|
| 382 |
+
if not ingest.frames:
|
| 383 |
+
return Pose2DResult(keypoints=[], fps=ingest.fps, confidence=0.0, notes="no frames in ingest")
|
| 384 |
+
|
| 385 |
+
key = model_key or config.DEFAULT_POSE_MODEL
|
| 386 |
+
spec = config.POSE_MODELS.get(key)
|
| 387 |
+
if spec is None:
|
| 388 |
+
logger.warning("Unknown model_key %r — falling back to %s", key, config.DEFAULT_POSE_MODEL)
|
| 389 |
+
spec = config.POSE_MODELS[config.DEFAULT_POSE_MODEL]
|
| 390 |
+
|
| 391 |
+
backend = spec["backend"]
|
| 392 |
+
try:
|
| 393 |
+
if backend == "yolo":
|
| 394 |
+
kps_per_frame = _run_yolo(ingest.frames, spec["path"])
|
| 395 |
+
elif backend == "mediapipe":
|
| 396 |
+
kps_per_frame = _run_mediapipe(ingest.frames, spec["hf_id"])
|
| 397 |
+
elif backend == "sapiens2":
|
| 398 |
+
kps_per_frame = _run_sapiens2(ingest.frames, spec["hf_id"])
|
| 399 |
+
else:
|
| 400 |
+
return Pose2DResult(
|
| 401 |
+
keypoints=[{} for _ in ingest.frames],
|
| 402 |
+
fps=ingest.fps, confidence=0.0,
|
| 403 |
+
notes=f"unknown backend: {backend}",
|
| 404 |
+
)
|
| 405 |
+
except Exception as e:
|
| 406 |
+
return Pose2DResult(
|
| 407 |
+
keypoints=[{} for _ in ingest.frames],
|
| 408 |
+
fps=ingest.fps, confidence=0.0,
|
| 409 |
+
notes=str(e),
|
| 410 |
+
)
|
| 411 |
+
|
| 412 |
+
n_detected = sum(1 for f in kps_per_frame if f)
|
| 413 |
+
total_conf = sum(
|
| 414 |
+
sum(kp["conf"] for kp in f.values()) / len(f)
|
| 415 |
+
for f in kps_per_frame if f
|
| 416 |
+
)
|
| 417 |
+
overall_conf = (total_conf / n_detected) if n_detected > 0 else 0.0
|
| 418 |
+
notes = "" if n_detected > 0 else "no person detected in any frame"
|
| 419 |
+
|
| 420 |
+
return Pose2DResult(
|
| 421 |
+
keypoints=kps_per_frame,
|
| 422 |
+
fps=ingest.fps,
|
| 423 |
+
confidence=overall_conf,
|
| 424 |
+
notes=notes,
|
| 425 |
+
)
|
| 426 |
+
```
|
| 427 |
+
|
| 428 |
+
- [ ] **Step 4: Run the new signature test**
|
| 429 |
+
|
| 430 |
+
```bash
|
| 431 |
+
pytest tests/test_pose2d.py::TestPose2DAgent::test_run_accepts_model_key -v
|
| 432 |
+
```
|
| 433 |
+
|
| 434 |
+
Expected: PASS
|
| 435 |
+
|
| 436 |
+
- [ ] **Step 5: Run full existing pose2d test suite**
|
| 437 |
+
|
| 438 |
+
```bash
|
| 439 |
+
pytest tests/test_pose2d.py -v
|
| 440 |
+
```
|
| 441 |
+
|
| 442 |
+
Expected: all existing tests pass (they will skip if YOLO model unavailable in env — that's OK).
|
| 443 |
+
|
| 444 |
+
- [ ] **Step 6: Commit and push**
|
| 445 |
+
|
| 446 |
+
```bash
|
| 447 |
+
git add formscout/agents/pose2d.py tests/test_pose2d.py
|
| 448 |
+
git commit -m "feat: Pose2DAgent — three backends (yolo/mediapipe/sapiens2), model_key dispatch"
|
| 449 |
+
git push
|
| 450 |
+
```
|
| 451 |
+
|
| 452 |
+
---
|
| 453 |
+
|
| 454 |
+
## Task 3: Add `onnxruntime` to requirements
|
| 455 |
+
|
| 456 |
+
**Files:**
|
| 457 |
+
- Modify: `requirements.txt`
|
| 458 |
+
|
| 459 |
+
- [ ] **Step 1: Add onnxruntime**
|
| 460 |
+
|
| 461 |
+
Open `requirements.txt` and add after the existing `transformers` line:
|
| 462 |
+
|
| 463 |
+
```
|
| 464 |
+
onnxruntime>=1.18
|
| 465 |
+
```
|
| 466 |
+
|
| 467 |
+
- [ ] **Step 2: Verify it installs**
|
| 468 |
+
|
| 469 |
+
```bash
|
| 470 |
+
pip install onnxruntime --quiet && python3 -c "import onnxruntime; print(onnxruntime.__version__)"
|
| 471 |
+
```
|
| 472 |
+
|
| 473 |
+
Expected: version string printed, no errors.
|
| 474 |
+
|
| 475 |
+
- [ ] **Step 3: Commit and push**
|
| 476 |
+
|
| 477 |
+
```bash
|
| 478 |
+
git add requirements.txt
|
| 479 |
+
git commit -m "chore: add onnxruntime for MediaPipe ONNX backend"
|
| 480 |
+
git push
|
| 481 |
+
```
|
| 482 |
+
|
| 483 |
+
---
|
| 484 |
+
|
| 485 |
+
## Task 4: Update `Director.run()` — `pose_model_path` → `model_key`
|
| 486 |
+
|
| 487 |
+
**Files:**
|
| 488 |
+
- Modify: `formscout/pipeline.py`
|
| 489 |
+
|
| 490 |
+
- [ ] **Step 1: Update the signature and the `pose2d` call**
|
| 491 |
+
|
| 492 |
+
In `formscout/pipeline.py`, change `Director.run()`:
|
| 493 |
+
|
| 494 |
+
```python
|
| 495 |
+
def run(self, video_path: str, test_name: str = "deep_squat", side: str = "na", model_key: str | None = None) -> PipelineState:
|
| 496 |
+
"""
|
| 497 |
+
Run the full pipeline on a single video.
|
| 498 |
+
test_name/side serve as manual override when provided (skips classifier).
|
| 499 |
+
model_key selects the pose backend (see config.POSE_MODELS).
|
| 500 |
+
"""
|
| 501 |
+
state = PipelineState(video_path=video_path)
|
| 502 |
+
|
| 503 |
+
# ─── Ingest ───
|
| 504 |
+
state.ingest = self._ingest.run(video_path)
|
| 505 |
+
if state.ingest.confidence < config.MIN_CONFIDENCE:
|
| 506 |
+
state.errors.append("ingest: low confidence — video may be corrupt")
|
| 507 |
+
return state
|
| 508 |
+
|
| 509 |
+
# ─── Pose 2D ───
|
| 510 |
+
state.pose2d = self._pose2d.run(state.ingest, model_key=model_key)
|
| 511 |
+
# ... rest of method unchanged
|
| 512 |
+
```
|
| 513 |
+
|
| 514 |
+
(Only the signature line and the `self._pose2d.run(...)` call change — everything else stays the same.)
|
| 515 |
+
|
| 516 |
+
- [ ] **Step 2: Verify import is clean**
|
| 517 |
+
|
| 518 |
+
```bash
|
| 519 |
+
python3 -c "from formscout.pipeline import Director; d = Director(); print('ok')"
|
| 520 |
+
```
|
| 521 |
+
|
| 522 |
+
Expected: `ok` (models load lazily so no crash here).
|
| 523 |
+
|
| 524 |
+
- [ ] **Step 3: Commit and push**
|
| 525 |
+
|
| 526 |
+
```bash
|
| 527 |
+
git add formscout/pipeline.py
|
| 528 |
+
git commit -m "feat: Director.run() accepts model_key, threads to Pose2DAgent"
|
| 529 |
+
git push
|
| 530 |
+
```
|
| 531 |
+
|
| 532 |
+
---
|
| 533 |
+
|
| 534 |
+
## Task 5: Wire the UI — pose model dropdown in `app.py`
|
| 535 |
+
|
| 536 |
+
**Files:**
|
| 537 |
+
- Modify: `app.py`
|
| 538 |
+
|
| 539 |
+
- [ ] **Step 1: Update `process_video` to use `model_key` and the unified registry**
|
| 540 |
+
|
| 541 |
+
Replace the existing `process_video` function signature and the old `YOLO_POSE_MODELS.get()` lookup:
|
| 542 |
+
|
| 543 |
+
```python
|
| 544 |
+
def process_video(video_path: str, test_name: str, side: str, model_key: str):
|
| 545 |
+
"""Process an uploaded video through the FormScout pipeline."""
|
| 546 |
+
if not video_path:
|
| 547 |
+
return (
|
| 548 |
+
_render_empty_state(),
|
| 549 |
+
"Upload a video to begin analysis.",
|
| 550 |
+
"",
|
| 551 |
+
"",
|
| 552 |
+
)
|
| 553 |
+
|
| 554 |
+
director = Director()
|
| 555 |
+
state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
|
| 556 |
+
```
|
| 557 |
+
|
| 558 |
+
(Remove the `pose_model_path = config.YOLO_POSE_MODELS.get(...)` line entirely.)
|
| 559 |
+
|
| 560 |
+
- [ ] **Step 2: Add the `pose_model_dropdown` in `build_app()`**
|
| 561 |
+
|
| 562 |
+
Inside `build_app()`, after the `side_dropdown` block (around line 265) and before `submit_btn`, add:
|
| 563 |
+
|
| 564 |
+
```python
|
| 565 |
+
pose_model_dropdown = gr.Dropdown(
|
| 566 |
+
choices=list(config.POSE_MODELS.keys()),
|
| 567 |
+
value=config.DEFAULT_POSE_MODEL,
|
| 568 |
+
label="Pose Model",
|
| 569 |
+
)
|
| 570 |
+
```
|
| 571 |
+
|
| 572 |
+
- [ ] **Step 3: Update `_map_inputs` to pass the model key**
|
| 573 |
+
|
| 574 |
+
Replace the existing `_map_inputs` closure:
|
| 575 |
+
|
| 576 |
+
```python
|
| 577 |
+
def _map_inputs(video, test_display_name, side_display, pose_model_key):
|
| 578 |
+
"""Map UI display values to internal values."""
|
| 579 |
+
test_map = {name: val for name, val in FMS_TESTS}
|
| 580 |
+
test_name = test_map.get(test_display_name, "deep_squat")
|
| 581 |
+
side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
|
| 582 |
+
return process_video(video, test_name, side, pose_model_key)
|
| 583 |
+
```
|
| 584 |
+
|
| 585 |
+
- [ ] **Step 4: Update `submit_btn.click` to include `pose_model_dropdown`**
|
| 586 |
+
|
| 587 |
+
Replace the existing `.click(...)` call:
|
| 588 |
+
|
| 589 |
+
```python
|
| 590 |
+
submit_btn.click(
|
| 591 |
+
fn=_map_inputs,
|
| 592 |
+
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown],
|
| 593 |
+
outputs=[score_html, pipeline_md, score_details, alerts_md],
|
| 594 |
+
)
|
| 595 |
+
```
|
| 596 |
+
|
| 597 |
+
- [ ] **Step 5: Smoke-test the app starts**
|
| 598 |
+
|
| 599 |
+
```bash
|
| 600 |
+
python3 -c "from app import build_app; app = build_app(); print('app built ok')"
|
| 601 |
+
```
|
| 602 |
+
|
| 603 |
+
Expected: `app built ok` — no import or config errors.
|
| 604 |
+
|
| 605 |
+
- [ ] **Step 6: Commit and push**
|
| 606 |
+
|
| 607 |
+
```bash
|
| 608 |
+
git add app.py
|
| 609 |
+
git commit -m "feat: pose model dropdown in UI, wired through process_video → Director"
|
| 610 |
+
git push
|
| 611 |
+
```
|
| 612 |
+
|
| 613 |
+
---
|
| 614 |
+
|
| 615 |
+
## Task 6: Add mocked backend tests
|
| 616 |
+
|
| 617 |
+
**Files:**
|
| 618 |
+
- Modify: `tests/test_pose2d.py`
|
| 619 |
+
|
| 620 |
+
- [ ] **Step 1: Add mocked YOLO test**
|
| 621 |
+
|
| 622 |
+
Append to `tests/test_pose2d.py`:
|
| 623 |
+
|
| 624 |
+
```python
|
| 625 |
+
import unittest.mock as mock
|
| 626 |
+
import numpy as np
|
| 627 |
+
from formscout.types import IngestResult, Pose2DResult
|
| 628 |
+
|
| 629 |
+
|
| 630 |
+
def _blank_ingest_3():
|
| 631 |
+
frames = [np.zeros((480, 640, 3), dtype=np.uint8) for _ in range(3)]
|
| 632 |
+
return IngestResult(frames=frames, fps=30.0, duration=0.1, n_people=1, width=640, height=480)
|
| 633 |
+
|
| 634 |
+
|
| 635 |
+
class TestPose2DBackendsMocked:
|
| 636 |
+
"""Backend dispatch tests — no real model downloads."""
|
| 637 |
+
|
| 638 |
+
def test_yolo_backend_dispatches(self):
|
| 639 |
+
from formscout.agents.pose2d import Pose2DAgent, _run_yolo
|
| 640 |
+
fake_kps = [{0: {"x": 10.0, "y": 20.0, "conf": 0.9}} for _ in range(3)]
|
| 641 |
+
with mock.patch("formscout.agents.pose2d._run_yolo", return_value=fake_kps) as m:
|
| 642 |
+
agent = Pose2DAgent()
|
| 643 |
+
result = agent.run(_blank_ingest_3(), model_key="YOLO26n — nano (0.7M, fastest)")
|
| 644 |
+
m.assert_called_once()
|
| 645 |
+
assert isinstance(result, Pose2DResult)
|
| 646 |
+
assert len(result.keypoints) == 3
|
| 647 |
+
assert result.confidence > 0.0
|
| 648 |
+
|
| 649 |
+
def test_mediapipe_backend_dispatches(self):
|
| 650 |
+
from formscout.agents.pose2d import Pose2DAgent
|
| 651 |
+
fake_kps = [{i: {"x": float(i), "y": float(i), "conf": 0.8} for i in range(17)} for _ in range(3)]
|
| 652 |
+
with mock.patch("formscout.agents.pose2d._run_mediapipe", return_value=fake_kps) as m:
|
| 653 |
+
agent = Pose2DAgent()
|
| 654 |
+
result = agent.run(_blank_ingest_3(), model_key="MediaPipe-Pose ⬇ ~16 MB, CPU-friendly")
|
| 655 |
+
m.assert_called_once()
|
| 656 |
+
assert isinstance(result, Pose2DResult)
|
| 657 |
+
assert len(result.keypoints) == 3
|
| 658 |
+
assert all(len(f) == 17 for f in result.keypoints)
|
| 659 |
+
|
| 660 |
+
def test_sapiens2_backend_dispatches(self):
|
| 661 |
+
from formscout.agents.pose2d import Pose2DAgent
|
| 662 |
+
fake_kps = [{i: {"x": float(i), "y": float(i), "conf": 0.85} for i in range(17)} for _ in range(3)]
|
| 663 |
+
with mock.patch("formscout.agents.pose2d._run_sapiens2", return_value=fake_kps) as m:
|
| 664 |
+
agent = Pose2DAgent()
|
| 665 |
+
result = agent.run(_blank_ingest_3(), model_key="Sapiens2-0.4B ⬇ ~1.6 GB")
|
| 666 |
+
m.assert_called_once()
|
| 667 |
+
assert isinstance(result, Pose2DResult)
|
| 668 |
+
assert len(result.keypoints) == 3
|
| 669 |
+
|
| 670 |
+
def test_unknown_model_key_falls_back(self):
|
| 671 |
+
from formscout.agents.pose2d import Pose2DAgent
|
| 672 |
+
fake_kps = [{0: {"x": 1.0, "y": 2.0, "conf": 0.7}} for _ in range(3)]
|
| 673 |
+
with mock.patch("formscout.agents.pose2d._run_yolo", return_value=fake_kps):
|
| 674 |
+
agent = Pose2DAgent()
|
| 675 |
+
result = agent.run(_blank_ingest_3(), model_key="nonexistent-model-xyz")
|
| 676 |
+
assert isinstance(result, Pose2DResult) # graceful fallback, no crash
|
| 677 |
+
|
| 678 |
+
def test_confidence_zero_on_empty_keypoints(self):
|
| 679 |
+
from formscout.agents.pose2d import Pose2DAgent
|
| 680 |
+
with mock.patch("formscout.agents.pose2d._run_yolo", return_value=[{}, {}, {}]):
|
| 681 |
+
agent = Pose2DAgent()
|
| 682 |
+
result = agent.run(_blank_ingest_3(), model_key="YOLO26n — nano (0.7M, fastest)")
|
| 683 |
+
assert result.confidence == 0.0
|
| 684 |
+
assert "no person" in result.notes.lower()
|
| 685 |
+
```
|
| 686 |
+
|
| 687 |
+
- [ ] **Step 2: Run the new tests**
|
| 688 |
+
|
| 689 |
+
```bash
|
| 690 |
+
pytest tests/test_pose2d.py::TestPose2DBackendsMocked -v
|
| 691 |
+
```
|
| 692 |
+
|
| 693 |
+
Expected: all 5 tests PASS.
|
| 694 |
+
|
| 695 |
+
- [ ] **Step 3: Run the full test suite to check for regressions**
|
| 696 |
+
|
| 697 |
+
```bash
|
| 698 |
+
pytest tests/ -v --tb=short 2>&1 | tail -30
|
| 699 |
+
```
|
| 700 |
+
|
| 701 |
+
Expected: same pass/fail ratio as before (45/46 known passing). The one known failure (`test_unimplemented_test_returns_low_confidence`) is pre-existing — ignore it.
|
| 702 |
+
|
| 703 |
+
- [ ] **Step 4: Commit and push**
|
| 704 |
+
|
| 705 |
+
```bash
|
| 706 |
+
git add tests/test_pose2d.py
|
| 707 |
+
git commit -m "test: mocked backend dispatch tests for YOLO, MediaPipe, Sapiens2"
|
| 708 |
+
git push
|
| 709 |
+
```
|
| 710 |
+
|
| 711 |
+
---
|
| 712 |
+
|
| 713 |
+
## Self-review
|
| 714 |
+
|
| 715 |
+
**Spec coverage:**
|
| 716 |
+
- ✅ Unified `POSE_MODELS` registry (Task 1)
|
| 717 |
+
- ✅ `DEFAULT_POSE_MODEL = YOLO26n` (Task 1)
|
| 718 |
+
- ✅ Backward-compat `YOLO_POSE_MODEL` / `YOLO_POSE_MODEL_HQ` aliases (Task 1)
|
| 719 |
+
- ✅ `_run_yolo` sub-runner (Task 2)
|
| 720 |
+
- ✅ `_run_mediapipe` with ONNX Runtime + BlazePose→COCO-17 mapping (Task 2)
|
| 721 |
+
- ✅ `_run_sapiens2` with transformers pipeline + named-keypoint→COCO-17 mapping (Task 2)
|
| 722 |
+
- ✅ `Pose2DAgent.run(model_key)` dispatch + fallback on unknown key (Task 2)
|
| 723 |
+
- ✅ `onnxruntime` added to requirements (Task 3)
|
| 724 |
+
- ✅ `Director.run(model_key)` threads key to agent (Task 4)
|
| 725 |
+
- ✅ `pose_model_dropdown` in UI (Task 5)
|
| 726 |
+
- ✅ `_map_inputs` + `submit_btn.click` wired (Task 5)
|
| 727 |
+
- ✅ Error handling: unknown key → warning + fallback; download failure → confidence=0 (Task 2)
|
| 728 |
+
- ✅ Mocked tests for all three backends (Task 6)
|
| 729 |
+
|
| 730 |
+
**Placeholder scan:** None found.
|
| 731 |
+
|
| 732 |
+
**Type consistency:** `model_key: str | None` used consistently across `Pose2DAgent.run`, `Director.run`, `process_video`. `config.POSE_MODELS` and `config.DEFAULT_POSE_MODEL` referenced consistently.
|
| 733 |
+
|
| 734 |
+
**Note on Sapiens2 keypoint format:** The `_run_sapiens2` implementation uses **named keypoint lookup** (by label string) rather than assuming fixed indices 0–16 = COCO. This is the safe approach — the transformers pipeline returns labeled keypoints and the code maps by name. If the pipeline returns unnamed keypoints (index-only), the `kp_lookup` will be empty and the frame will gracefully return `{}`.
|
docs/superpowers/plans/2026-06-09-pose-visualizer.md
CHANGED
|
@@ -1,914 +1,914 @@
|
|
| 1 |
-
# Pose Overlay Visualizer Implementation Plan
|
| 2 |
-
|
| 3 |
-
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
| 4 |
-
|
| 5 |
-
**Goal:** Add a pose overlay video output to FormScout with skeleton, motion trails, and velocity arrows, plus a per-joint velocity summary table.
|
| 6 |
-
|
| 7 |
-
**Architecture:** A new `formscout/agents/visualizer.py` runs after `director.run()` in `process_video()`; it uses Kalman-filtered per-joint velocity and OpenCV rendering. `app.py` gains a `gr.CheckboxGroup` for layer selection, a new `gr.Video` output tab, and a `gr.Markdown` velocity summary.
|
| 8 |
-
|
| 9 |
-
**Tech Stack:** `opencv-python`, `numpy`, `colorsys` (stdlib), `gradio`.
|
| 10 |
-
|
| 11 |
-
---
|
| 12 |
-
|
| 13 |
-
## File map
|
| 14 |
-
|
| 15 |
-
| File | Change |
|
| 16 |
-
|---|---|
|
| 17 |
-
| `formscout/agents/visualizer.py` | Create — Kalman filter, velocity, PoseVisualizer, summary |
|
| 18 |
-
| `tests/test_visualizer.py` | Create — all visualizer tests |
|
| 19 |
-
| `app.py` | Modify — overlay_layers checkbox, new tab, wiring |
|
| 20 |
-
|
| 21 |
-
---
|
| 22 |
-
|
| 23 |
-
## Task 1: `SimpleKalmanFilter` + `compute_joint_velocity`
|
| 24 |
-
|
| 25 |
-
**Files:**
|
| 26 |
-
- Create: `formscout/agents/visualizer.py`
|
| 27 |
-
- Create: `tests/test_visualizer.py`
|
| 28 |
-
|
| 29 |
-
- [ ] **Step 1: Write failing tests**
|
| 30 |
-
|
| 31 |
-
Create `tests/test_visualizer.py`:
|
| 32 |
-
|
| 33 |
-
```python
|
| 34 |
-
"""Tests for PoseVisualizer — no GPU, no model downloads."""
|
| 35 |
-
import numpy as np
|
| 36 |
-
import pytest
|
| 37 |
-
from formscout.types import IngestResult, Pose2DResult
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
def _make_ingest(n=5, h=480, w=640, fps=30.0):
|
| 41 |
-
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 42 |
-
return IngestResult(frames=frames, fps=fps, duration=n/fps, n_people=1, width=w, height=h)
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
def _make_pose(n=5, w=640, h=480):
|
| 46 |
-
"""Synthetic Pose2DResult: 17 joints at fixed pixel positions, conf=0.9."""
|
| 47 |
-
kps_per_frame = []
|
| 48 |
-
for i in range(n):
|
| 49 |
-
frame_kps = {}
|
| 50 |
-
for j in range(17):
|
| 51 |
-
frame_kps[j] = {
|
| 52 |
-
"x": float(50 + j * 30 + i * 2), # slight movement each frame
|
| 53 |
-
"y": float(100 + j * 20),
|
| 54 |
-
"conf": 0.9,
|
| 55 |
-
}
|
| 56 |
-
kps_per_frame.append(frame_kps)
|
| 57 |
-
return Pose2DResult(keypoints=kps_per_frame, fps=30.0, confidence=0.9, notes="")
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
class TestComputeJointVelocity:
|
| 61 |
-
def test_returns_17_joints(self):
|
| 62 |
-
from formscout.agents.visualizer import compute_joint_velocity
|
| 63 |
-
pose = _make_pose(n=5)
|
| 64 |
-
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 65 |
-
assert len(result) == 17
|
| 66 |
-
|
| 67 |
-
def test_each_list_has_n_frames(self):
|
| 68 |
-
from formscout.agents.visualizer import compute_joint_velocity
|
| 69 |
-
pose = _make_pose(n=5)
|
| 70 |
-
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 71 |
-
for joint_idx, speeds in result.items():
|
| 72 |
-
assert len(speeds) == 5, f"joint {joint_idx} has {len(speeds)} speeds, expected 5"
|
| 73 |
-
|
| 74 |
-
def test_speeds_are_non_negative(self):
|
| 75 |
-
from formscout.agents.visualizer import compute_joint_velocity
|
| 76 |
-
pose = _make_pose(n=5)
|
| 77 |
-
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 78 |
-
for speeds in result.values():
|
| 79 |
-
assert all(s >= 0.0 for s in speeds)
|
| 80 |
-
|
| 81 |
-
def test_missing_keypoints_give_zero_speed(self):
|
| 82 |
-
from formscout.agents.visualizer import compute_joint_velocity
|
| 83 |
-
# All frames empty
|
| 84 |
-
empty_kps = [{} for _ in range(5)]
|
| 85 |
-
result = compute_joint_velocity(empty_kps, fps=30.0)
|
| 86 |
-
for speeds in result.values():
|
| 87 |
-
assert all(s == 0.0 for s in speeds)
|
| 88 |
-
```
|
| 89 |
-
|
| 90 |
-
- [ ] **Step 2: Run to confirm failure**
|
| 91 |
-
|
| 92 |
-
```bash
|
| 93 |
-
pytest tests/test_visualizer.py::TestComputeJointVelocity -v
|
| 94 |
-
```
|
| 95 |
-
|
| 96 |
-
Expected: `ERROR` — `ModuleNotFoundError: No module named 'formscout.agents.visualizer'`
|
| 97 |
-
|
| 98 |
-
- [ ] **Step 3: Create `formscout/agents/visualizer.py` with Kalman + velocity**
|
| 99 |
-
|
| 100 |
-
```python
|
| 101 |
-
"""
|
| 102 |
-
PoseVisualizer — annotated overlay video with skeleton, trails, velocity arrows.
|
| 103 |
-
|
| 104 |
-
Input: IngestResult + Pose2DResult
|
| 105 |
-
Output: .mp4 path (or None on failure/empty layers)
|
| 106 |
-
Failure: returns None, never raises.
|
| 107 |
-
"""
|
| 108 |
-
from __future__ import annotations
|
| 109 |
-
|
| 110 |
-
import colorsys
|
| 111 |
-
import logging
|
| 112 |
-
import math
|
| 113 |
-
import tempfile
|
| 114 |
-
from collections import deque
|
| 115 |
-
|
| 116 |
-
import cv2
|
| 117 |
-
import numpy as np
|
| 118 |
-
|
| 119 |
-
logger = logging.getLogger(__name__)
|
| 120 |
-
|
| 121 |
-
# ── COCO constants ────────────────────────────────────────────────────────────
|
| 122 |
-
|
| 123 |
-
COCO_KEYPOINTS = [
|
| 124 |
-
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 125 |
-
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 126 |
-
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 127 |
-
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 128 |
-
]
|
| 129 |
-
|
| 130 |
-
COCO_SKELETON = [
|
| 131 |
-
(0, 1), (0, 2), (1, 3), (2, 4), # face
|
| 132 |
-
(5, 6), (5, 7), (7, 9), (6, 8), (8, 10), # arms
|
| 133 |
-
(5, 11), (6, 12), (11, 12), # torso
|
| 134 |
-
(11, 13), (13, 15), (12, 14), (14, 16), # legs
|
| 135 |
-
]
|
| 136 |
-
|
| 137 |
-
TRAIL_LENGTH = 10
|
| 138 |
-
MAX_ARROW_PX = 40
|
| 139 |
-
CONF_THRESHOLD = 0.3
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
# ── Kalman filter ─────────────────────────────────────────────────────────────
|
| 143 |
-
|
| 144 |
-
class SimpleKalmanFilter:
|
| 145 |
-
"""4-state Kalman filter (x, y, vx, vy) for joint tracking."""
|
| 146 |
-
|
| 147 |
-
def __init__(self, process_noise: float = 0.01, measurement_noise: float = 0.1):
|
| 148 |
-
self.is_initialized = False
|
| 149 |
-
self.state = np.zeros(4)
|
| 150 |
-
self.cov = np.eye(4) * 0.1
|
| 151 |
-
self.Q = np.eye(4) * process_noise
|
| 152 |
-
self.R = np.eye(2) * measurement_noise
|
| 153 |
-
self.H = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], dtype=float)
|
| 154 |
-
|
| 155 |
-
def predict(self, dt: float = 1.0):
|
| 156 |
-
F = np.array([[1, 0, dt, 0], [0, 1, 0, dt], [0, 0, 1, 0], [0, 0, 0, 1]], dtype=float)
|
| 157 |
-
self.state = F @ self.state
|
| 158 |
-
self.cov = F @ self.cov @ F.T + self.Q
|
| 159 |
-
|
| 160 |
-
def update(self, x: float, y: float):
|
| 161 |
-
z = np.array([x, y])
|
| 162 |
-
if not self.is_initialized:
|
| 163 |
-
self.state[:2] = z
|
| 164 |
-
self.is_initialized = True
|
| 165 |
-
return
|
| 166 |
-
S = self.H @ self.cov @ self.H.T + self.R
|
| 167 |
-
K = self.cov @ self.H.T @ np.linalg.inv(S)
|
| 168 |
-
self.state = self.state + K @ (z - self.H @ self.state)
|
| 169 |
-
self.cov = (np.eye(4) - K @ self.H) @ self.cov
|
| 170 |
-
|
| 171 |
-
def velocity_magnitude(self) -> float:
|
| 172 |
-
vx, vy = self.state[2], self.state[3]
|
| 173 |
-
return math.sqrt(vx * vx + vy * vy)
|
| 174 |
-
|
| 175 |
-
def velocity_vector(self) -> tuple[float, float]:
|
| 176 |
-
return float(self.state[2]), float(self.state[3])
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
# ── Velocity computation ──────────────────────────────────────────────────────
|
| 180 |
-
|
| 181 |
-
def compute_joint_velocity(
|
| 182 |
-
keypoints_per_frame: list[dict],
|
| 183 |
-
fps: float,
|
| 184 |
-
) -> dict[int, list[float]]:
|
| 185 |
-
"""
|
| 186 |
-
Compute Kalman-filtered per-joint speed (px/s) for each frame.
|
| 187 |
-
|
| 188 |
-
Returns dict[joint_idx, [speed_frame0, speed_frame1, ...]] for all 17 COCO joints.
|
| 189 |
-
Missing/low-confidence keypoints yield speed=0.0 for that frame.
|
| 190 |
-
"""
|
| 191 |
-
dt = 1.0 / fps if fps > 0 else 1.0
|
| 192 |
-
filters: dict[int, SimpleKalmanFilter] = {j: SimpleKalmanFilter() for j in range(17)}
|
| 193 |
-
result: dict[int, list[float]] = {j: [] for j in range(17)}
|
| 194 |
-
|
| 195 |
-
for frame_kps in keypoints_per_frame:
|
| 196 |
-
for j in range(17):
|
| 197 |
-
kf = filters[j]
|
| 198 |
-
kp = frame_kps.get(j)
|
| 199 |
-
kf.predict(dt)
|
| 200 |
-
if kp and kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 201 |
-
kf.update(kp["x"], kp["y"])
|
| 202 |
-
speed = kf.velocity_magnitude()
|
| 203 |
-
else:
|
| 204 |
-
speed = 0.0
|
| 205 |
-
result[j].append(speed)
|
| 206 |
-
|
| 207 |
-
return result
|
| 208 |
-
```
|
| 209 |
-
|
| 210 |
-
- [ ] **Step 4: Run tests**
|
| 211 |
-
|
| 212 |
-
```bash
|
| 213 |
-
pytest tests/test_visualizer.py::TestComputeJointVelocity -v
|
| 214 |
-
```
|
| 215 |
-
|
| 216 |
-
Expected: 4 PASS
|
| 217 |
-
|
| 218 |
-
- [ ] **Step 5: Commit**
|
| 219 |
-
|
| 220 |
-
```bash
|
| 221 |
-
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 222 |
-
git commit -m "feat: SimpleKalmanFilter + compute_joint_velocity (4 tests pass)"
|
| 223 |
-
```
|
| 224 |
-
|
| 225 |
-
---
|
| 226 |
-
|
| 227 |
-
## Task 2: `PoseVisualizer._draw_skeleton`
|
| 228 |
-
|
| 229 |
-
**Files:**
|
| 230 |
-
- Modify: `formscout/agents/visualizer.py`
|
| 231 |
-
- Modify: `tests/test_visualizer.py`
|
| 232 |
-
|
| 233 |
-
- [ ] **Step 1: Write failing test**
|
| 234 |
-
|
| 235 |
-
Append to `tests/test_visualizer.py`:
|
| 236 |
-
|
| 237 |
-
```python
|
| 238 |
-
class TestDrawSkeleton:
|
| 239 |
-
def test_skeleton_draws_without_error(self):
|
| 240 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 241 |
-
vis = PoseVisualizer()
|
| 242 |
-
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 243 |
-
kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
|
| 244 |
-
for j in range(17)}
|
| 245 |
-
result = vis._draw_skeleton(frame.copy(), kps)
|
| 246 |
-
assert result.shape == frame.shape
|
| 247 |
-
# Frame must be modified (not all zeros after drawing)
|
| 248 |
-
assert not np.array_equal(result, frame)
|
| 249 |
-
|
| 250 |
-
def test_low_confidence_keypoints_not_drawn(self):
|
| 251 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 252 |
-
vis = PoseVisualizer()
|
| 253 |
-
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 254 |
-
# All keypoints below threshold
|
| 255 |
-
kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.1} for j in range(17)}
|
| 256 |
-
result = vis._draw_skeleton(frame.copy(), kps)
|
| 257 |
-
# Nothing drawn — frame stays all zeros
|
| 258 |
-
assert np.array_equal(result, frame)
|
| 259 |
-
```
|
| 260 |
-
|
| 261 |
-
- [ ] **Step 2: Run to confirm failure**
|
| 262 |
-
|
| 263 |
-
```bash
|
| 264 |
-
pytest tests/test_visualizer.py::TestDrawSkeleton -v
|
| 265 |
-
```
|
| 266 |
-
|
| 267 |
-
Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute '_draw_skeleton'`
|
| 268 |
-
|
| 269 |
-
- [ ] **Step 3: Add `PoseVisualizer` class with `_draw_skeleton` to `visualizer.py`**
|
| 270 |
-
|
| 271 |
-
Append after `compute_joint_velocity`:
|
| 272 |
-
|
| 273 |
-
```python
|
| 274 |
-
# ── Helpers ───────────────────────────────────────────────────────────────────
|
| 275 |
-
|
| 276 |
-
def _conf_to_bgr(conf: float) -> tuple[int, int, int]:
|
| 277 |
-
"""Map confidence 0→1 to BGR color red→green via HSV."""
|
| 278 |
-
hue = conf * 120.0 / 360.0
|
| 279 |
-
r, g, b = colorsys.hsv_to_rgb(hue, 1.0, 1.0)
|
| 280 |
-
return (int(b * 255), int(g * 255), int(r * 255))
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
# ── PoseVisualizer ────────────────────────────────────────────────────────────
|
| 284 |
-
|
| 285 |
-
class PoseVisualizer:
|
| 286 |
-
"""Renders skeleton, trails, and velocity arrows onto video frames."""
|
| 287 |
-
|
| 288 |
-
def __init__(self):
|
| 289 |
-
self.last_velocities: dict[int, list[float]] = {}
|
| 290 |
-
|
| 291 |
-
# ── Skeleton ──────────────────────────────────────────────────────────────
|
| 292 |
-
|
| 293 |
-
def _draw_skeleton(self, frame: np.ndarray, kps: dict) -> np.ndarray:
|
| 294 |
-
"""Draw COCO-17 bones (white) and joints (confidence-colored) onto frame."""
|
| 295 |
-
visible = {j: kp for j, kp in kps.items() if kp.get("conf", 0.0) >= CONF_THRESHOLD}
|
| 296 |
-
|
| 297 |
-
# Bones
|
| 298 |
-
for j1, j2 in COCO_SKELETON:
|
| 299 |
-
if j1 in visible and j2 in visible:
|
| 300 |
-
p1 = (int(visible[j1]["x"]), int(visible[j1]["y"]))
|
| 301 |
-
p2 = (int(visible[j2]["x"]), int(visible[j2]["y"]))
|
| 302 |
-
cv2.line(frame, p1, p2, (255, 255, 255), 2)
|
| 303 |
-
|
| 304 |
-
# Joints
|
| 305 |
-
for j, kp in visible.items():
|
| 306 |
-
pt = (int(kp["x"]), int(kp["y"]))
|
| 307 |
-
color = _conf_to_bgr(kp["conf"])
|
| 308 |
-
cv2.circle(frame, pt, 4, color, -1)
|
| 309 |
-
cv2.circle(frame, pt, 5, (255, 255, 255), 1)
|
| 310 |
-
|
| 311 |
-
return frame
|
| 312 |
-
```
|
| 313 |
-
|
| 314 |
-
- [ ] **Step 4: Run tests**
|
| 315 |
-
|
| 316 |
-
```bash
|
| 317 |
-
pytest tests/test_visualizer.py::TestDrawSkeleton -v
|
| 318 |
-
```
|
| 319 |
-
|
| 320 |
-
Expected: 2 PASS
|
| 321 |
-
|
| 322 |
-
- [ ] **Step 5: Commit**
|
| 323 |
-
|
| 324 |
-
```bash
|
| 325 |
-
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 326 |
-
git commit -m "feat: PoseVisualizer._draw_skeleton with confidence-colored joints"
|
| 327 |
-
```
|
| 328 |
-
|
| 329 |
-
---
|
| 330 |
-
|
| 331 |
-
## Task 3: `PoseVisualizer._draw_trails`
|
| 332 |
-
|
| 333 |
-
**Files:**
|
| 334 |
-
- Modify: `formscout/agents/visualizer.py`
|
| 335 |
-
- Modify: `tests/test_visualizer.py`
|
| 336 |
-
|
| 337 |
-
- [ ] **Step 1: Write failing test**
|
| 338 |
-
|
| 339 |
-
Append to `tests/test_visualizer.py`:
|
| 340 |
-
|
| 341 |
-
```python
|
| 342 |
-
class TestDrawTrails:
|
| 343 |
-
def test_trails_draw_without_error(self):
|
| 344 |
-
from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
|
| 345 |
-
from collections import deque
|
| 346 |
-
vis = PoseVisualizer()
|
| 347 |
-
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 348 |
-
# Build a trail history for joint 0 with 5 positions
|
| 349 |
-
trail_history = {
|
| 350 |
-
0: deque([(100 + i * 5, 200 + i * 3) for i in range(5)], maxlen=TRAIL_LENGTH)
|
| 351 |
-
}
|
| 352 |
-
result = vis._draw_trails(frame.copy(), trail_history)
|
| 353 |
-
assert result.shape == frame.shape
|
| 354 |
-
# Trail should modify at least some pixels
|
| 355 |
-
assert not np.array_equal(result, frame)
|
| 356 |
-
|
| 357 |
-
def test_short_trail_no_crash(self):
|
| 358 |
-
from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
|
| 359 |
-
from collections import deque
|
| 360 |
-
vis = PoseVisualizer()
|
| 361 |
-
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 362 |
-
# Only one point — no line possible
|
| 363 |
-
trail_history = {0: deque([(100, 200)], maxlen=TRAIL_LENGTH)}
|
| 364 |
-
result = vis._draw_trails(frame.copy(), trail_history)
|
| 365 |
-
# No crash, frame unchanged (single point = no segment)
|
| 366 |
-
assert np.array_equal(result, frame)
|
| 367 |
-
```
|
| 368 |
-
|
| 369 |
-
- [ ] **Step 2: Run to confirm failure**
|
| 370 |
-
|
| 371 |
-
```bash
|
| 372 |
-
pytest tests/test_visualizer.py::TestDrawTrails -v
|
| 373 |
-
```
|
| 374 |
-
|
| 375 |
-
Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute '_draw_trails'`
|
| 376 |
-
|
| 377 |
-
- [ ] **Step 3: Add `_draw_trails` to `PoseVisualizer`**
|
| 378 |
-
|
| 379 |
-
Inside the `PoseVisualizer` class, after `_draw_skeleton`:
|
| 380 |
-
|
| 381 |
-
```python
|
| 382 |
-
# ── Trails ───────────────────────────────────────────────────────────────
|
| 383 |
-
|
| 384 |
-
def _draw_trails(self, frame: np.ndarray, trail_history: dict) -> np.ndarray:
|
| 385 |
-
"""Draw fading motion trails for each joint."""
|
| 386 |
-
for joint_idx, trail in trail_history.items():
|
| 387 |
-
pts = list(trail)
|
| 388 |
-
if len(pts) < 2:
|
| 389 |
-
continue
|
| 390 |
-
for i in range(1, len(pts)):
|
| 391 |
-
alpha = i / len(pts)
|
| 392 |
-
brightness = int(255 * alpha)
|
| 393 |
-
color = (brightness, brightness, brightness)
|
| 394 |
-
thickness = max(1, int(3 * alpha))
|
| 395 |
-
p1 = (int(pts[i - 1][0]), int(pts[i - 1][1]))
|
| 396 |
-
p2 = (int(pts[i][0]), int(pts[i][1]))
|
| 397 |
-
cv2.line(frame, p1, p2, color, thickness)
|
| 398 |
-
return frame
|
| 399 |
-
```
|
| 400 |
-
|
| 401 |
-
- [ ] **Step 4: Run tests**
|
| 402 |
-
|
| 403 |
-
```bash
|
| 404 |
-
pytest tests/test_visualizer.py::TestDrawTrails -v
|
| 405 |
-
```
|
| 406 |
-
|
| 407 |
-
Expected: 2 PASS
|
| 408 |
-
|
| 409 |
-
- [ ] **Step 5: Commit**
|
| 410 |
-
|
| 411 |
-
```bash
|
| 412 |
-
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 413 |
-
git commit -m "feat: PoseVisualizer._draw_trails with fading alpha"
|
| 414 |
-
```
|
| 415 |
-
|
| 416 |
-
---
|
| 417 |
-
|
| 418 |
-
## Task 4: `PoseVisualizer._draw_velocity_arrows`
|
| 419 |
-
|
| 420 |
-
**Files:**
|
| 421 |
-
- Modify: `formscout/agents/visualizer.py`
|
| 422 |
-
- Modify: `tests/test_visualizer.py`
|
| 423 |
-
|
| 424 |
-
- [ ] **Step 1: Write failing test**
|
| 425 |
-
|
| 426 |
-
Append to `tests/test_visualizer.py`:
|
| 427 |
-
|
| 428 |
-
```python
|
| 429 |
-
class TestDrawVelocityArrows:
|
| 430 |
-
def test_arrows_draw_without_error(self):
|
| 431 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 432 |
-
vis = PoseVisualizer()
|
| 433 |
-
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 434 |
-
kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
|
| 435 |
-
for j in range(17)}
|
| 436 |
-
prev_kps = {j: {"x": float(48 + j * 30), "y": float(98 + j * 20), "conf": 0.9}
|
| 437 |
-
for j in range(17)}
|
| 438 |
-
# velocities: joint 5 moving fast
|
| 439 |
-
velocities = {j: [0.0] * 5 for j in range(17)}
|
| 440 |
-
velocities[5] = [0.0, 10.0, 50.0, 80.0, 120.0]
|
| 441 |
-
result = vis._draw_velocity_arrows(frame.copy(), kps, prev_kps, velocities, frame_idx=4)
|
| 442 |
-
assert result.shape == frame.shape
|
| 443 |
-
|
| 444 |
-
def test_no_prev_kps_no_crash(self):
|
| 445 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 446 |
-
vis = PoseVisualizer()
|
| 447 |
-
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 448 |
-
kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.9} for j in range(17)}
|
| 449 |
-
velocities = {j: [50.0] * 5 for j in range(17)}
|
| 450 |
-
# prev_kps is None — should skip without crash
|
| 451 |
-
result = vis._draw_velocity_arrows(frame.copy(), kps, None, velocities, frame_idx=0)
|
| 452 |
-
assert result.shape == frame.shape
|
| 453 |
-
```
|
| 454 |
-
|
| 455 |
-
- [ ] **Step 2: Run to confirm failure**
|
| 456 |
-
|
| 457 |
-
```bash
|
| 458 |
-
pytest tests/test_visualizer.py::TestDrawVelocityArrows -v
|
| 459 |
-
```
|
| 460 |
-
|
| 461 |
-
Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute '_draw_velocity_arrows'`
|
| 462 |
-
|
| 463 |
-
- [ ] **Step 3: Add `_draw_velocity_arrows` to `PoseVisualizer`**
|
| 464 |
-
|
| 465 |
-
Inside the `PoseVisualizer` class, after `_draw_trails`:
|
| 466 |
-
|
| 467 |
-
```python
|
| 468 |
-
# ── Velocity arrows ───────────────────────────────────────────────────────
|
| 469 |
-
|
| 470 |
-
def _draw_velocity_arrows(
|
| 471 |
-
self,
|
| 472 |
-
frame: np.ndarray,
|
| 473 |
-
kps: dict,
|
| 474 |
-
prev_kps: dict | None,
|
| 475 |
-
velocities: dict[int, list[float]],
|
| 476 |
-
frame_idx: int,
|
| 477 |
-
) -> np.ndarray:
|
| 478 |
-
"""Draw per-joint velocity arrows scaled by speed."""
|
| 479 |
-
if prev_kps is None:
|
| 480 |
-
return frame
|
| 481 |
-
|
| 482 |
-
all_speeds = [velocities[j][frame_idx] for j in range(17) if frame_idx < len(velocities.get(j, []))]
|
| 483 |
-
peak = max(all_speeds) if all_speeds else 1.0
|
| 484 |
-
if peak == 0.0:
|
| 485 |
-
return frame
|
| 486 |
-
|
| 487 |
-
for j in range(17):
|
| 488 |
-
kp = kps.get(j)
|
| 489 |
-
pk = prev_kps.get(j)
|
| 490 |
-
if not kp or not pk:
|
| 491 |
-
continue
|
| 492 |
-
if kp.get("conf", 0.0) < CONF_THRESHOLD:
|
| 493 |
-
continue
|
| 494 |
-
speeds = velocities.get(j, [])
|
| 495 |
-
if frame_idx >= len(speeds):
|
| 496 |
-
continue
|
| 497 |
-
speed = speeds[frame_idx]
|
| 498 |
-
if speed == 0.0:
|
| 499 |
-
continue
|
| 500 |
-
|
| 501 |
-
dx = kp["x"] - pk["x"]
|
| 502 |
-
dy = kp["y"] - pk["y"]
|
| 503 |
-
mag = math.sqrt(dx * dx + dy * dy)
|
| 504 |
-
if mag < 1e-6:
|
| 505 |
-
continue
|
| 506 |
-
|
| 507 |
-
# Normalize direction, scale to arrow length
|
| 508 |
-
length = min(speed / peak * MAX_ARROW_PX, MAX_ARROW_PX)
|
| 509 |
-
nx, ny = dx / mag, dy / mag
|
| 510 |
-
start = (int(kp["x"]), int(kp["y"]))
|
| 511 |
-
end = (int(kp["x"] + nx * length), int(kp["y"] + ny * length))
|
| 512 |
-
|
| 513 |
-
ratio = speed / peak
|
| 514 |
-
if ratio < 0.33:
|
| 515 |
-
color = (0, 200, 0) # green
|
| 516 |
-
elif ratio < 0.66:
|
| 517 |
-
color = (0, 140, 255) # orange
|
| 518 |
-
else:
|
| 519 |
-
color = (0, 0, 255) # red
|
| 520 |
-
|
| 521 |
-
cv2.arrowedLine(frame, start, end, color, 2, tipLength=0.35)
|
| 522 |
-
|
| 523 |
-
return frame
|
| 524 |
-
```
|
| 525 |
-
|
| 526 |
-
- [ ] **Step 4: Run tests**
|
| 527 |
-
|
| 528 |
-
```bash
|
| 529 |
-
pytest tests/test_visualizer.py::TestDrawVelocityArrows -v
|
| 530 |
-
```
|
| 531 |
-
|
| 532 |
-
Expected: 2 PASS
|
| 533 |
-
|
| 534 |
-
- [ ] **Step 5: Commit**
|
| 535 |
-
|
| 536 |
-
```bash
|
| 537 |
-
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 538 |
-
git commit -m "feat: PoseVisualizer._draw_velocity_arrows speed-colored"
|
| 539 |
-
```
|
| 540 |
-
|
| 541 |
-
---
|
| 542 |
-
|
| 543 |
-
## Task 5: `render_video` + `build_velocity_summary`
|
| 544 |
-
|
| 545 |
-
**Files:**
|
| 546 |
-
- Modify: `formscout/agents/visualizer.py`
|
| 547 |
-
- Modify: `tests/test_visualizer.py`
|
| 548 |
-
|
| 549 |
-
- [ ] **Step 1: Write failing tests**
|
| 550 |
-
|
| 551 |
-
Append to `tests/test_visualizer.py`:
|
| 552 |
-
|
| 553 |
-
```python
|
| 554 |
-
class TestRenderVideo:
|
| 555 |
-
def test_creates_mp4_file(self, tmp_path):
|
| 556 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 557 |
-
vis = PoseVisualizer()
|
| 558 |
-
ingest = _make_ingest(n=5)
|
| 559 |
-
pose = _make_pose(n=5)
|
| 560 |
-
out = str(tmp_path / "out.mp4")
|
| 561 |
-
result = vis.render_video(ingest, pose, {"skeleton"}, out)
|
| 562 |
-
assert result is not None
|
| 563 |
-
import os
|
| 564 |
-
assert os.path.exists(result)
|
| 565 |
-
assert os.path.getsize(result) > 0
|
| 566 |
-
|
| 567 |
-
def test_empty_layers_returns_none(self, tmp_path):
|
| 568 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 569 |
-
vis = PoseVisualizer()
|
| 570 |
-
out = str(tmp_path / "out.mp4")
|
| 571 |
-
result = vis.render_video(_make_ingest(), _make_pose(), set(), out)
|
| 572 |
-
assert result is None
|
| 573 |
-
|
| 574 |
-
def test_no_detections_returns_none(self, tmp_path):
|
| 575 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 576 |
-
vis = PoseVisualizer()
|
| 577 |
-
ingest = _make_ingest(n=5)
|
| 578 |
-
empty_pose = Pose2DResult(
|
| 579 |
-
keypoints=[{} for _ in range(5)], fps=30.0, confidence=0.0, notes=""
|
| 580 |
-
)
|
| 581 |
-
out = str(tmp_path / "out.mp4")
|
| 582 |
-
result = vis.render_video(ingest, empty_pose, {"skeleton"}, out)
|
| 583 |
-
assert result is None
|
| 584 |
-
|
| 585 |
-
def test_last_velocities_set_after_render(self, tmp_path):
|
| 586 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 587 |
-
vis = PoseVisualizer()
|
| 588 |
-
out = str(tmp_path / "out.mp4")
|
| 589 |
-
vis.render_video(_make_ingest(n=5), _make_pose(n=5), {"skeleton"}, out)
|
| 590 |
-
assert len(vis.last_velocities) == 17
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
class TestBuildVelocitySummary:
|
| 594 |
-
def test_returns_markdown_table(self):
|
| 595 |
-
from formscout.agents.visualizer import build_velocity_summary, compute_joint_velocity
|
| 596 |
-
pose = _make_pose(n=10)
|
| 597 |
-
vels = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 598 |
-
result = build_velocity_summary(pose.keypoints, vels)
|
| 599 |
-
assert "|" in result
|
| 600 |
-
# At least one COCO joint name appears
|
| 601 |
-
assert any(name in result for name in ["knee", "shoulder", "hip", "ankle"])
|
| 602 |
-
|
| 603 |
-
def test_empty_keypoints_returns_empty_string(self):
|
| 604 |
-
from formscout.agents.visualizer import build_velocity_summary
|
| 605 |
-
empty_kps = [{} for _ in range(5)]
|
| 606 |
-
vels = {j: [0.0] * 5 for j in range(17)}
|
| 607 |
-
result = build_velocity_summary(empty_kps, vels)
|
| 608 |
-
assert result == ""
|
| 609 |
-
```
|
| 610 |
-
|
| 611 |
-
- [ ] **Step 2: Run to confirm failure**
|
| 612 |
-
|
| 613 |
-
```bash
|
| 614 |
-
pytest tests/test_visualizer.py::TestRenderVideo tests/test_visualizer.py::TestBuildVelocitySummary -v
|
| 615 |
-
```
|
| 616 |
-
|
| 617 |
-
Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute 'render_video'`
|
| 618 |
-
|
| 619 |
-
- [ ] **Step 3: Add `render_video` to `PoseVisualizer`**
|
| 620 |
-
|
| 621 |
-
Inside the `PoseVisualizer` class, after `_draw_velocity_arrows`:
|
| 622 |
-
|
| 623 |
-
```python
|
| 624 |
-
# ── Public ────────────────────────────────────────────────────────────────
|
| 625 |
-
|
| 626 |
-
def render_video(
|
| 627 |
-
self,
|
| 628 |
-
ingest,
|
| 629 |
-
pose2d,
|
| 630 |
-
layers: set[str],
|
| 631 |
-
output_path: str,
|
| 632 |
-
) -> str | None:
|
| 633 |
-
"""
|
| 634 |
-
Render annotated video. Returns output_path on success, None otherwise.
|
| 635 |
-
layers: subset of {"skeleton", "trails", "velocity_arrows"}
|
| 636 |
-
"""
|
| 637 |
-
if not layers:
|
| 638 |
-
return None
|
| 639 |
-
|
| 640 |
-
# Require at least one detected frame
|
| 641 |
-
if not any(pose2d.keypoints):
|
| 642 |
-
return None
|
| 643 |
-
|
| 644 |
-
try:
|
| 645 |
-
velocities = compute_joint_velocity(pose2d.keypoints, ingest.fps)
|
| 646 |
-
self.last_velocities = velocities
|
| 647 |
-
|
| 648 |
-
frames = ingest.frames
|
| 649 |
-
h, w = frames[0].shape[:2]
|
| 650 |
-
fps = ingest.fps or 30.0
|
| 651 |
-
|
| 652 |
-
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
|
| 653 |
-
writer = cv2.VideoWriter(output_path, fourcc, fps, (w, h))
|
| 654 |
-
if not writer.isOpened():
|
| 655 |
-
logger.warning("VideoWriter failed to open: %s", output_path)
|
| 656 |
-
return None
|
| 657 |
-
|
| 658 |
-
trail_history: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
|
| 659 |
-
prev_kps: dict | None = None
|
| 660 |
-
|
| 661 |
-
for frame_idx, (frame, kps) in enumerate(zip(frames, pose2d.keypoints)):
|
| 662 |
-
out_frame = frame.copy()
|
| 663 |
-
|
| 664 |
-
if "trails" in layers:
|
| 665 |
-
# Update trail history before drawing
|
| 666 |
-
for j, kp in kps.items():
|
| 667 |
-
if kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 668 |
-
trail_history[j].append((kp["x"], kp["y"]))
|
| 669 |
-
out_frame = self._draw_trails(out_frame, trail_history)
|
| 670 |
-
|
| 671 |
-
if "skeleton" in layers:
|
| 672 |
-
out_frame = self._draw_skeleton(out_frame, kps)
|
| 673 |
-
|
| 674 |
-
if "velocity_arrows" in layers:
|
| 675 |
-
out_frame = self._draw_velocity_arrows(
|
| 676 |
-
out_frame, kps, prev_kps, velocities, frame_idx
|
| 677 |
-
)
|
| 678 |
-
|
| 679 |
-
writer.write(out_frame)
|
| 680 |
-
prev_kps = kps
|
| 681 |
-
|
| 682 |
-
writer.release()
|
| 683 |
-
return output_path
|
| 684 |
-
|
| 685 |
-
except Exception as e:
|
| 686 |
-
logger.warning("render_video failed: %s", e)
|
| 687 |
-
return None
|
| 688 |
-
```
|
| 689 |
-
|
| 690 |
-
- [ ] **Step 4: Add `build_velocity_summary` after the class**
|
| 691 |
-
|
| 692 |
-
After the `PoseVisualizer` class definition, add:
|
| 693 |
-
|
| 694 |
-
```python
|
| 695 |
-
# ── Velocity summary ──────────────────────────────────────────────────────────
|
| 696 |
-
|
| 697 |
-
def build_velocity_summary(
|
| 698 |
-
keypoints_per_frame: list[dict],
|
| 699 |
-
velocities: dict[int, list[float]],
|
| 700 |
-
) -> str:
|
| 701 |
-
"""Return markdown table of per-joint avg/peak velocity. Empty string if no valid joints."""
|
| 702 |
-
n_frames = len(keypoints_per_frame)
|
| 703 |
-
if n_frames == 0:
|
| 704 |
-
return ""
|
| 705 |
-
|
| 706 |
-
rows = []
|
| 707 |
-
for j in range(17):
|
| 708 |
-
# Count frames where this joint is detected
|
| 709 |
-
detected = sum(
|
| 710 |
-
1 for kps in keypoints_per_frame
|
| 711 |
-
if kps.get(j, {}).get("conf", 0.0) >= CONF_THRESHOLD
|
| 712 |
-
)
|
| 713 |
-
if detected < n_frames * 0.5:
|
| 714 |
-
continue # skip joints present in <50% of frames
|
| 715 |
-
|
| 716 |
-
speeds = velocities.get(j, [])
|
| 717 |
-
if not speeds:
|
| 718 |
-
continue
|
| 719 |
-
|
| 720 |
-
avg_speed = sum(speeds) / len(speeds)
|
| 721 |
-
peak_speed = max(speeds)
|
| 722 |
-
rows.append((COCO_KEYPOINTS[j], avg_speed, peak_speed))
|
| 723 |
-
|
| 724 |
-
if not rows:
|
| 725 |
-
return ""
|
| 726 |
-
|
| 727 |
-
rows.sort(key=lambda r: r[2], reverse=True) # sort by peak descending
|
| 728 |
-
lines = [
|
| 729 |
-
"| Joint | Avg (px/s) | Peak (px/s) |",
|
| 730 |
-
"|---|---|---|",
|
| 731 |
-
]
|
| 732 |
-
for name, avg, peak in rows:
|
| 733 |
-
lines.append(f"| {name} | {avg:.1f} | {peak:.1f} |")
|
| 734 |
-
return "\n".join(lines)
|
| 735 |
-
```
|
| 736 |
-
|
| 737 |
-
- [ ] **Step 5: Run all visualizer tests**
|
| 738 |
-
|
| 739 |
-
```bash
|
| 740 |
-
pytest tests/test_visualizer.py -v
|
| 741 |
-
```
|
| 742 |
-
|
| 743 |
-
Expected: all tests PASS (4 + 2 + 2 + 2 + 4 + 2 = 16 total)
|
| 744 |
-
|
| 745 |
-
- [ ] **Step 6: Commit**
|
| 746 |
-
|
| 747 |
-
```bash
|
| 748 |
-
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 749 |
-
git commit -m "feat: PoseVisualizer.render_video + build_velocity_summary (16 tests pass)"
|
| 750 |
-
```
|
| 751 |
-
|
| 752 |
-
---
|
| 753 |
-
|
| 754 |
-
## Task 6: Wire `app.py`
|
| 755 |
-
|
| 756 |
-
**Files:**
|
| 757 |
-
- Modify: `app.py`
|
| 758 |
-
|
| 759 |
-
- [ ] **Step 1: Add `import tempfile` if not present and import visualizer in `process_video`**
|
| 760 |
-
|
| 761 |
-
Check the top of `app.py` for `import tempfile`. If missing, add it alongside the other stdlib imports. (Look at the existing import block and add `import tempfile` there.)
|
| 762 |
-
|
| 763 |
-
- [ ] **Step 2: Update `process_video()` signature and body**
|
| 764 |
-
|
| 765 |
-
Replace the existing `process_video` function (lines 46–83) with:
|
| 766 |
-
|
| 767 |
-
```python
|
| 768 |
-
def process_video(video_path: str, test_name: str, side: str, model_key: str, layers: list[str]):
|
| 769 |
-
"""Process an uploaded video through the FormScout pipeline."""
|
| 770 |
-
if not video_path:
|
| 771 |
-
return (
|
| 772 |
-
_render_empty_state(),
|
| 773 |
-
"Upload a video to begin analysis.",
|
| 774 |
-
"",
|
| 775 |
-
"",
|
| 776 |
-
None,
|
| 777 |
-
"",
|
| 778 |
-
)
|
| 779 |
-
|
| 780 |
-
director = Director()
|
| 781 |
-
state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
|
| 782 |
-
|
| 783 |
-
# ─── Score card ───
|
| 784 |
-
score_html = _render_empty_state()
|
| 785 |
-
score_details = ""
|
| 786 |
-
|
| 787 |
-
if state.features:
|
| 788 |
-
result = score_test(state.features)
|
| 789 |
-
judge = state.judge
|
| 790 |
-
if judge and judge.score is not None:
|
| 791 |
-
score_html = _render_score_card(judge.score, judge.confidence, judge.needs_human)
|
| 792 |
-
score_details = _render_score_details_judge(judge, result, state.features)
|
| 793 |
-
elif judge and judge.needs_human:
|
| 794 |
-
score_html = _render_score_card(0, 0, True)
|
| 795 |
-
score_details = f"### Needs Clinician Review\n{judge.rationale}"
|
| 796 |
-
else:
|
| 797 |
-
score_html = _render_score_card(result.score, result.confidence, result.needs_human)
|
| 798 |
-
score_details = _render_score_details(result, state.features)
|
| 799 |
-
|
| 800 |
-
# ─── Pipeline info ───
|
| 801 |
-
pipeline_md = _render_pipeline_status(state)
|
| 802 |
-
|
| 803 |
-
# ─── Warnings/errors ───
|
| 804 |
-
alerts = _render_alerts(state)
|
| 805 |
-
|
| 806 |
-
# ─── Overlay video ───
|
| 807 |
-
overlay_path = None
|
| 808 |
-
vel_summary = ""
|
| 809 |
-
layer_set = {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
|
| 810 |
-
if layer_set and state.ingest and state.pose2d:
|
| 811 |
-
try:
|
| 812 |
-
from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
|
| 813 |
-
vis = PoseVisualizer()
|
| 814 |
-
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
|
| 815 |
-
out_path = f.name
|
| 816 |
-
overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
|
| 817 |
-
if overlay_path:
|
| 818 |
-
vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
|
| 819 |
-
except Exception as e:
|
| 820 |
-
alerts = (alerts or "") + f"\n⚠️ Visualizer error: {e}"
|
| 821 |
-
|
| 822 |
-
return score_html, pipeline_md, score_details, alerts, overlay_path, vel_summary
|
| 823 |
-
```
|
| 824 |
-
|
| 825 |
-
- [ ] **Step 3: Add `overlay_layers` CheckboxGroup in `build_app()`**
|
| 826 |
-
|
| 827 |
-
After the `pose_model_dropdown` block (around line 270), and before `submit_btn`:
|
| 828 |
-
|
| 829 |
-
```python
|
| 830 |
-
overlay_layers = gr.CheckboxGroup(
|
| 831 |
-
choices=["Skeleton", "Trails", "Velocity arrows"],
|
| 832 |
-
value=["Skeleton", "Trails"],
|
| 833 |
-
label="Overlay Layers",
|
| 834 |
-
)
|
| 835 |
-
```
|
| 836 |
-
|
| 837 |
-
- [ ] **Step 4: Add overlay tab in the results panel**
|
| 838 |
-
|
| 839 |
-
Inside the `with gr.Tabs():` block (after the `⚠️ Alerts` tab):
|
| 840 |
-
|
| 841 |
-
```python
|
| 842 |
-
with gr.TabItem("🎬 Overlay Video"):
|
| 843 |
-
overlay_video = gr.Video(label="Annotated Movement")
|
| 844 |
-
velocity_md = gr.Markdown("")
|
| 845 |
-
```
|
| 846 |
-
|
| 847 |
-
- [ ] **Step 5: Update `_map_inputs` and `submit_btn.click`**
|
| 848 |
-
|
| 849 |
-
Replace the `_map_inputs` closure and `submit_btn.click` call:
|
| 850 |
-
|
| 851 |
-
```python
|
| 852 |
-
def _map_inputs(video, test_display_name, side_display, pose_model_key, overlay_layers):
|
| 853 |
-
"""Map UI display values to internal values."""
|
| 854 |
-
test_map = {name: val for name, val in FMS_TESTS}
|
| 855 |
-
test_name = test_map.get(test_display_name, "deep_squat")
|
| 856 |
-
side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
|
| 857 |
-
return process_video(video, test_name, side, pose_model_key, overlay_layers)
|
| 858 |
-
|
| 859 |
-
submit_btn.click(
|
| 860 |
-
fn=_map_inputs,
|
| 861 |
-
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown, overlay_layers],
|
| 862 |
-
outputs=[score_html, pipeline_md, score_details, alerts_md, overlay_video, velocity_md],
|
| 863 |
-
)
|
| 864 |
-
```
|
| 865 |
-
|
| 866 |
-
- [ ] **Step 6: Smoke-test the app builds**
|
| 867 |
-
|
| 868 |
-
```bash
|
| 869 |
-
python3 -c "from app import build_app; build_app(); print('ok')"
|
| 870 |
-
```
|
| 871 |
-
|
| 872 |
-
Expected: `ok` (Gradio UserWarning about theme is fine, not an error)
|
| 873 |
-
|
| 874 |
-
- [ ] **Step 7: Run full test suite to check for regressions**
|
| 875 |
-
|
| 876 |
-
```bash
|
| 877 |
-
pytest tests/ -v --tb=short 2>&1 | tail -15
|
| 878 |
-
```
|
| 879 |
-
|
| 880 |
-
Expected: all previous tests still pass (62 passing, 1 pre-existing fail in biomechanics), plus 16 new visualizer tests = 78 passing.
|
| 881 |
-
|
| 882 |
-
- [ ] **Step 8: Commit**
|
| 883 |
-
|
| 884 |
-
```bash
|
| 885 |
-
git add app.py
|
| 886 |
-
git commit -m "feat: overlay video tab + velocity summary wired in Gradio UI"
|
| 887 |
-
```
|
| 888 |
-
|
| 889 |
-
---
|
| 890 |
-
|
| 891 |
-
## Self-review
|
| 892 |
-
|
| 893 |
-
**Spec coverage:**
|
| 894 |
-
- ✅ `SimpleKalmanFilter` 4-state (Task 1)
|
| 895 |
-
- ✅ `compute_joint_velocity` Kalman-filtered px/s (Task 1)
|
| 896 |
-
- ✅ `_draw_skeleton` COCO bones, confidence-colored joints (Task 2)
|
| 897 |
-
- ✅ `_draw_trails` fading deque-based trails (Task 3)
|
| 898 |
-
- ✅ `_draw_velocity_arrows` speed-colored, direction from consecutive frames (Task 4)
|
| 899 |
-
- ✅ `render_video` layer dispatch, trail history, VideoWriter (Task 5)
|
| 900 |
-
- ✅ `build_velocity_summary` markdown table, >50% detection filter (Task 5)
|
| 901 |
-
- ✅ `overlay_layers` CheckboxGroup in UI (Task 6)
|
| 902 |
-
- ✅ New `🎬 Overlay Video` tab with `gr.Video` + `gr.Markdown` (Task 6)
|
| 903 |
-
- ✅ `process_video` wired with layers param (Task 6)
|
| 904 |
-
- ✅ `vis.last_velocities` stored on instance after `render_video` (Task 5)
|
| 905 |
-
- ✅ Error handling: empty layers → None, empty detections → None, exception → alerts (Task 5 + 6)
|
| 906 |
-
- ✅ All 5 spec test cases covered across Tasks 1–5
|
| 907 |
-
|
| 908 |
-
**Placeholder scan:** None found. All code blocks are complete.
|
| 909 |
-
|
| 910 |
-
**Type consistency:**
|
| 911 |
-
- `compute_joint_velocity` returns `dict[int, list[float]]` — used identically in `render_video`, `_draw_velocity_arrows`, and `build_velocity_summary`. ✓
|
| 912 |
-
- `layers: set[str]` in `render_video`; converted from `list[str]` in `process_video` via set comprehension. ✓
|
| 913 |
-
- `vis.last_velocities` set in `render_video`, read in `process_video`. ✓
|
| 914 |
-
- `_draw_velocity_arrows(frame, kps, prev_kps, velocities, frame_idx)` — signature matches call in `render_video`. ✓
|
|
|
|
| 1 |
+
# Pose Overlay Visualizer Implementation Plan
|
| 2 |
+
|
| 3 |
+
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
| 4 |
+
|
| 5 |
+
**Goal:** Add a pose overlay video output to FormScout with skeleton, motion trails, and velocity arrows, plus a per-joint velocity summary table.
|
| 6 |
+
|
| 7 |
+
**Architecture:** A new `formscout/agents/visualizer.py` runs after `director.run()` in `process_video()`; it uses Kalman-filtered per-joint velocity and OpenCV rendering. `app.py` gains a `gr.CheckboxGroup` for layer selection, a new `gr.Video` output tab, and a `gr.Markdown` velocity summary.
|
| 8 |
+
|
| 9 |
+
**Tech Stack:** `opencv-python`, `numpy`, `colorsys` (stdlib), `gradio`.
|
| 10 |
+
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
## File map
|
| 14 |
+
|
| 15 |
+
| File | Change |
|
| 16 |
+
|---|---|
|
| 17 |
+
| `formscout/agents/visualizer.py` | Create — Kalman filter, velocity, PoseVisualizer, summary |
|
| 18 |
+
| `tests/test_visualizer.py` | Create — all visualizer tests |
|
| 19 |
+
| `app.py` | Modify — overlay_layers checkbox, new tab, wiring |
|
| 20 |
+
|
| 21 |
+
---
|
| 22 |
+
|
| 23 |
+
## Task 1: `SimpleKalmanFilter` + `compute_joint_velocity`
|
| 24 |
+
|
| 25 |
+
**Files:**
|
| 26 |
+
- Create: `formscout/agents/visualizer.py`
|
| 27 |
+
- Create: `tests/test_visualizer.py`
|
| 28 |
+
|
| 29 |
+
- [ ] **Step 1: Write failing tests**
|
| 30 |
+
|
| 31 |
+
Create `tests/test_visualizer.py`:
|
| 32 |
+
|
| 33 |
+
```python
|
| 34 |
+
"""Tests for PoseVisualizer — no GPU, no model downloads."""
|
| 35 |
+
import numpy as np
|
| 36 |
+
import pytest
|
| 37 |
+
from formscout.types import IngestResult, Pose2DResult
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _make_ingest(n=5, h=480, w=640, fps=30.0):
|
| 41 |
+
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 42 |
+
return IngestResult(frames=frames, fps=fps, duration=n/fps, n_people=1, width=w, height=h)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _make_pose(n=5, w=640, h=480):
|
| 46 |
+
"""Synthetic Pose2DResult: 17 joints at fixed pixel positions, conf=0.9."""
|
| 47 |
+
kps_per_frame = []
|
| 48 |
+
for i in range(n):
|
| 49 |
+
frame_kps = {}
|
| 50 |
+
for j in range(17):
|
| 51 |
+
frame_kps[j] = {
|
| 52 |
+
"x": float(50 + j * 30 + i * 2), # slight movement each frame
|
| 53 |
+
"y": float(100 + j * 20),
|
| 54 |
+
"conf": 0.9,
|
| 55 |
+
}
|
| 56 |
+
kps_per_frame.append(frame_kps)
|
| 57 |
+
return Pose2DResult(keypoints=kps_per_frame, fps=30.0, confidence=0.9, notes="")
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class TestComputeJointVelocity:
|
| 61 |
+
def test_returns_17_joints(self):
|
| 62 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 63 |
+
pose = _make_pose(n=5)
|
| 64 |
+
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 65 |
+
assert len(result) == 17
|
| 66 |
+
|
| 67 |
+
def test_each_list_has_n_frames(self):
|
| 68 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 69 |
+
pose = _make_pose(n=5)
|
| 70 |
+
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 71 |
+
for joint_idx, speeds in result.items():
|
| 72 |
+
assert len(speeds) == 5, f"joint {joint_idx} has {len(speeds)} speeds, expected 5"
|
| 73 |
+
|
| 74 |
+
def test_speeds_are_non_negative(self):
|
| 75 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 76 |
+
pose = _make_pose(n=5)
|
| 77 |
+
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 78 |
+
for speeds in result.values():
|
| 79 |
+
assert all(s >= 0.0 for s in speeds)
|
| 80 |
+
|
| 81 |
+
def test_missing_keypoints_give_zero_speed(self):
|
| 82 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 83 |
+
# All frames empty
|
| 84 |
+
empty_kps = [{} for _ in range(5)]
|
| 85 |
+
result = compute_joint_velocity(empty_kps, fps=30.0)
|
| 86 |
+
for speeds in result.values():
|
| 87 |
+
assert all(s == 0.0 for s in speeds)
|
| 88 |
+
```
|
| 89 |
+
|
| 90 |
+
- [ ] **Step 2: Run to confirm failure**
|
| 91 |
+
|
| 92 |
+
```bash
|
| 93 |
+
pytest tests/test_visualizer.py::TestComputeJointVelocity -v
|
| 94 |
+
```
|
| 95 |
+
|
| 96 |
+
Expected: `ERROR` — `ModuleNotFoundError: No module named 'formscout.agents.visualizer'`
|
| 97 |
+
|
| 98 |
+
- [ ] **Step 3: Create `formscout/agents/visualizer.py` with Kalman + velocity**
|
| 99 |
+
|
| 100 |
+
```python
|
| 101 |
+
"""
|
| 102 |
+
PoseVisualizer — annotated overlay video with skeleton, trails, velocity arrows.
|
| 103 |
+
|
| 104 |
+
Input: IngestResult + Pose2DResult
|
| 105 |
+
Output: .mp4 path (or None on failure/empty layers)
|
| 106 |
+
Failure: returns None, never raises.
|
| 107 |
+
"""
|
| 108 |
+
from __future__ import annotations
|
| 109 |
+
|
| 110 |
+
import colorsys
|
| 111 |
+
import logging
|
| 112 |
+
import math
|
| 113 |
+
import tempfile
|
| 114 |
+
from collections import deque
|
| 115 |
+
|
| 116 |
+
import cv2
|
| 117 |
+
import numpy as np
|
| 118 |
+
|
| 119 |
+
logger = logging.getLogger(__name__)
|
| 120 |
+
|
| 121 |
+
# ── COCO constants ────────────────────────────────────────────────────────────
|
| 122 |
+
|
| 123 |
+
COCO_KEYPOINTS = [
|
| 124 |
+
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 125 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 126 |
+
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 127 |
+
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 128 |
+
]
|
| 129 |
+
|
| 130 |
+
COCO_SKELETON = [
|
| 131 |
+
(0, 1), (0, 2), (1, 3), (2, 4), # face
|
| 132 |
+
(5, 6), (5, 7), (7, 9), (6, 8), (8, 10), # arms
|
| 133 |
+
(5, 11), (6, 12), (11, 12), # torso
|
| 134 |
+
(11, 13), (13, 15), (12, 14), (14, 16), # legs
|
| 135 |
+
]
|
| 136 |
+
|
| 137 |
+
TRAIL_LENGTH = 10
|
| 138 |
+
MAX_ARROW_PX = 40
|
| 139 |
+
CONF_THRESHOLD = 0.3
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
# ── Kalman filter ─────────────────────────────────────────────────────────────
|
| 143 |
+
|
| 144 |
+
class SimpleKalmanFilter:
|
| 145 |
+
"""4-state Kalman filter (x, y, vx, vy) for joint tracking."""
|
| 146 |
+
|
| 147 |
+
def __init__(self, process_noise: float = 0.01, measurement_noise: float = 0.1):
|
| 148 |
+
self.is_initialized = False
|
| 149 |
+
self.state = np.zeros(4)
|
| 150 |
+
self.cov = np.eye(4) * 0.1
|
| 151 |
+
self.Q = np.eye(4) * process_noise
|
| 152 |
+
self.R = np.eye(2) * measurement_noise
|
| 153 |
+
self.H = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], dtype=float)
|
| 154 |
+
|
| 155 |
+
def predict(self, dt: float = 1.0):
|
| 156 |
+
F = np.array([[1, 0, dt, 0], [0, 1, 0, dt], [0, 0, 1, 0], [0, 0, 0, 1]], dtype=float)
|
| 157 |
+
self.state = F @ self.state
|
| 158 |
+
self.cov = F @ self.cov @ F.T + self.Q
|
| 159 |
+
|
| 160 |
+
def update(self, x: float, y: float):
|
| 161 |
+
z = np.array([x, y])
|
| 162 |
+
if not self.is_initialized:
|
| 163 |
+
self.state[:2] = z
|
| 164 |
+
self.is_initialized = True
|
| 165 |
+
return
|
| 166 |
+
S = self.H @ self.cov @ self.H.T + self.R
|
| 167 |
+
K = self.cov @ self.H.T @ np.linalg.inv(S)
|
| 168 |
+
self.state = self.state + K @ (z - self.H @ self.state)
|
| 169 |
+
self.cov = (np.eye(4) - K @ self.H) @ self.cov
|
| 170 |
+
|
| 171 |
+
def velocity_magnitude(self) -> float:
|
| 172 |
+
vx, vy = self.state[2], self.state[3]
|
| 173 |
+
return math.sqrt(vx * vx + vy * vy)
|
| 174 |
+
|
| 175 |
+
def velocity_vector(self) -> tuple[float, float]:
|
| 176 |
+
return float(self.state[2]), float(self.state[3])
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
# ── Velocity computation ──────────────────────────────────────────────────────
|
| 180 |
+
|
| 181 |
+
def compute_joint_velocity(
|
| 182 |
+
keypoints_per_frame: list[dict],
|
| 183 |
+
fps: float,
|
| 184 |
+
) -> dict[int, list[float]]:
|
| 185 |
+
"""
|
| 186 |
+
Compute Kalman-filtered per-joint speed (px/s) for each frame.
|
| 187 |
+
|
| 188 |
+
Returns dict[joint_idx, [speed_frame0, speed_frame1, ...]] for all 17 COCO joints.
|
| 189 |
+
Missing/low-confidence keypoints yield speed=0.0 for that frame.
|
| 190 |
+
"""
|
| 191 |
+
dt = 1.0 / fps if fps > 0 else 1.0
|
| 192 |
+
filters: dict[int, SimpleKalmanFilter] = {j: SimpleKalmanFilter() for j in range(17)}
|
| 193 |
+
result: dict[int, list[float]] = {j: [] for j in range(17)}
|
| 194 |
+
|
| 195 |
+
for frame_kps in keypoints_per_frame:
|
| 196 |
+
for j in range(17):
|
| 197 |
+
kf = filters[j]
|
| 198 |
+
kp = frame_kps.get(j)
|
| 199 |
+
kf.predict(dt)
|
| 200 |
+
if kp and kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 201 |
+
kf.update(kp["x"], kp["y"])
|
| 202 |
+
speed = kf.velocity_magnitude()
|
| 203 |
+
else:
|
| 204 |
+
speed = 0.0
|
| 205 |
+
result[j].append(speed)
|
| 206 |
+
|
| 207 |
+
return result
|
| 208 |
+
```
|
| 209 |
+
|
| 210 |
+
- [ ] **Step 4: Run tests**
|
| 211 |
+
|
| 212 |
+
```bash
|
| 213 |
+
pytest tests/test_visualizer.py::TestComputeJointVelocity -v
|
| 214 |
+
```
|
| 215 |
+
|
| 216 |
+
Expected: 4 PASS
|
| 217 |
+
|
| 218 |
+
- [ ] **Step 5: Commit**
|
| 219 |
+
|
| 220 |
+
```bash
|
| 221 |
+
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 222 |
+
git commit -m "feat: SimpleKalmanFilter + compute_joint_velocity (4 tests pass)"
|
| 223 |
+
```
|
| 224 |
+
|
| 225 |
+
---
|
| 226 |
+
|
| 227 |
+
## Task 2: `PoseVisualizer._draw_skeleton`
|
| 228 |
+
|
| 229 |
+
**Files:**
|
| 230 |
+
- Modify: `formscout/agents/visualizer.py`
|
| 231 |
+
- Modify: `tests/test_visualizer.py`
|
| 232 |
+
|
| 233 |
+
- [ ] **Step 1: Write failing test**
|
| 234 |
+
|
| 235 |
+
Append to `tests/test_visualizer.py`:
|
| 236 |
+
|
| 237 |
+
```python
|
| 238 |
+
class TestDrawSkeleton:
|
| 239 |
+
def test_skeleton_draws_without_error(self):
|
| 240 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 241 |
+
vis = PoseVisualizer()
|
| 242 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 243 |
+
kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
|
| 244 |
+
for j in range(17)}
|
| 245 |
+
result = vis._draw_skeleton(frame.copy(), kps)
|
| 246 |
+
assert result.shape == frame.shape
|
| 247 |
+
# Frame must be modified (not all zeros after drawing)
|
| 248 |
+
assert not np.array_equal(result, frame)
|
| 249 |
+
|
| 250 |
+
def test_low_confidence_keypoints_not_drawn(self):
|
| 251 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 252 |
+
vis = PoseVisualizer()
|
| 253 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 254 |
+
# All keypoints below threshold
|
| 255 |
+
kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.1} for j in range(17)}
|
| 256 |
+
result = vis._draw_skeleton(frame.copy(), kps)
|
| 257 |
+
# Nothing drawn — frame stays all zeros
|
| 258 |
+
assert np.array_equal(result, frame)
|
| 259 |
+
```
|
| 260 |
+
|
| 261 |
+
- [ ] **Step 2: Run to confirm failure**
|
| 262 |
+
|
| 263 |
+
```bash
|
| 264 |
+
pytest tests/test_visualizer.py::TestDrawSkeleton -v
|
| 265 |
+
```
|
| 266 |
+
|
| 267 |
+
Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute '_draw_skeleton'`
|
| 268 |
+
|
| 269 |
+
- [ ] **Step 3: Add `PoseVisualizer` class with `_draw_skeleton` to `visualizer.py`**
|
| 270 |
+
|
| 271 |
+
Append after `compute_joint_velocity`:
|
| 272 |
+
|
| 273 |
+
```python
|
| 274 |
+
# ── Helpers ───────────────────────────────────────────────────────────────────
|
| 275 |
+
|
| 276 |
+
def _conf_to_bgr(conf: float) -> tuple[int, int, int]:
|
| 277 |
+
"""Map confidence 0→1 to BGR color red→green via HSV."""
|
| 278 |
+
hue = conf * 120.0 / 360.0
|
| 279 |
+
r, g, b = colorsys.hsv_to_rgb(hue, 1.0, 1.0)
|
| 280 |
+
return (int(b * 255), int(g * 255), int(r * 255))
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
# ── PoseVisualizer ────────────────────────────────────────────────────────────
|
| 284 |
+
|
| 285 |
+
class PoseVisualizer:
|
| 286 |
+
"""Renders skeleton, trails, and velocity arrows onto video frames."""
|
| 287 |
+
|
| 288 |
+
def __init__(self):
|
| 289 |
+
self.last_velocities: dict[int, list[float]] = {}
|
| 290 |
+
|
| 291 |
+
# ── Skeleton ──────────────────────────────────────────────────────────────
|
| 292 |
+
|
| 293 |
+
def _draw_skeleton(self, frame: np.ndarray, kps: dict) -> np.ndarray:
|
| 294 |
+
"""Draw COCO-17 bones (white) and joints (confidence-colored) onto frame."""
|
| 295 |
+
visible = {j: kp for j, kp in kps.items() if kp.get("conf", 0.0) >= CONF_THRESHOLD}
|
| 296 |
+
|
| 297 |
+
# Bones
|
| 298 |
+
for j1, j2 in COCO_SKELETON:
|
| 299 |
+
if j1 in visible and j2 in visible:
|
| 300 |
+
p1 = (int(visible[j1]["x"]), int(visible[j1]["y"]))
|
| 301 |
+
p2 = (int(visible[j2]["x"]), int(visible[j2]["y"]))
|
| 302 |
+
cv2.line(frame, p1, p2, (255, 255, 255), 2)
|
| 303 |
+
|
| 304 |
+
# Joints
|
| 305 |
+
for j, kp in visible.items():
|
| 306 |
+
pt = (int(kp["x"]), int(kp["y"]))
|
| 307 |
+
color = _conf_to_bgr(kp["conf"])
|
| 308 |
+
cv2.circle(frame, pt, 4, color, -1)
|
| 309 |
+
cv2.circle(frame, pt, 5, (255, 255, 255), 1)
|
| 310 |
+
|
| 311 |
+
return frame
|
| 312 |
+
```
|
| 313 |
+
|
| 314 |
+
- [ ] **Step 4: Run tests**
|
| 315 |
+
|
| 316 |
+
```bash
|
| 317 |
+
pytest tests/test_visualizer.py::TestDrawSkeleton -v
|
| 318 |
+
```
|
| 319 |
+
|
| 320 |
+
Expected: 2 PASS
|
| 321 |
+
|
| 322 |
+
- [ ] **Step 5: Commit**
|
| 323 |
+
|
| 324 |
+
```bash
|
| 325 |
+
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 326 |
+
git commit -m "feat: PoseVisualizer._draw_skeleton with confidence-colored joints"
|
| 327 |
+
```
|
| 328 |
+
|
| 329 |
+
---
|
| 330 |
+
|
| 331 |
+
## Task 3: `PoseVisualizer._draw_trails`
|
| 332 |
+
|
| 333 |
+
**Files:**
|
| 334 |
+
- Modify: `formscout/agents/visualizer.py`
|
| 335 |
+
- Modify: `tests/test_visualizer.py`
|
| 336 |
+
|
| 337 |
+
- [ ] **Step 1: Write failing test**
|
| 338 |
+
|
| 339 |
+
Append to `tests/test_visualizer.py`:
|
| 340 |
+
|
| 341 |
+
```python
|
| 342 |
+
class TestDrawTrails:
|
| 343 |
+
def test_trails_draw_without_error(self):
|
| 344 |
+
from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
|
| 345 |
+
from collections import deque
|
| 346 |
+
vis = PoseVisualizer()
|
| 347 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 348 |
+
# Build a trail history for joint 0 with 5 positions
|
| 349 |
+
trail_history = {
|
| 350 |
+
0: deque([(100 + i * 5, 200 + i * 3) for i in range(5)], maxlen=TRAIL_LENGTH)
|
| 351 |
+
}
|
| 352 |
+
result = vis._draw_trails(frame.copy(), trail_history)
|
| 353 |
+
assert result.shape == frame.shape
|
| 354 |
+
# Trail should modify at least some pixels
|
| 355 |
+
assert not np.array_equal(result, frame)
|
| 356 |
+
|
| 357 |
+
def test_short_trail_no_crash(self):
|
| 358 |
+
from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
|
| 359 |
+
from collections import deque
|
| 360 |
+
vis = PoseVisualizer()
|
| 361 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 362 |
+
# Only one point — no line possible
|
| 363 |
+
trail_history = {0: deque([(100, 200)], maxlen=TRAIL_LENGTH)}
|
| 364 |
+
result = vis._draw_trails(frame.copy(), trail_history)
|
| 365 |
+
# No crash, frame unchanged (single point = no segment)
|
| 366 |
+
assert np.array_equal(result, frame)
|
| 367 |
+
```
|
| 368 |
+
|
| 369 |
+
- [ ] **Step 2: Run to confirm failure**
|
| 370 |
+
|
| 371 |
+
```bash
|
| 372 |
+
pytest tests/test_visualizer.py::TestDrawTrails -v
|
| 373 |
+
```
|
| 374 |
+
|
| 375 |
+
Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute '_draw_trails'`
|
| 376 |
+
|
| 377 |
+
- [ ] **Step 3: Add `_draw_trails` to `PoseVisualizer`**
|
| 378 |
+
|
| 379 |
+
Inside the `PoseVisualizer` class, after `_draw_skeleton`:
|
| 380 |
+
|
| 381 |
+
```python
|
| 382 |
+
# ── Trails ───────────────────────────────────────────────────────────────
|
| 383 |
+
|
| 384 |
+
def _draw_trails(self, frame: np.ndarray, trail_history: dict) -> np.ndarray:
|
| 385 |
+
"""Draw fading motion trails for each joint."""
|
| 386 |
+
for joint_idx, trail in trail_history.items():
|
| 387 |
+
pts = list(trail)
|
| 388 |
+
if len(pts) < 2:
|
| 389 |
+
continue
|
| 390 |
+
for i in range(1, len(pts)):
|
| 391 |
+
alpha = i / len(pts)
|
| 392 |
+
brightness = int(255 * alpha)
|
| 393 |
+
color = (brightness, brightness, brightness)
|
| 394 |
+
thickness = max(1, int(3 * alpha))
|
| 395 |
+
p1 = (int(pts[i - 1][0]), int(pts[i - 1][1]))
|
| 396 |
+
p2 = (int(pts[i][0]), int(pts[i][1]))
|
| 397 |
+
cv2.line(frame, p1, p2, color, thickness)
|
| 398 |
+
return frame
|
| 399 |
+
```
|
| 400 |
+
|
| 401 |
+
- [ ] **Step 4: Run tests**
|
| 402 |
+
|
| 403 |
+
```bash
|
| 404 |
+
pytest tests/test_visualizer.py::TestDrawTrails -v
|
| 405 |
+
```
|
| 406 |
+
|
| 407 |
+
Expected: 2 PASS
|
| 408 |
+
|
| 409 |
+
- [ ] **Step 5: Commit**
|
| 410 |
+
|
| 411 |
+
```bash
|
| 412 |
+
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 413 |
+
git commit -m "feat: PoseVisualizer._draw_trails with fading alpha"
|
| 414 |
+
```
|
| 415 |
+
|
| 416 |
+
---
|
| 417 |
+
|
| 418 |
+
## Task 4: `PoseVisualizer._draw_velocity_arrows`
|
| 419 |
+
|
| 420 |
+
**Files:**
|
| 421 |
+
- Modify: `formscout/agents/visualizer.py`
|
| 422 |
+
- Modify: `tests/test_visualizer.py`
|
| 423 |
+
|
| 424 |
+
- [ ] **Step 1: Write failing test**
|
| 425 |
+
|
| 426 |
+
Append to `tests/test_visualizer.py`:
|
| 427 |
+
|
| 428 |
+
```python
|
| 429 |
+
class TestDrawVelocityArrows:
|
| 430 |
+
def test_arrows_draw_without_error(self):
|
| 431 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 432 |
+
vis = PoseVisualizer()
|
| 433 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 434 |
+
kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
|
| 435 |
+
for j in range(17)}
|
| 436 |
+
prev_kps = {j: {"x": float(48 + j * 30), "y": float(98 + j * 20), "conf": 0.9}
|
| 437 |
+
for j in range(17)}
|
| 438 |
+
# velocities: joint 5 moving fast
|
| 439 |
+
velocities = {j: [0.0] * 5 for j in range(17)}
|
| 440 |
+
velocities[5] = [0.0, 10.0, 50.0, 80.0, 120.0]
|
| 441 |
+
result = vis._draw_velocity_arrows(frame.copy(), kps, prev_kps, velocities, frame_idx=4)
|
| 442 |
+
assert result.shape == frame.shape
|
| 443 |
+
|
| 444 |
+
def test_no_prev_kps_no_crash(self):
|
| 445 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 446 |
+
vis = PoseVisualizer()
|
| 447 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 448 |
+
kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.9} for j in range(17)}
|
| 449 |
+
velocities = {j: [50.0] * 5 for j in range(17)}
|
| 450 |
+
# prev_kps is None — should skip without crash
|
| 451 |
+
result = vis._draw_velocity_arrows(frame.copy(), kps, None, velocities, frame_idx=0)
|
| 452 |
+
assert result.shape == frame.shape
|
| 453 |
+
```
|
| 454 |
+
|
| 455 |
+
- [ ] **Step 2: Run to confirm failure**
|
| 456 |
+
|
| 457 |
+
```bash
|
| 458 |
+
pytest tests/test_visualizer.py::TestDrawVelocityArrows -v
|
| 459 |
+
```
|
| 460 |
+
|
| 461 |
+
Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute '_draw_velocity_arrows'`
|
| 462 |
+
|
| 463 |
+
- [ ] **Step 3: Add `_draw_velocity_arrows` to `PoseVisualizer`**
|
| 464 |
+
|
| 465 |
+
Inside the `PoseVisualizer` class, after `_draw_trails`:
|
| 466 |
+
|
| 467 |
+
```python
|
| 468 |
+
# ── Velocity arrows ───────────────────────────────────────────────────────
|
| 469 |
+
|
| 470 |
+
def _draw_velocity_arrows(
|
| 471 |
+
self,
|
| 472 |
+
frame: np.ndarray,
|
| 473 |
+
kps: dict,
|
| 474 |
+
prev_kps: dict | None,
|
| 475 |
+
velocities: dict[int, list[float]],
|
| 476 |
+
frame_idx: int,
|
| 477 |
+
) -> np.ndarray:
|
| 478 |
+
"""Draw per-joint velocity arrows scaled by speed."""
|
| 479 |
+
if prev_kps is None:
|
| 480 |
+
return frame
|
| 481 |
+
|
| 482 |
+
all_speeds = [velocities[j][frame_idx] for j in range(17) if frame_idx < len(velocities.get(j, []))]
|
| 483 |
+
peak = max(all_speeds) if all_speeds else 1.0
|
| 484 |
+
if peak == 0.0:
|
| 485 |
+
return frame
|
| 486 |
+
|
| 487 |
+
for j in range(17):
|
| 488 |
+
kp = kps.get(j)
|
| 489 |
+
pk = prev_kps.get(j)
|
| 490 |
+
if not kp or not pk:
|
| 491 |
+
continue
|
| 492 |
+
if kp.get("conf", 0.0) < CONF_THRESHOLD:
|
| 493 |
+
continue
|
| 494 |
+
speeds = velocities.get(j, [])
|
| 495 |
+
if frame_idx >= len(speeds):
|
| 496 |
+
continue
|
| 497 |
+
speed = speeds[frame_idx]
|
| 498 |
+
if speed == 0.0:
|
| 499 |
+
continue
|
| 500 |
+
|
| 501 |
+
dx = kp["x"] - pk["x"]
|
| 502 |
+
dy = kp["y"] - pk["y"]
|
| 503 |
+
mag = math.sqrt(dx * dx + dy * dy)
|
| 504 |
+
if mag < 1e-6:
|
| 505 |
+
continue
|
| 506 |
+
|
| 507 |
+
# Normalize direction, scale to arrow length
|
| 508 |
+
length = min(speed / peak * MAX_ARROW_PX, MAX_ARROW_PX)
|
| 509 |
+
nx, ny = dx / mag, dy / mag
|
| 510 |
+
start = (int(kp["x"]), int(kp["y"]))
|
| 511 |
+
end = (int(kp["x"] + nx * length), int(kp["y"] + ny * length))
|
| 512 |
+
|
| 513 |
+
ratio = speed / peak
|
| 514 |
+
if ratio < 0.33:
|
| 515 |
+
color = (0, 200, 0) # green
|
| 516 |
+
elif ratio < 0.66:
|
| 517 |
+
color = (0, 140, 255) # orange
|
| 518 |
+
else:
|
| 519 |
+
color = (0, 0, 255) # red
|
| 520 |
+
|
| 521 |
+
cv2.arrowedLine(frame, start, end, color, 2, tipLength=0.35)
|
| 522 |
+
|
| 523 |
+
return frame
|
| 524 |
+
```
|
| 525 |
+
|
| 526 |
+
- [ ] **Step 4: Run tests**
|
| 527 |
+
|
| 528 |
+
```bash
|
| 529 |
+
pytest tests/test_visualizer.py::TestDrawVelocityArrows -v
|
| 530 |
+
```
|
| 531 |
+
|
| 532 |
+
Expected: 2 PASS
|
| 533 |
+
|
| 534 |
+
- [ ] **Step 5: Commit**
|
| 535 |
+
|
| 536 |
+
```bash
|
| 537 |
+
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 538 |
+
git commit -m "feat: PoseVisualizer._draw_velocity_arrows speed-colored"
|
| 539 |
+
```
|
| 540 |
+
|
| 541 |
+
---
|
| 542 |
+
|
| 543 |
+
## Task 5: `render_video` + `build_velocity_summary`
|
| 544 |
+
|
| 545 |
+
**Files:**
|
| 546 |
+
- Modify: `formscout/agents/visualizer.py`
|
| 547 |
+
- Modify: `tests/test_visualizer.py`
|
| 548 |
+
|
| 549 |
+
- [ ] **Step 1: Write failing tests**
|
| 550 |
+
|
| 551 |
+
Append to `tests/test_visualizer.py`:
|
| 552 |
+
|
| 553 |
+
```python
|
| 554 |
+
class TestRenderVideo:
|
| 555 |
+
def test_creates_mp4_file(self, tmp_path):
|
| 556 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 557 |
+
vis = PoseVisualizer()
|
| 558 |
+
ingest = _make_ingest(n=5)
|
| 559 |
+
pose = _make_pose(n=5)
|
| 560 |
+
out = str(tmp_path / "out.mp4")
|
| 561 |
+
result = vis.render_video(ingest, pose, {"skeleton"}, out)
|
| 562 |
+
assert result is not None
|
| 563 |
+
import os
|
| 564 |
+
assert os.path.exists(result)
|
| 565 |
+
assert os.path.getsize(result) > 0
|
| 566 |
+
|
| 567 |
+
def test_empty_layers_returns_none(self, tmp_path):
|
| 568 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 569 |
+
vis = PoseVisualizer()
|
| 570 |
+
out = str(tmp_path / "out.mp4")
|
| 571 |
+
result = vis.render_video(_make_ingest(), _make_pose(), set(), out)
|
| 572 |
+
assert result is None
|
| 573 |
+
|
| 574 |
+
def test_no_detections_returns_none(self, tmp_path):
|
| 575 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 576 |
+
vis = PoseVisualizer()
|
| 577 |
+
ingest = _make_ingest(n=5)
|
| 578 |
+
empty_pose = Pose2DResult(
|
| 579 |
+
keypoints=[{} for _ in range(5)], fps=30.0, confidence=0.0, notes=""
|
| 580 |
+
)
|
| 581 |
+
out = str(tmp_path / "out.mp4")
|
| 582 |
+
result = vis.render_video(ingest, empty_pose, {"skeleton"}, out)
|
| 583 |
+
assert result is None
|
| 584 |
+
|
| 585 |
+
def test_last_velocities_set_after_render(self, tmp_path):
|
| 586 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 587 |
+
vis = PoseVisualizer()
|
| 588 |
+
out = str(tmp_path / "out.mp4")
|
| 589 |
+
vis.render_video(_make_ingest(n=5), _make_pose(n=5), {"skeleton"}, out)
|
| 590 |
+
assert len(vis.last_velocities) == 17
|
| 591 |
+
|
| 592 |
+
|
| 593 |
+
class TestBuildVelocitySummary:
|
| 594 |
+
def test_returns_markdown_table(self):
|
| 595 |
+
from formscout.agents.visualizer import build_velocity_summary, compute_joint_velocity
|
| 596 |
+
pose = _make_pose(n=10)
|
| 597 |
+
vels = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 598 |
+
result = build_velocity_summary(pose.keypoints, vels)
|
| 599 |
+
assert "|" in result
|
| 600 |
+
# At least one COCO joint name appears
|
| 601 |
+
assert any(name in result for name in ["knee", "shoulder", "hip", "ankle"])
|
| 602 |
+
|
| 603 |
+
def test_empty_keypoints_returns_empty_string(self):
|
| 604 |
+
from formscout.agents.visualizer import build_velocity_summary
|
| 605 |
+
empty_kps = [{} for _ in range(5)]
|
| 606 |
+
vels = {j: [0.0] * 5 for j in range(17)}
|
| 607 |
+
result = build_velocity_summary(empty_kps, vels)
|
| 608 |
+
assert result == ""
|
| 609 |
+
```
|
| 610 |
+
|
| 611 |
+
- [ ] **Step 2: Run to confirm failure**
|
| 612 |
+
|
| 613 |
+
```bash
|
| 614 |
+
pytest tests/test_visualizer.py::TestRenderVideo tests/test_visualizer.py::TestBuildVelocitySummary -v
|
| 615 |
+
```
|
| 616 |
+
|
| 617 |
+
Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute 'render_video'`
|
| 618 |
+
|
| 619 |
+
- [ ] **Step 3: Add `render_video` to `PoseVisualizer`**
|
| 620 |
+
|
| 621 |
+
Inside the `PoseVisualizer` class, after `_draw_velocity_arrows`:
|
| 622 |
+
|
| 623 |
+
```python
|
| 624 |
+
# ── Public ────────────────────────────────────────────────────────────────
|
| 625 |
+
|
| 626 |
+
def render_video(
|
| 627 |
+
self,
|
| 628 |
+
ingest,
|
| 629 |
+
pose2d,
|
| 630 |
+
layers: set[str],
|
| 631 |
+
output_path: str,
|
| 632 |
+
) -> str | None:
|
| 633 |
+
"""
|
| 634 |
+
Render annotated video. Returns output_path on success, None otherwise.
|
| 635 |
+
layers: subset of {"skeleton", "trails", "velocity_arrows"}
|
| 636 |
+
"""
|
| 637 |
+
if not layers:
|
| 638 |
+
return None
|
| 639 |
+
|
| 640 |
+
# Require at least one detected frame
|
| 641 |
+
if not any(pose2d.keypoints):
|
| 642 |
+
return None
|
| 643 |
+
|
| 644 |
+
try:
|
| 645 |
+
velocities = compute_joint_velocity(pose2d.keypoints, ingest.fps)
|
| 646 |
+
self.last_velocities = velocities
|
| 647 |
+
|
| 648 |
+
frames = ingest.frames
|
| 649 |
+
h, w = frames[0].shape[:2]
|
| 650 |
+
fps = ingest.fps or 30.0
|
| 651 |
+
|
| 652 |
+
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
|
| 653 |
+
writer = cv2.VideoWriter(output_path, fourcc, fps, (w, h))
|
| 654 |
+
if not writer.isOpened():
|
| 655 |
+
logger.warning("VideoWriter failed to open: %s", output_path)
|
| 656 |
+
return None
|
| 657 |
+
|
| 658 |
+
trail_history: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
|
| 659 |
+
prev_kps: dict | None = None
|
| 660 |
+
|
| 661 |
+
for frame_idx, (frame, kps) in enumerate(zip(frames, pose2d.keypoints)):
|
| 662 |
+
out_frame = frame.copy()
|
| 663 |
+
|
| 664 |
+
if "trails" in layers:
|
| 665 |
+
# Update trail history before drawing
|
| 666 |
+
for j, kp in kps.items():
|
| 667 |
+
if kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 668 |
+
trail_history[j].append((kp["x"], kp["y"]))
|
| 669 |
+
out_frame = self._draw_trails(out_frame, trail_history)
|
| 670 |
+
|
| 671 |
+
if "skeleton" in layers:
|
| 672 |
+
out_frame = self._draw_skeleton(out_frame, kps)
|
| 673 |
+
|
| 674 |
+
if "velocity_arrows" in layers:
|
| 675 |
+
out_frame = self._draw_velocity_arrows(
|
| 676 |
+
out_frame, kps, prev_kps, velocities, frame_idx
|
| 677 |
+
)
|
| 678 |
+
|
| 679 |
+
writer.write(out_frame)
|
| 680 |
+
prev_kps = kps
|
| 681 |
+
|
| 682 |
+
writer.release()
|
| 683 |
+
return output_path
|
| 684 |
+
|
| 685 |
+
except Exception as e:
|
| 686 |
+
logger.warning("render_video failed: %s", e)
|
| 687 |
+
return None
|
| 688 |
+
```
|
| 689 |
+
|
| 690 |
+
- [ ] **Step 4: Add `build_velocity_summary` after the class**
|
| 691 |
+
|
| 692 |
+
After the `PoseVisualizer` class definition, add:
|
| 693 |
+
|
| 694 |
+
```python
|
| 695 |
+
# ── Velocity summary ──────────────────────────────────────────────────────────
|
| 696 |
+
|
| 697 |
+
def build_velocity_summary(
|
| 698 |
+
keypoints_per_frame: list[dict],
|
| 699 |
+
velocities: dict[int, list[float]],
|
| 700 |
+
) -> str:
|
| 701 |
+
"""Return markdown table of per-joint avg/peak velocity. Empty string if no valid joints."""
|
| 702 |
+
n_frames = len(keypoints_per_frame)
|
| 703 |
+
if n_frames == 0:
|
| 704 |
+
return ""
|
| 705 |
+
|
| 706 |
+
rows = []
|
| 707 |
+
for j in range(17):
|
| 708 |
+
# Count frames where this joint is detected
|
| 709 |
+
detected = sum(
|
| 710 |
+
1 for kps in keypoints_per_frame
|
| 711 |
+
if kps.get(j, {}).get("conf", 0.0) >= CONF_THRESHOLD
|
| 712 |
+
)
|
| 713 |
+
if detected < n_frames * 0.5:
|
| 714 |
+
continue # skip joints present in <50% of frames
|
| 715 |
+
|
| 716 |
+
speeds = velocities.get(j, [])
|
| 717 |
+
if not speeds:
|
| 718 |
+
continue
|
| 719 |
+
|
| 720 |
+
avg_speed = sum(speeds) / len(speeds)
|
| 721 |
+
peak_speed = max(speeds)
|
| 722 |
+
rows.append((COCO_KEYPOINTS[j], avg_speed, peak_speed))
|
| 723 |
+
|
| 724 |
+
if not rows:
|
| 725 |
+
return ""
|
| 726 |
+
|
| 727 |
+
rows.sort(key=lambda r: r[2], reverse=True) # sort by peak descending
|
| 728 |
+
lines = [
|
| 729 |
+
"| Joint | Avg (px/s) | Peak (px/s) |",
|
| 730 |
+
"|---|---|---|",
|
| 731 |
+
]
|
| 732 |
+
for name, avg, peak in rows:
|
| 733 |
+
lines.append(f"| {name} | {avg:.1f} | {peak:.1f} |")
|
| 734 |
+
return "\n".join(lines)
|
| 735 |
+
```
|
| 736 |
+
|
| 737 |
+
- [ ] **Step 5: Run all visualizer tests**
|
| 738 |
+
|
| 739 |
+
```bash
|
| 740 |
+
pytest tests/test_visualizer.py -v
|
| 741 |
+
```
|
| 742 |
+
|
| 743 |
+
Expected: all tests PASS (4 + 2 + 2 + 2 + 4 + 2 = 16 total)
|
| 744 |
+
|
| 745 |
+
- [ ] **Step 6: Commit**
|
| 746 |
+
|
| 747 |
+
```bash
|
| 748 |
+
git add formscout/agents/visualizer.py tests/test_visualizer.py
|
| 749 |
+
git commit -m "feat: PoseVisualizer.render_video + build_velocity_summary (16 tests pass)"
|
| 750 |
+
```
|
| 751 |
+
|
| 752 |
+
---
|
| 753 |
+
|
| 754 |
+
## Task 6: Wire `app.py`
|
| 755 |
+
|
| 756 |
+
**Files:**
|
| 757 |
+
- Modify: `app.py`
|
| 758 |
+
|
| 759 |
+
- [ ] **Step 1: Add `import tempfile` if not present and import visualizer in `process_video`**
|
| 760 |
+
|
| 761 |
+
Check the top of `app.py` for `import tempfile`. If missing, add it alongside the other stdlib imports. (Look at the existing import block and add `import tempfile` there.)
|
| 762 |
+
|
| 763 |
+
- [ ] **Step 2: Update `process_video()` signature and body**
|
| 764 |
+
|
| 765 |
+
Replace the existing `process_video` function (lines 46–83) with:
|
| 766 |
+
|
| 767 |
+
```python
|
| 768 |
+
def process_video(video_path: str, test_name: str, side: str, model_key: str, layers: list[str]):
|
| 769 |
+
"""Process an uploaded video through the FormScout pipeline."""
|
| 770 |
+
if not video_path:
|
| 771 |
+
return (
|
| 772 |
+
_render_empty_state(),
|
| 773 |
+
"Upload a video to begin analysis.",
|
| 774 |
+
"",
|
| 775 |
+
"",
|
| 776 |
+
None,
|
| 777 |
+
"",
|
| 778 |
+
)
|
| 779 |
+
|
| 780 |
+
director = Director()
|
| 781 |
+
state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
|
| 782 |
+
|
| 783 |
+
# ─── Score card ───
|
| 784 |
+
score_html = _render_empty_state()
|
| 785 |
+
score_details = ""
|
| 786 |
+
|
| 787 |
+
if state.features:
|
| 788 |
+
result = score_test(state.features)
|
| 789 |
+
judge = state.judge
|
| 790 |
+
if judge and judge.score is not None:
|
| 791 |
+
score_html = _render_score_card(judge.score, judge.confidence, judge.needs_human)
|
| 792 |
+
score_details = _render_score_details_judge(judge, result, state.features)
|
| 793 |
+
elif judge and judge.needs_human:
|
| 794 |
+
score_html = _render_score_card(0, 0, True)
|
| 795 |
+
score_details = f"### Needs Clinician Review\n{judge.rationale}"
|
| 796 |
+
else:
|
| 797 |
+
score_html = _render_score_card(result.score, result.confidence, result.needs_human)
|
| 798 |
+
score_details = _render_score_details(result, state.features)
|
| 799 |
+
|
| 800 |
+
# ─── Pipeline info ───
|
| 801 |
+
pipeline_md = _render_pipeline_status(state)
|
| 802 |
+
|
| 803 |
+
# ─── Warnings/errors ───
|
| 804 |
+
alerts = _render_alerts(state)
|
| 805 |
+
|
| 806 |
+
# ─── Overlay video ───
|
| 807 |
+
overlay_path = None
|
| 808 |
+
vel_summary = ""
|
| 809 |
+
layer_set = {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
|
| 810 |
+
if layer_set and state.ingest and state.pose2d:
|
| 811 |
+
try:
|
| 812 |
+
from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
|
| 813 |
+
vis = PoseVisualizer()
|
| 814 |
+
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
|
| 815 |
+
out_path = f.name
|
| 816 |
+
overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
|
| 817 |
+
if overlay_path:
|
| 818 |
+
vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
|
| 819 |
+
except Exception as e:
|
| 820 |
+
alerts = (alerts or "") + f"\n⚠️ Visualizer error: {e}"
|
| 821 |
+
|
| 822 |
+
return score_html, pipeline_md, score_details, alerts, overlay_path, vel_summary
|
| 823 |
+
```
|
| 824 |
+
|
| 825 |
+
- [ ] **Step 3: Add `overlay_layers` CheckboxGroup in `build_app()`**
|
| 826 |
+
|
| 827 |
+
After the `pose_model_dropdown` block (around line 270), and before `submit_btn`:
|
| 828 |
+
|
| 829 |
+
```python
|
| 830 |
+
overlay_layers = gr.CheckboxGroup(
|
| 831 |
+
choices=["Skeleton", "Trails", "Velocity arrows"],
|
| 832 |
+
value=["Skeleton", "Trails"],
|
| 833 |
+
label="Overlay Layers",
|
| 834 |
+
)
|
| 835 |
+
```
|
| 836 |
+
|
| 837 |
+
- [ ] **Step 4: Add overlay tab in the results panel**
|
| 838 |
+
|
| 839 |
+
Inside the `with gr.Tabs():` block (after the `⚠️ Alerts` tab):
|
| 840 |
+
|
| 841 |
+
```python
|
| 842 |
+
with gr.TabItem("🎬 Overlay Video"):
|
| 843 |
+
overlay_video = gr.Video(label="Annotated Movement")
|
| 844 |
+
velocity_md = gr.Markdown("")
|
| 845 |
+
```
|
| 846 |
+
|
| 847 |
+
- [ ] **Step 5: Update `_map_inputs` and `submit_btn.click`**
|
| 848 |
+
|
| 849 |
+
Replace the `_map_inputs` closure and `submit_btn.click` call:
|
| 850 |
+
|
| 851 |
+
```python
|
| 852 |
+
def _map_inputs(video, test_display_name, side_display, pose_model_key, overlay_layers):
|
| 853 |
+
"""Map UI display values to internal values."""
|
| 854 |
+
test_map = {name: val for name, val in FMS_TESTS}
|
| 855 |
+
test_name = test_map.get(test_display_name, "deep_squat")
|
| 856 |
+
side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
|
| 857 |
+
return process_video(video, test_name, side, pose_model_key, overlay_layers)
|
| 858 |
+
|
| 859 |
+
submit_btn.click(
|
| 860 |
+
fn=_map_inputs,
|
| 861 |
+
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown, overlay_layers],
|
| 862 |
+
outputs=[score_html, pipeline_md, score_details, alerts_md, overlay_video, velocity_md],
|
| 863 |
+
)
|
| 864 |
+
```
|
| 865 |
+
|
| 866 |
+
- [ ] **Step 6: Smoke-test the app builds**
|
| 867 |
+
|
| 868 |
+
```bash
|
| 869 |
+
python3 -c "from app import build_app; build_app(); print('ok')"
|
| 870 |
+
```
|
| 871 |
+
|
| 872 |
+
Expected: `ok` (Gradio UserWarning about theme is fine, not an error)
|
| 873 |
+
|
| 874 |
+
- [ ] **Step 7: Run full test suite to check for regressions**
|
| 875 |
+
|
| 876 |
+
```bash
|
| 877 |
+
pytest tests/ -v --tb=short 2>&1 | tail -15
|
| 878 |
+
```
|
| 879 |
+
|
| 880 |
+
Expected: all previous tests still pass (62 passing, 1 pre-existing fail in biomechanics), plus 16 new visualizer tests = 78 passing.
|
| 881 |
+
|
| 882 |
+
- [ ] **Step 8: Commit**
|
| 883 |
+
|
| 884 |
+
```bash
|
| 885 |
+
git add app.py
|
| 886 |
+
git commit -m "feat: overlay video tab + velocity summary wired in Gradio UI"
|
| 887 |
+
```
|
| 888 |
+
|
| 889 |
+
---
|
| 890 |
+
|
| 891 |
+
## Self-review
|
| 892 |
+
|
| 893 |
+
**Spec coverage:**
|
| 894 |
+
- ✅ `SimpleKalmanFilter` 4-state (Task 1)
|
| 895 |
+
- ✅ `compute_joint_velocity` Kalman-filtered px/s (Task 1)
|
| 896 |
+
- ✅ `_draw_skeleton` COCO bones, confidence-colored joints (Task 2)
|
| 897 |
+
- ✅ `_draw_trails` fading deque-based trails (Task 3)
|
| 898 |
+
- ✅ `_draw_velocity_arrows` speed-colored, direction from consecutive frames (Task 4)
|
| 899 |
+
- ✅ `render_video` layer dispatch, trail history, VideoWriter (Task 5)
|
| 900 |
+
- ✅ `build_velocity_summary` markdown table, >50% detection filter (Task 5)
|
| 901 |
+
- ✅ `overlay_layers` CheckboxGroup in UI (Task 6)
|
| 902 |
+
- ✅ New `🎬 Overlay Video` tab with `gr.Video` + `gr.Markdown` (Task 6)
|
| 903 |
+
- ✅ `process_video` wired with layers param (Task 6)
|
| 904 |
+
- ✅ `vis.last_velocities` stored on instance after `render_video` (Task 5)
|
| 905 |
+
- ✅ Error handling: empty layers → None, empty detections → None, exception → alerts (Task 5 + 6)
|
| 906 |
+
- ✅ All 5 spec test cases covered across Tasks 1–5
|
| 907 |
+
|
| 908 |
+
**Placeholder scan:** None found. All code blocks are complete.
|
| 909 |
+
|
| 910 |
+
**Type consistency:**
|
| 911 |
+
- `compute_joint_velocity` returns `dict[int, list[float]]` — used identically in `render_video`, `_draw_velocity_arrows`, and `build_velocity_summary`. ✓
|
| 912 |
+
- `layers: set[str]` in `render_video`; converted from `list[str]` in `process_video` via set comprehension. ✓
|
| 913 |
+
- `vis.last_velocities` set in `render_video`, read in `process_video`. ✓
|
| 914 |
+
- `_draw_velocity_arrows(frame, kps, prev_kps, velocities, frame_idx)` — signature matches call in `render_video`. ✓
|
docs/superpowers/plans/2026-06-13-full-fms-session-pdf.md
CHANGED
|
@@ -1,1209 +1,1209 @@
|
|
| 1 |
-
# Full FMS Session + PDF Report — Implementation Plan
|
| 2 |
-
|
| 3 |
-
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
| 4 |
-
|
| 5 |
-
**Goal:** Turn FormScout's one-clip scorer into a screening session that accumulates analyzed clips into a composite 0–21 report and exports a branded PDF with annotated worst-moment key-frame stills.
|
| 6 |
-
|
| 7 |
-
**Architecture:** A new `formscout/session.py` accumulates typed `SessionEntry` objects (one per analyzed clip), persisting each to a temp session dir. `PoseVisualizer.render_frame()` captures the governing frame (already computed by `BiomechanicsAgent` and stored in `features.timing`) as an annotated PNG. On "Finish", the existing `ReportAgent` computes composite + asymmetries, and a new `PdfReportAgent` renders a ReportLab PDF. The UI (`app.py`) gains `gr.State` session accumulation with "Analyse new clip" / "Finish & generate PDF" buttons.
|
| 8 |
-
|
| 9 |
-
**Tech Stack:** Python 3.13, ReportLab (new dep), OpenCV (existing), Gradio 5, pytest. No model downloads in tests.
|
| 10 |
-
|
| 11 |
-
---
|
| 12 |
-
|
| 13 |
-
## File Structure
|
| 14 |
-
|
| 15 |
-
- `requirements.txt` — add `reportlab`.
|
| 16 |
-
- `formscout/types.py` — add `SessionEntry` frozen dataclass.
|
| 17 |
-
- `formscout/agents/biomechanics.py` — add `max_sag_frame` to `trunk_stability_pushup` timing (rotary already has `peak_extension_frame`).
|
| 18 |
-
- `formscout/agents/visualizer.py` — add `PoseVisualizer.render_frame()`.
|
| 19 |
-
- `formscout/session.py` — **new**: session accumulator (new/add/finish + persistence + key-frame helpers).
|
| 20 |
-
- `formscout/agents/pdf_report.py` — **new**: `PdfReportAgent` (ReportLab).
|
| 21 |
-
- `app.py` — wire `gr.State`, two buttons, "Session so far" table, finish handler.
|
| 22 |
-
- `tests/test_session.py`, `tests/test_keyframe.py`, `tests/test_pdf_report.py` — **new**.
|
| 23 |
-
|
| 24 |
-
---
|
| 25 |
-
|
| 26 |
-
## Task 1: Add ReportLab dependency
|
| 27 |
-
|
| 28 |
-
**Files:**
|
| 29 |
-
- Modify: `requirements.txt`
|
| 30 |
-
|
| 31 |
-
- [ ] **Step 1: Add the dependency**
|
| 32 |
-
|
| 33 |
-
Add this line to `requirements.txt` (after `pillow>=10.3`):
|
| 34 |
-
|
| 35 |
-
```
|
| 36 |
-
reportlab>=4.0
|
| 37 |
-
```
|
| 38 |
-
|
| 39 |
-
- [ ] **Step 2: Install it**
|
| 40 |
-
|
| 41 |
-
Run: `pip install 'reportlab>=4.0'`
|
| 42 |
-
Expected: `Successfully installed reportlab-4.x.x`
|
| 43 |
-
|
| 44 |
-
- [ ] **Step 3: Verify import**
|
| 45 |
-
|
| 46 |
-
Run: `python3 -c "import reportlab; print(reportlab.Version)"`
|
| 47 |
-
Expected: prints a version like `4.x.x`
|
| 48 |
-
|
| 49 |
-
- [ ] **Step 4: Commit**
|
| 50 |
-
|
| 51 |
-
```bash
|
| 52 |
-
git add requirements.txt
|
| 53 |
-
git commit -m "build: add reportlab for PDF report generation"
|
| 54 |
-
```
|
| 55 |
-
|
| 56 |
-
---
|
| 57 |
-
|
| 58 |
-
## Task 2: Add `SessionEntry` dataclass
|
| 59 |
-
|
| 60 |
-
**Files:**
|
| 61 |
-
- Modify: `formscout/types.py` (after `ReportResult`, before `PipelineState`)
|
| 62 |
-
- Test: `tests/test_session.py`
|
| 63 |
-
|
| 64 |
-
- [ ] **Step 1: Write the failing test**
|
| 65 |
-
|
| 66 |
-
Create `tests/test_session.py` with:
|
| 67 |
-
|
| 68 |
-
```python
|
| 69 |
-
"""Tests for the FMS session accumulator — no GPU, no model downloads."""
|
| 70 |
-
import numpy as np
|
| 71 |
-
|
| 72 |
-
from formscout.types import (
|
| 73 |
-
IngestResult, Pose2DResult, BiomechFeatures, ScoreResult, JudgeResult,
|
| 74 |
-
MovementResult, SessionEntry,
|
| 75 |
-
)
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
def test_session_entry_holds_typed_objects():
|
| 79 |
-
movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
|
| 80 |
-
features = BiomechFeatures(
|
| 81 |
-
test_name="deep_squat", view="2d", side="na",
|
| 82 |
-
angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": True},
|
| 83 |
-
symmetry_delta=None, timing={"deepest_frame": 2}, confidence=0.9,
|
| 84 |
-
)
|
| 85 |
-
rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
|
| 86 |
-
judge = JudgeResult(score=2, rationale="ok", compensation_tags=["heels elevated"],
|
| 87 |
-
corrective_hint="ankle mobility", confidence=0.85)
|
| 88 |
-
entry = SessionEntry(
|
| 89 |
-
test_name="deep_squat", side="na", score=2, needs_human=False,
|
| 90 |
-
rationale="ok", compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 91 |
-
measurements={"left_knee_flexion_deg": 95.0}, confidence=0.85, view="2d",
|
| 92 |
-
keyframe_path=None, movement=movement, features=features,
|
| 93 |
-
rubric_score=rubric, judge=judge,
|
| 94 |
-
)
|
| 95 |
-
assert entry.score == 2
|
| 96 |
-
assert entry.movement.test_name == "deep_squat"
|
| 97 |
-
assert entry.rubric_score.score == 2
|
| 98 |
-
assert entry.judge.compensation_tags == ["heels elevated"]
|
| 99 |
-
```
|
| 100 |
-
|
| 101 |
-
- [ ] **Step 2: Run test to verify it fails**
|
| 102 |
-
|
| 103 |
-
Run: `pytest tests/test_session.py::test_session_entry_holds_typed_objects -v`
|
| 104 |
-
Expected: FAIL with `ImportError: cannot import name 'SessionEntry'`
|
| 105 |
-
|
| 106 |
-
- [ ] **Step 3: Add the dataclass**
|
| 107 |
-
|
| 108 |
-
In `formscout/types.py`, insert after the `ReportResult` class (line ~142) and before `PipelineState`:
|
| 109 |
-
|
| 110 |
-
```python
|
| 111 |
-
@dataclass(frozen=True)
|
| 112 |
-
class SessionEntry:
|
| 113 |
-
"""One accumulated analysis in a screening session.
|
| 114 |
-
|
| 115 |
-
Display fields (test_name…keyframe_path) feed the PDF/JSON/MD artifacts;
|
| 116 |
-
the trailing typed objects (movement…judge) feed ReportAgent.run().
|
| 117 |
-
"""
|
| 118 |
-
test_name: str
|
| 119 |
-
side: str
|
| 120 |
-
score: int | None
|
| 121 |
-
needs_human: bool
|
| 122 |
-
rationale: str
|
| 123 |
-
compensation_tags: list
|
| 124 |
-
corrective_hint: str
|
| 125 |
-
measurements: dict
|
| 126 |
-
confidence: float
|
| 127 |
-
view: str
|
| 128 |
-
keyframe_path: str | None
|
| 129 |
-
movement: MovementResult
|
| 130 |
-
features: BiomechFeatures
|
| 131 |
-
rubric_score: ScoreResult
|
| 132 |
-
judge: JudgeResult | None
|
| 133 |
-
```
|
| 134 |
-
|
| 135 |
-
- [ ] **Step 4: Run test to verify it passes**
|
| 136 |
-
|
| 137 |
-
Run: `pytest tests/test_session.py::test_session_entry_holds_typed_objects -v`
|
| 138 |
-
Expected: PASS
|
| 139 |
-
|
| 140 |
-
- [ ] **Step 5: Commit**
|
| 141 |
-
|
| 142 |
-
```bash
|
| 143 |
-
git add formscout/types.py tests/test_session.py
|
| 144 |
-
git commit -m "feat: add SessionEntry typed contract for screening sessions"
|
| 145 |
-
```
|
| 146 |
-
|
| 147 |
-
---
|
| 148 |
-
|
| 149 |
-
## Task 3: Add governing-frame index to push-up biomechanics
|
| 150 |
-
|
| 151 |
-
**Files:**
|
| 152 |
-
- Modify: `formscout/agents/biomechanics.py:468-529` (`_trunk_stability_pushup`)
|
| 153 |
-
- Test: `tests/test_biomechanics.py` (append a test)
|
| 154 |
-
|
| 155 |
-
The other six tests already store a governing frame index in `features.timing`
|
| 156 |
-
(`deepest_frame`, `peak_step_frame`, `deepest_lunge_frame`, `measure_frame`,
|
| 157 |
-
`peak_raise_frame`, `peak_extension_frame`). Only `trunk_stability_pushup` is missing one.
|
| 158 |
-
|
| 159 |
-
- [ ] **Step 1: Write the failing test**
|
| 160 |
-
|
| 161 |
-
Append to `tests/test_biomechanics.py`:
|
| 162 |
-
|
| 163 |
-
```python
|
| 164 |
-
def test_pushup_timing_has_max_sag_frame():
|
| 165 |
-
from formscout.agents.biomechanics import BiomechanicsAgent
|
| 166 |
-
from formscout.types import Pose2DResult, Body3DResult, MovementResult
|
| 167 |
-
|
| 168 |
-
# 4 frames; frame 2 has the largest hip sag (hip far below shoulder/ankle midline)
|
| 169 |
-
def kps(hip_y):
|
| 170 |
-
base = {
|
| 171 |
-
5: {"x": 200, "y": 200, "conf": 0.9}, # L shoulder
|
| 172 |
-
6: {"x": 220, "y": 200, "conf": 0.9}, # R shoulder
|
| 173 |
-
11: {"x": 300, "y": hip_y, "conf": 0.9}, # L hip
|
| 174 |
-
12: {"x": 320, "y": hip_y, "conf": 0.9}, # R hip
|
| 175 |
-
15: {"x": 400, "y": 200, "conf": 0.9}, # L ankle
|
| 176 |
-
16: {"x": 420, "y": 200, "conf": 0.9}, # R ankle
|
| 177 |
-
}
|
| 178 |
-
return base
|
| 179 |
-
|
| 180 |
-
frames = [kps(200), kps(210), kps(260), kps(205)]
|
| 181 |
-
pose = Pose2DResult(keypoints=frames, fps=30.0, confidence=0.9)
|
| 182 |
-
body3d = Body3DResult(used=False, joints_3d=[])
|
| 183 |
-
movement = MovementResult(test_name="trunk_stability_pushup", side="na", confidence=1.0)
|
| 184 |
-
|
| 185 |
-
feats = BiomechanicsAgent().run(pose, body3d, movement)
|
| 186 |
-
assert "max_sag_frame" in feats.timing
|
| 187 |
-
assert feats.timing["max_sag_frame"] == 2
|
| 188 |
-
```
|
| 189 |
-
|
| 190 |
-
- [ ] **Step 2: Run test to verify it fails**
|
| 191 |
-
|
| 192 |
-
Run: `pytest tests/test_biomechanics.py::test_pushup_timing_has_max_sag_frame -v`
|
| 193 |
-
Expected: FAIL with `assert 'max_sag_frame' in {...}` (KeyError-style assertion failure)
|
| 194 |
-
|
| 195 |
-
- [ ] **Step 3: Track the max-sag frame index**
|
| 196 |
-
|
| 197 |
-
In `formscout/agents/biomechanics.py`, replace the body of `_trunk_stability_pushup` from the
|
| 198 |
-
`trunk_angles_over_time = []` loop through the `if trunk_angles_over_time:` block. Replace:
|
| 199 |
-
|
| 200 |
-
```python
|
| 201 |
-
# Analyze multiple frames to detect sag/lag
|
| 202 |
-
trunk_angles_over_time = []
|
| 203 |
-
for i, kps in enumerate(pose2d.keypoints):
|
| 204 |
-
```
|
| 205 |
-
|
| 206 |
-
…down to and including the `alignments["no_sag"] = max_sag < 30` line, with:
|
| 207 |
-
|
| 208 |
-
```python
|
| 209 |
-
# Analyze multiple frames to detect sag/lag
|
| 210 |
-
trunk_sags: list[tuple[int, float]] = [] # (frame_idx, sag_px)
|
| 211 |
-
for i, kps in enumerate(pose2d.keypoints):
|
| 212 |
-
l_sh = _get_joint(kps, L_SHOULDER)
|
| 213 |
-
r_sh = _get_joint(kps, R_SHOULDER)
|
| 214 |
-
l_hip = _get_joint(kps, L_HIP)
|
| 215 |
-
r_hip = _get_joint(kps, R_HIP)
|
| 216 |
-
l_ankle = _get_joint(kps, L_ANKLE)
|
| 217 |
-
r_ankle = _get_joint(kps, R_ANKLE)
|
| 218 |
-
|
| 219 |
-
if l_sh and r_sh and l_hip and r_hip and l_ankle and r_ankle:
|
| 220 |
-
sh_y = (l_sh[1] + r_sh[1]) / 2
|
| 221 |
-
hip_y = (l_hip[1] + r_hip[1]) / 2
|
| 222 |
-
ankle_y = (l_ankle[1] + r_ankle[1]) / 2
|
| 223 |
-
expected_hip_y = (sh_y + ankle_y) / 2
|
| 224 |
-
sag_px = hip_y - expected_hip_y
|
| 225 |
-
trunk_sags.append((i, sag_px))
|
| 226 |
-
|
| 227 |
-
max_sag_frame = 0
|
| 228 |
-
if trunk_sags:
|
| 229 |
-
sags = [s for _, s in trunk_sags]
|
| 230 |
-
max_sag_frame = max(trunk_sags, key=lambda t: t[1])[0]
|
| 231 |
-
mean = sum(sags) / len(sags)
|
| 232 |
-
variance = (sum((x - mean) ** 2 for x in sags) / len(sags)) ** 0.5
|
| 233 |
-
max_sag = max(sags)
|
| 234 |
-
angles["max_sag_px"] = max_sag
|
| 235 |
-
angles["trunk_variance_px"] = variance
|
| 236 |
-
alignments["body_rigid"] = max_sag < 30 and variance < 15
|
| 237 |
-
alignments["no_sag"] = max_sag < 30
|
| 238 |
-
else:
|
| 239 |
-
notes_parts.append("insufficient landmarks for trunk analysis")
|
| 240 |
-
```
|
| 241 |
-
|
| 242 |
-
Then update the `return BiomechFeatures(...)` `timing=` argument at the end of the method from:
|
| 243 |
-
|
| 244 |
-
```python
|
| 245 |
-
timing={"n_frames_analyzed": len(trunk_angles_over_time)},
|
| 246 |
-
```
|
| 247 |
-
|
| 248 |
-
to:
|
| 249 |
-
|
| 250 |
-
```python
|
| 251 |
-
timing={"n_frames_analyzed": len(trunk_sags), "max_sag_frame": max_sag_frame},
|
| 252 |
-
```
|
| 253 |
-
|
| 254 |
-
- [ ] **Step 4: Run test to verify it passes**
|
| 255 |
-
|
| 256 |
-
Run: `pytest tests/test_biomechanics.py::test_pushup_timing_has_max_sag_frame -v`
|
| 257 |
-
Expected: PASS
|
| 258 |
-
|
| 259 |
-
- [ ] **Step 5: Run the full biomechanics suite (no regressions)**
|
| 260 |
-
|
| 261 |
-
Run: `pytest tests/test_biomechanics.py -v`
|
| 262 |
-
Expected: all previously-passing tests still pass (the pre-existing `test_unimplemented_test_returns_low_confidence` known-failure may remain failing — that is unrelated and documented in CLAUDE.md).
|
| 263 |
-
|
| 264 |
-
- [ ] **Step 6: Commit**
|
| 265 |
-
|
| 266 |
-
```bash
|
| 267 |
-
git add formscout/agents/biomechanics.py tests/test_biomechanics.py
|
| 268 |
-
git commit -m "feat: track max-sag frame index in push-up biomechanics for key-frame capture"
|
| 269 |
-
```
|
| 270 |
-
|
| 271 |
-
---
|
| 272 |
-
|
| 273 |
-
## Task 4: Add `PoseVisualizer.render_frame()`
|
| 274 |
-
|
| 275 |
-
**Files:**
|
| 276 |
-
- Modify: `formscout/agents/visualizer.py` (add method to `PoseVisualizer`, after `render_video`)
|
| 277 |
-
- Test: `tests/test_keyframe.py`
|
| 278 |
-
|
| 279 |
-
- [ ] **Step 1: Write the failing test**
|
| 280 |
-
|
| 281 |
-
Create `tests/test_keyframe.py`:
|
| 282 |
-
|
| 283 |
-
```python
|
| 284 |
-
"""Tests for PoseVisualizer.render_frame — single annotated still."""
|
| 285 |
-
import os
|
| 286 |
-
import numpy as np
|
| 287 |
-
|
| 288 |
-
from formscout.types import IngestResult, Pose2DResult
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
def _ingest(n=5, h=480, w=640):
|
| 292 |
-
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 293 |
-
return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
def _pose(n=5):
|
| 297 |
-
kps = []
|
| 298 |
-
for i in range(n):
|
| 299 |
-
kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
|
| 300 |
-
for j in range(17)})
|
| 301 |
-
return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
def test_render_frame_writes_png(tmp_path):
|
| 305 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 306 |
-
out = str(tmp_path / "key.png")
|
| 307 |
-
path = PoseVisualizer().render_frame(_ingest(), _pose(), frame_idx=2,
|
| 308 |
-
layers={"skeleton"}, caption="Deep Squat — heels elevated",
|
| 309 |
-
out_png=out)
|
| 310 |
-
assert path == out
|
| 311 |
-
assert os.path.exists(out)
|
| 312 |
-
assert os.path.getsize(out) > 0
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
def test_render_frame_bad_index_returns_none(tmp_path):
|
| 316 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 317 |
-
out = str(tmp_path / "key.png")
|
| 318 |
-
path = PoseVisualizer().render_frame(_ingest(n=3), _pose(n=3), frame_idx=99,
|
| 319 |
-
layers={"skeleton"}, caption="", out_png=out)
|
| 320 |
-
assert path is None
|
| 321 |
-
```
|
| 322 |
-
|
| 323 |
-
- [ ] **Step 2: Run test to verify it fails**
|
| 324 |
-
|
| 325 |
-
Run: `pytest tests/test_keyframe.py -v`
|
| 326 |
-
Expected: FAIL with `AttributeError: 'PoseVisualizer' object has no attribute 'render_frame'`
|
| 327 |
-
|
| 328 |
-
- [ ] **Step 3: Add the method**
|
| 329 |
-
|
| 330 |
-
In `formscout/agents/visualizer.py`, inside the `PoseVisualizer` class, add this method
|
| 331 |
-
immediately after `render_video` (before the closing of the class / the module-level
|
| 332 |
-
`build_velocity_summary`):
|
| 333 |
-
|
| 334 |
-
```python
|
| 335 |
-
def render_frame(
|
| 336 |
-
self,
|
| 337 |
-
ingest,
|
| 338 |
-
pose2d,
|
| 339 |
-
frame_idx: int,
|
| 340 |
-
layers: set[str],
|
| 341 |
-
caption: str = "",
|
| 342 |
-
out_png: str | None = None,
|
| 343 |
-
) -> str | None:
|
| 344 |
-
"""Render a single annotated still (skeleton + optional trails + caption).
|
| 345 |
-
|
| 346 |
-
frame_idx is typically the governing frame from BiomechFeatures.timing.
|
| 347 |
-
Returns the PNG path on success, None on any failure. Never raises.
|
| 348 |
-
"""
|
| 349 |
-
try:
|
| 350 |
-
if not (0 <= frame_idx < len(ingest.frames)) or frame_idx >= len(pose2d.keypoints):
|
| 351 |
-
return None
|
| 352 |
-
|
| 353 |
-
frame = ingest.frames[frame_idx].copy()
|
| 354 |
-
kps = pose2d.keypoints[frame_idx]
|
| 355 |
-
|
| 356 |
-
if "trails" in layers:
|
| 357 |
-
trail: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
|
| 358 |
-
start = max(0, frame_idx - TRAIL_LENGTH)
|
| 359 |
-
for fi in range(start, frame_idx + 1):
|
| 360 |
-
for j, kp in pose2d.keypoints[fi].items():
|
| 361 |
-
if kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 362 |
-
trail[j].append((kp["x"], kp["y"]))
|
| 363 |
-
frame = self._draw_trails(frame, trail)
|
| 364 |
-
|
| 365 |
-
if "skeleton" in layers:
|
| 366 |
-
frame = self._draw_skeleton(frame, kps)
|
| 367 |
-
|
| 368 |
-
if caption:
|
| 369 |
-
cv2.rectangle(frame, (0, 0), (frame.shape[1], 28), (0, 0, 0), -1)
|
| 370 |
-
cv2.putText(frame, caption[:80], (8, 20), cv2.FONT_HERSHEY_SIMPLEX,
|
| 371 |
-
0.55, (255, 255, 255), 1, cv2.LINE_AA)
|
| 372 |
-
|
| 373 |
-
if out_png is None:
|
| 374 |
-
out_png = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
|
| 375 |
-
|
| 376 |
-
ok = cv2.imwrite(out_png, frame)
|
| 377 |
-
return out_png if ok else None
|
| 378 |
-
except Exception as e:
|
| 379 |
-
logger.warning("render_frame failed: %s", e)
|
| 380 |
-
return None
|
| 381 |
-
```
|
| 382 |
-
|
| 383 |
-
(`deque`, `cv2`, `tempfile`, `logger`, `TRAIL_LENGTH`, `CONF_THRESHOLD` are all already imported at the top of this file.)
|
| 384 |
-
|
| 385 |
-
- [ ] **Step 4: Run test to verify it passes**
|
| 386 |
-
|
| 387 |
-
Run: `pytest tests/test_keyframe.py -v`
|
| 388 |
-
Expected: both tests PASS
|
| 389 |
-
|
| 390 |
-
- [ ] **Step 5: Commit**
|
| 391 |
-
|
| 392 |
-
```bash
|
| 393 |
-
git add formscout/agents/visualizer.py tests/test_keyframe.py
|
| 394 |
-
git commit -m "feat: add PoseVisualizer.render_frame for annotated key-frame stills"
|
| 395 |
-
```
|
| 396 |
-
|
| 397 |
-
---
|
| 398 |
-
|
| 399 |
-
## Task 5: Create the session accumulator
|
| 400 |
-
|
| 401 |
-
**Files:**
|
| 402 |
-
- Create: `formscout/session.py`
|
| 403 |
-
- Test: `tests/test_session.py` (append tests)
|
| 404 |
-
|
| 405 |
-
- [ ] **Step 1: Write the failing tests**
|
| 406 |
-
|
| 407 |
-
Append to `tests/test_session.py`:
|
| 408 |
-
|
| 409 |
-
```python
|
| 410 |
-
def _ingest(n=5, h=480, w=640):
|
| 411 |
-
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 412 |
-
return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
def _pose(n=5):
|
| 416 |
-
kps = []
|
| 417 |
-
for i in range(n):
|
| 418 |
-
kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
|
| 419 |
-
for j in range(17)})
|
| 420 |
-
return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
def _features(test_name="deep_squat", side="na", frame_key="deepest_frame"):
|
| 424 |
-
return BiomechFeatures(
|
| 425 |
-
test_name=test_name, view="2d", side=side,
|
| 426 |
-
angles={"left_knee_flexion_deg": 95.0},
|
| 427 |
-
alignments={"knees_tracking_over_feet": False},
|
| 428 |
-
symmetry_delta=None, timing={frame_key: 2}, confidence=0.9,
|
| 429 |
-
)
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
def _judge(score=2, needs_human=False):
|
| 433 |
-
return JudgeResult(
|
| 434 |
-
score=None if needs_human else score, rationale="r",
|
| 435 |
-
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 436 |
-
confidence=0.85, needs_human=needs_human,
|
| 437 |
-
)
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
def test_add_analysis_appends_entry_and_writes_files():
|
| 441 |
-
import os
|
| 442 |
-
from formscout import session as S
|
| 443 |
-
sess = S.new_session()
|
| 444 |
-
entry = S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
|
| 445 |
-
features=_features(), judge=_judge(), test_name="deep_squat", side="na")
|
| 446 |
-
assert len(sess.entries) == 1
|
| 447 |
-
assert entry.score == 2
|
| 448 |
-
assert os.path.exists(os.path.join(sess.session_dir, "session.json"))
|
| 449 |
-
assert os.path.exists(os.path.join(sess.session_dir, "analysis.md"))
|
| 450 |
-
# key-frame still written (deepest_frame=2 is valid)
|
| 451 |
-
assert entry.keyframe_path and os.path.exists(entry.keyframe_path)
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
def test_finish_composite_null_when_needs_human():
|
| 455 |
-
from formscout import session as S
|
| 456 |
-
sess = S.new_session()
|
| 457 |
-
S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(), features=_features(),
|
| 458 |
-
judge=_judge(score=3), test_name="deep_squat", side="na")
|
| 459 |
-
S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
|
| 460 |
-
features=_features("trunk_stability_pushup", frame_key="max_sag_frame"),
|
| 461 |
-
judge=_judge(needs_human=True), test_name="trunk_stability_pushup", side="na")
|
| 462 |
-
report, pdf_path = S.finish_session(sess)
|
| 463 |
-
assert report is not None
|
| 464 |
-
assert report.composite is None # one test needs_human
|
| 465 |
-
|
| 466 |
-
|
| 467 |
-
def test_finish_empty_session_returns_none():
|
| 468 |
-
from formscout import session as S
|
| 469 |
-
sess = S.new_session()
|
| 470 |
-
report, pdf_path = S.finish_session(sess)
|
| 471 |
-
assert report is None and pdf_path is None
|
| 472 |
-
```
|
| 473 |
-
|
| 474 |
-
- [ ] **Step 2: Run tests to verify they fail**
|
| 475 |
-
|
| 476 |
-
Run: `pytest tests/test_session.py -v`
|
| 477 |
-
Expected: the three new tests FAIL with `ModuleNotFoundError: No module named 'formscout.session'`
|
| 478 |
-
|
| 479 |
-
- [ ] **Step 3: Create the module**
|
| 480 |
-
|
| 481 |
-
Create `formscout/session.py`:
|
| 482 |
-
|
| 483 |
-
```python
|
| 484 |
-
"""
|
| 485 |
-
Screening-session accumulator.
|
| 486 |
-
|
| 487 |
-
Accumulates one SessionEntry per analyzed clip, persists each to a temp session
|
| 488 |
-
dir (session.json + analysis.md + key-frame PNGs), and on finish builds a
|
| 489 |
-
ReportResult (via ReportAgent) + a PDF (via PdfReportAgent).
|
| 490 |
-
|
| 491 |
-
Pure orchestration — no Gradio imports. Disk writes tolerate failure with a
|
| 492 |
-
logged warning and never block scoring.
|
| 493 |
-
"""
|
| 494 |
-
from __future__ import annotations
|
| 495 |
-
|
| 496 |
-
import json
|
| 497 |
-
import logging
|
| 498 |
-
import os
|
| 499 |
-
import tempfile
|
| 500 |
-
import uuid
|
| 501 |
-
from dataclasses import dataclass, replace
|
| 502 |
-
|
| 503 |
-
from formscout.rubric import score_test
|
| 504 |
-
from formscout.types import MovementResult, ReportResult, SessionEntry
|
| 505 |
-
|
| 506 |
-
logger = logging.getLogger(__name__)
|
| 507 |
-
|
| 508 |
-
# Maps each test to the BiomechFeatures.timing key holding its governing frame.
|
| 509 |
-
TIMING_KEY = {
|
| 510 |
-
"deep_squat": "deepest_frame",
|
| 511 |
-
"hurdle_step": "peak_step_frame",
|
| 512 |
-
"inline_lunge": "deepest_lunge_frame",
|
| 513 |
-
"shoulder_mobility": "measure_frame",
|
| 514 |
-
"active_slr": "peak_raise_frame",
|
| 515 |
-
"trunk_stability_pushup": "max_sag_frame",
|
| 516 |
-
"rotary_stability": "peak_extension_frame",
|
| 517 |
-
}
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
@dataclass
|
| 521 |
-
class Session:
|
| 522 |
-
"""Mutable session: an id, its temp dir, and accumulated entries."""
|
| 523 |
-
session_id: str
|
| 524 |
-
session_dir: str
|
| 525 |
-
entries: list # list[SessionEntry]
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
def new_session() -> Session:
|
| 529 |
-
sid = uuid.uuid4().hex[:12]
|
| 530 |
-
base = os.path.join(tempfile.gettempdir(), "formscout_sessions", sid)
|
| 531 |
-
try:
|
| 532 |
-
os.makedirs(os.path.join(base, "keyframes"), exist_ok=True)
|
| 533 |
-
except Exception as e:
|
| 534 |
-
logger.warning("session dir create failed: %s", e)
|
| 535 |
-
return Session(session_id=sid, session_dir=base, entries=[])
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
def governing_frame_index(features) -> int | None:
|
| 539 |
-
"""Return the governing frame index for this test, or None."""
|
| 540 |
-
key = TIMING_KEY.get(features.test_name)
|
| 541 |
-
if key is None:
|
| 542 |
-
return None
|
| 543 |
-
idx = features.timing.get(key)
|
| 544 |
-
return int(idx) if isinstance(idx, (int, float)) else None
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
def worst_compensation_caption(judge, features) -> str:
|
| 548 |
-
"""Short caption naming the worst compensation for the key-frame still."""
|
| 549 |
-
if judge and getattr(judge, "compensation_tags", None):
|
| 550 |
-
return ", ".join(judge.compensation_tags)
|
| 551 |
-
failed = [k.replace("_", " ") for k, v in features.alignments.items() if v is False]
|
| 552 |
-
return ("compensation: " + ", ".join(failed)) if failed else "key position"
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
def add_analysis(session, *, ingest, pose2d, features, judge, test_name, side,
|
| 556 |
-
draw_trails: bool = False) -> SessionEntry:
|
| 557 |
-
"""Build a SessionEntry from a completed analysis, render its key-frame,
|
| 558 |
-
persist the session, append, and return the entry."""
|
| 559 |
-
movement = MovementResult(test_name=test_name, side=side, confidence=1.0)
|
| 560 |
-
rubric = score_test(features)
|
| 561 |
-
|
| 562 |
-
needs_human = bool((judge and judge.needs_human) or rubric.needs_human)
|
| 563 |
-
if needs_human:
|
| 564 |
-
score = None
|
| 565 |
-
elif judge and judge.score is not None:
|
| 566 |
-
score = judge.score
|
| 567 |
-
else:
|
| 568 |
-
score = rubric.score
|
| 569 |
-
|
| 570 |
-
keyframe_path = None
|
| 571 |
-
idx = governing_frame_index(features)
|
| 572 |
-
if idx is not None and 0 <= idx < len(pose2d.keypoints):
|
| 573 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 574 |
-
caption = (f"{test_name.replace('_', ' ').title()} "
|
| 575 |
-
f"({side}) — {worst_compensation_caption(judge, features)}")
|
| 576 |
-
layers = {"skeleton", "trails"} if draw_trails else {"skeleton"}
|
| 577 |
-
out_png = os.path.join(session.session_dir, "keyframes", f"{test_name}_{side}.png")
|
| 578 |
-
try:
|
| 579 |
-
keyframe_path = PoseVisualizer().render_frame(ingest, pose2d, idx, layers, caption, out_png)
|
| 580 |
-
except Exception as e:
|
| 581 |
-
logger.warning("keyframe render failed: %s", e)
|
| 582 |
-
|
| 583 |
-
measurements = {}
|
| 584 |
-
measurements.update(features.angles)
|
| 585 |
-
measurements.update(features.alignments)
|
| 586 |
-
|
| 587 |
-
entry = SessionEntry(
|
| 588 |
-
test_name=test_name, side=side, score=score, needs_human=needs_human,
|
| 589 |
-
rationale=(judge.rationale if judge else rubric.rationale),
|
| 590 |
-
compensation_tags=list(judge.compensation_tags) if judge else [],
|
| 591 |
-
corrective_hint=(judge.corrective_hint if judge else ""),
|
| 592 |
-
measurements=measurements,
|
| 593 |
-
confidence=(judge.confidence if judge else rubric.confidence),
|
| 594 |
-
view=features.view,
|
| 595 |
-
keyframe_path=keyframe_path,
|
| 596 |
-
movement=movement, features=features, rubric_score=rubric, judge=judge,
|
| 597 |
-
)
|
| 598 |
-
session.entries.append(entry)
|
| 599 |
-
_persist(session)
|
| 600 |
-
return entry
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
def finish_session(session) -> tuple[ReportResult | None, str | None]:
|
| 604 |
-
"""Build the composite report + PDF. Returns (report, pdf_path).
|
| 605 |
-
Returns (None, None) for an empty session."""
|
| 606 |
-
if not session.entries:
|
| 607 |
-
return None, None
|
| 608 |
-
|
| 609 |
-
from formscout.agents.report import ReportAgent
|
| 610 |
-
report_inputs = [{
|
| 611 |
-
"movement": e.movement, "features": e.features,
|
| 612 |
-
"rubric_score": e.rubric_score, "judge": e.judge, "side": e.side,
|
| 613 |
-
} for e in session.entries]
|
| 614 |
-
report = ReportAgent().run(report_inputs)
|
| 615 |
-
|
| 616 |
-
pdf_path = None
|
| 617 |
-
try:
|
| 618 |
-
from formscout.agents.pdf_report import PdfReportAgent
|
| 619 |
-
pdf_path = PdfReportAgent().run(report, session.entries, session.session_dir)
|
| 620 |
-
except Exception as e:
|
| 621 |
-
logger.warning("pdf generation failed: %s", e)
|
| 622 |
-
|
| 623 |
-
report = replace(report, pdf_path=pdf_path)
|
| 624 |
-
return report, pdf_path
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
# ── Persistence ───────────────────────────────────────────────────────────────
|
| 628 |
-
|
| 629 |
-
def _jsonable(d: dict) -> dict:
|
| 630 |
-
out = {}
|
| 631 |
-
for k, v in d.items():
|
| 632 |
-
if isinstance(v, float):
|
| 633 |
-
out[k] = round(v, 2)
|
| 634 |
-
elif isinstance(v, (int, str, bool)) or v is None:
|
| 635 |
-
out[k] = v
|
| 636 |
-
else:
|
| 637 |
-
out[k] = str(v)
|
| 638 |
-
return out
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
def _entry_display(e: SessionEntry) -> dict:
|
| 642 |
-
return {
|
| 643 |
-
"test_name": e.test_name, "side": e.side, "score": e.score,
|
| 644 |
-
"needs_human": e.needs_human, "rationale": e.rationale,
|
| 645 |
-
"compensation_tags": list(e.compensation_tags), "corrective_hint": e.corrective_hint,
|
| 646 |
-
"measurements": _jsonable(e.measurements), "confidence": round(e.confidence, 2),
|
| 647 |
-
"view": e.view, "keyframe_path": e.keyframe_path,
|
| 648 |
-
}
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
def _render_markdown(session: Session) -> str:
|
| 652 |
-
lines = ["# FormScout — Session Log", ""]
|
| 653 |
-
for e in session.entries:
|
| 654 |
-
title = e.test_name.replace("_", " ").title()
|
| 655 |
-
if e.side in ("left", "right"):
|
| 656 |
-
title += f" ({e.side})"
|
| 657 |
-
score = "Clinician review required" if e.needs_human else f"{e.score}/3"
|
| 658 |
-
lines.append(f"## {title}
|
| 659 |
-
lines.append(e.rationale or "")
|
| 660 |
-
if e.compensation_tags:
|
| 661 |
-
lines.append(f"- Compensations: {', '.join(e.compensation_tags)}")
|
| 662 |
-
if e.corrective_hint:
|
| 663 |
-
lines.append(f"- Corrective: {e.corrective_hint}")
|
| 664 |
-
if e.keyframe_path:
|
| 665 |
-
lines.append(f"- Key frame: `{e.keyframe_path}`")
|
| 666 |
-
lines.append("")
|
| 667 |
-
return "\n".join(lines)
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
def _persist(session: Session) -> None:
|
| 671 |
-
try:
|
| 672 |
-
with open(os.path.join(session.session_dir, "session.json"), "w") as f:
|
| 673 |
-
json.dump([_entry_display(e) for e in session.entries], f, indent=2)
|
| 674 |
-
with open(os.path.join(session.session_dir, "analysis.md"), "w") as f:
|
| 675 |
-
f.write(_render_markdown(session))
|
| 676 |
-
except Exception as e:
|
| 677 |
-
logger.warning("session persist failed: %s", e)
|
| 678 |
-
```
|
| 679 |
-
|
| 680 |
-
- [ ] **Step 4: Run tests to verify they pass**
|
| 681 |
-
|
| 682 |
-
Run: `pytest tests/test_session.py -v`
|
| 683 |
-
Expected: all session tests PASS (Task 6 provides `PdfReportAgent`; `finish_session` tolerates its
|
| 684 |
-
absence via the try/except, so these pass now — `pdf_path` may be `None` until Task 6).
|
| 685 |
-
|
| 686 |
-
- [ ] **Step 5: Commit**
|
| 687 |
-
|
| 688 |
-
```bash
|
| 689 |
-
git add formscout/session.py tests/test_session.py
|
| 690 |
-
git commit -m "feat: add screening-session accumulator with key-frame capture and persistence"
|
| 691 |
-
```
|
| 692 |
-
|
| 693 |
-
---
|
| 694 |
-
|
| 695 |
-
## Task 6: Create `PdfReportAgent`
|
| 696 |
-
|
| 697 |
-
**Files:**
|
| 698 |
-
- Create: `formscout/agents/pdf_report.py`
|
| 699 |
-
- Test: `tests/test_pdf_report.py`
|
| 700 |
-
|
| 701 |
-
- [ ] **Step 1: Write the failing test**
|
| 702 |
-
|
| 703 |
-
Create `tests/test_pdf_report.py`:
|
| 704 |
-
|
| 705 |
-
```python
|
| 706 |
-
"""Tests for PdfReportAgent — no GPU, no model downloads."""
|
| 707 |
-
import os
|
| 708 |
-
|
| 709 |
-
from formscout.types import (
|
| 710 |
-
ReportResult, SessionEntry, MovementResult, BiomechFeatures, ScoreResult, JudgeResult,
|
| 711 |
-
)
|
| 712 |
-
|
| 713 |
-
|
| 714 |
-
def _entry(test_name="deep_squat", score=2, needs_human=False):
|
| 715 |
-
movement = MovementResult(test_name=test_name, side="na", confidence=1.0)
|
| 716 |
-
features = BiomechFeatures(
|
| 717 |
-
test_name=test_name, view="2d", side="na",
|
| 718 |
-
angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": False},
|
| 719 |
-
symmetry_delta=None, timing={"deepest_frame": 1}, confidence=0.9,
|
| 720 |
-
)
|
| 721 |
-
rubric = ScoreResult(score=2, rationale="rubric ok", confidence=0.8)
|
| 722 |
-
judge = JudgeResult(score=None if needs_human else score, rationale="judge rationale",
|
| 723 |
-
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 724 |
-
confidence=0.85, needs_human=needs_human)
|
| 725 |
-
return SessionEntry(
|
| 726 |
-
test_name=test_name, side="na", score=None if needs_human else score,
|
| 727 |
-
needs_human=needs_human, rationale="judge rationale",
|
| 728 |
-
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 729 |
-
measurements={"left_knee_flexion_deg": 95.0, "knees_tracking_over_feet": False},
|
| 730 |
-
confidence=0.85, view="2d", keyframe_path=None,
|
| 731 |
-
movement=movement, features=features, rubric_score=rubric, judge=judge,
|
| 732 |
-
)
|
| 733 |
-
|
| 734 |
-
|
| 735 |
-
def _report(composite=2):
|
| 736 |
-
return ReportResult(
|
| 737 |
-
per_test=[], composite=composite, asymmetries=[],
|
| 738 |
-
overlay_video_path=None, pdf_path=None,
|
| 739 |
-
low_confidence_flags=[], disagreement_flags=[],
|
| 740 |
-
)
|
| 741 |
-
|
| 742 |
-
|
| 743 |
-
def test_pdf_is_created(tmp_path):
|
| 744 |
-
from formscout.agents.pdf_report import PdfReportAgent
|
| 745 |
-
path = PdfReportAgent().run(_report(2), [_entry()], str(tmp_path))
|
| 746 |
-
assert path is not None
|
| 747 |
-
assert os.path.exists(path)
|
| 748 |
-
assert os.path.getsize(path) > 1000 # a real PDF, not an empty file
|
| 749 |
-
with open(path, "rb") as f:
|
| 750 |
-
assert f.read(5) == b"%PDF-"
|
| 751 |
-
|
| 752 |
-
|
| 753 |
-
def test_pdf_handles_incomplete_composite(tmp_path):
|
| 754 |
-
from formscout.agents.pdf_report import PdfReportAgent
|
| 755 |
-
path = PdfReportAgent().run(_report(None), [_entry(needs_human=True)], str(tmp_path))
|
| 756 |
-
assert path is not None and os.path.exists(path)
|
| 757 |
-
```
|
| 758 |
-
|
| 759 |
-
- [ ] **Step 2: Run test to verify it fails**
|
| 760 |
-
|
| 761 |
-
Run: `pytest tests/test_pdf_report.py -v`
|
| 762 |
-
Expected: FAIL with `ModuleNotFoundError: No module named 'formscout.agents.pdf_report'`
|
| 763 |
-
|
| 764 |
-
- [ ] **Step 3: Create the agent**
|
| 765 |
-
|
| 766 |
-
Create `formscout/agents/pdf_report.py`:
|
| 767 |
-
|
| 768 |
-
```python
|
| 769 |
-
"""
|
| 770 |
-
PdfReportAgent — renders a ReportResult + session entries to a branded PDF.
|
| 771 |
-
|
| 772 |
-
Input: ReportResult, list[SessionEntry], session_dir (str)
|
| 773 |
-
Output: path to the written PDF (str), or None on failure.
|
| 774 |
-
Failure: returns None, never raises.
|
| 775 |
-
Params: 0 (pure rendering — no model).
|
| 776 |
-
License: n/a.
|
| 777 |
-
Gated: no.
|
| 778 |
-
"""
|
| 779 |
-
from __future__ import annotations
|
| 780 |
-
|
| 781 |
-
import logging
|
| 782 |
-
import os
|
| 783 |
-
|
| 784 |
-
from formscout.types import ReportResult
|
| 785 |
-
|
| 786 |
-
logger = logging.getLogger(__name__)
|
| 787 |
-
|
| 788 |
-
DISCLAIMER = "Screening aid — not a diagnosis. Pain or clearing tests require a clinician."
|
| 789 |
-
|
| 790 |
-
|
| 791 |
-
class PdfReportAgent:
|
| 792 |
-
"""Assembles the screening-session PDF via ReportLab."""
|
| 793 |
-
|
| 794 |
-
def run(self, report: ReportResult, entries: list, session_dir: str) -> str | None:
|
| 795 |
-
try:
|
| 796 |
-
from reportlab.lib import colors
|
| 797 |
-
from reportlab.lib.pagesizes import LETTER
|
| 798 |
-
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
| 799 |
-
from reportlab.lib.units import inch
|
| 800 |
-
from reportlab.platypus import (
|
| 801 |
-
Image, Paragraph, SimpleDocTemplate, Spacer, Table, TableStyle,
|
| 802 |
-
)
|
| 803 |
-
except Exception as e:
|
| 804 |
-
logger.warning("reportlab unavailable: %s", e)
|
| 805 |
-
return None
|
| 806 |
-
|
| 807 |
-
out_path = os.path.join(session_dir, "formscout_report.pdf")
|
| 808 |
-
try:
|
| 809 |
-
styles = getSampleStyleSheet()
|
| 810 |
-
banner = ParagraphStyle(
|
| 811 |
-
"banner", parent=styles["Normal"], fontSize=9, textColor=colors.white,
|
| 812 |
-
backColor=colors.HexColor("#b45309"), alignment=1, borderPadding=6, spaceAfter=12,
|
| 813 |
-
)
|
| 814 |
-
story = []
|
| 815 |
-
story.append(Paragraph(f"<b>⚠ {DISCLAIMER}</b>", banner))
|
| 816 |
-
story.append(Paragraph("FormScout — FMS Screening Report", styles["Title"]))
|
| 817 |
-
|
| 818 |
-
if report.composite is not None:
|
| 819 |
-
comp = f"Composite: <b>{report.composite} / 21</b>"
|
| 820 |
-
else:
|
| 821 |
-
comp = f"Composite: <b>Incomplete</b> — {len(entries)}/7 tests scored"
|
| 822 |
-
story.append(Paragraph(comp, styles["Heading2"]))
|
| 823 |
-
story.append(Spacer(1, 0.2 * inch))
|
| 824 |
-
|
| 825 |
-
for e in entries:
|
| 826 |
-
title = e.test_name.replace("_", " ").title()
|
| 827 |
-
if e.side in ("left", "right"):
|
| 828 |
-
title += f" ({e.side})"
|
| 829 |
-
score_txt = "Clinician review required" if e.needs_human else f"Score: {e.score}/3"
|
| 830 |
-
story.append(Paragraph(f"<b>{title}</b> — {score_txt}", styles["Heading3"]))
|
| 831 |
-
if e.rationale:
|
| 832 |
-
story.append(Paragraph(e.rationale, styles["Normal"]))
|
| 833 |
-
if e.compensation_tags:
|
| 834 |
-
story.append(Paragraph("Compensations: " + ", ".join(e.compensation_tags),
|
| 835 |
-
styles["Normal"]))
|
| 836 |
-
if e.corrective_hint:
|
| 837 |
-
story.append(Paragraph("Corrective: " + e.corrective_hint, styles["Normal"]))
|
| 838 |
-
|
| 839 |
-
items = list(e.measurements.items())[:6]
|
| 840 |
-
if items:
|
| 841 |
-
rows = [[k.replace("_", " "),
|
| 842 |
-
(f"{v:.1f}" if isinstance(v, float) else str(v))] for k, v in items]
|
| 843 |
-
tbl = Table(rows, colWidths=[3 * inch, 1.5 * inch])
|
| 844 |
-
tbl.setStyle(TableStyle([
|
| 845 |
-
("FONTSIZE", (0, 0), (-1, -1), 8),
|
| 846 |
-
("TEXTCOLOR", (0, 0), (-1, -1), colors.HexColor("#334155")),
|
| 847 |
-
]))
|
| 848 |
-
story.append(tbl)
|
| 849 |
-
|
| 850 |
-
if e.keyframe_path and os.path.exists(e.keyframe_path):
|
| 851 |
-
try:
|
| 852 |
-
story.append(Image(e.keyframe_path, width=3.0 * inch, height=2.25 * inch))
|
| 853 |
-
except Exception:
|
| 854 |
-
story.append(Paragraph("<i>(key-frame image unavailable)</i>", styles["Normal"]))
|
| 855 |
-
else:
|
| 856 |
-
story.append(Paragraph("<i>(key-frame image unavailable)</i>", styles["Normal"]))
|
| 857 |
-
|
| 858 |
-
story.append(Spacer(1, 0.2 * inch))
|
| 859 |
-
|
| 860 |
-
if report.asymmetries:
|
| 861 |
-
story.append(Paragraph("Asymmetries", styles["Heading2"]))
|
| 862 |
-
for a in report.asymmetries:
|
| 863 |
-
story.append(Paragraph(
|
| 864 |
-
f"{a['test'].replace('_', ' ').title()}: "
|
| 865 |
-
f"L={a['left_score']} R={a['right_score']} (Δ {a['delta']})",
|
| 866 |
-
styles["Normal"]))
|
| 867 |
-
|
| 868 |
-
flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
|
| 869 |
-
if flags:
|
| 870 |
-
story.append(Paragraph("Flags", styles["Heading2"]))
|
| 871 |
-
for fl in flags:
|
| 872 |
-
story.append(Paragraph(fl, styles["Normal"]))
|
| 873 |
-
|
| 874 |
-
story.append(Spacer(1, 0.3 * inch))
|
| 875 |
-
story.append(Paragraph(f"<b>⚠ {DISCLAIMER}</b>", banner))
|
| 876 |
-
|
| 877 |
-
doc = SimpleDocTemplate(out_path, pagesize=LETTER,
|
| 878 |
-
topMargin=0.6 * inch, bottomMargin=0.6 * inch)
|
| 879 |
-
doc.build(story)
|
| 880 |
-
return out_path
|
| 881 |
-
except Exception as e:
|
| 882 |
-
logger.warning("pdf build failed: %s", e)
|
| 883 |
-
return None
|
| 884 |
-
```
|
| 885 |
-
|
| 886 |
-
- [ ] **Step 4: Run test to verify it passes**
|
| 887 |
-
|
| 888 |
-
Run: `pytest tests/test_pdf_report.py -v`
|
| 889 |
-
Expected: both tests PASS
|
| 890 |
-
|
| 891 |
-
- [ ] **Step 5: Re-run the session suite (pdf_path now populated)**
|
| 892 |
-
|
| 893 |
-
Run: `pytest tests/test_session.py -v`
|
| 894 |
-
Expected: all PASS (now `finish_session` returns a real `pdf_path`).
|
| 895 |
-
|
| 896 |
-
- [ ] **Step 6: Commit**
|
| 897 |
-
|
| 898 |
-
```bash
|
| 899 |
-
git add formscout/agents/pdf_report.py tests/test_pdf_report.py
|
| 900 |
-
git commit -m "feat: add PdfReportAgent — branded ReportLab session PDF"
|
| 901 |
-
```
|
| 902 |
-
|
| 903 |
-
---
|
| 904 |
-
|
| 905 |
-
## Task 7: Wire the session UI in `app.py`
|
| 906 |
-
|
| 907 |
-
**Files:**
|
| 908 |
-
- Modify: `app.py` (`process_video`, `build_app`, event wiring)
|
| 909 |
-
|
| 910 |
-
This task is verified by running the app (Gradio event wiring is not unit-tested; the
|
| 911 |
-
orchestration it calls is already covered by `tests/test_session.py`).
|
| 912 |
-
|
| 913 |
-
- [ ] **Step 1: Import the session module**
|
| 914 |
-
|
| 915 |
-
In `app.py`, add to the imports block (after `from formscout.startup import ensure_checkpoints`):
|
| 916 |
-
|
| 917 |
-
```python
|
| 918 |
-
from formscout import session as session_mod
|
| 919 |
-
```
|
| 920 |
-
|
| 921 |
-
- [ ] **Step 2: Refactor `process_video` to accumulate into a session**
|
| 922 |
-
|
| 923 |
-
Replace the entire `process_video` function (lines ~51-105) with a version that takes and
|
| 924 |
-
returns the session, appends an entry on success, and builds the "Session so far" table.
|
| 925 |
-
Replace from `def process_video(` through its final `return ...` with:
|
| 926 |
-
|
| 927 |
-
```python
|
| 928 |
-
def process_video(video_path: str, test_name: str, side: str, model_key: str,
|
| 929 |
-
layers: list[str], session_state):
|
| 930 |
-
"""Analyse one clip and accumulate it into the screening session."""
|
| 931 |
-
if not video_path:
|
| 932 |
-
return (
|
| 933 |
-
session_state, _render_empty_state(), "Upload a video to begin analysis.",
|
| 934 |
-
"", "", None, "", _render_session_table(session_state),
|
| 935 |
-
gr.update(visible=False), gr.update(visible=False),
|
| 936 |
-
)
|
| 937 |
-
|
| 938 |
-
if session_state is None:
|
| 939 |
-
session_state = session_mod.new_session()
|
| 940 |
-
|
| 941 |
-
director = Director()
|
| 942 |
-
state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
|
| 943 |
-
|
| 944 |
-
score_html = _render_empty_state()
|
| 945 |
-
score_details = ""
|
| 946 |
-
|
| 947 |
-
if state.features:
|
| 948 |
-
result = score_test(state.features)
|
| 949 |
-
judge = state.judge
|
| 950 |
-
if judge and judge.score is not None:
|
| 951 |
-
score_html = _render_score_card(judge.score, judge.confidence, judge.needs_human)
|
| 952 |
-
score_details = _render_score_details_judge(judge, result, state.features)
|
| 953 |
-
elif judge and judge.needs_human:
|
| 954 |
-
score_html = _render_score_card(0, 0, True)
|
| 955 |
-
score_details = f"### Needs Clinician Review\n{judge.rationale}"
|
| 956 |
-
else:
|
| 957 |
-
score_html = _render_score_card(result.score, result.confidence, result.needs_human)
|
| 958 |
-
score_details = _render_score_details(result, state.features)
|
| 959 |
-
|
| 960 |
-
# Accumulate into the session (only when we have a real analysis)
|
| 961 |
-
if state.ingest and state.pose2d and state.judge:
|
| 962 |
-
draw_trails = "trails" in {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
|
| 963 |
-
try:
|
| 964 |
-
session_mod.add_analysis(
|
| 965 |
-
session_state, ingest=state.ingest, pose2d=state.pose2d,
|
| 966 |
-
features=state.features, judge=state.judge,
|
| 967 |
-
test_name=test_name, side=side, draw_trails=draw_trails,
|
| 968 |
-
)
|
| 969 |
-
except Exception as e:
|
| 970 |
-
state.warnings.append(f"session accumulation failed: {e}")
|
| 971 |
-
|
| 972 |
-
pipeline_md = _render_pipeline_status(state)
|
| 973 |
-
alerts = _render_alerts(state)
|
| 974 |
-
|
| 975 |
-
overlay_path = None
|
| 976 |
-
vel_summary = ""
|
| 977 |
-
layer_set = {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
|
| 978 |
-
if layer_set and state.ingest and state.pose2d:
|
| 979 |
-
try:
|
| 980 |
-
from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
|
| 981 |
-
vis = PoseVisualizer()
|
| 982 |
-
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
|
| 983 |
-
out_path = f.name
|
| 984 |
-
overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
|
| 985 |
-
if overlay_path:
|
| 986 |
-
vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
|
| 987 |
-
except Exception as e:
|
| 988 |
-
alerts = (alerts or "") + f"\n⚠️ Visualizer error: {e}"
|
| 989 |
-
|
| 990 |
-
has_entries = bool(session_state and session_state.entries)
|
| 991 |
-
return (
|
| 992 |
-
session_state, score_html, pipeline_md, score_details, alerts,
|
| 993 |
-
overlay_path, vel_summary, _render_session_table(session_state),
|
| 994 |
-
gr.update(visible=has_entries), gr.update(visible=has_entries),
|
| 995 |
-
)
|
| 996 |
-
```
|
| 997 |
-
|
| 998 |
-
- [ ] **Step 3: Add the session-table renderer and finish handler**
|
| 999 |
-
|
| 1000 |
-
In `app.py`, add these two functions just before `def build_app()`:
|
| 1001 |
-
|
| 1002 |
-
```python
|
| 1003 |
-
def _render_session_table(session_state) -> str:
|
| 1004 |
-
"""Render the accumulated 'Session so far' table as markdown."""
|
| 1005 |
-
if not session_state or not session_state.entries:
|
| 1006 |
-
return "*No clips analysed yet.*"
|
| 1007 |
-
lines = ["| Test | Side | Score | Status |", "|---|---|---|---|"]
|
| 1008 |
-
for e in session_state.entries:
|
| 1009 |
-
test = e.test_name.replace("_", " ").title()
|
| 1010 |
-
side = e.side if e.side in ("left", "right") else "—"
|
| 1011 |
-
if e.needs_human:
|
| 1012 |
-
score, status = "—", "⚠️ Clinician review"
|
| 1013 |
-
else:
|
| 1014 |
-
score, status = f"{e.score}/3", "✓ scored"
|
| 1015 |
-
lines.append(f"| {test} | {side} | {score} | {status} |")
|
| 1016 |
-
return "\n".join(lines)
|
| 1017 |
-
|
| 1018 |
-
|
| 1019 |
-
def _finish_session(session_state):
|
| 1020 |
-
"""Build the composite report + PDF for the whole session."""
|
| 1021 |
-
if not session_state or not session_state.entries:
|
| 1022 |
-
return ("⚠️ No clips analysed yet — analyse at least one clip first.",
|
| 1023 |
-
None, None)
|
| 1024 |
-
|
| 1025 |
-
report, pdf_path = session_mod.finish_session(session_state)
|
| 1026 |
-
if report is None:
|
| 1027 |
-
return ("⚠️ Nothing to report.", None, None)
|
| 1028 |
-
|
| 1029 |
-
if report.composite is not None:
|
| 1030 |
-
summary = [f"## Composite: {report.composite} / 21"]
|
| 1031 |
-
else:
|
| 1032 |
-
n = len(session_state.entries)
|
| 1033 |
-
summary = [f"## Composite: Incomplete — {n}/7 tests scored",
|
| 1034 |
-
"*(One or more tests need clinician review or were unscored.)*"]
|
| 1035 |
-
|
| 1036 |
-
if report.asymmetries:
|
| 1037 |
-
summary.append("\n### Asymmetries")
|
| 1038 |
-
for a in report.asymmetries:
|
| 1039 |
-
test = a["test"].replace("_", " ").title()
|
| 1040 |
-
summary.append(f"- **{test}:** L={a['left_score']} R={a['right_score']} (Δ {a['delta']})")
|
| 1041 |
-
|
| 1042 |
-
flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
|
| 1043 |
-
if flags:
|
| 1044 |
-
summary.append("\n### Flags")
|
| 1045 |
-
for fl in flags:
|
| 1046 |
-
summary.append(f"- {fl}")
|
| 1047 |
-
|
| 1048 |
-
md_path = os.path.join(session_state.session_dir, "analysis.md")
|
| 1049 |
-
md_out = md_path if os.path.exists(md_path) else None
|
| 1050 |
-
return "\n".join(summary), pdf_path, md_out
|
| 1051 |
-
```
|
| 1052 |
-
|
| 1053 |
-
Also add `import os` to the top of `app.py` if not already present (it currently imports only
|
| 1054 |
-
`tempfile` and `gradio`). Add after `import tempfile`:
|
| 1055 |
-
|
| 1056 |
-
```python
|
| 1057 |
-
import os
|
| 1058 |
-
```
|
| 1059 |
-
|
| 1060 |
-
- [ ] **Step 4: Add the session state, buttons, and outputs to `build_app`**
|
| 1061 |
-
|
| 1062 |
-
In `build_app`, inside the `with gr.Blocks(...) as app:` block, immediately after the line
|
| 1063 |
-
`with gr.Blocks(title="FormScout — FMS Screening Aid") as app:` add:
|
| 1064 |
-
|
| 1065 |
-
```python
|
| 1066 |
-
session_state = gr.State(None)
|
| 1067 |
-
```
|
| 1068 |
-
|
| 1069 |
-
Then, in the left input column, replace the single submit button block:
|
| 1070 |
-
|
| 1071 |
-
```python
|
| 1072 |
-
submit_btn = gr.Button(
|
| 1073 |
-
"🎯 Score Movement",
|
| 1074 |
-
variant="primary",
|
| 1075 |
-
size="lg",
|
| 1076 |
-
)
|
| 1077 |
-
```
|
| 1078 |
-
|
| 1079 |
-
with:
|
| 1080 |
-
|
| 1081 |
-
```python
|
| 1082 |
-
submit_btn = gr.Button(
|
| 1083 |
-
"🎯 Score Movement",
|
| 1084 |
-
variant="primary",
|
| 1085 |
-
size="lg",
|
| 1086 |
-
)
|
| 1087 |
-
with gr.Row():
|
| 1088 |
-
new_clip_btn = gr.Button("➕ Analyse new clip", visible=False)
|
| 1089 |
-
finish_btn = gr.Button("✅ Finish & generate PDF",
|
| 1090 |
-
variant="primary", visible=False)
|
| 1091 |
-
```
|
| 1092 |
-
|
| 1093 |
-
In the right results column, add a "Session" tab and a finish-output area. Inside `with gr.Tabs():`
|
| 1094 |
-
add a new tab after the "🎬 Overlay Video" tab:
|
| 1095 |
-
|
| 1096 |
-
```python
|
| 1097 |
-
with gr.TabItem("🗂️ Session"):
|
| 1098 |
-
session_table = gr.Markdown("*No clips analysed yet.*")
|
| 1099 |
-
finish_summary = gr.Markdown("")
|
| 1100 |
-
pdf_file = gr.File(label="Screening Report (PDF)", visible=True)
|
| 1101 |
-
md_file = gr.File(label="Analysis Log (Markdown)", visible=True)
|
| 1102 |
-
```
|
| 1103 |
-
|
| 1104 |
-
- [ ] **Step 5: Update event wiring**
|
| 1105 |
-
|
| 1106 |
-
Replace the `_map_inputs` function and `submit_btn.click(...)` block at the bottom of `build_app`
|
| 1107 |
-
with:
|
| 1108 |
-
|
| 1109 |
-
```python
|
| 1110 |
-
def _map_inputs(video, test_display_name, side_display, pose_model_key, overlay_layers, sess):
|
| 1111 |
-
"""Map UI display values to internal values and accumulate into the session."""
|
| 1112 |
-
test_map = {name: val for name, val in FMS_TESTS}
|
| 1113 |
-
test_name = test_map.get(test_display_name, "deep_squat")
|
| 1114 |
-
side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
|
| 1115 |
-
return process_video(video, test_name, side, pose_model_key, overlay_layers, sess)
|
| 1116 |
-
|
| 1117 |
-
submit_btn.click(
|
| 1118 |
-
fn=_map_inputs,
|
| 1119 |
-
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown,
|
| 1120 |
-
overlay_layers, session_state],
|
| 1121 |
-
outputs=[session_state, score_html, pipeline_md, score_details, alerts_md,
|
| 1122 |
-
overlay_video, velocity_md, session_table, new_clip_btn, finish_btn],
|
| 1123 |
-
)
|
| 1124 |
-
|
| 1125 |
-
def _new_clip():
|
| 1126 |
-
"""Clear inputs for the next clip; keep the session intact."""
|
| 1127 |
-
return None, _render_empty_state(), ""
|
| 1128 |
-
|
| 1129 |
-
new_clip_btn.click(
|
| 1130 |
-
fn=_new_clip,
|
| 1131 |
-
inputs=[],
|
| 1132 |
-
outputs=[video_input, score_html, score_details],
|
| 1133 |
-
)
|
| 1134 |
-
|
| 1135 |
-
finish_btn.click(
|
| 1136 |
-
fn=_finish_session,
|
| 1137 |
-
inputs=[session_state],
|
| 1138 |
-
outputs=[finish_summary, pdf_file, md_file],
|
| 1139 |
-
)
|
| 1140 |
-
```
|
| 1141 |
-
|
| 1142 |
-
- [ ] **Step 6: Verify the full test suite still passes**
|
| 1143 |
-
|
| 1144 |
-
Run: `pytest tests/ -q`
|
| 1145 |
-
Expected: all tests pass except the single pre-existing known failure documented in CLAUDE.md
|
| 1146 |
-
(`test_unimplemented_test_returns_low_confidence`). No new failures.
|
| 1147 |
-
|
| 1148 |
-
- [ ] **Step 7: Manually verify the app**
|
| 1149 |
-
|
| 1150 |
-
Run: `python3 app.py`
|
| 1151 |
-
Then in the browser:
|
| 1152 |
-
1. Upload a clip, pick a test, click **Score Movement** → score card appears; the **Session** tab
|
| 1153 |
-
shows one row; the two new buttons appear.
|
| 1154 |
-
2. Click **➕ Analyse new clip** → the video input clears, the session row persists.
|
| 1155 |
-
3. Analyse a second test → a second row appears.
|
| 1156 |
-
4. Click **✅ Finish & generate PDF** → the Session tab shows the composite summary and a
|
| 1157 |
-
downloadable PDF (open it: disclaimer top + bottom, per-test blocks with key-frame images,
|
| 1158 |
-
composite or "Incomplete"). The Markdown log is also downloadable.
|
| 1159 |
-
|
| 1160 |
-
Expected: all four steps work; PDF opens and contains the disclaimer, composite, and per-test sections.
|
| 1161 |
-
|
| 1162 |
-
- [ ] **Step 8: Commit**
|
| 1163 |
-
|
| 1164 |
-
```bash
|
| 1165 |
-
git add app.py
|
| 1166 |
-
git commit -m "feat: accumulate FMS clips into a session with composite report + PDF export"
|
| 1167 |
-
```
|
| 1168 |
-
|
| 1169 |
-
---
|
| 1170 |
-
|
| 1171 |
-
## Task 8: Update docs
|
| 1172 |
-
|
| 1173 |
-
**Files:**
|
| 1174 |
-
- Modify: `CLAUDE.md` (Build phases / status)
|
| 1175 |
-
- Modify: `MODEL_BUDGET.md` (no param change — note PDF agent adds 0 params, for completeness)
|
| 1176 |
-
|
| 1177 |
-
- [ ] **Step 1: Update the Phase 4 line in CLAUDE.md**
|
| 1178 |
-
|
| 1179 |
-
In `CLAUDE.md`, in the "Build phases" section, update the Phase 4 line from:
|
| 1180 |
-
|
| 1181 |
-
```
|
| 1182 |
-
4. **Phase 4 — Polish + ship:** Custom Svelte UI components, PDF export, agent trace to Hub, blog post. (Overlay video already done via `PoseVisualizer`.)
|
| 1183 |
-
```
|
| 1184 |
-
|
| 1185 |
-
to:
|
| 1186 |
-
|
| 1187 |
-
```
|
| 1188 |
-
4. **Phase 4 — Polish + ship:** Custom Svelte UI components, agent trace to Hub, blog post. (Overlay video done via `PoseVisualizer`; full 7-test session + PDF export done via `formscout/session.py` + `PdfReportAgent`.)
|
| 1189 |
-
```
|
| 1190 |
-
|
| 1191 |
-
- [ ] **Step 2: Note the PDF agent in the architecture section**
|
| 1192 |
-
|
| 1193 |
-
In `CLAUDE.md`, under "### Rubric scorers" or near the ReportAgent description, this is optional
|
| 1194 |
-
context; no required change. Skip if no natural home.
|
| 1195 |
-
|
| 1196 |
-
- [ ] **Step 3: Commit**
|
| 1197 |
-
|
| 1198 |
-
```bash
|
| 1199 |
-
git add CLAUDE.md
|
| 1200 |
-
git commit -m "docs: mark full FMS session + PDF export complete in build phases"
|
| 1201 |
-
```
|
| 1202 |
-
|
| 1203 |
-
---
|
| 1204 |
-
|
| 1205 |
-
## Self-Review Notes (already applied)
|
| 1206 |
-
|
| 1207 |
-
- **Spec coverage:** session accumulation (Task 5), two-button UX (Task 7), on-disk MD/JSON/keyframes (Task 5), key-frame from `features.timing` (Tasks 3–5), ReportLab PDF top/bottom disclaimer + composite + per-test + asymmetry + flags (Task 6), `SessionEntry` type (Task 2), `ReportAgent` reuse (Task 5 `finish_session`), composite-null-on-needs-human (Task 5 test), error tolerance / never-raise (Tasks 4–6). All covered.
|
| 1208 |
-
- **Type consistency:** `SessionEntry` field names are identical across Tasks 2, 5, 6, 7. `finish_session` returns `(ReportResult | None, str | None)` and is consumed that way in Task 7. `render_frame(ingest, pose2d, frame_idx, layers, caption, out_png)` signature matches its callers.
|
| 1209 |
-
- **No placeholders:** every code step shows complete code; every run step states the exact command + expected outcome.
|
|
|
|
| 1 |
+
# Full FMS Session + PDF Report — Implementation Plan
|
| 2 |
+
|
| 3 |
+
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
| 4 |
+
|
| 5 |
+
**Goal:** Turn FormScout's one-clip scorer into a screening session that accumulates analyzed clips into a composite 0–21 report and exports a branded PDF with annotated worst-moment key-frame stills.
|
| 6 |
+
|
| 7 |
+
**Architecture:** A new `formscout/session.py` accumulates typed `SessionEntry` objects (one per analyzed clip), persisting each to a temp session dir. `PoseVisualizer.render_frame()` captures the governing frame (already computed by `BiomechanicsAgent` and stored in `features.timing`) as an annotated PNG. On "Finish", the existing `ReportAgent` computes composite + asymmetries, and a new `PdfReportAgent` renders a ReportLab PDF. The UI (`app.py`) gains `gr.State` session accumulation with "Analyse new clip" / "Finish & generate PDF" buttons.
|
| 8 |
+
|
| 9 |
+
**Tech Stack:** Python 3.13, ReportLab (new dep), OpenCV (existing), Gradio 5, pytest. No model downloads in tests.
|
| 10 |
+
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
## File Structure
|
| 14 |
+
|
| 15 |
+
- `requirements.txt` — add `reportlab`.
|
| 16 |
+
- `formscout/types.py` — add `SessionEntry` frozen dataclass.
|
| 17 |
+
- `formscout/agents/biomechanics.py` — add `max_sag_frame` to `trunk_stability_pushup` timing (rotary already has `peak_extension_frame`).
|
| 18 |
+
- `formscout/agents/visualizer.py` — add `PoseVisualizer.render_frame()`.
|
| 19 |
+
- `formscout/session.py` — **new**: session accumulator (new/add/finish + persistence + key-frame helpers).
|
| 20 |
+
- `formscout/agents/pdf_report.py` — **new**: `PdfReportAgent` (ReportLab).
|
| 21 |
+
- `app.py` — wire `gr.State`, two buttons, "Session so far" table, finish handler.
|
| 22 |
+
- `tests/test_session.py`, `tests/test_keyframe.py`, `tests/test_pdf_report.py` — **new**.
|
| 23 |
+
|
| 24 |
+
---
|
| 25 |
+
|
| 26 |
+
## Task 1: Add ReportLab dependency
|
| 27 |
+
|
| 28 |
+
**Files:**
|
| 29 |
+
- Modify: `requirements.txt`
|
| 30 |
+
|
| 31 |
+
- [ ] **Step 1: Add the dependency**
|
| 32 |
+
|
| 33 |
+
Add this line to `requirements.txt` (after `pillow>=10.3`):
|
| 34 |
+
|
| 35 |
+
```
|
| 36 |
+
reportlab>=4.0
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
- [ ] **Step 2: Install it**
|
| 40 |
+
|
| 41 |
+
Run: `pip install 'reportlab>=4.0'`
|
| 42 |
+
Expected: `Successfully installed reportlab-4.x.x`
|
| 43 |
+
|
| 44 |
+
- [ ] **Step 3: Verify import**
|
| 45 |
+
|
| 46 |
+
Run: `python3 -c "import reportlab; print(reportlab.Version)"`
|
| 47 |
+
Expected: prints a version like `4.x.x`
|
| 48 |
+
|
| 49 |
+
- [ ] **Step 4: Commit**
|
| 50 |
+
|
| 51 |
+
```bash
|
| 52 |
+
git add requirements.txt
|
| 53 |
+
git commit -m "build: add reportlab for PDF report generation"
|
| 54 |
+
```
|
| 55 |
+
|
| 56 |
+
---
|
| 57 |
+
|
| 58 |
+
## Task 2: Add `SessionEntry` dataclass
|
| 59 |
+
|
| 60 |
+
**Files:**
|
| 61 |
+
- Modify: `formscout/types.py` (after `ReportResult`, before `PipelineState`)
|
| 62 |
+
- Test: `tests/test_session.py`
|
| 63 |
+
|
| 64 |
+
- [ ] **Step 1: Write the failing test**
|
| 65 |
+
|
| 66 |
+
Create `tests/test_session.py` with:
|
| 67 |
+
|
| 68 |
+
```python
|
| 69 |
+
"""Tests for the FMS session accumulator — no GPU, no model downloads."""
|
| 70 |
+
import numpy as np
|
| 71 |
+
|
| 72 |
+
from formscout.types import (
|
| 73 |
+
IngestResult, Pose2DResult, BiomechFeatures, ScoreResult, JudgeResult,
|
| 74 |
+
MovementResult, SessionEntry,
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def test_session_entry_holds_typed_objects():
|
| 79 |
+
movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
|
| 80 |
+
features = BiomechFeatures(
|
| 81 |
+
test_name="deep_squat", view="2d", side="na",
|
| 82 |
+
angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": True},
|
| 83 |
+
symmetry_delta=None, timing={"deepest_frame": 2}, confidence=0.9,
|
| 84 |
+
)
|
| 85 |
+
rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
|
| 86 |
+
judge = JudgeResult(score=2, rationale="ok", compensation_tags=["heels elevated"],
|
| 87 |
+
corrective_hint="ankle mobility", confidence=0.85)
|
| 88 |
+
entry = SessionEntry(
|
| 89 |
+
test_name="deep_squat", side="na", score=2, needs_human=False,
|
| 90 |
+
rationale="ok", compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 91 |
+
measurements={"left_knee_flexion_deg": 95.0}, confidence=0.85, view="2d",
|
| 92 |
+
keyframe_path=None, movement=movement, features=features,
|
| 93 |
+
rubric_score=rubric, judge=judge,
|
| 94 |
+
)
|
| 95 |
+
assert entry.score == 2
|
| 96 |
+
assert entry.movement.test_name == "deep_squat"
|
| 97 |
+
assert entry.rubric_score.score == 2
|
| 98 |
+
assert entry.judge.compensation_tags == ["heels elevated"]
|
| 99 |
+
```
|
| 100 |
+
|
| 101 |
+
- [ ] **Step 2: Run test to verify it fails**
|
| 102 |
+
|
| 103 |
+
Run: `pytest tests/test_session.py::test_session_entry_holds_typed_objects -v`
|
| 104 |
+
Expected: FAIL with `ImportError: cannot import name 'SessionEntry'`
|
| 105 |
+
|
| 106 |
+
- [ ] **Step 3: Add the dataclass**
|
| 107 |
+
|
| 108 |
+
In `formscout/types.py`, insert after the `ReportResult` class (line ~142) and before `PipelineState`:
|
| 109 |
+
|
| 110 |
+
```python
|
| 111 |
+
@dataclass(frozen=True)
|
| 112 |
+
class SessionEntry:
|
| 113 |
+
"""One accumulated analysis in a screening session.
|
| 114 |
+
|
| 115 |
+
Display fields (test_name…keyframe_path) feed the PDF/JSON/MD artifacts;
|
| 116 |
+
the trailing typed objects (movement…judge) feed ReportAgent.run().
|
| 117 |
+
"""
|
| 118 |
+
test_name: str
|
| 119 |
+
side: str
|
| 120 |
+
score: int | None
|
| 121 |
+
needs_human: bool
|
| 122 |
+
rationale: str
|
| 123 |
+
compensation_tags: list
|
| 124 |
+
corrective_hint: str
|
| 125 |
+
measurements: dict
|
| 126 |
+
confidence: float
|
| 127 |
+
view: str
|
| 128 |
+
keyframe_path: str | None
|
| 129 |
+
movement: MovementResult
|
| 130 |
+
features: BiomechFeatures
|
| 131 |
+
rubric_score: ScoreResult
|
| 132 |
+
judge: JudgeResult | None
|
| 133 |
+
```
|
| 134 |
+
|
| 135 |
+
- [ ] **Step 4: Run test to verify it passes**
|
| 136 |
+
|
| 137 |
+
Run: `pytest tests/test_session.py::test_session_entry_holds_typed_objects -v`
|
| 138 |
+
Expected: PASS
|
| 139 |
+
|
| 140 |
+
- [ ] **Step 5: Commit**
|
| 141 |
+
|
| 142 |
+
```bash
|
| 143 |
+
git add formscout/types.py tests/test_session.py
|
| 144 |
+
git commit -m "feat: add SessionEntry typed contract for screening sessions"
|
| 145 |
+
```
|
| 146 |
+
|
| 147 |
+
---
|
| 148 |
+
|
| 149 |
+
## Task 3: Add governing-frame index to push-up biomechanics
|
| 150 |
+
|
| 151 |
+
**Files:**
|
| 152 |
+
- Modify: `formscout/agents/biomechanics.py:468-529` (`_trunk_stability_pushup`)
|
| 153 |
+
- Test: `tests/test_biomechanics.py` (append a test)
|
| 154 |
+
|
| 155 |
+
The other six tests already store a governing frame index in `features.timing`
|
| 156 |
+
(`deepest_frame`, `peak_step_frame`, `deepest_lunge_frame`, `measure_frame`,
|
| 157 |
+
`peak_raise_frame`, `peak_extension_frame`). Only `trunk_stability_pushup` is missing one.
|
| 158 |
+
|
| 159 |
+
- [ ] **Step 1: Write the failing test**
|
| 160 |
+
|
| 161 |
+
Append to `tests/test_biomechanics.py`:
|
| 162 |
+
|
| 163 |
+
```python
|
| 164 |
+
def test_pushup_timing_has_max_sag_frame():
|
| 165 |
+
from formscout.agents.biomechanics import BiomechanicsAgent
|
| 166 |
+
from formscout.types import Pose2DResult, Body3DResult, MovementResult
|
| 167 |
+
|
| 168 |
+
# 4 frames; frame 2 has the largest hip sag (hip far below shoulder/ankle midline)
|
| 169 |
+
def kps(hip_y):
|
| 170 |
+
base = {
|
| 171 |
+
5: {"x": 200, "y": 200, "conf": 0.9}, # L shoulder
|
| 172 |
+
6: {"x": 220, "y": 200, "conf": 0.9}, # R shoulder
|
| 173 |
+
11: {"x": 300, "y": hip_y, "conf": 0.9}, # L hip
|
| 174 |
+
12: {"x": 320, "y": hip_y, "conf": 0.9}, # R hip
|
| 175 |
+
15: {"x": 400, "y": 200, "conf": 0.9}, # L ankle
|
| 176 |
+
16: {"x": 420, "y": 200, "conf": 0.9}, # R ankle
|
| 177 |
+
}
|
| 178 |
+
return base
|
| 179 |
+
|
| 180 |
+
frames = [kps(200), kps(210), kps(260), kps(205)]
|
| 181 |
+
pose = Pose2DResult(keypoints=frames, fps=30.0, confidence=0.9)
|
| 182 |
+
body3d = Body3DResult(used=False, joints_3d=[])
|
| 183 |
+
movement = MovementResult(test_name="trunk_stability_pushup", side="na", confidence=1.0)
|
| 184 |
+
|
| 185 |
+
feats = BiomechanicsAgent().run(pose, body3d, movement)
|
| 186 |
+
assert "max_sag_frame" in feats.timing
|
| 187 |
+
assert feats.timing["max_sag_frame"] == 2
|
| 188 |
+
```
|
| 189 |
+
|
| 190 |
+
- [ ] **Step 2: Run test to verify it fails**
|
| 191 |
+
|
| 192 |
+
Run: `pytest tests/test_biomechanics.py::test_pushup_timing_has_max_sag_frame -v`
|
| 193 |
+
Expected: FAIL with `assert 'max_sag_frame' in {...}` (KeyError-style assertion failure)
|
| 194 |
+
|
| 195 |
+
- [ ] **Step 3: Track the max-sag frame index**
|
| 196 |
+
|
| 197 |
+
In `formscout/agents/biomechanics.py`, replace the body of `_trunk_stability_pushup` from the
|
| 198 |
+
`trunk_angles_over_time = []` loop through the `if trunk_angles_over_time:` block. Replace:
|
| 199 |
+
|
| 200 |
+
```python
|
| 201 |
+
# Analyze multiple frames to detect sag/lag
|
| 202 |
+
trunk_angles_over_time = []
|
| 203 |
+
for i, kps in enumerate(pose2d.keypoints):
|
| 204 |
+
```
|
| 205 |
+
|
| 206 |
+
…down to and including the `alignments["no_sag"] = max_sag < 30` line, with:
|
| 207 |
+
|
| 208 |
+
```python
|
| 209 |
+
# Analyze multiple frames to detect sag/lag
|
| 210 |
+
trunk_sags: list[tuple[int, float]] = [] # (frame_idx, sag_px)
|
| 211 |
+
for i, kps in enumerate(pose2d.keypoints):
|
| 212 |
+
l_sh = _get_joint(kps, L_SHOULDER)
|
| 213 |
+
r_sh = _get_joint(kps, R_SHOULDER)
|
| 214 |
+
l_hip = _get_joint(kps, L_HIP)
|
| 215 |
+
r_hip = _get_joint(kps, R_HIP)
|
| 216 |
+
l_ankle = _get_joint(kps, L_ANKLE)
|
| 217 |
+
r_ankle = _get_joint(kps, R_ANKLE)
|
| 218 |
+
|
| 219 |
+
if l_sh and r_sh and l_hip and r_hip and l_ankle and r_ankle:
|
| 220 |
+
sh_y = (l_sh[1] + r_sh[1]) / 2
|
| 221 |
+
hip_y = (l_hip[1] + r_hip[1]) / 2
|
| 222 |
+
ankle_y = (l_ankle[1] + r_ankle[1]) / 2
|
| 223 |
+
expected_hip_y = (sh_y + ankle_y) / 2
|
| 224 |
+
sag_px = hip_y - expected_hip_y
|
| 225 |
+
trunk_sags.append((i, sag_px))
|
| 226 |
+
|
| 227 |
+
max_sag_frame = 0
|
| 228 |
+
if trunk_sags:
|
| 229 |
+
sags = [s for _, s in trunk_sags]
|
| 230 |
+
max_sag_frame = max(trunk_sags, key=lambda t: t[1])[0]
|
| 231 |
+
mean = sum(sags) / len(sags)
|
| 232 |
+
variance = (sum((x - mean) ** 2 for x in sags) / len(sags)) ** 0.5
|
| 233 |
+
max_sag = max(sags)
|
| 234 |
+
angles["max_sag_px"] = max_sag
|
| 235 |
+
angles["trunk_variance_px"] = variance
|
| 236 |
+
alignments["body_rigid"] = max_sag < 30 and variance < 15
|
| 237 |
+
alignments["no_sag"] = max_sag < 30
|
| 238 |
+
else:
|
| 239 |
+
notes_parts.append("insufficient landmarks for trunk analysis")
|
| 240 |
+
```
|
| 241 |
+
|
| 242 |
+
Then update the `return BiomechFeatures(...)` `timing=` argument at the end of the method from:
|
| 243 |
+
|
| 244 |
+
```python
|
| 245 |
+
timing={"n_frames_analyzed": len(trunk_angles_over_time)},
|
| 246 |
+
```
|
| 247 |
+
|
| 248 |
+
to:
|
| 249 |
+
|
| 250 |
+
```python
|
| 251 |
+
timing={"n_frames_analyzed": len(trunk_sags), "max_sag_frame": max_sag_frame},
|
| 252 |
+
```
|
| 253 |
+
|
| 254 |
+
- [ ] **Step 4: Run test to verify it passes**
|
| 255 |
+
|
| 256 |
+
Run: `pytest tests/test_biomechanics.py::test_pushup_timing_has_max_sag_frame -v`
|
| 257 |
+
Expected: PASS
|
| 258 |
+
|
| 259 |
+
- [ ] **Step 5: Run the full biomechanics suite (no regressions)**
|
| 260 |
+
|
| 261 |
+
Run: `pytest tests/test_biomechanics.py -v`
|
| 262 |
+
Expected: all previously-passing tests still pass (the pre-existing `test_unimplemented_test_returns_low_confidence` known-failure may remain failing — that is unrelated and documented in CLAUDE.md).
|
| 263 |
+
|
| 264 |
+
- [ ] **Step 6: Commit**
|
| 265 |
+
|
| 266 |
+
```bash
|
| 267 |
+
git add formscout/agents/biomechanics.py tests/test_biomechanics.py
|
| 268 |
+
git commit -m "feat: track max-sag frame index in push-up biomechanics for key-frame capture"
|
| 269 |
+
```
|
| 270 |
+
|
| 271 |
+
---
|
| 272 |
+
|
| 273 |
+
## Task 4: Add `PoseVisualizer.render_frame()`
|
| 274 |
+
|
| 275 |
+
**Files:**
|
| 276 |
+
- Modify: `formscout/agents/visualizer.py` (add method to `PoseVisualizer`, after `render_video`)
|
| 277 |
+
- Test: `tests/test_keyframe.py`
|
| 278 |
+
|
| 279 |
+
- [ ] **Step 1: Write the failing test**
|
| 280 |
+
|
| 281 |
+
Create `tests/test_keyframe.py`:
|
| 282 |
+
|
| 283 |
+
```python
|
| 284 |
+
"""Tests for PoseVisualizer.render_frame — single annotated still."""
|
| 285 |
+
import os
|
| 286 |
+
import numpy as np
|
| 287 |
+
|
| 288 |
+
from formscout.types import IngestResult, Pose2DResult
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def _ingest(n=5, h=480, w=640):
|
| 292 |
+
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 293 |
+
return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def _pose(n=5):
|
| 297 |
+
kps = []
|
| 298 |
+
for i in range(n):
|
| 299 |
+
kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
|
| 300 |
+
for j in range(17)})
|
| 301 |
+
return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def test_render_frame_writes_png(tmp_path):
|
| 305 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 306 |
+
out = str(tmp_path / "key.png")
|
| 307 |
+
path = PoseVisualizer().render_frame(_ingest(), _pose(), frame_idx=2,
|
| 308 |
+
layers={"skeleton"}, caption="Deep Squat — heels elevated",
|
| 309 |
+
out_png=out)
|
| 310 |
+
assert path == out
|
| 311 |
+
assert os.path.exists(out)
|
| 312 |
+
assert os.path.getsize(out) > 0
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
def test_render_frame_bad_index_returns_none(tmp_path):
|
| 316 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 317 |
+
out = str(tmp_path / "key.png")
|
| 318 |
+
path = PoseVisualizer().render_frame(_ingest(n=3), _pose(n=3), frame_idx=99,
|
| 319 |
+
layers={"skeleton"}, caption="", out_png=out)
|
| 320 |
+
assert path is None
|
| 321 |
+
```
|
| 322 |
+
|
| 323 |
+
- [ ] **Step 2: Run test to verify it fails**
|
| 324 |
+
|
| 325 |
+
Run: `pytest tests/test_keyframe.py -v`
|
| 326 |
+
Expected: FAIL with `AttributeError: 'PoseVisualizer' object has no attribute 'render_frame'`
|
| 327 |
+
|
| 328 |
+
- [ ] **Step 3: Add the method**
|
| 329 |
+
|
| 330 |
+
In `formscout/agents/visualizer.py`, inside the `PoseVisualizer` class, add this method
|
| 331 |
+
immediately after `render_video` (before the closing of the class / the module-level
|
| 332 |
+
`build_velocity_summary`):
|
| 333 |
+
|
| 334 |
+
```python
|
| 335 |
+
def render_frame(
|
| 336 |
+
self,
|
| 337 |
+
ingest,
|
| 338 |
+
pose2d,
|
| 339 |
+
frame_idx: int,
|
| 340 |
+
layers: set[str],
|
| 341 |
+
caption: str = "",
|
| 342 |
+
out_png: str | None = None,
|
| 343 |
+
) -> str | None:
|
| 344 |
+
"""Render a single annotated still (skeleton + optional trails + caption).
|
| 345 |
+
|
| 346 |
+
frame_idx is typically the governing frame from BiomechFeatures.timing.
|
| 347 |
+
Returns the PNG path on success, None on any failure. Never raises.
|
| 348 |
+
"""
|
| 349 |
+
try:
|
| 350 |
+
if not (0 <= frame_idx < len(ingest.frames)) or frame_idx >= len(pose2d.keypoints):
|
| 351 |
+
return None
|
| 352 |
+
|
| 353 |
+
frame = ingest.frames[frame_idx].copy()
|
| 354 |
+
kps = pose2d.keypoints[frame_idx]
|
| 355 |
+
|
| 356 |
+
if "trails" in layers:
|
| 357 |
+
trail: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
|
| 358 |
+
start = max(0, frame_idx - TRAIL_LENGTH)
|
| 359 |
+
for fi in range(start, frame_idx + 1):
|
| 360 |
+
for j, kp in pose2d.keypoints[fi].items():
|
| 361 |
+
if kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 362 |
+
trail[j].append((kp["x"], kp["y"]))
|
| 363 |
+
frame = self._draw_trails(frame, trail)
|
| 364 |
+
|
| 365 |
+
if "skeleton" in layers:
|
| 366 |
+
frame = self._draw_skeleton(frame, kps)
|
| 367 |
+
|
| 368 |
+
if caption:
|
| 369 |
+
cv2.rectangle(frame, (0, 0), (frame.shape[1], 28), (0, 0, 0), -1)
|
| 370 |
+
cv2.putText(frame, caption[:80], (8, 20), cv2.FONT_HERSHEY_SIMPLEX,
|
| 371 |
+
0.55, (255, 255, 255), 1, cv2.LINE_AA)
|
| 372 |
+
|
| 373 |
+
if out_png is None:
|
| 374 |
+
out_png = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
|
| 375 |
+
|
| 376 |
+
ok = cv2.imwrite(out_png, frame)
|
| 377 |
+
return out_png if ok else None
|
| 378 |
+
except Exception as e:
|
| 379 |
+
logger.warning("render_frame failed: %s", e)
|
| 380 |
+
return None
|
| 381 |
+
```
|
| 382 |
+
|
| 383 |
+
(`deque`, `cv2`, `tempfile`, `logger`, `TRAIL_LENGTH`, `CONF_THRESHOLD` are all already imported at the top of this file.)
|
| 384 |
+
|
| 385 |
+
- [ ] **Step 4: Run test to verify it passes**
|
| 386 |
+
|
| 387 |
+
Run: `pytest tests/test_keyframe.py -v`
|
| 388 |
+
Expected: both tests PASS
|
| 389 |
+
|
| 390 |
+
- [ ] **Step 5: Commit**
|
| 391 |
+
|
| 392 |
+
```bash
|
| 393 |
+
git add formscout/agents/visualizer.py tests/test_keyframe.py
|
| 394 |
+
git commit -m "feat: add PoseVisualizer.render_frame for annotated key-frame stills"
|
| 395 |
+
```
|
| 396 |
+
|
| 397 |
+
---
|
| 398 |
+
|
| 399 |
+
## Task 5: Create the session accumulator
|
| 400 |
+
|
| 401 |
+
**Files:**
|
| 402 |
+
- Create: `formscout/session.py`
|
| 403 |
+
- Test: `tests/test_session.py` (append tests)
|
| 404 |
+
|
| 405 |
+
- [ ] **Step 1: Write the failing tests**
|
| 406 |
+
|
| 407 |
+
Append to `tests/test_session.py`:
|
| 408 |
+
|
| 409 |
+
```python
|
| 410 |
+
def _ingest(n=5, h=480, w=640):
|
| 411 |
+
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 412 |
+
return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
def _pose(n=5):
|
| 416 |
+
kps = []
|
| 417 |
+
for i in range(n):
|
| 418 |
+
kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
|
| 419 |
+
for j in range(17)})
|
| 420 |
+
return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
def _features(test_name="deep_squat", side="na", frame_key="deepest_frame"):
|
| 424 |
+
return BiomechFeatures(
|
| 425 |
+
test_name=test_name, view="2d", side=side,
|
| 426 |
+
angles={"left_knee_flexion_deg": 95.0},
|
| 427 |
+
alignments={"knees_tracking_over_feet": False},
|
| 428 |
+
symmetry_delta=None, timing={frame_key: 2}, confidence=0.9,
|
| 429 |
+
)
|
| 430 |
+
|
| 431 |
+
|
| 432 |
+
def _judge(score=2, needs_human=False):
|
| 433 |
+
return JudgeResult(
|
| 434 |
+
score=None if needs_human else score, rationale="r",
|
| 435 |
+
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 436 |
+
confidence=0.85, needs_human=needs_human,
|
| 437 |
+
)
|
| 438 |
+
|
| 439 |
+
|
| 440 |
+
def test_add_analysis_appends_entry_and_writes_files():
|
| 441 |
+
import os
|
| 442 |
+
from formscout import session as S
|
| 443 |
+
sess = S.new_session()
|
| 444 |
+
entry = S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
|
| 445 |
+
features=_features(), judge=_judge(), test_name="deep_squat", side="na")
|
| 446 |
+
assert len(sess.entries) == 1
|
| 447 |
+
assert entry.score == 2
|
| 448 |
+
assert os.path.exists(os.path.join(sess.session_dir, "session.json"))
|
| 449 |
+
assert os.path.exists(os.path.join(sess.session_dir, "analysis.md"))
|
| 450 |
+
# key-frame still written (deepest_frame=2 is valid)
|
| 451 |
+
assert entry.keyframe_path and os.path.exists(entry.keyframe_path)
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
def test_finish_composite_null_when_needs_human():
|
| 455 |
+
from formscout import session as S
|
| 456 |
+
sess = S.new_session()
|
| 457 |
+
S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(), features=_features(),
|
| 458 |
+
judge=_judge(score=3), test_name="deep_squat", side="na")
|
| 459 |
+
S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
|
| 460 |
+
features=_features("trunk_stability_pushup", frame_key="max_sag_frame"),
|
| 461 |
+
judge=_judge(needs_human=True), test_name="trunk_stability_pushup", side="na")
|
| 462 |
+
report, pdf_path = S.finish_session(sess)
|
| 463 |
+
assert report is not None
|
| 464 |
+
assert report.composite is None # one test needs_human
|
| 465 |
+
|
| 466 |
+
|
| 467 |
+
def test_finish_empty_session_returns_none():
|
| 468 |
+
from formscout import session as S
|
| 469 |
+
sess = S.new_session()
|
| 470 |
+
report, pdf_path = S.finish_session(sess)
|
| 471 |
+
assert report is None and pdf_path is None
|
| 472 |
+
```
|
| 473 |
+
|
| 474 |
+
- [ ] **Step 2: Run tests to verify they fail**
|
| 475 |
+
|
| 476 |
+
Run: `pytest tests/test_session.py -v`
|
| 477 |
+
Expected: the three new tests FAIL with `ModuleNotFoundError: No module named 'formscout.session'`
|
| 478 |
+
|
| 479 |
+
- [ ] **Step 3: Create the module**
|
| 480 |
+
|
| 481 |
+
Create `formscout/session.py`:
|
| 482 |
+
|
| 483 |
+
```python
|
| 484 |
+
"""
|
| 485 |
+
Screening-session accumulator.
|
| 486 |
+
|
| 487 |
+
Accumulates one SessionEntry per analyzed clip, persists each to a temp session
|
| 488 |
+
dir (session.json + analysis.md + key-frame PNGs), and on finish builds a
|
| 489 |
+
ReportResult (via ReportAgent) + a PDF (via PdfReportAgent).
|
| 490 |
+
|
| 491 |
+
Pure orchestration — no Gradio imports. Disk writes tolerate failure with a
|
| 492 |
+
logged warning and never block scoring.
|
| 493 |
+
"""
|
| 494 |
+
from __future__ import annotations
|
| 495 |
+
|
| 496 |
+
import json
|
| 497 |
+
import logging
|
| 498 |
+
import os
|
| 499 |
+
import tempfile
|
| 500 |
+
import uuid
|
| 501 |
+
from dataclasses import dataclass, replace
|
| 502 |
+
|
| 503 |
+
from formscout.rubric import score_test
|
| 504 |
+
from formscout.types import MovementResult, ReportResult, SessionEntry
|
| 505 |
+
|
| 506 |
+
logger = logging.getLogger(__name__)
|
| 507 |
+
|
| 508 |
+
# Maps each test to the BiomechFeatures.timing key holding its governing frame.
|
| 509 |
+
TIMING_KEY = {
|
| 510 |
+
"deep_squat": "deepest_frame",
|
| 511 |
+
"hurdle_step": "peak_step_frame",
|
| 512 |
+
"inline_lunge": "deepest_lunge_frame",
|
| 513 |
+
"shoulder_mobility": "measure_frame",
|
| 514 |
+
"active_slr": "peak_raise_frame",
|
| 515 |
+
"trunk_stability_pushup": "max_sag_frame",
|
| 516 |
+
"rotary_stability": "peak_extension_frame",
|
| 517 |
+
}
|
| 518 |
+
|
| 519 |
+
|
| 520 |
+
@dataclass
|
| 521 |
+
class Session:
|
| 522 |
+
"""Mutable session: an id, its temp dir, and accumulated entries."""
|
| 523 |
+
session_id: str
|
| 524 |
+
session_dir: str
|
| 525 |
+
entries: list # list[SessionEntry]
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
def new_session() -> Session:
|
| 529 |
+
sid = uuid.uuid4().hex[:12]
|
| 530 |
+
base = os.path.join(tempfile.gettempdir(), "formscout_sessions", sid)
|
| 531 |
+
try:
|
| 532 |
+
os.makedirs(os.path.join(base, "keyframes"), exist_ok=True)
|
| 533 |
+
except Exception as e:
|
| 534 |
+
logger.warning("session dir create failed: %s", e)
|
| 535 |
+
return Session(session_id=sid, session_dir=base, entries=[])
|
| 536 |
+
|
| 537 |
+
|
| 538 |
+
def governing_frame_index(features) -> int | None:
|
| 539 |
+
"""Return the governing frame index for this test, or None."""
|
| 540 |
+
key = TIMING_KEY.get(features.test_name)
|
| 541 |
+
if key is None:
|
| 542 |
+
return None
|
| 543 |
+
idx = features.timing.get(key)
|
| 544 |
+
return int(idx) if isinstance(idx, (int, float)) else None
|
| 545 |
+
|
| 546 |
+
|
| 547 |
+
def worst_compensation_caption(judge, features) -> str:
|
| 548 |
+
"""Short caption naming the worst compensation for the key-frame still."""
|
| 549 |
+
if judge and getattr(judge, "compensation_tags", None):
|
| 550 |
+
return ", ".join(judge.compensation_tags)
|
| 551 |
+
failed = [k.replace("_", " ") for k, v in features.alignments.items() if v is False]
|
| 552 |
+
return ("compensation: " + ", ".join(failed)) if failed else "key position"
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
def add_analysis(session, *, ingest, pose2d, features, judge, test_name, side,
|
| 556 |
+
draw_trails: bool = False) -> SessionEntry:
|
| 557 |
+
"""Build a SessionEntry from a completed analysis, render its key-frame,
|
| 558 |
+
persist the session, append, and return the entry."""
|
| 559 |
+
movement = MovementResult(test_name=test_name, side=side, confidence=1.0)
|
| 560 |
+
rubric = score_test(features)
|
| 561 |
+
|
| 562 |
+
needs_human = bool((judge and judge.needs_human) or rubric.needs_human)
|
| 563 |
+
if needs_human:
|
| 564 |
+
score = None
|
| 565 |
+
elif judge and judge.score is not None:
|
| 566 |
+
score = judge.score
|
| 567 |
+
else:
|
| 568 |
+
score = rubric.score
|
| 569 |
+
|
| 570 |
+
keyframe_path = None
|
| 571 |
+
idx = governing_frame_index(features)
|
| 572 |
+
if idx is not None and 0 <= idx < len(pose2d.keypoints):
|
| 573 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 574 |
+
caption = (f"{test_name.replace('_', ' ').title()} "
|
| 575 |
+
f"({side}) — {worst_compensation_caption(judge, features)}")
|
| 576 |
+
layers = {"skeleton", "trails"} if draw_trails else {"skeleton"}
|
| 577 |
+
out_png = os.path.join(session.session_dir, "keyframes", f"{test_name}_{side}.png")
|
| 578 |
+
try:
|
| 579 |
+
keyframe_path = PoseVisualizer().render_frame(ingest, pose2d, idx, layers, caption, out_png)
|
| 580 |
+
except Exception as e:
|
| 581 |
+
logger.warning("keyframe render failed: %s", e)
|
| 582 |
+
|
| 583 |
+
measurements = {}
|
| 584 |
+
measurements.update(features.angles)
|
| 585 |
+
measurements.update(features.alignments)
|
| 586 |
+
|
| 587 |
+
entry = SessionEntry(
|
| 588 |
+
test_name=test_name, side=side, score=score, needs_human=needs_human,
|
| 589 |
+
rationale=(judge.rationale if judge else rubric.rationale),
|
| 590 |
+
compensation_tags=list(judge.compensation_tags) if judge else [],
|
| 591 |
+
corrective_hint=(judge.corrective_hint if judge else ""),
|
| 592 |
+
measurements=measurements,
|
| 593 |
+
confidence=(judge.confidence if judge else rubric.confidence),
|
| 594 |
+
view=features.view,
|
| 595 |
+
keyframe_path=keyframe_path,
|
| 596 |
+
movement=movement, features=features, rubric_score=rubric, judge=judge,
|
| 597 |
+
)
|
| 598 |
+
session.entries.append(entry)
|
| 599 |
+
_persist(session)
|
| 600 |
+
return entry
|
| 601 |
+
|
| 602 |
+
|
| 603 |
+
def finish_session(session) -> tuple[ReportResult | None, str | None]:
|
| 604 |
+
"""Build the composite report + PDF. Returns (report, pdf_path).
|
| 605 |
+
Returns (None, None) for an empty session."""
|
| 606 |
+
if not session.entries:
|
| 607 |
+
return None, None
|
| 608 |
+
|
| 609 |
+
from formscout.agents.report import ReportAgent
|
| 610 |
+
report_inputs = [{
|
| 611 |
+
"movement": e.movement, "features": e.features,
|
| 612 |
+
"rubric_score": e.rubric_score, "judge": e.judge, "side": e.side,
|
| 613 |
+
} for e in session.entries]
|
| 614 |
+
report = ReportAgent().run(report_inputs)
|
| 615 |
+
|
| 616 |
+
pdf_path = None
|
| 617 |
+
try:
|
| 618 |
+
from formscout.agents.pdf_report import PdfReportAgent
|
| 619 |
+
pdf_path = PdfReportAgent().run(report, session.entries, session.session_dir)
|
| 620 |
+
except Exception as e:
|
| 621 |
+
logger.warning("pdf generation failed: %s", e)
|
| 622 |
+
|
| 623 |
+
report = replace(report, pdf_path=pdf_path)
|
| 624 |
+
return report, pdf_path
|
| 625 |
+
|
| 626 |
+
|
| 627 |
+
# ── Persistence ───────────────────────────────────────────────────────────────
|
| 628 |
+
|
| 629 |
+
def _jsonable(d: dict) -> dict:
|
| 630 |
+
out = {}
|
| 631 |
+
for k, v in d.items():
|
| 632 |
+
if isinstance(v, float):
|
| 633 |
+
out[k] = round(v, 2)
|
| 634 |
+
elif isinstance(v, (int, str, bool)) or v is None:
|
| 635 |
+
out[k] = v
|
| 636 |
+
else:
|
| 637 |
+
out[k] = str(v)
|
| 638 |
+
return out
|
| 639 |
+
|
| 640 |
+
|
| 641 |
+
def _entry_display(e: SessionEntry) -> dict:
|
| 642 |
+
return {
|
| 643 |
+
"test_name": e.test_name, "side": e.side, "score": e.score,
|
| 644 |
+
"needs_human": e.needs_human, "rationale": e.rationale,
|
| 645 |
+
"compensation_tags": list(e.compensation_tags), "corrective_hint": e.corrective_hint,
|
| 646 |
+
"measurements": _jsonable(e.measurements), "confidence": round(e.confidence, 2),
|
| 647 |
+
"view": e.view, "keyframe_path": e.keyframe_path,
|
| 648 |
+
}
|
| 649 |
+
|
| 650 |
+
|
| 651 |
+
def _render_markdown(session: Session) -> str:
|
| 652 |
+
lines = ["# FormScout — Session Log", ""]
|
| 653 |
+
for e in session.entries:
|
| 654 |
+
title = e.test_name.replace("_", " ").title()
|
| 655 |
+
if e.side in ("left", "right"):
|
| 656 |
+
title += f" ({e.side})"
|
| 657 |
+
score = "Clinician review required" if e.needs_human else f"{e.score}/3"
|
| 658 |
+
lines.append(f"## {title} — {score}")
|
| 659 |
+
lines.append(e.rationale or "")
|
| 660 |
+
if e.compensation_tags:
|
| 661 |
+
lines.append(f"- Compensations: {', '.join(e.compensation_tags)}")
|
| 662 |
+
if e.corrective_hint:
|
| 663 |
+
lines.append(f"- Corrective: {e.corrective_hint}")
|
| 664 |
+
if e.keyframe_path:
|
| 665 |
+
lines.append(f"- Key frame: `{e.keyframe_path}`")
|
| 666 |
+
lines.append("")
|
| 667 |
+
return "\n".join(lines)
|
| 668 |
+
|
| 669 |
+
|
| 670 |
+
def _persist(session: Session) -> None:
|
| 671 |
+
try:
|
| 672 |
+
with open(os.path.join(session.session_dir, "session.json"), "w") as f:
|
| 673 |
+
json.dump([_entry_display(e) for e in session.entries], f, indent=2)
|
| 674 |
+
with open(os.path.join(session.session_dir, "analysis.md"), "w") as f:
|
| 675 |
+
f.write(_render_markdown(session))
|
| 676 |
+
except Exception as e:
|
| 677 |
+
logger.warning("session persist failed: %s", e)
|
| 678 |
+
```
|
| 679 |
+
|
| 680 |
+
- [ ] **Step 4: Run tests to verify they pass**
|
| 681 |
+
|
| 682 |
+
Run: `pytest tests/test_session.py -v`
|
| 683 |
+
Expected: all session tests PASS (Task 6 provides `PdfReportAgent`; `finish_session` tolerates its
|
| 684 |
+
absence via the try/except, so these pass now — `pdf_path` may be `None` until Task 6).
|
| 685 |
+
|
| 686 |
+
- [ ] **Step 5: Commit**
|
| 687 |
+
|
| 688 |
+
```bash
|
| 689 |
+
git add formscout/session.py tests/test_session.py
|
| 690 |
+
git commit -m "feat: add screening-session accumulator with key-frame capture and persistence"
|
| 691 |
+
```
|
| 692 |
+
|
| 693 |
+
---
|
| 694 |
+
|
| 695 |
+
## Task 6: Create `PdfReportAgent`
|
| 696 |
+
|
| 697 |
+
**Files:**
|
| 698 |
+
- Create: `formscout/agents/pdf_report.py`
|
| 699 |
+
- Test: `tests/test_pdf_report.py`
|
| 700 |
+
|
| 701 |
+
- [ ] **Step 1: Write the failing test**
|
| 702 |
+
|
| 703 |
+
Create `tests/test_pdf_report.py`:
|
| 704 |
+
|
| 705 |
+
```python
|
| 706 |
+
"""Tests for PdfReportAgent — no GPU, no model downloads."""
|
| 707 |
+
import os
|
| 708 |
+
|
| 709 |
+
from formscout.types import (
|
| 710 |
+
ReportResult, SessionEntry, MovementResult, BiomechFeatures, ScoreResult, JudgeResult,
|
| 711 |
+
)
|
| 712 |
+
|
| 713 |
+
|
| 714 |
+
def _entry(test_name="deep_squat", score=2, needs_human=False):
|
| 715 |
+
movement = MovementResult(test_name=test_name, side="na", confidence=1.0)
|
| 716 |
+
features = BiomechFeatures(
|
| 717 |
+
test_name=test_name, view="2d", side="na",
|
| 718 |
+
angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": False},
|
| 719 |
+
symmetry_delta=None, timing={"deepest_frame": 1}, confidence=0.9,
|
| 720 |
+
)
|
| 721 |
+
rubric = ScoreResult(score=2, rationale="rubric ok", confidence=0.8)
|
| 722 |
+
judge = JudgeResult(score=None if needs_human else score, rationale="judge rationale",
|
| 723 |
+
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 724 |
+
confidence=0.85, needs_human=needs_human)
|
| 725 |
+
return SessionEntry(
|
| 726 |
+
test_name=test_name, side="na", score=None if needs_human else score,
|
| 727 |
+
needs_human=needs_human, rationale="judge rationale",
|
| 728 |
+
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 729 |
+
measurements={"left_knee_flexion_deg": 95.0, "knees_tracking_over_feet": False},
|
| 730 |
+
confidence=0.85, view="2d", keyframe_path=None,
|
| 731 |
+
movement=movement, features=features, rubric_score=rubric, judge=judge,
|
| 732 |
+
)
|
| 733 |
+
|
| 734 |
+
|
| 735 |
+
def _report(composite=2):
|
| 736 |
+
return ReportResult(
|
| 737 |
+
per_test=[], composite=composite, asymmetries=[],
|
| 738 |
+
overlay_video_path=None, pdf_path=None,
|
| 739 |
+
low_confidence_flags=[], disagreement_flags=[],
|
| 740 |
+
)
|
| 741 |
+
|
| 742 |
+
|
| 743 |
+
def test_pdf_is_created(tmp_path):
|
| 744 |
+
from formscout.agents.pdf_report import PdfReportAgent
|
| 745 |
+
path = PdfReportAgent().run(_report(2), [_entry()], str(tmp_path))
|
| 746 |
+
assert path is not None
|
| 747 |
+
assert os.path.exists(path)
|
| 748 |
+
assert os.path.getsize(path) > 1000 # a real PDF, not an empty file
|
| 749 |
+
with open(path, "rb") as f:
|
| 750 |
+
assert f.read(5) == b"%PDF-"
|
| 751 |
+
|
| 752 |
+
|
| 753 |
+
def test_pdf_handles_incomplete_composite(tmp_path):
|
| 754 |
+
from formscout.agents.pdf_report import PdfReportAgent
|
| 755 |
+
path = PdfReportAgent().run(_report(None), [_entry(needs_human=True)], str(tmp_path))
|
| 756 |
+
assert path is not None and os.path.exists(path)
|
| 757 |
+
```
|
| 758 |
+
|
| 759 |
+
- [ ] **Step 2: Run test to verify it fails**
|
| 760 |
+
|
| 761 |
+
Run: `pytest tests/test_pdf_report.py -v`
|
| 762 |
+
Expected: FAIL with `ModuleNotFoundError: No module named 'formscout.agents.pdf_report'`
|
| 763 |
+
|
| 764 |
+
- [ ] **Step 3: Create the agent**
|
| 765 |
+
|
| 766 |
+
Create `formscout/agents/pdf_report.py`:
|
| 767 |
+
|
| 768 |
+
```python
|
| 769 |
+
"""
|
| 770 |
+
PdfReportAgent — renders a ReportResult + session entries to a branded PDF.
|
| 771 |
+
|
| 772 |
+
Input: ReportResult, list[SessionEntry], session_dir (str)
|
| 773 |
+
Output: path to the written PDF (str), or None on failure.
|
| 774 |
+
Failure: returns None, never raises.
|
| 775 |
+
Params: 0 (pure rendering — no model).
|
| 776 |
+
License: n/a.
|
| 777 |
+
Gated: no.
|
| 778 |
+
"""
|
| 779 |
+
from __future__ import annotations
|
| 780 |
+
|
| 781 |
+
import logging
|
| 782 |
+
import os
|
| 783 |
+
|
| 784 |
+
from formscout.types import ReportResult
|
| 785 |
+
|
| 786 |
+
logger = logging.getLogger(__name__)
|
| 787 |
+
|
| 788 |
+
DISCLAIMER = "Screening aid — not a diagnosis. Pain or clearing tests require a clinician."
|
| 789 |
+
|
| 790 |
+
|
| 791 |
+
class PdfReportAgent:
|
| 792 |
+
"""Assembles the screening-session PDF via ReportLab."""
|
| 793 |
+
|
| 794 |
+
def run(self, report: ReportResult, entries: list, session_dir: str) -> str | None:
|
| 795 |
+
try:
|
| 796 |
+
from reportlab.lib import colors
|
| 797 |
+
from reportlab.lib.pagesizes import LETTER
|
| 798 |
+
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
| 799 |
+
from reportlab.lib.units import inch
|
| 800 |
+
from reportlab.platypus import (
|
| 801 |
+
Image, Paragraph, SimpleDocTemplate, Spacer, Table, TableStyle,
|
| 802 |
+
)
|
| 803 |
+
except Exception as e:
|
| 804 |
+
logger.warning("reportlab unavailable: %s", e)
|
| 805 |
+
return None
|
| 806 |
+
|
| 807 |
+
out_path = os.path.join(session_dir, "formscout_report.pdf")
|
| 808 |
+
try:
|
| 809 |
+
styles = getSampleStyleSheet()
|
| 810 |
+
banner = ParagraphStyle(
|
| 811 |
+
"banner", parent=styles["Normal"], fontSize=9, textColor=colors.white,
|
| 812 |
+
backColor=colors.HexColor("#b45309"), alignment=1, borderPadding=6, spaceAfter=12,
|
| 813 |
+
)
|
| 814 |
+
story = []
|
| 815 |
+
story.append(Paragraph(f"<b>⚠ {DISCLAIMER}</b>", banner))
|
| 816 |
+
story.append(Paragraph("FormScout — FMS Screening Report", styles["Title"]))
|
| 817 |
+
|
| 818 |
+
if report.composite is not None:
|
| 819 |
+
comp = f"Composite: <b>{report.composite} / 21</b>"
|
| 820 |
+
else:
|
| 821 |
+
comp = f"Composite: <b>Incomplete</b> — {len(entries)}/7 tests scored"
|
| 822 |
+
story.append(Paragraph(comp, styles["Heading2"]))
|
| 823 |
+
story.append(Spacer(1, 0.2 * inch))
|
| 824 |
+
|
| 825 |
+
for e in entries:
|
| 826 |
+
title = e.test_name.replace("_", " ").title()
|
| 827 |
+
if e.side in ("left", "right"):
|
| 828 |
+
title += f" ({e.side})"
|
| 829 |
+
score_txt = "Clinician review required" if e.needs_human else f"Score: {e.score}/3"
|
| 830 |
+
story.append(Paragraph(f"<b>{title}</b> — {score_txt}", styles["Heading3"]))
|
| 831 |
+
if e.rationale:
|
| 832 |
+
story.append(Paragraph(e.rationale, styles["Normal"]))
|
| 833 |
+
if e.compensation_tags:
|
| 834 |
+
story.append(Paragraph("Compensations: " + ", ".join(e.compensation_tags),
|
| 835 |
+
styles["Normal"]))
|
| 836 |
+
if e.corrective_hint:
|
| 837 |
+
story.append(Paragraph("Corrective: " + e.corrective_hint, styles["Normal"]))
|
| 838 |
+
|
| 839 |
+
items = list(e.measurements.items())[:6]
|
| 840 |
+
if items:
|
| 841 |
+
rows = [[k.replace("_", " "),
|
| 842 |
+
(f"{v:.1f}" if isinstance(v, float) else str(v))] for k, v in items]
|
| 843 |
+
tbl = Table(rows, colWidths=[3 * inch, 1.5 * inch])
|
| 844 |
+
tbl.setStyle(TableStyle([
|
| 845 |
+
("FONTSIZE", (0, 0), (-1, -1), 8),
|
| 846 |
+
("TEXTCOLOR", (0, 0), (-1, -1), colors.HexColor("#334155")),
|
| 847 |
+
]))
|
| 848 |
+
story.append(tbl)
|
| 849 |
+
|
| 850 |
+
if e.keyframe_path and os.path.exists(e.keyframe_path):
|
| 851 |
+
try:
|
| 852 |
+
story.append(Image(e.keyframe_path, width=3.0 * inch, height=2.25 * inch))
|
| 853 |
+
except Exception:
|
| 854 |
+
story.append(Paragraph("<i>(key-frame image unavailable)</i>", styles["Normal"]))
|
| 855 |
+
else:
|
| 856 |
+
story.append(Paragraph("<i>(key-frame image unavailable)</i>", styles["Normal"]))
|
| 857 |
+
|
| 858 |
+
story.append(Spacer(1, 0.2 * inch))
|
| 859 |
+
|
| 860 |
+
if report.asymmetries:
|
| 861 |
+
story.append(Paragraph("Asymmetries", styles["Heading2"]))
|
| 862 |
+
for a in report.asymmetries:
|
| 863 |
+
story.append(Paragraph(
|
| 864 |
+
f"{a['test'].replace('_', ' ').title()}: "
|
| 865 |
+
f"L={a['left_score']} R={a['right_score']} (Δ {a['delta']})",
|
| 866 |
+
styles["Normal"]))
|
| 867 |
+
|
| 868 |
+
flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
|
| 869 |
+
if flags:
|
| 870 |
+
story.append(Paragraph("Flags", styles["Heading2"]))
|
| 871 |
+
for fl in flags:
|
| 872 |
+
story.append(Paragraph(fl, styles["Normal"]))
|
| 873 |
+
|
| 874 |
+
story.append(Spacer(1, 0.3 * inch))
|
| 875 |
+
story.append(Paragraph(f"<b>⚠ {DISCLAIMER}</b>", banner))
|
| 876 |
+
|
| 877 |
+
doc = SimpleDocTemplate(out_path, pagesize=LETTER,
|
| 878 |
+
topMargin=0.6 * inch, bottomMargin=0.6 * inch)
|
| 879 |
+
doc.build(story)
|
| 880 |
+
return out_path
|
| 881 |
+
except Exception as e:
|
| 882 |
+
logger.warning("pdf build failed: %s", e)
|
| 883 |
+
return None
|
| 884 |
+
```
|
| 885 |
+
|
| 886 |
+
- [ ] **Step 4: Run test to verify it passes**
|
| 887 |
+
|
| 888 |
+
Run: `pytest tests/test_pdf_report.py -v`
|
| 889 |
+
Expected: both tests PASS
|
| 890 |
+
|
| 891 |
+
- [ ] **Step 5: Re-run the session suite (pdf_path now populated)**
|
| 892 |
+
|
| 893 |
+
Run: `pytest tests/test_session.py -v`
|
| 894 |
+
Expected: all PASS (now `finish_session` returns a real `pdf_path`).
|
| 895 |
+
|
| 896 |
+
- [ ] **Step 6: Commit**
|
| 897 |
+
|
| 898 |
+
```bash
|
| 899 |
+
git add formscout/agents/pdf_report.py tests/test_pdf_report.py
|
| 900 |
+
git commit -m "feat: add PdfReportAgent — branded ReportLab session PDF"
|
| 901 |
+
```
|
| 902 |
+
|
| 903 |
+
---
|
| 904 |
+
|
| 905 |
+
## Task 7: Wire the session UI in `app.py`
|
| 906 |
+
|
| 907 |
+
**Files:**
|
| 908 |
+
- Modify: `app.py` (`process_video`, `build_app`, event wiring)
|
| 909 |
+
|
| 910 |
+
This task is verified by running the app (Gradio event wiring is not unit-tested; the
|
| 911 |
+
orchestration it calls is already covered by `tests/test_session.py`).
|
| 912 |
+
|
| 913 |
+
- [ ] **Step 1: Import the session module**
|
| 914 |
+
|
| 915 |
+
In `app.py`, add to the imports block (after `from formscout.startup import ensure_checkpoints`):
|
| 916 |
+
|
| 917 |
+
```python
|
| 918 |
+
from formscout import session as session_mod
|
| 919 |
+
```
|
| 920 |
+
|
| 921 |
+
- [ ] **Step 2: Refactor `process_video` to accumulate into a session**
|
| 922 |
+
|
| 923 |
+
Replace the entire `process_video` function (lines ~51-105) with a version that takes and
|
| 924 |
+
returns the session, appends an entry on success, and builds the "Session so far" table.
|
| 925 |
+
Replace from `def process_video(` through its final `return ...` with:
|
| 926 |
+
|
| 927 |
+
```python
|
| 928 |
+
def process_video(video_path: str, test_name: str, side: str, model_key: str,
|
| 929 |
+
layers: list[str], session_state):
|
| 930 |
+
"""Analyse one clip and accumulate it into the screening session."""
|
| 931 |
+
if not video_path:
|
| 932 |
+
return (
|
| 933 |
+
session_state, _render_empty_state(), "Upload a video to begin analysis.",
|
| 934 |
+
"", "", None, "", _render_session_table(session_state),
|
| 935 |
+
gr.update(visible=False), gr.update(visible=False),
|
| 936 |
+
)
|
| 937 |
+
|
| 938 |
+
if session_state is None:
|
| 939 |
+
session_state = session_mod.new_session()
|
| 940 |
+
|
| 941 |
+
director = Director()
|
| 942 |
+
state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
|
| 943 |
+
|
| 944 |
+
score_html = _render_empty_state()
|
| 945 |
+
score_details = ""
|
| 946 |
+
|
| 947 |
+
if state.features:
|
| 948 |
+
result = score_test(state.features)
|
| 949 |
+
judge = state.judge
|
| 950 |
+
if judge and judge.score is not None:
|
| 951 |
+
score_html = _render_score_card(judge.score, judge.confidence, judge.needs_human)
|
| 952 |
+
score_details = _render_score_details_judge(judge, result, state.features)
|
| 953 |
+
elif judge and judge.needs_human:
|
| 954 |
+
score_html = _render_score_card(0, 0, True)
|
| 955 |
+
score_details = f"### Needs Clinician Review\n{judge.rationale}"
|
| 956 |
+
else:
|
| 957 |
+
score_html = _render_score_card(result.score, result.confidence, result.needs_human)
|
| 958 |
+
score_details = _render_score_details(result, state.features)
|
| 959 |
+
|
| 960 |
+
# Accumulate into the session (only when we have a real analysis)
|
| 961 |
+
if state.ingest and state.pose2d and state.judge:
|
| 962 |
+
draw_trails = "trails" in {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
|
| 963 |
+
try:
|
| 964 |
+
session_mod.add_analysis(
|
| 965 |
+
session_state, ingest=state.ingest, pose2d=state.pose2d,
|
| 966 |
+
features=state.features, judge=state.judge,
|
| 967 |
+
test_name=test_name, side=side, draw_trails=draw_trails,
|
| 968 |
+
)
|
| 969 |
+
except Exception as e:
|
| 970 |
+
state.warnings.append(f"session accumulation failed: {e}")
|
| 971 |
+
|
| 972 |
+
pipeline_md = _render_pipeline_status(state)
|
| 973 |
+
alerts = _render_alerts(state)
|
| 974 |
+
|
| 975 |
+
overlay_path = None
|
| 976 |
+
vel_summary = ""
|
| 977 |
+
layer_set = {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
|
| 978 |
+
if layer_set and state.ingest and state.pose2d:
|
| 979 |
+
try:
|
| 980 |
+
from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
|
| 981 |
+
vis = PoseVisualizer()
|
| 982 |
+
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
|
| 983 |
+
out_path = f.name
|
| 984 |
+
overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
|
| 985 |
+
if overlay_path:
|
| 986 |
+
vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
|
| 987 |
+
except Exception as e:
|
| 988 |
+
alerts = (alerts or "") + f"\n⚠️ Visualizer error: {e}"
|
| 989 |
+
|
| 990 |
+
has_entries = bool(session_state and session_state.entries)
|
| 991 |
+
return (
|
| 992 |
+
session_state, score_html, pipeline_md, score_details, alerts,
|
| 993 |
+
overlay_path, vel_summary, _render_session_table(session_state),
|
| 994 |
+
gr.update(visible=has_entries), gr.update(visible=has_entries),
|
| 995 |
+
)
|
| 996 |
+
```
|
| 997 |
+
|
| 998 |
+
- [ ] **Step 3: Add the session-table renderer and finish handler**
|
| 999 |
+
|
| 1000 |
+
In `app.py`, add these two functions just before `def build_app()`:
|
| 1001 |
+
|
| 1002 |
+
```python
|
| 1003 |
+
def _render_session_table(session_state) -> str:
|
| 1004 |
+
"""Render the accumulated 'Session so far' table as markdown."""
|
| 1005 |
+
if not session_state or not session_state.entries:
|
| 1006 |
+
return "*No clips analysed yet.*"
|
| 1007 |
+
lines = ["| Test | Side | Score | Status |", "|---|---|---|---|"]
|
| 1008 |
+
for e in session_state.entries:
|
| 1009 |
+
test = e.test_name.replace("_", " ").title()
|
| 1010 |
+
side = e.side if e.side in ("left", "right") else "—"
|
| 1011 |
+
if e.needs_human:
|
| 1012 |
+
score, status = "—", "⚠️ Clinician review"
|
| 1013 |
+
else:
|
| 1014 |
+
score, status = f"{e.score}/3", "✓ scored"
|
| 1015 |
+
lines.append(f"| {test} | {side} | {score} | {status} |")
|
| 1016 |
+
return "\n".join(lines)
|
| 1017 |
+
|
| 1018 |
+
|
| 1019 |
+
def _finish_session(session_state):
|
| 1020 |
+
"""Build the composite report + PDF for the whole session."""
|
| 1021 |
+
if not session_state or not session_state.entries:
|
| 1022 |
+
return ("⚠️ No clips analysed yet — analyse at least one clip first.",
|
| 1023 |
+
None, None)
|
| 1024 |
+
|
| 1025 |
+
report, pdf_path = session_mod.finish_session(session_state)
|
| 1026 |
+
if report is None:
|
| 1027 |
+
return ("⚠️ Nothing to report.", None, None)
|
| 1028 |
+
|
| 1029 |
+
if report.composite is not None:
|
| 1030 |
+
summary = [f"## Composite: {report.composite} / 21"]
|
| 1031 |
+
else:
|
| 1032 |
+
n = len(session_state.entries)
|
| 1033 |
+
summary = [f"## Composite: Incomplete — {n}/7 tests scored",
|
| 1034 |
+
"*(One or more tests need clinician review or were unscored.)*"]
|
| 1035 |
+
|
| 1036 |
+
if report.asymmetries:
|
| 1037 |
+
summary.append("\n### Asymmetries")
|
| 1038 |
+
for a in report.asymmetries:
|
| 1039 |
+
test = a["test"].replace("_", " ").title()
|
| 1040 |
+
summary.append(f"- **{test}:** L={a['left_score']} R={a['right_score']} (Δ {a['delta']})")
|
| 1041 |
+
|
| 1042 |
+
flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
|
| 1043 |
+
if flags:
|
| 1044 |
+
summary.append("\n### Flags")
|
| 1045 |
+
for fl in flags:
|
| 1046 |
+
summary.append(f"- {fl}")
|
| 1047 |
+
|
| 1048 |
+
md_path = os.path.join(session_state.session_dir, "analysis.md")
|
| 1049 |
+
md_out = md_path if os.path.exists(md_path) else None
|
| 1050 |
+
return "\n".join(summary), pdf_path, md_out
|
| 1051 |
+
```
|
| 1052 |
+
|
| 1053 |
+
Also add `import os` to the top of `app.py` if not already present (it currently imports only
|
| 1054 |
+
`tempfile` and `gradio`). Add after `import tempfile`:
|
| 1055 |
+
|
| 1056 |
+
```python
|
| 1057 |
+
import os
|
| 1058 |
+
```
|
| 1059 |
+
|
| 1060 |
+
- [ ] **Step 4: Add the session state, buttons, and outputs to `build_app`**
|
| 1061 |
+
|
| 1062 |
+
In `build_app`, inside the `with gr.Blocks(...) as app:` block, immediately after the line
|
| 1063 |
+
`with gr.Blocks(title="FormScout — FMS Screening Aid") as app:` add:
|
| 1064 |
+
|
| 1065 |
+
```python
|
| 1066 |
+
session_state = gr.State(None)
|
| 1067 |
+
```
|
| 1068 |
+
|
| 1069 |
+
Then, in the left input column, replace the single submit button block:
|
| 1070 |
+
|
| 1071 |
+
```python
|
| 1072 |
+
submit_btn = gr.Button(
|
| 1073 |
+
"🎯 Score Movement",
|
| 1074 |
+
variant="primary",
|
| 1075 |
+
size="lg",
|
| 1076 |
+
)
|
| 1077 |
+
```
|
| 1078 |
+
|
| 1079 |
+
with:
|
| 1080 |
+
|
| 1081 |
+
```python
|
| 1082 |
+
submit_btn = gr.Button(
|
| 1083 |
+
"🎯 Score Movement",
|
| 1084 |
+
variant="primary",
|
| 1085 |
+
size="lg",
|
| 1086 |
+
)
|
| 1087 |
+
with gr.Row():
|
| 1088 |
+
new_clip_btn = gr.Button("➕ Analyse new clip", visible=False)
|
| 1089 |
+
finish_btn = gr.Button("✅ Finish & generate PDF",
|
| 1090 |
+
variant="primary", visible=False)
|
| 1091 |
+
```
|
| 1092 |
+
|
| 1093 |
+
In the right results column, add a "Session" tab and a finish-output area. Inside `with gr.Tabs():`
|
| 1094 |
+
add a new tab after the "🎬 Overlay Video" tab:
|
| 1095 |
+
|
| 1096 |
+
```python
|
| 1097 |
+
with gr.TabItem("🗂️ Session"):
|
| 1098 |
+
session_table = gr.Markdown("*No clips analysed yet.*")
|
| 1099 |
+
finish_summary = gr.Markdown("")
|
| 1100 |
+
pdf_file = gr.File(label="Screening Report (PDF)", visible=True)
|
| 1101 |
+
md_file = gr.File(label="Analysis Log (Markdown)", visible=True)
|
| 1102 |
+
```
|
| 1103 |
+
|
| 1104 |
+
- [ ] **Step 5: Update event wiring**
|
| 1105 |
+
|
| 1106 |
+
Replace the `_map_inputs` function and `submit_btn.click(...)` block at the bottom of `build_app`
|
| 1107 |
+
with:
|
| 1108 |
+
|
| 1109 |
+
```python
|
| 1110 |
+
def _map_inputs(video, test_display_name, side_display, pose_model_key, overlay_layers, sess):
|
| 1111 |
+
"""Map UI display values to internal values and accumulate into the session."""
|
| 1112 |
+
test_map = {name: val for name, val in FMS_TESTS}
|
| 1113 |
+
test_name = test_map.get(test_display_name, "deep_squat")
|
| 1114 |
+
side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
|
| 1115 |
+
return process_video(video, test_name, side, pose_model_key, overlay_layers, sess)
|
| 1116 |
+
|
| 1117 |
+
submit_btn.click(
|
| 1118 |
+
fn=_map_inputs,
|
| 1119 |
+
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown,
|
| 1120 |
+
overlay_layers, session_state],
|
| 1121 |
+
outputs=[session_state, score_html, pipeline_md, score_details, alerts_md,
|
| 1122 |
+
overlay_video, velocity_md, session_table, new_clip_btn, finish_btn],
|
| 1123 |
+
)
|
| 1124 |
+
|
| 1125 |
+
def _new_clip():
|
| 1126 |
+
"""Clear inputs for the next clip; keep the session intact."""
|
| 1127 |
+
return None, _render_empty_state(), ""
|
| 1128 |
+
|
| 1129 |
+
new_clip_btn.click(
|
| 1130 |
+
fn=_new_clip,
|
| 1131 |
+
inputs=[],
|
| 1132 |
+
outputs=[video_input, score_html, score_details],
|
| 1133 |
+
)
|
| 1134 |
+
|
| 1135 |
+
finish_btn.click(
|
| 1136 |
+
fn=_finish_session,
|
| 1137 |
+
inputs=[session_state],
|
| 1138 |
+
outputs=[finish_summary, pdf_file, md_file],
|
| 1139 |
+
)
|
| 1140 |
+
```
|
| 1141 |
+
|
| 1142 |
+
- [ ] **Step 6: Verify the full test suite still passes**
|
| 1143 |
+
|
| 1144 |
+
Run: `pytest tests/ -q`
|
| 1145 |
+
Expected: all tests pass except the single pre-existing known failure documented in CLAUDE.md
|
| 1146 |
+
(`test_unimplemented_test_returns_low_confidence`). No new failures.
|
| 1147 |
+
|
| 1148 |
+
- [ ] **Step 7: Manually verify the app**
|
| 1149 |
+
|
| 1150 |
+
Run: `python3 app.py`
|
| 1151 |
+
Then in the browser:
|
| 1152 |
+
1. Upload a clip, pick a test, click **Score Movement** → score card appears; the **Session** tab
|
| 1153 |
+
shows one row; the two new buttons appear.
|
| 1154 |
+
2. Click **➕ Analyse new clip** → the video input clears, the session row persists.
|
| 1155 |
+
3. Analyse a second test → a second row appears.
|
| 1156 |
+
4. Click **✅ Finish & generate PDF** → the Session tab shows the composite summary and a
|
| 1157 |
+
downloadable PDF (open it: disclaimer top + bottom, per-test blocks with key-frame images,
|
| 1158 |
+
composite or "Incomplete"). The Markdown log is also downloadable.
|
| 1159 |
+
|
| 1160 |
+
Expected: all four steps work; PDF opens and contains the disclaimer, composite, and per-test sections.
|
| 1161 |
+
|
| 1162 |
+
- [ ] **Step 8: Commit**
|
| 1163 |
+
|
| 1164 |
+
```bash
|
| 1165 |
+
git add app.py
|
| 1166 |
+
git commit -m "feat: accumulate FMS clips into a session with composite report + PDF export"
|
| 1167 |
+
```
|
| 1168 |
+
|
| 1169 |
+
---
|
| 1170 |
+
|
| 1171 |
+
## Task 8: Update docs
|
| 1172 |
+
|
| 1173 |
+
**Files:**
|
| 1174 |
+
- Modify: `CLAUDE.md` (Build phases / status)
|
| 1175 |
+
- Modify: `MODEL_BUDGET.md` (no param change — note PDF agent adds 0 params, for completeness)
|
| 1176 |
+
|
| 1177 |
+
- [ ] **Step 1: Update the Phase 4 line in CLAUDE.md**
|
| 1178 |
+
|
| 1179 |
+
In `CLAUDE.md`, in the "Build phases" section, update the Phase 4 line from:
|
| 1180 |
+
|
| 1181 |
+
```
|
| 1182 |
+
4. **Phase 4 — Polish + ship:** Custom Svelte UI components, PDF export, agent trace to Hub, blog post. (Overlay video already done via `PoseVisualizer`.)
|
| 1183 |
+
```
|
| 1184 |
+
|
| 1185 |
+
to:
|
| 1186 |
+
|
| 1187 |
+
```
|
| 1188 |
+
4. **Phase 4 — Polish + ship:** Custom Svelte UI components, agent trace to Hub, blog post. (Overlay video done via `PoseVisualizer`; full 7-test session + PDF export done via `formscout/session.py` + `PdfReportAgent`.)
|
| 1189 |
+
```
|
| 1190 |
+
|
| 1191 |
+
- [ ] **Step 2: Note the PDF agent in the architecture section**
|
| 1192 |
+
|
| 1193 |
+
In `CLAUDE.md`, under "### Rubric scorers" or near the ReportAgent description, this is optional
|
| 1194 |
+
context; no required change. Skip if no natural home.
|
| 1195 |
+
|
| 1196 |
+
- [ ] **Step 3: Commit**
|
| 1197 |
+
|
| 1198 |
+
```bash
|
| 1199 |
+
git add CLAUDE.md
|
| 1200 |
+
git commit -m "docs: mark full FMS session + PDF export complete in build phases"
|
| 1201 |
+
```
|
| 1202 |
+
|
| 1203 |
+
---
|
| 1204 |
+
|
| 1205 |
+
## Self-Review Notes (already applied)
|
| 1206 |
+
|
| 1207 |
+
- **Spec coverage:** session accumulation (Task 5), two-button UX (Task 7), on-disk MD/JSON/keyframes (Task 5), key-frame from `features.timing` (Tasks 3–5), ReportLab PDF top/bottom disclaimer + composite + per-test + asymmetry + flags (Task 6), `SessionEntry` type (Task 2), `ReportAgent` reuse (Task 5 `finish_session`), composite-null-on-needs-human (Task 5 test), error tolerance / never-raise (Tasks 4–6). All covered.
|
| 1208 |
+
- **Type consistency:** `SessionEntry` field names are identical across Tasks 2, 5, 6, 7. `finish_session` returns `(ReportResult | None, str | None)` and is consumed that way in Task 7. `render_frame(ingest, pose2d, frame_idx, layers, caption, out_png)` signature matches its callers.
|
| 1209 |
+
- **No placeholders:** every code step shows complete code; every run step states the exact command + expected outcome.
|
docs/superpowers/specs/2026-06-09-pose-model-selector-design.md
CHANGED
|
@@ -1,171 +1,171 @@
|
|
| 1 |
-
# Pose Model Selector — Design Spec
|
| 2 |
-
|
| 3 |
-
**Date:** 2026-06-09
|
| 4 |
-
**Status:** Approved
|
| 5 |
-
|
| 6 |
-
## Goal
|
| 7 |
-
|
| 8 |
-
Expose all available pose estimation models as a selectable dropdown in the Gradio UI, replacing the hard-coded YOLO26l default. Supported families: MediaPipe (Qualcomm HF/ONNX), YOLO26 n→x (local), Sapiens2 0.4B→5B (HF/transformers).
|
| 9 |
-
|
| 10 |
-
---
|
| 11 |
-
|
| 12 |
-
## Architecture
|
| 13 |
-
|
| 14 |
-
### Unified model registry (`config.py`)
|
| 15 |
-
|
| 16 |
-
Replace `YOLO_POSE_MODELS` with a single `POSE_MODELS` dict. Each entry:
|
| 17 |
-
|
| 18 |
-
```python
|
| 19 |
-
{
|
| 20 |
-
"backend": "yolo" | "mediapipe" | "sapiens2",
|
| 21 |
-
"path": str, # yolo only — absolute path to local .pt
|
| 22 |
-
"hf_id": str, # mediapipe + sapiens2 — HuggingFace repo id
|
| 23 |
-
"params_m": float, # millions of parameters
|
| 24 |
-
}
|
| 25 |
-
```
|
| 26 |
-
|
| 27 |
-
Ordered as displayed in the UI:
|
| 28 |
-
|
| 29 |
-
| Label | backend | source |
|
| 30 |
-
|---|---|---|
|
| 31 |
-
| `MediaPipe-Pose ⬇ ~16 MB, CPU-friendly` | mediapipe | `qualcomm/MediaPipe-Pose-Estimation` |
|
| 32 |
-
| `YOLO26n — nano (0.7M, fastest)` ★ default | yolo | local checkpoint |
|
| 33 |
-
| `YOLO26s — small (3.5M)` | yolo | local checkpoint |
|
| 34 |
-
| `YOLO26m — medium (9M)` | yolo | local checkpoint |
|
| 35 |
-
| `YOLO26l — large (25.9M)` | yolo | local checkpoint |
|
| 36 |
-
| `YOLO26x — extra-large (57.6M)` | yolo | local checkpoint |
|
| 37 |
-
| `Sapiens2-0.4B ⬇ ~1.6 GB` | sapiens2 | `facebook/sapiens2-pose-0.4b` |
|
| 38 |
-
| `Sapiens2-0.8B ⬇ ~3.2 GB` | sapiens2 | `facebook/sapiens2-pose-0.8b` |
|
| 39 |
-
| `Sapiens2-1B ⬇ ~4 GB` | sapiens2 | `facebook/sapiens2-pose-1b` |
|
| 40 |
-
| `Sapiens2-5B ⬇ ~20 GB, large GPU` | sapiens2 | `facebook/sapiens2-pose-5b` |
|
| 41 |
-
|
| 42 |
-
```python
|
| 43 |
-
DEFAULT_POSE_MODEL = "YOLO26n — nano (0.7M, fastest)"
|
| 44 |
-
```
|
| 45 |
-
|
| 46 |
-
Keep `YOLO_POSE_MODEL` and `YOLO_POSE_MODEL_HQ` as string aliases for backward compat with any direct references outside the agent.
|
| 47 |
-
|
| 48 |
-
---
|
| 49 |
-
|
| 50 |
-
### Pose2DAgent (`formscout/agents/pose2d.py`)
|
| 51 |
-
|
| 52 |
-
Three private sub-runners, all returning `list[dict[int, dict]]` (COCO 17 keypoints per frame, same format as today):
|
| 53 |
-
|
| 54 |
-
#### `_run_yolo(frames, path) -> list[dict]`
|
| 55 |
-
Existing logic, lifted into a named function. Model cached in `_model_cache[path]`.
|
| 56 |
-
|
| 57 |
-
#### `_run_mediapipe(frames, hf_id) -> list[dict]`
|
| 58 |
-
- Download repo snapshot via `huggingface_hub.snapshot_download(hf_id)`
|
| 59 |
-
- Locate the pose landmark `.onnx` file in the snapshot
|
| 60 |
-
- Load with `onnxruntime.InferenceSession`
|
| 61 |
-
- Preprocess each frame: resize to 256×256, normalize
|
| 62 |
-
- Run inference → 33 BlazePose landmarks
|
| 63 |
-
- Map BlazePose 33 → COCO 17 via fixed index table:
|
| 64 |
-
```
|
| 65 |
-
COCO 0=nose → BlazePose 0
|
| 66 |
-
COCO 1=left_eye → BlazePose 2
|
| 67 |
-
COCO 2=right_eye → BlazePose 5
|
| 68 |
-
COCO 3=left_ear → BlazePose 7
|
| 69 |
-
COCO 4=right_ear → BlazePose 8
|
| 70 |
-
COCO 5=left_shld → BlazePose 11
|
| 71 |
-
COCO 6=right_shld → BlazePose 12
|
| 72 |
-
COCO 7=left_elbow → BlazePose 13
|
| 73 |
-
COCO 8=right_elbow → BlazePose 14
|
| 74 |
-
COCO 9=left_wrist → BlazePose 15
|
| 75 |
-
COCO 10=right_wrist → BlazePose 16
|
| 76 |
-
COCO 11=left_hip → BlazePose 23
|
| 77 |
-
COCO 12=right_hip → BlazePose 24
|
| 78 |
-
COCO 13=left_knee → BlazePose 25
|
| 79 |
-
COCO 14=right_knee → BlazePose 26
|
| 80 |
-
COCO 15=left_ankle → BlazePose 27
|
| 81 |
-
COCO 16=right_ankle → BlazePose 28
|
| 82 |
-
```
|
| 83 |
-
- Session cached in `_model_cache[hf_id]`
|
| 84 |
-
|
| 85 |
-
#### `_run_sapiens2(frames, hf_id) -> list[dict]`
|
| 86 |
-
- Load via `transformers.pipeline("pose-estimation", model=hf_id)`
|
| 87 |
-
- Sapiens2 outputs 308 whole-body keypoints; map first 17 (indices 0–16) to COCO 17 — Sapiens2 preserves COCO ordering for the body subset
|
| 88 |
-
- Pipeline cached in `_model_cache[hf_id]`
|
| 89 |
-
|
| 90 |
-
#### `Pose2DAgent.run(ingest, model_key)`
|
| 91 |
-
- `model_key: str` replaces `model_path: str` (old param)
|
| 92 |
-
- Looks up `config.POSE_MODELS[model_key]` (falls back to `DEFAULT_POSE_MODEL` if key missing)
|
| 93 |
-
- Dispatches to the appropriate sub-runner
|
| 94 |
-
- Returns `Pose2DResult` — identical contract as today
|
| 95 |
-
|
| 96 |
-
---
|
| 97 |
-
|
| 98 |
-
### UI (`app.py`)
|
| 99 |
-
|
| 100 |
-
Add `gr.Dropdown` for pose model in the input column, below the test/side row:
|
| 101 |
-
|
| 102 |
-
```python
|
| 103 |
-
pose_model_dropdown = gr.Dropdown(
|
| 104 |
-
choices=list(config.POSE_MODELS.keys()),
|
| 105 |
-
value=config.DEFAULT_POSE_MODEL,
|
| 106 |
-
label="Pose Model",
|
| 107 |
-
)
|
| 108 |
-
```
|
| 109 |
-
|
| 110 |
-
Update `_map_inputs` to accept and forward `pose_model_key`:
|
| 111 |
-
|
| 112 |
-
```python
|
| 113 |
-
def _map_inputs(video, test_display_name, side_display, pose_model_key):
|
| 114 |
-
...
|
| 115 |
-
return process_video(video, test_name, side, pose_model_key)
|
| 116 |
-
```
|
| 117 |
-
|
| 118 |
-
Update `submit_btn.click` inputs to include `pose_model_dropdown`.
|
| 119 |
-
|
| 120 |
-
`process_video(video_path, test_name, side, pose_model_key)` passes `pose_model_key` through to `director.run()`, which passes it to `Pose2DAgent.run()`. Remove the old `YOLO_POSE_MODELS.get()` lookup from `process_video`.
|
| 121 |
-
|
| 122 |
-
---
|
| 123 |
-
|
| 124 |
-
## Data flow
|
| 125 |
-
|
| 126 |
-
```
|
| 127 |
-
UI dropdown (pose_model_key: str)
|
| 128 |
-
→ process_video()
|
| 129 |
-
→ Director.run(pose_model_key=...)
|
| 130 |
-
→ Pose2DAgent.run(ingest, model_key=pose_model_key)
|
| 131 |
-
→ config.POSE_MODELS[model_key] → {backend, path|hf_id}
|
| 132 |
-
→ _run_yolo / _run_mediapipe / _run_sapiens2
|
| 133 |
-
→ list[dict[int, {x, y, conf}]] (COCO 17, same contract)
|
| 134 |
-
|
| 135 |
-
```
|
| 136 |
-
|
| 137 |
-
---
|
| 138 |
-
|
| 139 |
-
## Error handling
|
| 140 |
-
|
| 141 |
-
- Unknown `model_key`: log warning, fall back to `DEFAULT_POSE_MODEL`
|
| 142 |
-
- ONNX file not found in MediaPipe snapshot: `Pose2DResult(confidence=0.0, notes="mediapipe onnx not found")`
|
| 143 |
-
- Sapiens2 / MediaPipe download failure: `Pose2DResult(confidence=0.0, notes=str(e))`
|
| 144 |
-
- All failures are non-fatal; pipeline continues with 0-confidence result and surfaces alert in UI
|
| 145 |
-
|
| 146 |
-
---
|
| 147 |
-
|
| 148 |
-
## Dependencies to add (`requirements.txt`)
|
| 149 |
-
|
| 150 |
-
- `onnxruntime` — MediaPipe ONNX inference
|
| 151 |
-
- `huggingface_hub` — snapshot download for MediaPipe (already likely present via transformers)
|
| 152 |
-
|
| 153 |
-
Sapiens2 uses `transformers`, already a dependency.
|
| 154 |
-
|
| 155 |
-
---
|
| 156 |
-
|
| 157 |
-
## Testing
|
| 158 |
-
|
| 159 |
-
Each new backend gets a pytest in `tests/test_pose2d.py` that:
|
| 160 |
-
- Mocks the model load (no actual HF download in CI)
|
| 161 |
-
- Passes a 3-frame synthetic IngestResult
|
| 162 |
-
- Asserts `Pose2DResult.keypoints` has 3 entries, each a dict with at most 17 int keys
|
| 163 |
-
- Asserts `confidence` is a float in [0, 1]
|
| 164 |
-
|
| 165 |
-
---
|
| 166 |
-
|
| 167 |
-
## Out of scope
|
| 168 |
-
|
| 169 |
-
- Sapiens2 / MediaPipe accuracy benchmarking
|
| 170 |
-
- Automatic backend selection based on hardware
|
| 171 |
-
- Downloading Sapiens2/MediaPipe checkpoints to local `checkpoints/` directory
|
|
|
|
| 1 |
+
# Pose Model Selector — Design Spec
|
| 2 |
+
|
| 3 |
+
**Date:** 2026-06-09
|
| 4 |
+
**Status:** Approved
|
| 5 |
+
|
| 6 |
+
## Goal
|
| 7 |
+
|
| 8 |
+
Expose all available pose estimation models as a selectable dropdown in the Gradio UI, replacing the hard-coded YOLO26l default. Supported families: MediaPipe (Qualcomm HF/ONNX), YOLO26 n→x (local), Sapiens2 0.4B→5B (HF/transformers).
|
| 9 |
+
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
## Architecture
|
| 13 |
+
|
| 14 |
+
### Unified model registry (`config.py`)
|
| 15 |
+
|
| 16 |
+
Replace `YOLO_POSE_MODELS` with a single `POSE_MODELS` dict. Each entry:
|
| 17 |
+
|
| 18 |
+
```python
|
| 19 |
+
{
|
| 20 |
+
"backend": "yolo" | "mediapipe" | "sapiens2",
|
| 21 |
+
"path": str, # yolo only — absolute path to local .pt
|
| 22 |
+
"hf_id": str, # mediapipe + sapiens2 — HuggingFace repo id
|
| 23 |
+
"params_m": float, # millions of parameters
|
| 24 |
+
}
|
| 25 |
+
```
|
| 26 |
+
|
| 27 |
+
Ordered as displayed in the UI:
|
| 28 |
+
|
| 29 |
+
| Label | backend | source |
|
| 30 |
+
|---|---|---|
|
| 31 |
+
| `MediaPipe-Pose ⬇ ~16 MB, CPU-friendly` | mediapipe | `qualcomm/MediaPipe-Pose-Estimation` |
|
| 32 |
+
| `YOLO26n — nano (0.7M, fastest)` ★ default | yolo | local checkpoint |
|
| 33 |
+
| `YOLO26s — small (3.5M)` | yolo | local checkpoint |
|
| 34 |
+
| `YOLO26m — medium (9M)` | yolo | local checkpoint |
|
| 35 |
+
| `YOLO26l — large (25.9M)` | yolo | local checkpoint |
|
| 36 |
+
| `YOLO26x — extra-large (57.6M)` | yolo | local checkpoint |
|
| 37 |
+
| `Sapiens2-0.4B ⬇ ~1.6 GB` | sapiens2 | `facebook/sapiens2-pose-0.4b` |
|
| 38 |
+
| `Sapiens2-0.8B ⬇ ~3.2 GB` | sapiens2 | `facebook/sapiens2-pose-0.8b` |
|
| 39 |
+
| `Sapiens2-1B ⬇ ~4 GB` | sapiens2 | `facebook/sapiens2-pose-1b` |
|
| 40 |
+
| `Sapiens2-5B ⬇ ~20 GB, large GPU` | sapiens2 | `facebook/sapiens2-pose-5b` |
|
| 41 |
+
|
| 42 |
+
```python
|
| 43 |
+
DEFAULT_POSE_MODEL = "YOLO26n — nano (0.7M, fastest)"
|
| 44 |
+
```
|
| 45 |
+
|
| 46 |
+
Keep `YOLO_POSE_MODEL` and `YOLO_POSE_MODEL_HQ` as string aliases for backward compat with any direct references outside the agent.
|
| 47 |
+
|
| 48 |
+
---
|
| 49 |
+
|
| 50 |
+
### Pose2DAgent (`formscout/agents/pose2d.py`)
|
| 51 |
+
|
| 52 |
+
Three private sub-runners, all returning `list[dict[int, dict]]` (COCO 17 keypoints per frame, same format as today):
|
| 53 |
+
|
| 54 |
+
#### `_run_yolo(frames, path) -> list[dict]`
|
| 55 |
+
Existing logic, lifted into a named function. Model cached in `_model_cache[path]`.
|
| 56 |
+
|
| 57 |
+
#### `_run_mediapipe(frames, hf_id) -> list[dict]`
|
| 58 |
+
- Download repo snapshot via `huggingface_hub.snapshot_download(hf_id)`
|
| 59 |
+
- Locate the pose landmark `.onnx` file in the snapshot
|
| 60 |
+
- Load with `onnxruntime.InferenceSession`
|
| 61 |
+
- Preprocess each frame: resize to 256×256, normalize
|
| 62 |
+
- Run inference → 33 BlazePose landmarks
|
| 63 |
+
- Map BlazePose 33 → COCO 17 via fixed index table:
|
| 64 |
+
```
|
| 65 |
+
COCO 0=nose → BlazePose 0
|
| 66 |
+
COCO 1=left_eye → BlazePose 2
|
| 67 |
+
COCO 2=right_eye → BlazePose 5
|
| 68 |
+
COCO 3=left_ear → BlazePose 7
|
| 69 |
+
COCO 4=right_ear → BlazePose 8
|
| 70 |
+
COCO 5=left_shld → BlazePose 11
|
| 71 |
+
COCO 6=right_shld → BlazePose 12
|
| 72 |
+
COCO 7=left_elbow → BlazePose 13
|
| 73 |
+
COCO 8=right_elbow → BlazePose 14
|
| 74 |
+
COCO 9=left_wrist → BlazePose 15
|
| 75 |
+
COCO 10=right_wrist → BlazePose 16
|
| 76 |
+
COCO 11=left_hip → BlazePose 23
|
| 77 |
+
COCO 12=right_hip → BlazePose 24
|
| 78 |
+
COCO 13=left_knee → BlazePose 25
|
| 79 |
+
COCO 14=right_knee → BlazePose 26
|
| 80 |
+
COCO 15=left_ankle → BlazePose 27
|
| 81 |
+
COCO 16=right_ankle → BlazePose 28
|
| 82 |
+
```
|
| 83 |
+
- Session cached in `_model_cache[hf_id]`
|
| 84 |
+
|
| 85 |
+
#### `_run_sapiens2(frames, hf_id) -> list[dict]`
|
| 86 |
+
- Load via `transformers.pipeline("pose-estimation", model=hf_id)`
|
| 87 |
+
- Sapiens2 outputs 308 whole-body keypoints; map first 17 (indices 0–16) to COCO 17 — Sapiens2 preserves COCO ordering for the body subset
|
| 88 |
+
- Pipeline cached in `_model_cache[hf_id]`
|
| 89 |
+
|
| 90 |
+
#### `Pose2DAgent.run(ingest, model_key)`
|
| 91 |
+
- `model_key: str` replaces `model_path: str` (old param)
|
| 92 |
+
- Looks up `config.POSE_MODELS[model_key]` (falls back to `DEFAULT_POSE_MODEL` if key missing)
|
| 93 |
+
- Dispatches to the appropriate sub-runner
|
| 94 |
+
- Returns `Pose2DResult` — identical contract as today
|
| 95 |
+
|
| 96 |
+
---
|
| 97 |
+
|
| 98 |
+
### UI (`app.py`)
|
| 99 |
+
|
| 100 |
+
Add `gr.Dropdown` for pose model in the input column, below the test/side row:
|
| 101 |
+
|
| 102 |
+
```python
|
| 103 |
+
pose_model_dropdown = gr.Dropdown(
|
| 104 |
+
choices=list(config.POSE_MODELS.keys()),
|
| 105 |
+
value=config.DEFAULT_POSE_MODEL,
|
| 106 |
+
label="Pose Model",
|
| 107 |
+
)
|
| 108 |
+
```
|
| 109 |
+
|
| 110 |
+
Update `_map_inputs` to accept and forward `pose_model_key`:
|
| 111 |
+
|
| 112 |
+
```python
|
| 113 |
+
def _map_inputs(video, test_display_name, side_display, pose_model_key):
|
| 114 |
+
...
|
| 115 |
+
return process_video(video, test_name, side, pose_model_key)
|
| 116 |
+
```
|
| 117 |
+
|
| 118 |
+
Update `submit_btn.click` inputs to include `pose_model_dropdown`.
|
| 119 |
+
|
| 120 |
+
`process_video(video_path, test_name, side, pose_model_key)` passes `pose_model_key` through to `director.run()`, which passes it to `Pose2DAgent.run()`. Remove the old `YOLO_POSE_MODELS.get()` lookup from `process_video`.
|
| 121 |
+
|
| 122 |
+
---
|
| 123 |
+
|
| 124 |
+
## Data flow
|
| 125 |
+
|
| 126 |
+
```
|
| 127 |
+
UI dropdown (pose_model_key: str)
|
| 128 |
+
→ process_video()
|
| 129 |
+
→ Director.run(pose_model_key=...)
|
| 130 |
+
→ Pose2DAgent.run(ingest, model_key=pose_model_key)
|
| 131 |
+
→ config.POSE_MODELS[model_key] → {backend, path|hf_id}
|
| 132 |
+
→ _run_yolo / _run_mediapipe / _run_sapiens2
|
| 133 |
+
→ list[dict[int, {x, y, conf}]] (COCO 17, same contract)
|
| 134 |
+
→ Pose2DResult
|
| 135 |
+
```
|
| 136 |
+
|
| 137 |
+
---
|
| 138 |
+
|
| 139 |
+
## Error handling
|
| 140 |
+
|
| 141 |
+
- Unknown `model_key`: log warning, fall back to `DEFAULT_POSE_MODEL`
|
| 142 |
+
- ONNX file not found in MediaPipe snapshot: `Pose2DResult(confidence=0.0, notes="mediapipe onnx not found")`
|
| 143 |
+
- Sapiens2 / MediaPipe download failure: `Pose2DResult(confidence=0.0, notes=str(e))`
|
| 144 |
+
- All failures are non-fatal; pipeline continues with 0-confidence result and surfaces alert in UI
|
| 145 |
+
|
| 146 |
+
---
|
| 147 |
+
|
| 148 |
+
## Dependencies to add (`requirements.txt`)
|
| 149 |
+
|
| 150 |
+
- `onnxruntime` — MediaPipe ONNX inference
|
| 151 |
+
- `huggingface_hub` — snapshot download for MediaPipe (already likely present via transformers)
|
| 152 |
+
|
| 153 |
+
Sapiens2 uses `transformers`, already a dependency.
|
| 154 |
+
|
| 155 |
+
---
|
| 156 |
+
|
| 157 |
+
## Testing
|
| 158 |
+
|
| 159 |
+
Each new backend gets a pytest in `tests/test_pose2d.py` that:
|
| 160 |
+
- Mocks the model load (no actual HF download in CI)
|
| 161 |
+
- Passes a 3-frame synthetic IngestResult
|
| 162 |
+
- Asserts `Pose2DResult.keypoints` has 3 entries, each a dict with at most 17 int keys
|
| 163 |
+
- Asserts `confidence` is a float in [0, 1]
|
| 164 |
+
|
| 165 |
+
---
|
| 166 |
+
|
| 167 |
+
## Out of scope
|
| 168 |
+
|
| 169 |
+
- Sapiens2 / MediaPipe accuracy benchmarking
|
| 170 |
+
- Automatic backend selection based on hardware
|
| 171 |
+
- Downloading Sapiens2/MediaPipe checkpoints to local `checkpoints/` directory
|
docs/superpowers/specs/2026-06-09-pose-visualizer-design.md
CHANGED
|
@@ -1,197 +1,197 @@
|
|
| 1 |
-
# Pose Overlay Visualizer — Design Spec
|
| 2 |
-
|
| 3 |
-
**Date:** 2026-06-09
|
| 4 |
-
**Status:** Approved
|
| 5 |
-
|
| 6 |
-
## Goal
|
| 7 |
-
|
| 8 |
-
Add an annotated overlay video output to the FormScout UI showing skeleton, motion trails, and velocity arrows on top of the original footage, alongside a per-joint velocity summary table. Overlay layers are user-selectable via checkboxes. Adapted from the Laban Movement Analysis project.
|
| 9 |
-
|
| 10 |
-
---
|
| 11 |
-
|
| 12 |
-
## Architecture
|
| 13 |
-
|
| 14 |
-
Three files change or are created. No changes to `pipeline.py`, `types.py`, or any existing agent.
|
| 15 |
-
|
| 16 |
-
```
|
| 17 |
-
formscout/agents/visualizer.py ← new
|
| 18 |
-
tests/test_visualizer.py ← new
|
| 19 |
-
app.py ← overlay_layers checkbox, new tab, wiring
|
| 20 |
-
```
|
| 21 |
-
|
| 22 |
-
The visualizer runs **after** `director.run()` returns in `process_video()` — it is a pure post-processing step, never on the critical scoring path.
|
| 23 |
-
|
| 24 |
-
---
|
| 25 |
-
|
| 26 |
-
## Module: `formscout/agents/visualizer.py`
|
| 27 |
-
|
| 28 |
-
### `compute_joint_velocity(keypoints_per_frame, fps) → dict[int, list[float]]`
|
| 29 |
-
|
| 30 |
-
- Input: `list[dict[int, {x, y, conf}]]` (COCO-17 pixel coords per frame), `fps: float`
|
| 31 |
-
- Output: `dict[int, list[float]]` — per-joint per-frame speed in **px/s**
|
| 32 |
-
- Method: for each joint index, run a `SimpleKalmanFilter` (1D per axis, constant-velocity model, same structure as Laban's engine) over the (x, y) series. Speed = `sqrt(vx² + vy²)` from the filter's velocity state.
|
| 33 |
-
- Missing keypoints (conf < 0.3 or absent) → speed = 0.0 for that frame, filter state held.
|
| 34 |
-
|
| 35 |
-
### `SimpleKalmanFilter`
|
| 36 |
-
|
| 37 |
-
Minimal 4-state Kalman (x, y, vx, vy), identical in structure to the Laban `SimpleKalmanFilter`:
|
| 38 |
-
- Transition: constant-velocity model
|
| 39 |
-
- Measurement: position only (x, y)
|
| 40 |
-
- One instance per joint per video run
|
| 41 |
-
|
| 42 |
-
### `PoseVisualizer`
|
| 43 |
-
|
| 44 |
-
#### Constants
|
| 45 |
-
```python
|
| 46 |
-
COCO_SKELETON = [
|
| 47 |
-
(0,1),(0,2),(1,3),(2,4), # face
|
| 48 |
-
(5,6),(5,7),(7,9),(6,8),(8,10), # arms
|
| 49 |
-
(5,11),(6,12),(11,12), # torso
|
| 50 |
-
(11,13),(13,15),(12,14),(14,16), # legs
|
| 51 |
-
]
|
| 52 |
-
TRAIL_LENGTH = 10 # frames of trail history
|
| 53 |
-
MAX_ARROW_PX = 40 # arrow scaled so peak velocity → 40px length
|
| 54 |
-
CONF_THRESHOLD = 0.3 # min confidence to draw a keypoint
|
| 55 |
-
```
|
| 56 |
-
|
| 57 |
-
#### Private methods
|
| 58 |
-
|
| 59 |
-
**`_draw_skeleton(frame, kps)`**
|
| 60 |
-
- Draw each COCO bone as a line if both endpoints have conf > CONF_THRESHOLD
|
| 61 |
-
- Joint dots: color green→red by confidence using HSV (same as Laban `_confidence_to_color`)
|
| 62 |
-
- Bone color: white
|
| 63 |
-
|
| 64 |
-
**`_draw_trails(frame, trail_history, frame_idx)`**
|
| 65 |
-
- `trail_history: dict[int, deque(maxlen=TRAIL_LENGTH)]` keyed by joint index
|
| 66 |
-
- Each deque holds `(x, y)` pixel positions from previous frames
|
| 67 |
-
- Draw fading line segments: alpha = segment_position / TRAIL_LENGTH, color white
|
| 68 |
-
|
| 69 |
-
**`_draw_velocity_arrows(frame, kps, velocities, frame_idx)`**
|
| 70 |
-
- `velocities: dict[int, list[float]]` — speeds per joint per frame
|
| 71 |
-
- Direction vector from consecutive keypoint positions (x[t] - x[t-1], y[t] - y[t-1])
|
| 72 |
-
- Arrow length = `speed / peak_speed * MAX_ARROW_PX` (clamped)
|
| 73 |
-
- Drawn only for joints with conf > CONF_THRESHOLD and speed > 0
|
| 74 |
-
- Color: green=slow, orange=medium, red=fast (same thresholds as Laban intensity)
|
| 75 |
-
|
| 76 |
-
#### Public method
|
| 77 |
-
|
| 78 |
-
**`render_video(ingest, pose2d, layers: set[str], output_path: str) → str | None`**
|
| 79 |
-
- `layers`: subset of `{"skeleton", "trails", "velocity_arrows"}`
|
| 80 |
-
- If `layers` is empty → return `None` immediately
|
| 81 |
-
- Pre-computes `compute_joint_velocity(pose2d.keypoints, ingest.fps)`
|
| 82 |
-
- Iterates frames, updates `trail_history`, calls selected `_draw_*` methods
|
| 83 |
-
- Writes output via `cv2.VideoWriter` (codec: `mp4v`, same fps as ingest)
|
| 84 |
-
- Returns output path on success; `None` on any exception (logs warning)
|
| 85 |
-
|
| 86 |
-
#### Velocity summary
|
| 87 |
-
|
| 88 |
-
**`build_velocity_summary(keypoints_per_frame, velocities) → str`**
|
| 89 |
-
- For each joint with conf > 0.3 in >50% of frames:
|
| 90 |
-
- Compute avg and peak speed (px/s)
|
| 91 |
-
- Return markdown table sorted by peak speed descending:
|
| 92 |
-
```
|
| 93 |
-
| Joint | Avg (px/s) | Peak (px/s) |
|
| 94 |
-
|---------------|-----------|-------------|
|
| 95 |
-
| left_knee | 42.3 | 118.7 |
|
| 96 |
-
```
|
| 97 |
-
- Returns empty string if no valid joints
|
| 98 |
-
|
| 99 |
-
---
|
| 100 |
-
|
| 101 |
-
## UI changes: `app.py`
|
| 102 |
-
|
| 103 |
-
### Input column — overlay layer checkboxes
|
| 104 |
-
|
| 105 |
-
Below `pose_model_dropdown`, add:
|
| 106 |
-
|
| 107 |
-
```python
|
| 108 |
-
overlay_layers = gr.CheckboxGroup(
|
| 109 |
-
choices=["Skeleton", "Trails", "Velocity arrows"],
|
| 110 |
-
value=["Skeleton", "Trails"],
|
| 111 |
-
label="Overlay Layers",
|
| 112 |
-
)
|
| 113 |
-
```
|
| 114 |
-
|
| 115 |
-
### Results panel — new tab
|
| 116 |
-
|
| 117 |
-
Inside the existing `gr.Tabs()` block, add a fourth tab:
|
| 118 |
-
|
| 119 |
-
```python
|
| 120 |
-
with gr.TabItem("🎬 Overlay Video"):
|
| 121 |
-
overlay_video = gr.Video(label="Annotated Movement")
|
| 122 |
-
velocity_md = gr.Markdown("")
|
| 123 |
-
```
|
| 124 |
-
|
| 125 |
-
### `process_video()` signature
|
| 126 |
-
|
| 127 |
-
```python
|
| 128 |
-
def process_video(video_path, test_name, side, model_key, layers: list[str]):
|
| 129 |
-
```
|
| 130 |
-
|
| 131 |
-
After `director.run()`:
|
| 132 |
-
```python
|
| 133 |
-
from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
|
| 134 |
-
layer_set = {l.lower().replace(" ", "_") for l in layers}
|
| 135 |
-
# map UI labels to internal names:
|
| 136 |
-
# "Skeleton" → "skeleton", "Trails" → "trails", "Velocity arrows" → "velocity_arrows"
|
| 137 |
-
overlay_path = None
|
| 138 |
-
vel_summary = ""
|
| 139 |
-
if layer_set and state.ingest and state.pose2d:
|
| 140 |
-
try:
|
| 141 |
-
vis = PoseVisualizer()
|
| 142 |
-
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
|
| 143 |
-
out_path = f.name
|
| 144 |
-
overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
|
| 145 |
-
if overlay_path:
|
| 146 |
-
vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
|
| 147 |
-
except Exception as e:
|
| 148 |
-
alerts += f"\n⚠️ Visualizer error: {e}"
|
| 149 |
-
return score_html, pipeline_md, score_details, alerts, overlay_path, vel_summary
|
| 150 |
-
```
|
| 151 |
-
|
| 152 |
-
`vis.last_velocities` is stored on the instance after `render_video()` to avoid recomputing.
|
| 153 |
-
|
| 154 |
-
### Event wiring
|
| 155 |
-
|
| 156 |
-
```python
|
| 157 |
-
submit_btn.click(
|
| 158 |
-
fn=_map_inputs,
|
| 159 |
-
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown, overlay_layers],
|
| 160 |
-
outputs=[score_html, pipeline_md, score_details, alerts_md, overlay_video, velocity_md],
|
| 161 |
-
)
|
| 162 |
-
```
|
| 163 |
-
|
| 164 |
-
`_map_inputs` gains `overlay_layers` as fifth parameter.
|
| 165 |
-
|
| 166 |
-
---
|
| 167 |
-
|
| 168 |
-
## Error handling
|
| 169 |
-
|
| 170 |
-
| Failure | Behaviour |
|
| 171 |
-
|---|---|
|
| 172 |
-
| All frames have no detections | `render_video()` returns `None`, tab empty, no crash |
|
| 173 |
-
| `cv2.VideoWriter` fails | logs warning, returns `None` |
|
| 174 |
-
| Any exception in visualizer | caught in `process_video()`, appended to alerts, `overlay_path = None` |
|
| 175 |
-
| `layers` is empty | returns `None` immediately, no processing |
|
| 176 |
-
|
| 177 |
-
The score is always returned regardless of visualizer outcome.
|
| 178 |
-
|
| 179 |
-
---
|
| 180 |
-
|
| 181 |
-
## Testing: `tests/test_visualizer.py`
|
| 182 |
-
|
| 183 |
-
- Synthetic `IngestResult`: 5 blank 480×640 BGR frames, fps=30
|
| 184 |
-
- Synthetic `Pose2DResult`: 17 keypoints per frame at fixed positions with conf=0.9
|
| 185 |
-
- `test_render_video_creates_file`: assert output `.mp4` exists and size > 0
|
| 186 |
-
- `test_compute_joint_velocity_shape`: assert 17-key dict, each list length == 5
|
| 187 |
-
- `test_empty_layers_returns_none`: assert `render_video(..., layers=set())` returns `None`
|
| 188 |
-
- `test_no_detections_returns_none`: all-empty keypoints → `None`
|
| 189 |
-
- `test_velocity_summary_markdown`: assert output contains `|` (table) and at least one joint name
|
| 190 |
-
|
| 191 |
-
---
|
| 192 |
-
|
| 193 |
-
## Out of scope
|
| 194 |
-
|
| 195 |
-
- Frame-by-frame metrics synced to video playback (Phase 4 / custom Svelte)
|
| 196 |
-
- Multi-person tracking
|
| 197 |
-
- Saving overlay video to Hugging Face Hub (tracing feature, Phase 4)
|
|
|
|
| 1 |
+
# Pose Overlay Visualizer — Design Spec
|
| 2 |
+
|
| 3 |
+
**Date:** 2026-06-09
|
| 4 |
+
**Status:** Approved
|
| 5 |
+
|
| 6 |
+
## Goal
|
| 7 |
+
|
| 8 |
+
Add an annotated overlay video output to the FormScout UI showing skeleton, motion trails, and velocity arrows on top of the original footage, alongside a per-joint velocity summary table. Overlay layers are user-selectable via checkboxes. Adapted from the Laban Movement Analysis project.
|
| 9 |
+
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
## Architecture
|
| 13 |
+
|
| 14 |
+
Three files change or are created. No changes to `pipeline.py`, `types.py`, or any existing agent.
|
| 15 |
+
|
| 16 |
+
```
|
| 17 |
+
formscout/agents/visualizer.py ← new
|
| 18 |
+
tests/test_visualizer.py ← new
|
| 19 |
+
app.py ← overlay_layers checkbox, new tab, wiring
|
| 20 |
+
```
|
| 21 |
+
|
| 22 |
+
The visualizer runs **after** `director.run()` returns in `process_video()` — it is a pure post-processing step, never on the critical scoring path.
|
| 23 |
+
|
| 24 |
+
---
|
| 25 |
+
|
| 26 |
+
## Module: `formscout/agents/visualizer.py`
|
| 27 |
+
|
| 28 |
+
### `compute_joint_velocity(keypoints_per_frame, fps) → dict[int, list[float]]`
|
| 29 |
+
|
| 30 |
+
- Input: `list[dict[int, {x, y, conf}]]` (COCO-17 pixel coords per frame), `fps: float`
|
| 31 |
+
- Output: `dict[int, list[float]]` — per-joint per-frame speed in **px/s**
|
| 32 |
+
- Method: for each joint index, run a `SimpleKalmanFilter` (1D per axis, constant-velocity model, same structure as Laban's engine) over the (x, y) series. Speed = `sqrt(vx² + vy²)` from the filter's velocity state.
|
| 33 |
+
- Missing keypoints (conf < 0.3 or absent) → speed = 0.0 for that frame, filter state held.
|
| 34 |
+
|
| 35 |
+
### `SimpleKalmanFilter`
|
| 36 |
+
|
| 37 |
+
Minimal 4-state Kalman (x, y, vx, vy), identical in structure to the Laban `SimpleKalmanFilter`:
|
| 38 |
+
- Transition: constant-velocity model
|
| 39 |
+
- Measurement: position only (x, y)
|
| 40 |
+
- One instance per joint per video run
|
| 41 |
+
|
| 42 |
+
### `PoseVisualizer`
|
| 43 |
+
|
| 44 |
+
#### Constants
|
| 45 |
+
```python
|
| 46 |
+
COCO_SKELETON = [
|
| 47 |
+
(0,1),(0,2),(1,3),(2,4), # face
|
| 48 |
+
(5,6),(5,7),(7,9),(6,8),(8,10), # arms
|
| 49 |
+
(5,11),(6,12),(11,12), # torso
|
| 50 |
+
(11,13),(13,15),(12,14),(14,16), # legs
|
| 51 |
+
]
|
| 52 |
+
TRAIL_LENGTH = 10 # frames of trail history
|
| 53 |
+
MAX_ARROW_PX = 40 # arrow scaled so peak velocity → 40px length
|
| 54 |
+
CONF_THRESHOLD = 0.3 # min confidence to draw a keypoint
|
| 55 |
+
```
|
| 56 |
+
|
| 57 |
+
#### Private methods
|
| 58 |
+
|
| 59 |
+
**`_draw_skeleton(frame, kps)`**
|
| 60 |
+
- Draw each COCO bone as a line if both endpoints have conf > CONF_THRESHOLD
|
| 61 |
+
- Joint dots: color green→red by confidence using HSV (same as Laban `_confidence_to_color`)
|
| 62 |
+
- Bone color: white
|
| 63 |
+
|
| 64 |
+
**`_draw_trails(frame, trail_history, frame_idx)`**
|
| 65 |
+
- `trail_history: dict[int, deque(maxlen=TRAIL_LENGTH)]` keyed by joint index
|
| 66 |
+
- Each deque holds `(x, y)` pixel positions from previous frames
|
| 67 |
+
- Draw fading line segments: alpha = segment_position / TRAIL_LENGTH, color white
|
| 68 |
+
|
| 69 |
+
**`_draw_velocity_arrows(frame, kps, velocities, frame_idx)`**
|
| 70 |
+
- `velocities: dict[int, list[float]]` — speeds per joint per frame
|
| 71 |
+
- Direction vector from consecutive keypoint positions (x[t] - x[t-1], y[t] - y[t-1])
|
| 72 |
+
- Arrow length = `speed / peak_speed * MAX_ARROW_PX` (clamped)
|
| 73 |
+
- Drawn only for joints with conf > CONF_THRESHOLD and speed > 0
|
| 74 |
+
- Color: green=slow, orange=medium, red=fast (same thresholds as Laban intensity)
|
| 75 |
+
|
| 76 |
+
#### Public method
|
| 77 |
+
|
| 78 |
+
**`render_video(ingest, pose2d, layers: set[str], output_path: str) → str | None`**
|
| 79 |
+
- `layers`: subset of `{"skeleton", "trails", "velocity_arrows"}`
|
| 80 |
+
- If `layers` is empty → return `None` immediately
|
| 81 |
+
- Pre-computes `compute_joint_velocity(pose2d.keypoints, ingest.fps)`
|
| 82 |
+
- Iterates frames, updates `trail_history`, calls selected `_draw_*` methods
|
| 83 |
+
- Writes output via `cv2.VideoWriter` (codec: `mp4v`, same fps as ingest)
|
| 84 |
+
- Returns output path on success; `None` on any exception (logs warning)
|
| 85 |
+
|
| 86 |
+
#### Velocity summary
|
| 87 |
+
|
| 88 |
+
**`build_velocity_summary(keypoints_per_frame, velocities) → str`**
|
| 89 |
+
- For each joint with conf > 0.3 in >50% of frames:
|
| 90 |
+
- Compute avg and peak speed (px/s)
|
| 91 |
+
- Return markdown table sorted by peak speed descending:
|
| 92 |
+
```
|
| 93 |
+
| Joint | Avg (px/s) | Peak (px/s) |
|
| 94 |
+
|---------------|-----------|-------------|
|
| 95 |
+
| left_knee | 42.3 | 118.7 |
|
| 96 |
+
```
|
| 97 |
+
- Returns empty string if no valid joints
|
| 98 |
+
|
| 99 |
+
---
|
| 100 |
+
|
| 101 |
+
## UI changes: `app.py`
|
| 102 |
+
|
| 103 |
+
### Input column — overlay layer checkboxes
|
| 104 |
+
|
| 105 |
+
Below `pose_model_dropdown`, add:
|
| 106 |
+
|
| 107 |
+
```python
|
| 108 |
+
overlay_layers = gr.CheckboxGroup(
|
| 109 |
+
choices=["Skeleton", "Trails", "Velocity arrows"],
|
| 110 |
+
value=["Skeleton", "Trails"],
|
| 111 |
+
label="Overlay Layers",
|
| 112 |
+
)
|
| 113 |
+
```
|
| 114 |
+
|
| 115 |
+
### Results panel — new tab
|
| 116 |
+
|
| 117 |
+
Inside the existing `gr.Tabs()` block, add a fourth tab:
|
| 118 |
+
|
| 119 |
+
```python
|
| 120 |
+
with gr.TabItem("🎬 Overlay Video"):
|
| 121 |
+
overlay_video = gr.Video(label="Annotated Movement")
|
| 122 |
+
velocity_md = gr.Markdown("")
|
| 123 |
+
```
|
| 124 |
+
|
| 125 |
+
### `process_video()` signature
|
| 126 |
+
|
| 127 |
+
```python
|
| 128 |
+
def process_video(video_path, test_name, side, model_key, layers: list[str]):
|
| 129 |
+
```
|
| 130 |
+
|
| 131 |
+
After `director.run()`:
|
| 132 |
+
```python
|
| 133 |
+
from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
|
| 134 |
+
layer_set = {l.lower().replace(" ", "_") for l in layers}
|
| 135 |
+
# map UI labels to internal names:
|
| 136 |
+
# "Skeleton" → "skeleton", "Trails" → "trails", "Velocity arrows" → "velocity_arrows"
|
| 137 |
+
overlay_path = None
|
| 138 |
+
vel_summary = ""
|
| 139 |
+
if layer_set and state.ingest and state.pose2d:
|
| 140 |
+
try:
|
| 141 |
+
vis = PoseVisualizer()
|
| 142 |
+
with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
|
| 143 |
+
out_path = f.name
|
| 144 |
+
overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
|
| 145 |
+
if overlay_path:
|
| 146 |
+
vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
|
| 147 |
+
except Exception as e:
|
| 148 |
+
alerts += f"\n⚠️ Visualizer error: {e}"
|
| 149 |
+
return score_html, pipeline_md, score_details, alerts, overlay_path, vel_summary
|
| 150 |
+
```
|
| 151 |
+
|
| 152 |
+
`vis.last_velocities` is stored on the instance after `render_video()` to avoid recomputing.
|
| 153 |
+
|
| 154 |
+
### Event wiring
|
| 155 |
+
|
| 156 |
+
```python
|
| 157 |
+
submit_btn.click(
|
| 158 |
+
fn=_map_inputs,
|
| 159 |
+
inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown, overlay_layers],
|
| 160 |
+
outputs=[score_html, pipeline_md, score_details, alerts_md, overlay_video, velocity_md],
|
| 161 |
+
)
|
| 162 |
+
```
|
| 163 |
+
|
| 164 |
+
`_map_inputs` gains `overlay_layers` as fifth parameter.
|
| 165 |
+
|
| 166 |
+
---
|
| 167 |
+
|
| 168 |
+
## Error handling
|
| 169 |
+
|
| 170 |
+
| Failure | Behaviour |
|
| 171 |
+
|---|---|
|
| 172 |
+
| All frames have no detections | `render_video()` returns `None`, tab empty, no crash |
|
| 173 |
+
| `cv2.VideoWriter` fails | logs warning, returns `None` |
|
| 174 |
+
| Any exception in visualizer | caught in `process_video()`, appended to alerts, `overlay_path = None` |
|
| 175 |
+
| `layers` is empty | returns `None` immediately, no processing |
|
| 176 |
+
|
| 177 |
+
The score is always returned regardless of visualizer outcome.
|
| 178 |
+
|
| 179 |
+
---
|
| 180 |
+
|
| 181 |
+
## Testing: `tests/test_visualizer.py`
|
| 182 |
+
|
| 183 |
+
- Synthetic `IngestResult`: 5 blank 480×640 BGR frames, fps=30
|
| 184 |
+
- Synthetic `Pose2DResult`: 17 keypoints per frame at fixed positions with conf=0.9
|
| 185 |
+
- `test_render_video_creates_file`: assert output `.mp4` exists and size > 0
|
| 186 |
+
- `test_compute_joint_velocity_shape`: assert 17-key dict, each list length == 5
|
| 187 |
+
- `test_empty_layers_returns_none`: assert `render_video(..., layers=set())` returns `None`
|
| 188 |
+
- `test_no_detections_returns_none`: all-empty keypoints → `None`
|
| 189 |
+
- `test_velocity_summary_markdown`: assert output contains `|` (table) and at least one joint name
|
| 190 |
+
|
| 191 |
+
---
|
| 192 |
+
|
| 193 |
+
## Out of scope
|
| 194 |
+
|
| 195 |
+
- Frame-by-frame metrics synced to video playback (Phase 4 / custom Svelte)
|
| 196 |
+
- Multi-person tracking
|
| 197 |
+
- Saving overlay video to Hugging Face Hub (tracing feature, Phase 4)
|
docs/superpowers/specs/2026-06-13-full-fms-session-pdf-design.md
CHANGED
|
@@ -1,154 +1,154 @@
|
|
| 1 |
-
# Full FMS Session + PDF Report — Design
|
| 2 |
-
|
| 3 |
-
**Date:** 2026-06-13
|
| 4 |
-
**Status:** Approved (brainstorming) — pending implementation plan
|
| 5 |
-
**Owner:** FormScout
|
| 6 |
-
**Related:** `formscout/agents/report.py`, `formscout/agents/visualizer.py`, `formscout/agents/biomechanics.py`, `app.py`, `formscout/types.py`
|
| 7 |
-
|
| 8 |
-
## Problem
|
| 9 |
-
|
| 10 |
-
FormScout today scores **one** FMS test per upload. A real Functional Movement Screen is **all 7 tests** producing a single **composite 0–21** with asymmetry flags. `ReportAgent` and `ReportResult.composite` already support a multi-test report, but the UI never accumulates more than one test, and `ReportResult.pdf_path` is a hardcoded `None` stub.
|
| 11 |
-
|
| 12 |
-
This feature turns the one-clip scorer into a **screening session**: each analyzed clip accumulates; a "Finish" action produces the composite report plus a downloadable, brand-consistent **PDF**. Each clip's worst-moment frame is captured as an annotated still embedded in both an on-disk log and the PDF.
|
| 13 |
-
|
| 14 |
-
## Goals
|
| 15 |
-
|
| 16 |
-
- Accumulate multiple analyzed clips into one session, then emit a composite 0–21 report.
|
| 17 |
-
- Generate a clinician/client-facing **PDF** handout (ReportLab) with scores, rationale, asymmetries, key-frame images, and the safety disclaimer.
|
| 18 |
-
- Capture and annotate the **worst-moment frame** per test (the governing/peak frame already computed by `BiomechanicsAgent`).
|
| 19 |
-
- Persist each analysis incrementally to disk (`session.json`, `analysis.md`, key-frame PNGs) until "Finish" is clicked.
|
| 20 |
-
|
| 21 |
-
## Non-goals (YAGNI)
|
| 22 |
-
|
| 23 |
-
- No cross-restart session reload — the session lives in `gr.State` + a temp dir for the browser session.
|
| 24 |
-
- No PDF styling beyond a clean branded layout (no HTML/CSS engine; ReportLab only).
|
| 25 |
-
- No RAG / exemplar-clip citations (separate future spec).
|
| 26 |
-
- No changes to the scoring pipeline, rubric functions, or Director flow.
|
| 27 |
-
|
| 28 |
-
## UX
|
| 29 |
-
|
| 30 |
-
The current one-clip-at-a-time flow is preserved. Two new buttons appear after an analysis completes:
|
| 31 |
-
|
| 32 |
-
- **➕ Analyse new clip** — clears the video/test inputs for the next upload; **keeps** the session.
|
| 33 |
-
- **✅ Finish & generate PDF** — runs the report + PDF over everything accumulated so far.
|
| 34 |
-
|
| 35 |
-
After each analysis, a **"Session so far"** table updates: `test · side · score · status`. Finish renders an on-screen composite scorecard + asymmetry summary and exposes the PDF (and `analysis.md`) via `gr.File` for download.
|
| 36 |
-
|
| 37 |
-
Guard: Finish with zero analyses → warning, no PDF.
|
| 38 |
-
|
| 39 |
-
## Components
|
| 40 |
-
|
| 41 |
-
### 1. Session state + on-disk store
|
| 42 |
-
|
| 43 |
-
A per-session temp directory `<tmpdir>/formscout_sessions/<session_id>/`:
|
| 44 |
-
|
| 45 |
-
- `session.json` — structured list of entries; **source of truth** for the PDF.
|
| 46 |
-
- `analysis.md` — human-readable log, appended after each clip.
|
| 47 |
-
- `keyframes/<test_name>_<side>.png` — annotated worst-frame stills.
|
| 48 |
-
|
| 49 |
-
Session identity lives in a `gr.State`. Each entry carries:
|
| 50 |
-
|
| 51 |
-
- `test_name`, `side`, `score` (judge score, else rubric), `needs_human`
|
| 52 |
-
- `rationale`, `compensation_tags`, `corrective_hint`
|
| 53 |
-
- key measurements (selected `angles` / `alignments`)
|
| 54 |
-
- `confidence`, `view` (`"2d"`/`"3d"`)
|
| 55 |
-
- `keyframe_path`
|
| 56 |
-
- the `movement` / `features` / `rubric_score` / `judge` objects that `ReportAgent.run()` consumes
|
| 57 |
-
|
| 58 |
-
Persistence lasts until Finish; files are kept afterward for download. Cross-restart cleanup is best-effort and out of scope.
|
| 59 |
-
|
| 60 |
-
### 2. Key-frame capture
|
| 61 |
-
|
| 62 |
-
New method on `PoseVisualizer`:
|
| 63 |
-
|
| 64 |
-
```python
|
| 65 |
-
def render_frame(self, ingest, pose2d, frame_idx: int,
|
| 66 |
-
layers: set[str], caption: str, out_png: str) -> str | None
|
| 67 |
-
```
|
| 68 |
-
|
| 69 |
-
- `frame_idx` comes from `features.timing`, which already stores the governing frame per test:
|
| 70 |
-
`deep_squat → deepest_frame`, `hurdle_step → peak_step_frame`,
|
| 71 |
-
`inline_lunge → deepest_lunge_frame`, `shoulder_mobility → measure_frame`,
|
| 72 |
-
`active_slr → peak_raise_frame`.
|
| 73 |
-
- `trunk_stability_pushup` and `rotary_stability` currently store only counts in `timing`. Add the worst-sag-frame and peak-extension-frame index to their `timing` dicts (one-line change in each `BiomechanicsAgent` method).
|
| 74 |
-
- Reuses `_draw_skeleton` (+ optional `_draw_trails`) on the single frame, overlays a caption naming the worst compensation, writes a PNG.
|
| 75 |
-
- Returns `None` on any failure — never raises, never blocks the entry.
|
| 76 |
-
|
| 77 |
-
The "worst compensation" caption is derived from `judge.compensation_tags` (preferred) or the failed `alignments` (fallback).
|
| 78 |
-
|
| 79 |
-
### 3. PDF generator
|
| 80 |
-
|
| 81 |
-
New module `formscout/agents/pdf_report.py`:
|
| 82 |
-
|
| 83 |
-
```python
|
| 84 |
-
class PdfReportAgent:
|
| 85 |
-
def run(self, report_result: ReportResult,
|
| 86 |
-
entries: list[SessionEntry], session_dir: str) -> str | None
|
| 87 |
-
```
|
| 88 |
-
|
| 89 |
-
Uses **ReportLab** (pure-Python, no system deps — safe on HF Spaces/ZeroGPU). Layout:
|
| 90 |
-
|
| 91 |
-
- Safety disclaimer banner at **top and bottom** (mirrors the UI invariant).
|
| 92 |
-
- Title/brand header + date.
|
| 93 |
-
- Composite **0–21** badge, or "Incomplete — N/7 tests scored" when `composite is None`.
|
| 94 |
-
- Per-test block: score, rationale, key measurements, compensation tags, corrective hint, the annotated key-frame image, asymmetry delta (bilateral).
|
| 95 |
-
- Flags section: low-confidence, rubric↔judge disagreement, needs-human.
|
| 96 |
-
- Populates `ReportResult.pdf_path`.
|
| 97 |
-
|
| 98 |
-
Returns the PDF path, or `None` on failure (UI surfaces the error and keeps the session for retry). Image embedding tolerates a missing/`None` `keyframe_path` with a placeholder line.
|
| 99 |
-
|
| 100 |
-
### 4. ReportAgent reuse
|
| 101 |
-
|
| 102 |
-
At Finish, build the entry list and call the existing `ReportAgent.run()` for composite + asymmetries + flags. The bilateral lower-score + asymmetry-delta logic and the null-composite rule already exist and are not rewritten. A small adapter converts `SessionEntry` objects to the dict schema `ReportAgent.run()` expects (or `ReportAgent` gains overload tolerance — implementer's choice, keep it minimal).
|
| 103 |
-
|
| 104 |
-
### 5. Types
|
| 105 |
-
|
| 106 |
-
Add a `SessionEntry` frozen dataclass to `formscout/types.py` (consistent with the "every agent I/O is a typed dataclass" standard), including `keyframe_path: str | None`. Populate the existing `ReportResult.pdf_path` (and optionally `overlay_video_path`). No other type changes.
|
| 107 |
-
|
| 108 |
-
### 6. UI (`app.py`)
|
| 109 |
-
|
| 110 |
-
- Add a `gr.State` holding the session (id + entries).
|
| 111 |
-
- After each analysis: render the scorecard as today, append the entry, write `session.json`/`analysis.md`/keyframe PNG, refresh the "Session so far" table, and reveal the two buttons.
|
| 112 |
-
- **Analyse new clip**: reset the video/test/side inputs; keep session state.
|
| 113 |
-
- **Finish & generate PDF**: `ReportAgent.run` → `PdfReportAgent.run` → display composite + asymmetry summary + `gr.File` downloads (PDF + `analysis.md`).
|
| 114 |
-
- Guard: Finish with zero analyses → warning.
|
| 115 |
-
|
| 116 |
-
## Data flow
|
| 117 |
-
|
| 118 |
-
```
|
| 119 |
-
upload → Director.run → score
|
| 120 |
-
→ build SessionEntry (+ render_frame keyframe png)
|
| 121 |
-
→ append to gr.State + write session.json / analysis.md / keyframe png
|
| 122 |
-
→ refresh "Session so far" table
|
| 123 |
-
|
| 124 |
-
Finish → ReportAgent.run(entries) → composite / asymmetries / flags
|
| 125 |
-
→ PdfReportAgent.run(...) → pdf_path
|
| 126 |
-
→ on-screen composite + gr.File (PDF, analysis.md)
|
| 127 |
-
```
|
| 128 |
-
|
| 129 |
-
## Error handling
|
| 130 |
-
|
| 131 |
-
- Key-frame render fails → entry still saved; PDF shows an image placeholder.
|
| 132 |
-
- PDF generation fails → surface the error, keep the session intact for retry.
|
| 133 |
-
- `needs_human` entry → no numeric score; PDF shows "Clinician review required"; composite null.
|
| 134 |
-
- Composite is `None` whenever any test is unscored or needs human review (existing rule — never show a partial 0–21 as complete).
|
| 135 |
-
- All disk writes tolerate failure with a logged warning; a write failure degrades the artifact but never blocks scoring.
|
| 136 |
-
|
| 137 |
-
## Testing (must run without model downloads)
|
| 138 |
-
|
| 139 |
-
- `tests/test_pdf_report.py` — synthetic `ReportResult` + entries → PDF file created, non-zero size, contains the disclaimer text and composite line.
|
| 140 |
-
- `tests/test_session.py` — accumulation; composite math; bilateral lower-score + asymmetry delta; null composite when one entry `needs_human`.
|
| 141 |
-
- `tests/test_keyframe.py` — `render_frame` returns a real PNG path (file exists) for a synthetic frame; returns `None` gracefully on bad input.
|
| 142 |
-
|
| 143 |
-
## Invariants preserved
|
| 144 |
-
|
| 145 |
-
- Pipeline stays headless — no Gradio imports in agent files (`PdfReportAgent` is a pure agent; key-frame capture stays in `visualizer.py`, the existing UI-layer component).
|
| 146 |
-
- Safety disclaimer present top and bottom of the PDF, mirroring the UI.
|
| 147 |
-
- Pain / clearing / needs-human is never auto-scored; composite null when any test unscored.
|
| 148 |
-
- New code follows the engineering standards: one public entrypoint per agent, typed dataclass I/O, `confidence`/`notes` where applicable, module docstring stating purpose/inputs/outputs/failure/params/license/gated.
|
| 149 |
-
|
| 150 |
-
## Open implementation choices (left to the plan)
|
| 151 |
-
|
| 152 |
-
- Exact `SessionEntry` → `ReportAgent` dict adapter shape.
|
| 153 |
-
- Which measurements to surface per test in the PDF (a curated subset, not the full `angles` dump).
|
| 154 |
-
- PDF assertion strategy in tests (text extraction vs. size/smoke).
|
|
|
|
| 1 |
+
# Full FMS Session + PDF Report — Design
|
| 2 |
+
|
| 3 |
+
**Date:** 2026-06-13
|
| 4 |
+
**Status:** Approved (brainstorming) — pending implementation plan
|
| 5 |
+
**Owner:** FormScout
|
| 6 |
+
**Related:** `formscout/agents/report.py`, `formscout/agents/visualizer.py`, `formscout/agents/biomechanics.py`, `app.py`, `formscout/types.py`
|
| 7 |
+
|
| 8 |
+
## Problem
|
| 9 |
+
|
| 10 |
+
FormScout today scores **one** FMS test per upload. A real Functional Movement Screen is **all 7 tests** producing a single **composite 0–21** with asymmetry flags. `ReportAgent` and `ReportResult.composite` already support a multi-test report, but the UI never accumulates more than one test, and `ReportResult.pdf_path` is a hardcoded `None` stub.
|
| 11 |
+
|
| 12 |
+
This feature turns the one-clip scorer into a **screening session**: each analyzed clip accumulates; a "Finish" action produces the composite report plus a downloadable, brand-consistent **PDF**. Each clip's worst-moment frame is captured as an annotated still embedded in both an on-disk log and the PDF.
|
| 13 |
+
|
| 14 |
+
## Goals
|
| 15 |
+
|
| 16 |
+
- Accumulate multiple analyzed clips into one session, then emit a composite 0–21 report.
|
| 17 |
+
- Generate a clinician/client-facing **PDF** handout (ReportLab) with scores, rationale, asymmetries, key-frame images, and the safety disclaimer.
|
| 18 |
+
- Capture and annotate the **worst-moment frame** per test (the governing/peak frame already computed by `BiomechanicsAgent`).
|
| 19 |
+
- Persist each analysis incrementally to disk (`session.json`, `analysis.md`, key-frame PNGs) until "Finish" is clicked.
|
| 20 |
+
|
| 21 |
+
## Non-goals (YAGNI)
|
| 22 |
+
|
| 23 |
+
- No cross-restart session reload — the session lives in `gr.State` + a temp dir for the browser session.
|
| 24 |
+
- No PDF styling beyond a clean branded layout (no HTML/CSS engine; ReportLab only).
|
| 25 |
+
- No RAG / exemplar-clip citations (separate future spec).
|
| 26 |
+
- No changes to the scoring pipeline, rubric functions, or Director flow.
|
| 27 |
+
|
| 28 |
+
## UX
|
| 29 |
+
|
| 30 |
+
The current one-clip-at-a-time flow is preserved. Two new buttons appear after an analysis completes:
|
| 31 |
+
|
| 32 |
+
- **➕ Analyse new clip** — clears the video/test inputs for the next upload; **keeps** the session.
|
| 33 |
+
- **✅ Finish & generate PDF** — runs the report + PDF over everything accumulated so far.
|
| 34 |
+
|
| 35 |
+
After each analysis, a **"Session so far"** table updates: `test · side · score · status`. Finish renders an on-screen composite scorecard + asymmetry summary and exposes the PDF (and `analysis.md`) via `gr.File` for download.
|
| 36 |
+
|
| 37 |
+
Guard: Finish with zero analyses → warning, no PDF.
|
| 38 |
+
|
| 39 |
+
## Components
|
| 40 |
+
|
| 41 |
+
### 1. Session state + on-disk store
|
| 42 |
+
|
| 43 |
+
A per-session temp directory `<tmpdir>/formscout_sessions/<session_id>/`:
|
| 44 |
+
|
| 45 |
+
- `session.json` — structured list of entries; **source of truth** for the PDF.
|
| 46 |
+
- `analysis.md` — human-readable log, appended after each clip.
|
| 47 |
+
- `keyframes/<test_name>_<side>.png` — annotated worst-frame stills.
|
| 48 |
+
|
| 49 |
+
Session identity lives in a `gr.State`. Each entry carries:
|
| 50 |
+
|
| 51 |
+
- `test_name`, `side`, `score` (judge score, else rubric), `needs_human`
|
| 52 |
+
- `rationale`, `compensation_tags`, `corrective_hint`
|
| 53 |
+
- key measurements (selected `angles` / `alignments`)
|
| 54 |
+
- `confidence`, `view` (`"2d"`/`"3d"`)
|
| 55 |
+
- `keyframe_path`
|
| 56 |
+
- the `movement` / `features` / `rubric_score` / `judge` objects that `ReportAgent.run()` consumes
|
| 57 |
+
|
| 58 |
+
Persistence lasts until Finish; files are kept afterward for download. Cross-restart cleanup is best-effort and out of scope.
|
| 59 |
+
|
| 60 |
+
### 2. Key-frame capture
|
| 61 |
+
|
| 62 |
+
New method on `PoseVisualizer`:
|
| 63 |
+
|
| 64 |
+
```python
|
| 65 |
+
def render_frame(self, ingest, pose2d, frame_idx: int,
|
| 66 |
+
layers: set[str], caption: str, out_png: str) -> str | None
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
- `frame_idx` comes from `features.timing`, which already stores the governing frame per test:
|
| 70 |
+
`deep_squat → deepest_frame`, `hurdle_step → peak_step_frame`,
|
| 71 |
+
`inline_lunge → deepest_lunge_frame`, `shoulder_mobility → measure_frame`,
|
| 72 |
+
`active_slr → peak_raise_frame`.
|
| 73 |
+
- `trunk_stability_pushup` and `rotary_stability` currently store only counts in `timing`. Add the worst-sag-frame and peak-extension-frame index to their `timing` dicts (one-line change in each `BiomechanicsAgent` method).
|
| 74 |
+
- Reuses `_draw_skeleton` (+ optional `_draw_trails`) on the single frame, overlays a caption naming the worst compensation, writes a PNG.
|
| 75 |
+
- Returns `None` on any failure — never raises, never blocks the entry.
|
| 76 |
+
|
| 77 |
+
The "worst compensation" caption is derived from `judge.compensation_tags` (preferred) or the failed `alignments` (fallback).
|
| 78 |
+
|
| 79 |
+
### 3. PDF generator
|
| 80 |
+
|
| 81 |
+
New module `formscout/agents/pdf_report.py`:
|
| 82 |
+
|
| 83 |
+
```python
|
| 84 |
+
class PdfReportAgent:
|
| 85 |
+
def run(self, report_result: ReportResult,
|
| 86 |
+
entries: list[SessionEntry], session_dir: str) -> str | None
|
| 87 |
+
```
|
| 88 |
+
|
| 89 |
+
Uses **ReportLab** (pure-Python, no system deps — safe on HF Spaces/ZeroGPU). Layout:
|
| 90 |
+
|
| 91 |
+
- Safety disclaimer banner at **top and bottom** (mirrors the UI invariant).
|
| 92 |
+
- Title/brand header + date.
|
| 93 |
+
- Composite **0–21** badge, or "Incomplete — N/7 tests scored" when `composite is None`.
|
| 94 |
+
- Per-test block: score, rationale, key measurements, compensation tags, corrective hint, the annotated key-frame image, asymmetry delta (bilateral).
|
| 95 |
+
- Flags section: low-confidence, rubric↔judge disagreement, needs-human.
|
| 96 |
+
- Populates `ReportResult.pdf_path`.
|
| 97 |
+
|
| 98 |
+
Returns the PDF path, or `None` on failure (UI surfaces the error and keeps the session for retry). Image embedding tolerates a missing/`None` `keyframe_path` with a placeholder line.
|
| 99 |
+
|
| 100 |
+
### 4. ReportAgent reuse
|
| 101 |
+
|
| 102 |
+
At Finish, build the entry list and call the existing `ReportAgent.run()` for composite + asymmetries + flags. The bilateral lower-score + asymmetry-delta logic and the null-composite rule already exist and are not rewritten. A small adapter converts `SessionEntry` objects to the dict schema `ReportAgent.run()` expects (or `ReportAgent` gains overload tolerance — implementer's choice, keep it minimal).
|
| 103 |
+
|
| 104 |
+
### 5. Types
|
| 105 |
+
|
| 106 |
+
Add a `SessionEntry` frozen dataclass to `formscout/types.py` (consistent with the "every agent I/O is a typed dataclass" standard), including `keyframe_path: str | None`. Populate the existing `ReportResult.pdf_path` (and optionally `overlay_video_path`). No other type changes.
|
| 107 |
+
|
| 108 |
+
### 6. UI (`app.py`)
|
| 109 |
+
|
| 110 |
+
- Add a `gr.State` holding the session (id + entries).
|
| 111 |
+
- After each analysis: render the scorecard as today, append the entry, write `session.json`/`analysis.md`/keyframe PNG, refresh the "Session so far" table, and reveal the two buttons.
|
| 112 |
+
- **Analyse new clip**: reset the video/test/side inputs; keep session state.
|
| 113 |
+
- **Finish & generate PDF**: `ReportAgent.run` → `PdfReportAgent.run` → display composite + asymmetry summary + `gr.File` downloads (PDF + `analysis.md`).
|
| 114 |
+
- Guard: Finish with zero analyses → warning.
|
| 115 |
+
|
| 116 |
+
## Data flow
|
| 117 |
+
|
| 118 |
+
```
|
| 119 |
+
upload → Director.run → score
|
| 120 |
+
→ build SessionEntry (+ render_frame keyframe png)
|
| 121 |
+
→ append to gr.State + write session.json / analysis.md / keyframe png
|
| 122 |
+
→ refresh "Session so far" table
|
| 123 |
+
|
| 124 |
+
Finish → ReportAgent.run(entries) → composite / asymmetries / flags
|
| 125 |
+
→ PdfReportAgent.run(...) → pdf_path
|
| 126 |
+
→ on-screen composite + gr.File (PDF, analysis.md)
|
| 127 |
+
```
|
| 128 |
+
|
| 129 |
+
## Error handling
|
| 130 |
+
|
| 131 |
+
- Key-frame render fails → entry still saved; PDF shows an image placeholder.
|
| 132 |
+
- PDF generation fails → surface the error, keep the session intact for retry.
|
| 133 |
+
- `needs_human` entry → no numeric score; PDF shows "Clinician review required"; composite null.
|
| 134 |
+
- Composite is `None` whenever any test is unscored or needs human review (existing rule — never show a partial 0–21 as complete).
|
| 135 |
+
- All disk writes tolerate failure with a logged warning; a write failure degrades the artifact but never blocks scoring.
|
| 136 |
+
|
| 137 |
+
## Testing (must run without model downloads)
|
| 138 |
+
|
| 139 |
+
- `tests/test_pdf_report.py` — synthetic `ReportResult` + entries → PDF file created, non-zero size, contains the disclaimer text and composite line.
|
| 140 |
+
- `tests/test_session.py` — accumulation; composite math; bilateral lower-score + asymmetry delta; null composite when one entry `needs_human`.
|
| 141 |
+
- `tests/test_keyframe.py` — `render_frame` returns a real PNG path (file exists) for a synthetic frame; returns `None` gracefully on bad input.
|
| 142 |
+
|
| 143 |
+
## Invariants preserved
|
| 144 |
+
|
| 145 |
+
- Pipeline stays headless — no Gradio imports in agent files (`PdfReportAgent` is a pure agent; key-frame capture stays in `visualizer.py`, the existing UI-layer component).
|
| 146 |
+
- Safety disclaimer present top and bottom of the PDF, mirroring the UI.
|
| 147 |
+
- Pain / clearing / needs-human is never auto-scored; composite null when any test unscored.
|
| 148 |
+
- New code follows the engineering standards: one public entrypoint per agent, typed dataclass I/O, `confidence`/`notes` where applicable, module docstring stating purpose/inputs/outputs/failure/params/license/gated.
|
| 149 |
+
|
| 150 |
+
## Open implementation choices (left to the plan)
|
| 151 |
+
|
| 152 |
+
- Exact `SessionEntry` → `ReportAgent` dict adapter shape.
|
| 153 |
+
- Which measurements to surface per test in the PDF (a curated subset, not the full `angles` dump).
|
| 154 |
+
- PDF assertion strategy in tests (text extraction vs. size/smoke).
|
formscout/agents/classifier.py
CHANGED
|
@@ -1,102 +1,102 @@
|
|
| 1 |
-
"""
|
| 2 |
-
MovementClassifierAgent — identifies which FMS test is in the clip.
|
| 3 |
-
|
| 4 |
-
Input: IngestResult (keyframes), Pose2DResult (skeleton context)
|
| 5 |
-
Output: MovementResult(test_name, side, confidence)
|
| 6 |
-
Failure: returns MovementResult(test_name="unknown") — pipeline stops and asks for manual override.
|
| 7 |
-
Model: Qwen3-VL-8B-Instruct via llama.cpp (8B params, Apache-2.0).
|
| 8 |
-
Gated: No.
|
| 9 |
-
"""
|
| 10 |
-
from __future__ import annotations
|
| 11 |
-
|
| 12 |
-
import logging
|
| 13 |
-
from pathlib import Path
|
| 14 |
-
|
| 15 |
-
from formscout import config
|
| 16 |
-
from formscout.types import IngestResult, Pose2DResult, MovementResult
|
| 17 |
-
from formscout.serving.llama_cpp import LlamaCppClient
|
| 18 |
-
|
| 19 |
-
logger = logging.getLogger(__name__)
|
| 20 |
-
|
| 21 |
-
_PROMPT_PATH = Path(__file__).parent / "prompts" / "c1_classifier.md"
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
class MovementClassifierAgent:
|
| 25 |
-
"""Classifies which FMS test is being performed via VLM or manual override."""
|
| 26 |
-
|
| 27 |
-
def __init__(self):
|
| 28 |
-
self._client = LlamaCppClient(port=config.LLAMA_CPP_PORT_VLM)
|
| 29 |
-
self._system_prompt = _PROMPT_PATH.read_text(encoding="utf-8")
|
| 30 |
-
|
| 31 |
-
def run(
|
| 32 |
-
self,
|
| 33 |
-
ingest: IngestResult,
|
| 34 |
-
pose2d: Pose2DResult | None = None,
|
| 35 |
-
manual_override: str | None = None,
|
| 36 |
-
) -> MovementResult:
|
| 37 |
-
"""
|
| 38 |
-
Classify the movement. If manual_override is provided, use it directly.
|
| 39 |
-
Otherwise, use VLM inference on keyframes.
|
| 40 |
-
"""
|
| 41 |
-
if manual_override and manual_override != "unknown":
|
| 42 |
-
return MovementResult(
|
| 43 |
-
test_name=manual_override, side="na",
|
| 44 |
-
confidence=1.0, notes="manual override",
|
| 45 |
-
)
|
| 46 |
-
|
| 47 |
-
if not self._client.available:
|
| 48 |
-
return MovementResult(
|
| 49 |
-
test_name="unknown", side="na", confidence=0.0,
|
| 50 |
-
notes="VLM server unavailable — use manual override",
|
| 51 |
-
)
|
| 52 |
-
|
| 53 |
-
# Select keyframes for classification (3 evenly spaced)
|
| 54 |
-
n = len(ingest.frames)
|
| 55 |
-
indices = [0, n // 2, n - 1] if n >= 3 else list(range(n))
|
| 56 |
-
images = self._encode_frames(ingest.frames, indices)
|
| 57 |
-
|
| 58 |
-
prompt = f"{self._system_prompt}\n\nClassify this movement from the keyframes shown."
|
| 59 |
-
result = self._client.complete(prompt, images=images, max_tokens=256, temperature=0.1)
|
| 60 |
-
|
| 61 |
-
return self._parse_response(result)
|
| 62 |
-
|
| 63 |
-
def _encode_frames(self, frames: list, indices: list[int]) -> list[str]:
|
| 64 |
-
"""Encode selected frames as base64 JPEG for the VLM."""
|
| 65 |
-
import cv2
|
| 66 |
-
import base64
|
| 67 |
-
|
| 68 |
-
encoded = []
|
| 69 |
-
for idx in indices:
|
| 70 |
-
if idx < len(frames):
|
| 71 |
-
_, buf = cv2.imencode(".jpg", frames[idx], [cv2.IMWRITE_JPEG_QUALITY, 80])
|
| 72 |
-
encoded.append(base64.b64encode(buf.tobytes()).decode())
|
| 73 |
-
return encoded
|
| 74 |
-
|
| 75 |
-
def _parse_response(self, result: dict) -> MovementResult:
|
| 76 |
-
"""Parse VLM JSON response into MovementResult."""
|
| 77 |
-
if "error" in result:
|
| 78 |
-
return MovementResult(
|
| 79 |
-
test_name="unknown", side="na", confidence=0.0,
|
| 80 |
-
notes=f"VLM error: {result['error']}",
|
| 81 |
-
)
|
| 82 |
-
|
| 83 |
-
test = result.get("test", "unknown")
|
| 84 |
-
side = result.get("side", "na")
|
| 85 |
-
confidence = float(result.get("confidence", 0.0))
|
| 86 |
-
reason = result.get("reason", "")
|
| 87 |
-
|
| 88 |
-
valid_tests = {
|
| 89 |
-
"deep_squat", "hurdle_step", "inline_lunge",
|
| 90 |
-
"shoulder_mobility", "active_slr",
|
| 91 |
-
"trunk_stability_pushup", "rotary_stability", "unknown",
|
| 92 |
-
}
|
| 93 |
-
if test not in valid_tests:
|
| 94 |
-
test = "unknown"
|
| 95 |
-
|
| 96 |
-
if side not in ("left", "right", "na"):
|
| 97 |
-
side = "na"
|
| 98 |
-
|
| 99 |
-
return MovementResult(
|
| 100 |
-
test_name=test, side=side,
|
| 101 |
-
confidence=confidence, notes=reason,
|
| 102 |
-
)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
MovementClassifierAgent — identifies which FMS test is in the clip.
|
| 3 |
+
|
| 4 |
+
Input: IngestResult (keyframes), Pose2DResult (skeleton context)
|
| 5 |
+
Output: MovementResult(test_name, side, confidence)
|
| 6 |
+
Failure: returns MovementResult(test_name="unknown") — pipeline stops and asks for manual override.
|
| 7 |
+
Model: Qwen3-VL-8B-Instruct via llama.cpp (8B params, Apache-2.0).
|
| 8 |
+
Gated: No.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import logging
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
from formscout import config
|
| 16 |
+
from formscout.types import IngestResult, Pose2DResult, MovementResult
|
| 17 |
+
from formscout.serving.llama_cpp import LlamaCppClient
|
| 18 |
+
|
| 19 |
+
logger = logging.getLogger(__name__)
|
| 20 |
+
|
| 21 |
+
_PROMPT_PATH = Path(__file__).parent / "prompts" / "c1_classifier.md"
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class MovementClassifierAgent:
|
| 25 |
+
"""Classifies which FMS test is being performed via VLM or manual override."""
|
| 26 |
+
|
| 27 |
+
def __init__(self):
|
| 28 |
+
self._client = LlamaCppClient(port=config.LLAMA_CPP_PORT_VLM)
|
| 29 |
+
self._system_prompt = _PROMPT_PATH.read_text(encoding="utf-8")
|
| 30 |
+
|
| 31 |
+
def run(
|
| 32 |
+
self,
|
| 33 |
+
ingest: IngestResult,
|
| 34 |
+
pose2d: Pose2DResult | None = None,
|
| 35 |
+
manual_override: str | None = None,
|
| 36 |
+
) -> MovementResult:
|
| 37 |
+
"""
|
| 38 |
+
Classify the movement. If manual_override is provided, use it directly.
|
| 39 |
+
Otherwise, use VLM inference on keyframes.
|
| 40 |
+
"""
|
| 41 |
+
if manual_override and manual_override != "unknown":
|
| 42 |
+
return MovementResult(
|
| 43 |
+
test_name=manual_override, side="na",
|
| 44 |
+
confidence=1.0, notes="manual override",
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
if not self._client.available:
|
| 48 |
+
return MovementResult(
|
| 49 |
+
test_name="unknown", side="na", confidence=0.0,
|
| 50 |
+
notes="VLM server unavailable — use manual override",
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
# Select keyframes for classification (3 evenly spaced)
|
| 54 |
+
n = len(ingest.frames)
|
| 55 |
+
indices = [0, n // 2, n - 1] if n >= 3 else list(range(n))
|
| 56 |
+
images = self._encode_frames(ingest.frames, indices)
|
| 57 |
+
|
| 58 |
+
prompt = f"{self._system_prompt}\n\nClassify this movement from the keyframes shown."
|
| 59 |
+
result = self._client.complete(prompt, images=images, max_tokens=256, temperature=0.1)
|
| 60 |
+
|
| 61 |
+
return self._parse_response(result)
|
| 62 |
+
|
| 63 |
+
def _encode_frames(self, frames: list, indices: list[int]) -> list[str]:
|
| 64 |
+
"""Encode selected frames as base64 JPEG for the VLM."""
|
| 65 |
+
import cv2
|
| 66 |
+
import base64
|
| 67 |
+
|
| 68 |
+
encoded = []
|
| 69 |
+
for idx in indices:
|
| 70 |
+
if idx < len(frames):
|
| 71 |
+
_, buf = cv2.imencode(".jpg", frames[idx], [cv2.IMWRITE_JPEG_QUALITY, 80])
|
| 72 |
+
encoded.append(base64.b64encode(buf.tobytes()).decode())
|
| 73 |
+
return encoded
|
| 74 |
+
|
| 75 |
+
def _parse_response(self, result: dict) -> MovementResult:
|
| 76 |
+
"""Parse VLM JSON response into MovementResult."""
|
| 77 |
+
if "error" in result:
|
| 78 |
+
return MovementResult(
|
| 79 |
+
test_name="unknown", side="na", confidence=0.0,
|
| 80 |
+
notes=f"VLM error: {result['error']}",
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
test = result.get("test", "unknown")
|
| 84 |
+
side = result.get("side", "na")
|
| 85 |
+
confidence = float(result.get("confidence", 0.0))
|
| 86 |
+
reason = result.get("reason", "")
|
| 87 |
+
|
| 88 |
+
valid_tests = {
|
| 89 |
+
"deep_squat", "hurdle_step", "inline_lunge",
|
| 90 |
+
"shoulder_mobility", "active_slr",
|
| 91 |
+
"trunk_stability_pushup", "rotary_stability", "unknown",
|
| 92 |
+
}
|
| 93 |
+
if test not in valid_tests:
|
| 94 |
+
test = "unknown"
|
| 95 |
+
|
| 96 |
+
if side not in ("left", "right", "na"):
|
| 97 |
+
side = "na"
|
| 98 |
+
|
| 99 |
+
return MovementResult(
|
| 100 |
+
test_name=test, side=side,
|
| 101 |
+
confidence=confidence, notes=reason,
|
| 102 |
+
)
|
formscout/agents/ingest.py
CHANGED
|
@@ -49,32 +49,22 @@ class IngestAgent:
|
|
| 49 |
total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
| 50 |
w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
| 51 |
h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
|
|
|
| 52 |
|
| 53 |
notes_parts: list[str] = []
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
-
#
|
| 56 |
-
|
| 57 |
-
# every frame up to the cap, subsampling only when the count is trusted.
|
| 58 |
-
count_reliable = total > 0
|
| 59 |
-
if count_reliable:
|
| 60 |
-
step = max(1, total // config.MAX_FRAMES)
|
| 61 |
-
else:
|
| 62 |
-
step = 1 # unknown length — keep frames, cap at MAX_FRAMES
|
| 63 |
-
|
| 64 |
-
# Read frames; tolerate a truncated/premature end gracefully.
|
| 65 |
frames: list = []
|
| 66 |
idx = 0
|
| 67 |
-
read_failures = 0
|
| 68 |
while True:
|
| 69 |
ret, frame = cap.read()
|
| 70 |
if not ret:
|
| 71 |
-
# A truncated final chunk yields ret=False; stop cleanly.
|
| 72 |
break
|
| 73 |
-
if frame is None:
|
| 74 |
-
read_failures += 1
|
| 75 |
-
if read_failures > 5:
|
| 76 |
-
break
|
| 77 |
-
continue
|
| 78 |
if idx % step == 0:
|
| 79 |
frames.append(frame)
|
| 80 |
idx += 1
|
|
@@ -82,17 +72,6 @@ class IngestAgent:
|
|
| 82 |
break
|
| 83 |
cap.release()
|
| 84 |
|
| 85 |
-
# Compute duration from what we actually decoded, not the (possibly
|
| 86 |
-
# bogus) header frame count.
|
| 87 |
-
decoded_total = idx if idx > 0 else len(frames)
|
| 88 |
-
duration = decoded_total / fps if fps > 0 else 0.0
|
| 89 |
-
if duration > config.MAX_DURATION_SEC:
|
| 90 |
-
notes_parts.append(
|
| 91 |
-
f"video is {duration:.1f}s (>{config.MAX_DURATION_SEC}s) — capping frames"
|
| 92 |
-
)
|
| 93 |
-
if not count_reliable and frames:
|
| 94 |
-
notes_parts.append("frame count unreliable (webm/mkv) — read sequentially")
|
| 95 |
-
|
| 96 |
if not frames:
|
| 97 |
return IngestResult(
|
| 98 |
frames=[], fps=fps, duration=duration, n_people=0,
|
|
|
|
| 49 |
total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
| 50 |
w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
| 51 |
h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
| 52 |
+
duration = total / fps if fps > 0 else 0.0
|
| 53 |
|
| 54 |
notes_parts: list[str] = []
|
| 55 |
+
if duration > config.MAX_DURATION_SEC:
|
| 56 |
+
notes_parts.append(
|
| 57 |
+
f"video is {duration:.1f}s (>{config.MAX_DURATION_SEC}s) — capping frames"
|
| 58 |
+
)
|
| 59 |
|
| 60 |
+
# Sample frames evenly, capped at MAX_FRAMES
|
| 61 |
+
step = max(1, total // config.MAX_FRAMES)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
frames: list = []
|
| 63 |
idx = 0
|
|
|
|
| 64 |
while True:
|
| 65 |
ret, frame = cap.read()
|
| 66 |
if not ret:
|
|
|
|
| 67 |
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
if idx % step == 0:
|
| 69 |
frames.append(frame)
|
| 70 |
idx += 1
|
|
|
|
| 72 |
break
|
| 73 |
cap.release()
|
| 74 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
if not frames:
|
| 76 |
return IngestResult(
|
| 77 |
frames=[], fps=fps, duration=duration, n_people=0,
|
formscout/agents/judge.py
CHANGED
|
@@ -1,136 +1,125 @@
|
|
| 1 |
-
"""
|
| 2 |
-
JudgeAgent — VLM-based final scorer with rationale, compensation tags, pain detection.
|
| 3 |
-
|
| 4 |
-
Input: BiomechFeatures, ScoreResult (rubric candidate), MovementResult, keyframes
|
| 5 |
-
Output: JudgeResult(score, rationale, compensation_tags, corrective_hint, needs_human)
|
| 6 |
-
Failure: returns JudgeResult(needs_human=True, score=None) when uncertain.
|
| 7 |
-
Model: Qwen3-VL-8B-Instruct via llama.cpp (8B params, Apache-2.0).
|
| 8 |
-
Gated: No.
|
| 9 |
-
|
| 10 |
-
Safety: NEVER auto-scores pain. If any indication of pain/clearing test,
|
| 11 |
-
sets needs_human=True and score=None.
|
| 12 |
-
"""
|
| 13 |
-
from __future__ import annotations
|
| 14 |
-
|
| 15 |
-
import json
|
| 16 |
-
import logging
|
| 17 |
-
from pathlib import Path
|
| 18 |
-
|
| 19 |
-
from formscout import config
|
| 20 |
-
from formscout.types import (
|
| 21 |
-
BiomechFeatures, ScoreResult, MovementResult,
|
| 22 |
-
IngestResult, JudgeResult,
|
| 23 |
-
)
|
| 24 |
-
from formscout.serving
|
| 25 |
-
|
| 26 |
-
logger = logging.getLogger(__name__)
|
| 27 |
-
|
| 28 |
-
_PROMPT_PATH = Path(__file__).parent / "prompts" / "c2_judge.md"
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
class JudgeAgent:
|
| 32 |
-
"""VLM judge that produces the final FMS score with rationale."""
|
| 33 |
-
|
| 34 |
-
def __init__(self):
|
| 35 |
-
self._client =
|
| 36 |
-
self._system_prompt = _PROMPT_PATH.read_text(encoding="utf-8")
|
| 37 |
-
|
| 38 |
-
def run(
|
| 39 |
-
self,
|
| 40 |
-
features: BiomechFeatures,
|
| 41 |
-
rubric_score: ScoreResult,
|
| 42 |
-
movement: MovementResult,
|
| 43 |
-
ingest: IngestResult | None = None,
|
| 44 |
-
) -> JudgeResult:
|
| 45 |
-
"""
|
| 46 |
-
Produce final score. Falls back to rubric score if VLM unavailable.
|
| 47 |
-
"""
|
| 48 |
-
if not config.ENABLE_JUDGE:
|
| 49 |
-
return self._fallback_from_rubric(rubric_score, features)
|
| 50 |
-
|
| 51 |
-
if not self._client.available:
|
| 52 |
-
logger.warning("JudgeAgent: VLM unavailable, using rubric score as final")
|
| 53 |
-
return self._fallback_from_rubric(rubric_score, features)
|
| 54 |
-
|
| 55 |
-
# Build context for the judge
|
| 56 |
-
context = {
|
| 57 |
-
"test": features.test_name,
|
| 58 |
-
"side": features.side,
|
| 59 |
-
"view": features.view,
|
| 60 |
-
"features": {"angles": features.angles, "alignments": features.alignments},
|
| 61 |
-
"candidate_score": rubric_score.score,
|
| 62 |
-
"candidate_confidence": rubric_score.confidence,
|
| 63 |
-
"exemplars": [], # Phase 3: populated by RetrievalAgent
|
| 64 |
-
}
|
| 65 |
-
|
| 66 |
-
prompt = f"{self._system_prompt}\n\n{json.dumps(context, indent=2)}"
|
| 67 |
-
|
| 68 |
-
# Optionally include keyframes
|
| 69 |
-
images = None
|
| 70 |
-
if ingest and ingest.frames:
|
| 71 |
-
images = self._encode_keyframes(ingest.frames)
|
| 72 |
-
|
| 73 |
-
result = self._client.complete(prompt, images=images, max_tokens=512, temperature=0.1)
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
)
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
return JudgeResult(
|
| 118 |
-
score=score,
|
| 119 |
-
rationale=
|
| 120 |
-
compensation_tags=
|
| 121 |
-
corrective_hint=
|
| 122 |
-
confidence=
|
| 123 |
-
needs_human=needs_human,
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
def _fallback_from_rubric(self, rubric: ScoreResult, features: BiomechFeatures) -> JudgeResult:
|
| 127 |
-
"""When VLM is unavailable, promote the rubric score as the final score."""
|
| 128 |
-
return JudgeResult(
|
| 129 |
-
score=rubric.score,
|
| 130 |
-
rationale=f"[rubric-only] {rubric.rationale}",
|
| 131 |
-
compensation_tags=[],
|
| 132 |
-
corrective_hint="",
|
| 133 |
-
confidence=rubric.confidence * 0.8,
|
| 134 |
-
needs_human=rubric.needs_human,
|
| 135 |
-
notes="VLM unavailable — rubric score used as final",
|
| 136 |
-
)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
JudgeAgent — VLM-based final scorer with rationale, compensation tags, pain detection.
|
| 3 |
+
|
| 4 |
+
Input: BiomechFeatures, ScoreResult (rubric candidate), MovementResult, keyframes
|
| 5 |
+
Output: JudgeResult(score, rationale, compensation_tags, corrective_hint, needs_human)
|
| 6 |
+
Failure: returns JudgeResult(needs_human=True, score=None) when uncertain.
|
| 7 |
+
Model: Qwen3-VL-8B-Instruct via llama.cpp (8B params, Apache-2.0).
|
| 8 |
+
Gated: No.
|
| 9 |
+
|
| 10 |
+
Safety: NEVER auto-scores pain. If any indication of pain/clearing test,
|
| 11 |
+
sets needs_human=True and score=None.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import json
|
| 16 |
+
import logging
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
from formscout import config
|
| 20 |
+
from formscout.types import (
|
| 21 |
+
BiomechFeatures, ScoreResult, MovementResult,
|
| 22 |
+
IngestResult, JudgeResult,
|
| 23 |
+
)
|
| 24 |
+
from formscout.serving import get_vlm_client
|
| 25 |
+
|
| 26 |
+
logger = logging.getLogger(__name__)
|
| 27 |
+
|
| 28 |
+
_PROMPT_PATH = Path(__file__).parent / "prompts" / "c2_judge.md"
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class JudgeAgent:
|
| 32 |
+
"""VLM judge that produces the final FMS score with rationale."""
|
| 33 |
+
|
| 34 |
+
def __init__(self):
|
| 35 |
+
self._client = get_vlm_client()
|
| 36 |
+
self._system_prompt = _PROMPT_PATH.read_text(encoding="utf-8")
|
| 37 |
+
|
| 38 |
+
def run(
|
| 39 |
+
self,
|
| 40 |
+
features: BiomechFeatures,
|
| 41 |
+
rubric_score: ScoreResult,
|
| 42 |
+
movement: MovementResult,
|
| 43 |
+
ingest: IngestResult | None = None,
|
| 44 |
+
) -> JudgeResult:
|
| 45 |
+
"""
|
| 46 |
+
Produce final score. Falls back to rubric score if VLM unavailable.
|
| 47 |
+
"""
|
| 48 |
+
if not config.ENABLE_JUDGE:
|
| 49 |
+
return self._fallback_from_rubric(rubric_score, features)
|
| 50 |
+
|
| 51 |
+
if not self._client.available:
|
| 52 |
+
logger.warning("JudgeAgent: VLM unavailable, using rubric score as final")
|
| 53 |
+
return self._fallback_from_rubric(rubric_score, features)
|
| 54 |
+
|
| 55 |
+
# Build context for the judge
|
| 56 |
+
context = {
|
| 57 |
+
"test": features.test_name,
|
| 58 |
+
"side": features.side,
|
| 59 |
+
"view": features.view,
|
| 60 |
+
"features": {"angles": features.angles, "alignments": features.alignments},
|
| 61 |
+
"candidate_score": rubric_score.score,
|
| 62 |
+
"candidate_confidence": rubric_score.confidence,
|
| 63 |
+
"exemplars": [], # Phase 3: populated by RetrievalAgent
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
prompt = f"{self._system_prompt}\n\n{json.dumps(context, indent=2)}"
|
| 67 |
+
|
| 68 |
+
# Optionally include keyframes
|
| 69 |
+
images = None
|
| 70 |
+
if ingest and ingest.frames:
|
| 71 |
+
images = self._encode_keyframes(ingest.frames)
|
| 72 |
+
|
| 73 |
+
result = self._client.complete(prompt, images=images, max_tokens=512, temperature=0.1)
|
| 74 |
+
if result.get("fallback"):
|
| 75 |
+
# transformers backend couldn't load/run — use the deterministic rubric
|
| 76 |
+
return self._fallback_from_rubric(rubric_score, features)
|
| 77 |
+
return self._parse_response(result)
|
| 78 |
+
|
| 79 |
+
def _encode_keyframes(self, frames: list) -> list[str]:
|
| 80 |
+
"""Encode 3 keyframes for VLM context."""
|
| 81 |
+
import cv2
|
| 82 |
+
import base64
|
| 83 |
+
|
| 84 |
+
n = len(frames)
|
| 85 |
+
indices = [0, n // 2, n - 1] if n >= 3 else list(range(n))
|
| 86 |
+
encoded = []
|
| 87 |
+
for idx in indices:
|
| 88 |
+
_, buf = cv2.imencode(".jpg", frames[idx], [cv2.IMWRITE_JPEG_QUALITY, 70])
|
| 89 |
+
encoded.append(base64.b64encode(buf.tobytes()).decode())
|
| 90 |
+
return encoded
|
| 91 |
+
|
| 92 |
+
def _parse_response(self, result: dict) -> JudgeResult:
|
| 93 |
+
"""Parse VLM JSON response into JudgeResult."""
|
| 94 |
+
if "error" in result:
|
| 95 |
+
return JudgeResult(
|
| 96 |
+
score=None, rationale=f"VLM error: {result['error']}",
|
| 97 |
+
compensation_tags=[], corrective_hint="",
|
| 98 |
+
confidence=0.0, needs_human=True,
|
| 99 |
+
)
|
| 100 |
+
|
| 101 |
+
needs_human = result.get("needs_human", False)
|
| 102 |
+
score = result.get("score") if not needs_human else None
|
| 103 |
+
if score is not None:
|
| 104 |
+
score = max(0, min(3, int(score)))
|
| 105 |
+
|
| 106 |
+
return JudgeResult(
|
| 107 |
+
score=score,
|
| 108 |
+
rationale=result.get("rationale", ""),
|
| 109 |
+
compensation_tags=result.get("compensation_tags", []),
|
| 110 |
+
corrective_hint=result.get("corrective_hint", ""),
|
| 111 |
+
confidence=float(result.get("confidence", 0.5)),
|
| 112 |
+
needs_human=needs_human,
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
def _fallback_from_rubric(self, rubric: ScoreResult, features: BiomechFeatures) -> JudgeResult:
|
| 116 |
+
"""When VLM is unavailable, promote the rubric score as the final score."""
|
| 117 |
+
return JudgeResult(
|
| 118 |
+
score=rubric.score,
|
| 119 |
+
rationale=f"[rubric-only] {rubric.rationale}",
|
| 120 |
+
compensation_tags=[],
|
| 121 |
+
corrective_hint="",
|
| 122 |
+
confidence=rubric.confidence * 0.8,
|
| 123 |
+
needs_human=rubric.needs_human,
|
| 124 |
+
notes="VLM unavailable — rubric score used as final",
|
| 125 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
formscout/agents/pdf_report.py
CHANGED
|
@@ -1,115 +1,175 @@
|
|
| 1 |
-
"""
|
| 2 |
-
PdfReportAgent — renders a ReportResult + session entries to a branded PDF.
|
| 3 |
-
|
| 4 |
-
Input: ReportResult, list[SessionEntry], session_dir (str)
|
| 5 |
-
Output: path to the written PDF (str), or None on failure.
|
| 6 |
-
Failure: returns None, never raises.
|
| 7 |
-
Params: 0 (pure rendering — no model).
|
| 8 |
-
License: n/a.
|
| 9 |
-
Gated: no.
|
| 10 |
-
"""
|
| 11 |
-
from __future__ import annotations
|
| 12 |
-
|
| 13 |
-
import logging
|
| 14 |
-
import os
|
| 15 |
-
|
| 16 |
-
from formscout.types import ReportResult
|
| 17 |
-
|
| 18 |
-
logger = logging.getLogger(__name__)
|
| 19 |
-
|
| 20 |
-
DISCLAIMER = "Screening aid — not a diagnosis. Pain or clearing tests require a clinician."
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
class PdfReportAgent:
|
| 24 |
-
"""Assembles the screening-session PDF via ReportLab."""
|
| 25 |
-
|
| 26 |
-
def run(self, report: ReportResult, entries: list, session_dir: str) -> str | None:
|
| 27 |
-
try:
|
| 28 |
-
from reportlab.lib import colors
|
| 29 |
-
from reportlab.lib.pagesizes import LETTER
|
| 30 |
-
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
| 31 |
-
from reportlab.lib.units import inch
|
| 32 |
-
from reportlab.platypus import (
|
| 33 |
-
Image, Paragraph, SimpleDocTemplate, Spacer, Table, TableStyle,
|
| 34 |
-
)
|
| 35 |
-
except Exception as e:
|
| 36 |
-
logger.warning("reportlab unavailable: %s", e)
|
| 37 |
-
return None
|
| 38 |
-
|
| 39 |
-
out_path = os.path.join(session_dir, "formscout_report.pdf")
|
| 40 |
-
try:
|
| 41 |
-
styles = getSampleStyleSheet()
|
| 42 |
-
banner = ParagraphStyle(
|
| 43 |
-
"banner", parent=styles["Normal"], fontSize=9, textColor=colors.white,
|
| 44 |
-
backColor=colors.HexColor("#
|
| 45 |
-
)
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
story.append(
|
| 81 |
-
|
| 82 |
-
if e.
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
PdfReportAgent — renders a ReportResult + session entries to a branded PDF.
|
| 3 |
+
|
| 4 |
+
Input: ReportResult, list[SessionEntry], session_dir (str)
|
| 5 |
+
Output: path to the written PDF (str), or None on failure.
|
| 6 |
+
Failure: returns None, never raises.
|
| 7 |
+
Params: 0 (pure rendering — no model).
|
| 8 |
+
License: n/a.
|
| 9 |
+
Gated: no.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import logging
|
| 14 |
+
import os
|
| 15 |
+
|
| 16 |
+
from formscout.types import ReportResult
|
| 17 |
+
|
| 18 |
+
logger = logging.getLogger(__name__)
|
| 19 |
+
|
| 20 |
+
DISCLAIMER = "Screening aid — not a diagnosis. Pain or clearing tests require a clinician."
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class PdfReportAgent:
|
| 24 |
+
"""Assembles the screening-session PDF via ReportLab."""
|
| 25 |
+
|
| 26 |
+
def run(self, report: ReportResult, entries: list, session_dir: str) -> str | None:
|
| 27 |
+
try:
|
| 28 |
+
from reportlab.lib import colors
|
| 29 |
+
from reportlab.lib.pagesizes import LETTER
|
| 30 |
+
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
| 31 |
+
from reportlab.lib.units import inch
|
| 32 |
+
from reportlab.platypus import (
|
| 33 |
+
Image, PageBreak, Paragraph, SimpleDocTemplate, Spacer, Table, TableStyle,
|
| 34 |
+
)
|
| 35 |
+
except Exception as e:
|
| 36 |
+
logger.warning("reportlab unavailable: %s", e)
|
| 37 |
+
return None
|
| 38 |
+
|
| 39 |
+
out_path = os.path.join(session_dir, "formscout_report.pdf")
|
| 40 |
+
try:
|
| 41 |
+
styles = getSampleStyleSheet()
|
| 42 |
+
banner = ParagraphStyle(
|
| 43 |
+
"banner", parent=styles["Normal"], fontSize=9, textColor=colors.white,
|
| 44 |
+
backColor=colors.HexColor("#cf922a"), alignment=1, borderPadding=6, spaceAfter=12,
|
| 45 |
+
)
|
| 46 |
+
ink = colors.HexColor("#243a34")
|
| 47 |
+
|
| 48 |
+
def _meas_table(pairs, col0=3.0, col1=1.6):
|
| 49 |
+
rows = [[str(k).replace("_", " "),
|
| 50 |
+
(f"{v:.2f}" if isinstance(v, float) else str(v))] for k, v in pairs]
|
| 51 |
+
tbl = Table(rows, colWidths=[col0 * inch, col1 * inch])
|
| 52 |
+
tbl.setStyle(TableStyle([
|
| 53 |
+
("FONTSIZE", (0, 0), (-1, -1), 8),
|
| 54 |
+
("TEXTCOLOR", (0, 0), (-1, -1), ink),
|
| 55 |
+
("ROWBACKGROUNDS", (0, 0), (-1, -1),
|
| 56 |
+
[colors.HexColor("#f7eedd"), colors.white]),
|
| 57 |
+
]))
|
| 58 |
+
return tbl
|
| 59 |
+
|
| 60 |
+
def _img(path, w=3.0, h=2.25):
|
| 61 |
+
if path and os.path.exists(path):
|
| 62 |
+
try:
|
| 63 |
+
return Image(path, width=w * inch, height=h * inch)
|
| 64 |
+
except Exception:
|
| 65 |
+
return None
|
| 66 |
+
return None
|
| 67 |
+
story = []
|
| 68 |
+
story.append(Paragraph(f"<b>⚠ {DISCLAIMER}</b>", banner))
|
| 69 |
+
story.append(Paragraph("FormScout — FMS Screening Report", styles["Title"]))
|
| 70 |
+
|
| 71 |
+
if report.composite is not None:
|
| 72 |
+
comp = f"Composite: <b>{report.composite} / 21</b>"
|
| 73 |
+
else:
|
| 74 |
+
comp = f"Composite: <b>Incomplete</b> — {len(entries)}/7 tests scored"
|
| 75 |
+
story.append(Paragraph(comp, styles["Heading2"]))
|
| 76 |
+
story.append(Spacer(1, 0.2 * inch))
|
| 77 |
+
|
| 78 |
+
for ei, e in enumerate(entries):
|
| 79 |
+
if ei > 0:
|
| 80 |
+
story.append(PageBreak())
|
| 81 |
+
title = e.test_name.replace("_", " ").title()
|
| 82 |
+
if e.side in ("left", "right"):
|
| 83 |
+
title += f" ({e.side})"
|
| 84 |
+
score_txt = "Clinician review required" if e.needs_human else f"Score: {e.score}/3"
|
| 85 |
+
story.append(Paragraph(f"<b>{title}</b> — {score_txt}", styles["Heading3"]))
|
| 86 |
+
story.append(Paragraph(f"<font size=8>view: {e.view} · confidence: "
|
| 87 |
+
f"{e.confidence:.0%}</font>", styles["Normal"]))
|
| 88 |
+
if e.rationale:
|
| 89 |
+
story.append(Paragraph(e.rationale, styles["Normal"]))
|
| 90 |
+
if e.compensation_tags:
|
| 91 |
+
story.append(Paragraph("<b>Compensations:</b> " + ", ".join(e.compensation_tags),
|
| 92 |
+
styles["Normal"]))
|
| 93 |
+
if e.corrective_hint:
|
| 94 |
+
story.append(Paragraph("<b>Corrective:</b> " + e.corrective_hint, styles["Normal"]))
|
| 95 |
+
|
| 96 |
+
# Key frame + flexion chart side by side
|
| 97 |
+
kf, fb = _img(e.keyframe_path), _img((e.chart_paths or {}).get("flexion"), w=3.2, h=2.0)
|
| 98 |
+
if kf or fb:
|
| 99 |
+
cells = [c for c in (kf, fb) if c] or [Paragraph("<i>(images unavailable)</i>",
|
| 100 |
+
styles["Normal"])]
|
| 101 |
+
story.append(Table([cells], hAlign="LEFT"))
|
| 102 |
+
|
| 103 |
+
# Relevant-joint flexion table
|
| 104 |
+
if e.flexion:
|
| 105 |
+
story.append(Paragraph("<b>Relevant joint flexion (key frame)</b>", styles["Normal"]))
|
| 106 |
+
story.append(_meas_table(
|
| 107 |
+
[(n, f"{v['deg']:.1f}° — {v['openness']}") for n, v in e.flexion.items()],
|
| 108 |
+
col0=2.6, col1=2.6))
|
| 109 |
+
|
| 110 |
+
# Laban Effort + radar
|
| 111 |
+
if e.laban:
|
| 112 |
+
eff, lab = e.laban.get("effort", {}), e.laban.get("labels", {})
|
| 113 |
+
story.append(Spacer(1, 0.08 * inch))
|
| 114 |
+
story.append(Paragraph("<b>Laban Effort (kinematic estimate)</b>", styles["Normal"]))
|
| 115 |
+
laban_tbl = _meas_table(
|
| 116 |
+
[(k.title(), f"{eff.get(k, 0):.2f} — {lab.get(k, '')}")
|
| 117 |
+
for k in ("space", "weight", "time", "flow")], col0=2.6, col1=2.6)
|
| 118 |
+
radar = _img((e.chart_paths or {}).get("radar"), w=2.6, h=2.6)
|
| 119 |
+
if radar:
|
| 120 |
+
story.append(Table([[laban_tbl, radar]], hAlign="LEFT"))
|
| 121 |
+
else:
|
| 122 |
+
story.append(laban_tbl)
|
| 123 |
+
if e.laban.get("body_emphasis"):
|
| 124 |
+
emph = ", ".join(f"{n}" for n, _ in e.laban["body_emphasis"])
|
| 125 |
+
story.append(Paragraph(f"<font size=8>Body emphasis: {emph} · "
|
| 126 |
+
f"{e.laban.get('notes', '')}</font>", styles["Normal"]))
|
| 127 |
+
|
| 128 |
+
# Angle + velocity charts
|
| 129 |
+
for kind in ("angle", "velocity"):
|
| 130 |
+
chart = _img((e.chart_paths or {}).get(kind), w=5.0, h=2.5)
|
| 131 |
+
if chart:
|
| 132 |
+
story.append(chart)
|
| 133 |
+
|
| 134 |
+
# Full measurement dump
|
| 135 |
+
if e.measurements:
|
| 136 |
+
story.append(Paragraph("<b>All measurements</b>", styles["Normal"]))
|
| 137 |
+
story.append(_meas_table(list(e.measurements.items())))
|
| 138 |
+
|
| 139 |
+
story.append(Spacer(1, 0.15 * inch))
|
| 140 |
+
|
| 141 |
+
if report.asymmetries:
|
| 142 |
+
story.append(PageBreak())
|
| 143 |
+
story.append(Paragraph("Asymmetries", styles["Heading2"]))
|
| 144 |
+
for a in report.asymmetries:
|
| 145 |
+
story.append(Paragraph(
|
| 146 |
+
f"{a['test'].replace('_', ' ').title()}: "
|
| 147 |
+
f"L={a['left_score']} R={a['right_score']} (Δ {a['delta']})",
|
| 148 |
+
styles["Normal"]))
|
| 149 |
+
try:
|
| 150 |
+
from formscout.analysis.charts import symmetry_bars
|
| 151 |
+
os.makedirs(os.path.join(session_dir, "charts"), exist_ok=True)
|
| 152 |
+
sym_png = symmetry_bars(report.asymmetries,
|
| 153 |
+
os.path.join(session_dir, "charts", "symmetry.png"))
|
| 154 |
+
sym_img = _img(sym_png, w=5.5, h=2.75)
|
| 155 |
+
if sym_img:
|
| 156 |
+
story.append(sym_img)
|
| 157 |
+
except Exception:
|
| 158 |
+
pass
|
| 159 |
+
|
| 160 |
+
flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
|
| 161 |
+
if flags:
|
| 162 |
+
story.append(Paragraph("Flags", styles["Heading2"]))
|
| 163 |
+
for fl in flags:
|
| 164 |
+
story.append(Paragraph(fl, styles["Normal"]))
|
| 165 |
+
|
| 166 |
+
story.append(Spacer(1, 0.3 * inch))
|
| 167 |
+
story.append(Paragraph(f"<b>⚠ {DISCLAIMER}</b>", banner))
|
| 168 |
+
|
| 169 |
+
doc = SimpleDocTemplate(out_path, pagesize=LETTER,
|
| 170 |
+
topMargin=0.6 * inch, bottomMargin=0.6 * inch)
|
| 171 |
+
doc.build(story)
|
| 172 |
+
return out_path
|
| 173 |
+
except Exception as e:
|
| 174 |
+
logger.warning("pdf build failed: %s", e)
|
| 175 |
+
return None
|
formscout/agents/pose2d.py
CHANGED
|
@@ -1,232 +1,232 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Pose2DAgent — 2D per-frame keypoint extraction.
|
| 3 |
-
|
| 4 |
-
Backends: yolo (local checkpoints, ultralytics), mediapipe (official Tasks API,
|
| 5 |
-
local .task checkpoint), sapiens2 (Meta HF/transformers).
|
| 6 |
-
All backends output COCO-17 keypoints: dict[int, {x, y, conf}] per frame.
|
| 7 |
-
|
| 8 |
-
Input: IngestResult
|
| 9 |
-
Output: Pose2DResult(keypoints per frame, fps, confidence)
|
| 10 |
-
Failure: Pose2DResult(confidence=0.0, notes=<reason>) — never raises.
|
| 11 |
-
Gated: yolo=no; mediapipe=no (local checkpoint); sapiens2=yes (access accepted).
|
| 12 |
-
"""
|
| 13 |
-
from __future__ import annotations
|
| 14 |
-
|
| 15 |
-
import logging
|
| 16 |
-
import numpy as np
|
| 17 |
-
|
| 18 |
-
from formscout import config
|
| 19 |
-
from formscout.types import IngestResult, Pose2DResult
|
| 20 |
-
|
| 21 |
-
logger = logging.getLogger(__name__)
|
| 22 |
-
|
| 23 |
-
COCO_KEYPOINTS = [
|
| 24 |
-
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 25 |
-
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 26 |
-
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 27 |
-
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 28 |
-
]
|
| 29 |
-
|
| 30 |
-
# BlazePose-33 source indices → COCO-17 target indices
|
| 31 |
-
# BlazePose: 0=nose, 2=left_eye, 5=right_eye, 7=left_ear, 8=right_ear,
|
| 32 |
-
# 11=left_shoulder, 12=right_shoulder, 13=left_elbow, 14=right_elbow,
|
| 33 |
-
# 15=left_wrist, 16=right_wrist, 23=left_hip, 24=right_hip,
|
| 34 |
-
# 25=left_knee, 26=right_knee, 27=left_ankle, 28=right_ankle
|
| 35 |
-
_BP_SRC = [0, 2, 5, 7, 8, 11, 12, 13, 14, 15, 16, 23, 24, 25, 26, 27, 28]
|
| 36 |
-
_BP_DST = list(range(17)) # COCO indices 0..16
|
| 37 |
-
|
| 38 |
-
_model_cache: dict[str, object] = {}
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
# ── YOLO backend ──────────────────────────────────────────────────────────────
|
| 42 |
-
|
| 43 |
-
def _get_yolo(path: str) -> object:
|
| 44 |
-
if path not in _model_cache:
|
| 45 |
-
from ultralytics import YOLO
|
| 46 |
-
_model_cache[path] = YOLO(path)
|
| 47 |
-
return _model_cache[path]
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
def _run_yolo(frames: list, path: str) -> list[dict]:
|
| 51 |
-
model = _get_yolo(path)
|
| 52 |
-
out = []
|
| 53 |
-
for frame in frames:
|
| 54 |
-
try:
|
| 55 |
-
results = model(frame, verbose=False)
|
| 56 |
-
kps: dict[int, dict] = {}
|
| 57 |
-
if results and results[0].keypoints is not None:
|
| 58 |
-
kp = results[0].keypoints
|
| 59 |
-
if kp.xy is not None and len(kp.xy) > 0:
|
| 60 |
-
xy = kp.xy[0].cpu().numpy()
|
| 61 |
-
conf = kp.conf[0].cpu().numpy()
|
| 62 |
-
for j in range(min(len(xy), 17)):
|
| 63 |
-
kps[j] = {"x": float(xy[j, 0]), "y": float(xy[j, 1]), "conf": float(conf[j])}
|
| 64 |
-
out.append(kps)
|
| 65 |
-
except Exception:
|
| 66 |
-
out.append({})
|
| 67 |
-
return out
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
# ── MediaPipe backend (official Tasks API, local .task checkpoint) ────────────
|
| 71 |
-
|
| 72 |
-
def _get_mediapipe_landmarker(path: str) -> object:
|
| 73 |
-
"""Return PoseLandmarker cached by model path."""
|
| 74 |
-
cache_key = f"mp:{path}"
|
| 75 |
-
if cache_key not in _model_cache:
|
| 76 |
-
from mediapipe.tasks import python as mp_tasks
|
| 77 |
-
from mediapipe.tasks.python import vision
|
| 78 |
-
|
| 79 |
-
options = vision.PoseLandmarkerOptions(
|
| 80 |
-
base_options=mp_tasks.BaseOptions(model_asset_path=path),
|
| 81 |
-
running_mode=vision.RunningMode.IMAGE,
|
| 82 |
-
num_poses=1,
|
| 83 |
-
min_pose_detection_confidence=0.4,
|
| 84 |
-
min_pose_presence_confidence=0.4,
|
| 85 |
-
min_tracking_confidence=0.4,
|
| 86 |
-
)
|
| 87 |
-
_model_cache[cache_key] = vision.PoseLandmarker.create_from_options(options)
|
| 88 |
-
return _model_cache[cache_key]
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
def _run_mediapipe(frames: list, path: str) -> list[dict]:
|
| 92 |
-
import cv2
|
| 93 |
-
import mediapipe as mp
|
| 94 |
-
|
| 95 |
-
try:
|
| 96 |
-
landmarker = _get_mediapipe_landmarker(path)
|
| 97 |
-
except Exception as e:
|
| 98 |
-
logger.warning("mediapipe load failed: %s", e)
|
| 99 |
-
return [{} for _ in frames]
|
| 100 |
-
|
| 101 |
-
out = []
|
| 102 |
-
for frame in frames:
|
| 103 |
-
try:
|
| 104 |
-
h, w = frame.shape[:2]
|
| 105 |
-
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 106 |
-
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
|
| 107 |
-
detection = landmarker.detect(mp_image)
|
| 108 |
-
|
| 109 |
-
kps: dict[int, dict] = {}
|
| 110 |
-
if detection.pose_landmarks:
|
| 111 |
-
lms = detection.pose_landmarks[0]
|
| 112 |
-
for coco_idx, bp_idx in zip(_BP_DST, _BP_SRC):
|
| 113 |
-
if bp_idx < len(lms):
|
| 114 |
-
lm = lms[bp_idx]
|
| 115 |
-
kps[coco_idx] = {
|
| 116 |
-
"x": float(lm.x * w),
|
| 117 |
-
"y": float(lm.y * h),
|
| 118 |
-
"conf": float(lm.visibility),
|
| 119 |
-
}
|
| 120 |
-
out.append(kps)
|
| 121 |
-
except Exception:
|
| 122 |
-
out.append({})
|
| 123 |
-
return out
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
# ── Sapiens2 backend (Meta HF, transformers) ──────────────────────────────────
|
| 127 |
-
|
| 128 |
-
def _get_sapiens2(hf_id: str) -> object:
|
| 129 |
-
if hf_id not in _model_cache:
|
| 130 |
-
from transformers import pipeline as hf_pipeline
|
| 131 |
-
_model_cache[hf_id] = hf_pipeline("pose-estimation", model=hf_id)
|
| 132 |
-
return _model_cache[hf_id]
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
def _run_sapiens2(frames: list, hf_id: str) -> list[dict]:
|
| 136 |
-
try:
|
| 137 |
-
pipe = _get_sapiens2(hf_id)
|
| 138 |
-
except Exception as e:
|
| 139 |
-
logger.warning("sapiens2 load failed: %s", e)
|
| 140 |
-
return [{} for _ in frames]
|
| 141 |
-
|
| 142 |
-
from PIL import Image
|
| 143 |
-
|
| 144 |
-
out = []
|
| 145 |
-
for frame in frames:
|
| 146 |
-
try:
|
| 147 |
-
pil_img = Image.fromarray(frame)
|
| 148 |
-
result = pipe(pil_img)
|
| 149 |
-
|
| 150 |
-
if not result:
|
| 151 |
-
out.append({})
|
| 152 |
-
continue
|
| 153 |
-
|
| 154 |
-
# Take highest-confidence person (first result)
|
| 155 |
-
person = result[0]
|
| 156 |
-
keypoints = person.get("keypoints", [])
|
| 157 |
-
scores = person.get("keypoint_scores", [])
|
| 158 |
-
|
| 159 |
-
# Build name→(x, y, score) lookup from pipeline output
|
| 160 |
-
kp_lookup: dict[str, tuple] = {}
|
| 161 |
-
for i, kp in enumerate(keypoints):
|
| 162 |
-
if isinstance(kp, dict):
|
| 163 |
-
name = kp.get("label", "")
|
| 164 |
-
x, y = kp.get("x", 0.0), kp.get("y", 0.0)
|
| 165 |
-
else:
|
| 166 |
-
name = ""
|
| 167 |
-
x, y = float(kp[0]), float(kp[1])
|
| 168 |
-
score = float(scores[i]) if i < len(scores) else 0.0
|
| 169 |
-
if name:
|
| 170 |
-
kp_lookup[name] = (x, y, score)
|
| 171 |
-
|
| 172 |
-
kps: dict[int, dict] = {}
|
| 173 |
-
for coco_idx, name in enumerate(COCO_KEYPOINTS):
|
| 174 |
-
if name in kp_lookup:
|
| 175 |
-
x, y, s = kp_lookup[name]
|
| 176 |
-
kps[coco_idx] = {"x": x, "y": y, "conf": s}
|
| 177 |
-
out.append(kps)
|
| 178 |
-
except Exception:
|
| 179 |
-
out.append({})
|
| 180 |
-
return out
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
# ── Agent ─────────────────────────────────────────────────────────────────────
|
| 184 |
-
|
| 185 |
-
class Pose2DAgent:
|
| 186 |
-
"""Extracts COCO-17 keypoints per frame; dispatches to YOLO, MediaPipe, or Sapiens2."""
|
| 187 |
-
|
| 188 |
-
def run(self, ingest: IngestResult, model_key: str | None = None) -> Pose2DResult:
|
| 189 |
-
if not ingest.frames:
|
| 190 |
-
return Pose2DResult(keypoints=[], fps=ingest.fps, confidence=0.0, notes="no frames in ingest")
|
| 191 |
-
|
| 192 |
-
key = model_key or config.DEFAULT_POSE_MODEL
|
| 193 |
-
spec = config.POSE_MODELS.get(key)
|
| 194 |
-
if spec is None:
|
| 195 |
-
logger.warning("Unknown model_key %r — falling back to %s", key, config.DEFAULT_POSE_MODEL)
|
| 196 |
-
spec = config.POSE_MODELS[config.DEFAULT_POSE_MODEL]
|
| 197 |
-
|
| 198 |
-
backend = spec["backend"]
|
| 199 |
-
try:
|
| 200 |
-
if backend == "yolo":
|
| 201 |
-
kps_per_frame = _run_yolo(ingest.frames, spec["path"])
|
| 202 |
-
elif backend == "mediapipe":
|
| 203 |
-
kps_per_frame = _run_mediapipe(ingest.frames, spec["path"])
|
| 204 |
-
elif backend == "sapiens2":
|
| 205 |
-
kps_per_frame = _run_sapiens2(ingest.frames, spec["hf_id"])
|
| 206 |
-
else:
|
| 207 |
-
return Pose2DResult(
|
| 208 |
-
keypoints=[{} for _ in ingest.frames],
|
| 209 |
-
fps=ingest.fps, confidence=0.0,
|
| 210 |
-
notes=f"unknown backend: {backend}",
|
| 211 |
-
)
|
| 212 |
-
except Exception as e:
|
| 213 |
-
return Pose2DResult(
|
| 214 |
-
keypoints=[{} for _ in ingest.frames],
|
| 215 |
-
fps=ingest.fps, confidence=0.0,
|
| 216 |
-
notes=str(e),
|
| 217 |
-
)
|
| 218 |
-
|
| 219 |
-
n_detected = sum(1 for f in kps_per_frame if f)
|
| 220 |
-
total_conf = sum(
|
| 221 |
-
sum(kp["conf"] for kp in f.values()) / len(f)
|
| 222 |
-
for f in kps_per_frame if f
|
| 223 |
-
)
|
| 224 |
-
overall_conf = (total_conf / n_detected) if n_detected > 0 else 0.0
|
| 225 |
-
notes = "" if n_detected > 0 else "no person detected in any frame"
|
| 226 |
-
|
| 227 |
-
return Pose2DResult(
|
| 228 |
-
keypoints=kps_per_frame,
|
| 229 |
-
fps=ingest.fps,
|
| 230 |
-
confidence=overall_conf,
|
| 231 |
-
notes=notes,
|
| 232 |
-
)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Pose2DAgent — 2D per-frame keypoint extraction.
|
| 3 |
+
|
| 4 |
+
Backends: yolo (local checkpoints, ultralytics), mediapipe (official Tasks API,
|
| 5 |
+
local .task checkpoint), sapiens2 (Meta HF/transformers).
|
| 6 |
+
All backends output COCO-17 keypoints: dict[int, {x, y, conf}] per frame.
|
| 7 |
+
|
| 8 |
+
Input: IngestResult
|
| 9 |
+
Output: Pose2DResult(keypoints per frame, fps, confidence)
|
| 10 |
+
Failure: Pose2DResult(confidence=0.0, notes=<reason>) — never raises.
|
| 11 |
+
Gated: yolo=no; mediapipe=no (local checkpoint); sapiens2=yes (access accepted).
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import logging
|
| 16 |
+
import numpy as np
|
| 17 |
+
|
| 18 |
+
from formscout import config
|
| 19 |
+
from formscout.types import IngestResult, Pose2DResult
|
| 20 |
+
|
| 21 |
+
logger = logging.getLogger(__name__)
|
| 22 |
+
|
| 23 |
+
COCO_KEYPOINTS = [
|
| 24 |
+
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 25 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 26 |
+
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 27 |
+
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 28 |
+
]
|
| 29 |
+
|
| 30 |
+
# BlazePose-33 source indices → COCO-17 target indices
|
| 31 |
+
# BlazePose: 0=nose, 2=left_eye, 5=right_eye, 7=left_ear, 8=right_ear,
|
| 32 |
+
# 11=left_shoulder, 12=right_shoulder, 13=left_elbow, 14=right_elbow,
|
| 33 |
+
# 15=left_wrist, 16=right_wrist, 23=left_hip, 24=right_hip,
|
| 34 |
+
# 25=left_knee, 26=right_knee, 27=left_ankle, 28=right_ankle
|
| 35 |
+
_BP_SRC = [0, 2, 5, 7, 8, 11, 12, 13, 14, 15, 16, 23, 24, 25, 26, 27, 28]
|
| 36 |
+
_BP_DST = list(range(17)) # COCO indices 0..16
|
| 37 |
+
|
| 38 |
+
_model_cache: dict[str, object] = {}
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# ── YOLO backend ──────────────────────────────────────────────────────────────
|
| 42 |
+
|
| 43 |
+
def _get_yolo(path: str) -> object:
|
| 44 |
+
if path not in _model_cache:
|
| 45 |
+
from ultralytics import YOLO
|
| 46 |
+
_model_cache[path] = YOLO(path)
|
| 47 |
+
return _model_cache[path]
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _run_yolo(frames: list, path: str) -> list[dict]:
|
| 51 |
+
model = _get_yolo(path)
|
| 52 |
+
out = []
|
| 53 |
+
for frame in frames:
|
| 54 |
+
try:
|
| 55 |
+
results = model(frame, verbose=False)
|
| 56 |
+
kps: dict[int, dict] = {}
|
| 57 |
+
if results and results[0].keypoints is not None:
|
| 58 |
+
kp = results[0].keypoints
|
| 59 |
+
if kp.xy is not None and len(kp.xy) > 0:
|
| 60 |
+
xy = kp.xy[0].cpu().numpy()
|
| 61 |
+
conf = kp.conf[0].cpu().numpy()
|
| 62 |
+
for j in range(min(len(xy), 17)):
|
| 63 |
+
kps[j] = {"x": float(xy[j, 0]), "y": float(xy[j, 1]), "conf": float(conf[j])}
|
| 64 |
+
out.append(kps)
|
| 65 |
+
except Exception:
|
| 66 |
+
out.append({})
|
| 67 |
+
return out
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
# ── MediaPipe backend (official Tasks API, local .task checkpoint) ────────────
|
| 71 |
+
|
| 72 |
+
def _get_mediapipe_landmarker(path: str) -> object:
|
| 73 |
+
"""Return PoseLandmarker cached by model path."""
|
| 74 |
+
cache_key = f"mp:{path}"
|
| 75 |
+
if cache_key not in _model_cache:
|
| 76 |
+
from mediapipe.tasks import python as mp_tasks
|
| 77 |
+
from mediapipe.tasks.python import vision
|
| 78 |
+
|
| 79 |
+
options = vision.PoseLandmarkerOptions(
|
| 80 |
+
base_options=mp_tasks.BaseOptions(model_asset_path=path),
|
| 81 |
+
running_mode=vision.RunningMode.IMAGE,
|
| 82 |
+
num_poses=1,
|
| 83 |
+
min_pose_detection_confidence=0.4,
|
| 84 |
+
min_pose_presence_confidence=0.4,
|
| 85 |
+
min_tracking_confidence=0.4,
|
| 86 |
+
)
|
| 87 |
+
_model_cache[cache_key] = vision.PoseLandmarker.create_from_options(options)
|
| 88 |
+
return _model_cache[cache_key]
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _run_mediapipe(frames: list, path: str) -> list[dict]:
|
| 92 |
+
import cv2
|
| 93 |
+
import mediapipe as mp
|
| 94 |
+
|
| 95 |
+
try:
|
| 96 |
+
landmarker = _get_mediapipe_landmarker(path)
|
| 97 |
+
except Exception as e:
|
| 98 |
+
logger.warning("mediapipe load failed: %s", e)
|
| 99 |
+
return [{} for _ in frames]
|
| 100 |
+
|
| 101 |
+
out = []
|
| 102 |
+
for frame in frames:
|
| 103 |
+
try:
|
| 104 |
+
h, w = frame.shape[:2]
|
| 105 |
+
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 106 |
+
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
|
| 107 |
+
detection = landmarker.detect(mp_image)
|
| 108 |
+
|
| 109 |
+
kps: dict[int, dict] = {}
|
| 110 |
+
if detection.pose_landmarks:
|
| 111 |
+
lms = detection.pose_landmarks[0]
|
| 112 |
+
for coco_idx, bp_idx in zip(_BP_DST, _BP_SRC):
|
| 113 |
+
if bp_idx < len(lms):
|
| 114 |
+
lm = lms[bp_idx]
|
| 115 |
+
kps[coco_idx] = {
|
| 116 |
+
"x": float(lm.x * w),
|
| 117 |
+
"y": float(lm.y * h),
|
| 118 |
+
"conf": float(lm.visibility),
|
| 119 |
+
}
|
| 120 |
+
out.append(kps)
|
| 121 |
+
except Exception:
|
| 122 |
+
out.append({})
|
| 123 |
+
return out
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
# ── Sapiens2 backend (Meta HF, transformers) ──────────────────────────────────
|
| 127 |
+
|
| 128 |
+
def _get_sapiens2(hf_id: str) -> object:
|
| 129 |
+
if hf_id not in _model_cache:
|
| 130 |
+
from transformers import pipeline as hf_pipeline
|
| 131 |
+
_model_cache[hf_id] = hf_pipeline("pose-estimation", model=hf_id)
|
| 132 |
+
return _model_cache[hf_id]
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _run_sapiens2(frames: list, hf_id: str) -> list[dict]:
|
| 136 |
+
try:
|
| 137 |
+
pipe = _get_sapiens2(hf_id)
|
| 138 |
+
except Exception as e:
|
| 139 |
+
logger.warning("sapiens2 load failed: %s", e)
|
| 140 |
+
return [{} for _ in frames]
|
| 141 |
+
|
| 142 |
+
from PIL import Image
|
| 143 |
+
|
| 144 |
+
out = []
|
| 145 |
+
for frame in frames:
|
| 146 |
+
try:
|
| 147 |
+
pil_img = Image.fromarray(frame)
|
| 148 |
+
result = pipe(pil_img)
|
| 149 |
+
|
| 150 |
+
if not result:
|
| 151 |
+
out.append({})
|
| 152 |
+
continue
|
| 153 |
+
|
| 154 |
+
# Take highest-confidence person (first result)
|
| 155 |
+
person = result[0]
|
| 156 |
+
keypoints = person.get("keypoints", [])
|
| 157 |
+
scores = person.get("keypoint_scores", [])
|
| 158 |
+
|
| 159 |
+
# Build name→(x, y, score) lookup from pipeline output
|
| 160 |
+
kp_lookup: dict[str, tuple] = {}
|
| 161 |
+
for i, kp in enumerate(keypoints):
|
| 162 |
+
if isinstance(kp, dict):
|
| 163 |
+
name = kp.get("label", "")
|
| 164 |
+
x, y = kp.get("x", 0.0), kp.get("y", 0.0)
|
| 165 |
+
else:
|
| 166 |
+
name = ""
|
| 167 |
+
x, y = float(kp[0]), float(kp[1])
|
| 168 |
+
score = float(scores[i]) if i < len(scores) else 0.0
|
| 169 |
+
if name:
|
| 170 |
+
kp_lookup[name] = (x, y, score)
|
| 171 |
+
|
| 172 |
+
kps: dict[int, dict] = {}
|
| 173 |
+
for coco_idx, name in enumerate(COCO_KEYPOINTS):
|
| 174 |
+
if name in kp_lookup:
|
| 175 |
+
x, y, s = kp_lookup[name]
|
| 176 |
+
kps[coco_idx] = {"x": x, "y": y, "conf": s}
|
| 177 |
+
out.append(kps)
|
| 178 |
+
except Exception:
|
| 179 |
+
out.append({})
|
| 180 |
+
return out
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
# ── Agent ─────────────────────────────────────────────────────────────────────
|
| 184 |
+
|
| 185 |
+
class Pose2DAgent:
|
| 186 |
+
"""Extracts COCO-17 keypoints per frame; dispatches to YOLO, MediaPipe, or Sapiens2."""
|
| 187 |
+
|
| 188 |
+
def run(self, ingest: IngestResult, model_key: str | None = None) -> Pose2DResult:
|
| 189 |
+
if not ingest.frames:
|
| 190 |
+
return Pose2DResult(keypoints=[], fps=ingest.fps, confidence=0.0, notes="no frames in ingest")
|
| 191 |
+
|
| 192 |
+
key = model_key or config.DEFAULT_POSE_MODEL
|
| 193 |
+
spec = config.POSE_MODELS.get(key)
|
| 194 |
+
if spec is None:
|
| 195 |
+
logger.warning("Unknown model_key %r — falling back to %s", key, config.DEFAULT_POSE_MODEL)
|
| 196 |
+
spec = config.POSE_MODELS[config.DEFAULT_POSE_MODEL]
|
| 197 |
+
|
| 198 |
+
backend = spec["backend"]
|
| 199 |
+
try:
|
| 200 |
+
if backend == "yolo":
|
| 201 |
+
kps_per_frame = _run_yolo(ingest.frames, spec["path"])
|
| 202 |
+
elif backend == "mediapipe":
|
| 203 |
+
kps_per_frame = _run_mediapipe(ingest.frames, spec["path"])
|
| 204 |
+
elif backend == "sapiens2":
|
| 205 |
+
kps_per_frame = _run_sapiens2(ingest.frames, spec["hf_id"])
|
| 206 |
+
else:
|
| 207 |
+
return Pose2DResult(
|
| 208 |
+
keypoints=[{} for _ in ingest.frames],
|
| 209 |
+
fps=ingest.fps, confidence=0.0,
|
| 210 |
+
notes=f"unknown backend: {backend}",
|
| 211 |
+
)
|
| 212 |
+
except Exception as e:
|
| 213 |
+
return Pose2DResult(
|
| 214 |
+
keypoints=[{} for _ in ingest.frames],
|
| 215 |
+
fps=ingest.fps, confidence=0.0,
|
| 216 |
+
notes=str(e),
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
n_detected = sum(1 for f in kps_per_frame if f)
|
| 220 |
+
total_conf = sum(
|
| 221 |
+
sum(kp["conf"] for kp in f.values()) / len(f)
|
| 222 |
+
for f in kps_per_frame if f
|
| 223 |
+
)
|
| 224 |
+
overall_conf = (total_conf / n_detected) if n_detected > 0 else 0.0
|
| 225 |
+
notes = "" if n_detected > 0 else "no person detected in any frame"
|
| 226 |
+
|
| 227 |
+
return Pose2DResult(
|
| 228 |
+
keypoints=kps_per_frame,
|
| 229 |
+
fps=ingest.fps,
|
| 230 |
+
confidence=overall_conf,
|
| 231 |
+
notes=notes,
|
| 232 |
+
)
|
formscout/agents/report.py
CHANGED
|
@@ -1,139 +1,139 @@
|
|
| 1 |
-
"""
|
| 2 |
-
ReportAgent — assembles per-test scorecard, composite, asymmetries.
|
| 3 |
-
|
| 4 |
-
Input: List of (MovementResult, BiomechFeatures, ScoreResult, JudgeResult) per test
|
| 5 |
-
Output: ReportResult(per_test, composite, asymmetries, overlay_video_path, pdf_path)
|
| 6 |
-
Failure: returns ReportResult with composite=None if any test unscored.
|
| 7 |
-
Params: 0 (pure assembly — no model).
|
| 8 |
-
License: n/a.
|
| 9 |
-
Gated: no.
|
| 10 |
-
"""
|
| 11 |
-
from __future__ import annotations
|
| 12 |
-
|
| 13 |
-
from formscout.types import (
|
| 14 |
-
MovementResult, BiomechFeatures, ScoreResult, JudgeResult, ReportResult,
|
| 15 |
-
)
|
| 16 |
-
from formscout import config
|
| 17 |
-
|
| 18 |
-
# Bilateral tests that need L/R scoring
|
| 19 |
-
BILATERAL_TESTS = {"hurdle_step", "inline_lunge", "shoulder_mobility", "active_slr"}
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
class ReportAgent:
|
| 23 |
-
"""Assembles the final screening report from all test results."""
|
| 24 |
-
|
| 25 |
-
def run(self, test_results: list[dict]) -> ReportResult:
|
| 26 |
-
"""
|
| 27 |
-
Assemble the report.
|
| 28 |
-
|
| 29 |
-
Args:
|
| 30 |
-
test_results: list of dicts with keys:
|
| 31 |
-
- movement: MovementResult
|
| 32 |
-
- features: BiomechFeatures
|
| 33 |
-
- rubric_score: ScoreResult
|
| 34 |
-
- judge: JudgeResult
|
| 35 |
-
- side: str (for bilateral: "left" or "right")
|
| 36 |
-
"""
|
| 37 |
-
per_test = []
|
| 38 |
-
asymmetries = []
|
| 39 |
-
low_confidence_flags = []
|
| 40 |
-
disagreement_flags = []
|
| 41 |
-
|
| 42 |
-
# Group bilateral tests by test_name
|
| 43 |
-
bilateral_groups: dict[str, list[dict]] = {}
|
| 44 |
-
unilateral: list[dict] = []
|
| 45 |
-
|
| 46 |
-
for entry in test_results:
|
| 47 |
-
test_name = entry["movement"].test_name
|
| 48 |
-
if test_name in BILATERAL_TESTS:
|
| 49 |
-
bilateral_groups.setdefault(test_name, []).append(entry)
|
| 50 |
-
else:
|
| 51 |
-
unilateral.append(entry)
|
| 52 |
-
|
| 53 |
-
# Process bilateral tests — take the lower score, emit asymmetry
|
| 54 |
-
for test_name, entries in bilateral_groups.items():
|
| 55 |
-
scores = []
|
| 56 |
-
for entry in entries:
|
| 57 |
-
judge = entry["judge"]
|
| 58 |
-
side = entry.get("side", entry["movement"].side)
|
| 59 |
-
score = judge.score if judge.score is not None else None
|
| 60 |
-
scores.append({"side": side, "score": score, "entry": entry})
|
| 61 |
-
|
| 62 |
-
# Find best entry per side
|
| 63 |
-
left = next((s for s in scores if s["side"] == "left"), None)
|
| 64 |
-
right = next((s for s in scores if s["side"] == "right"), None)
|
| 65 |
-
|
| 66 |
-
left_score = left["score"] if left else None
|
| 67 |
-
right_score = right["score"] if right else None
|
| 68 |
-
|
| 69 |
-
# Report lower
|
| 70 |
-
if left_score is not None and right_score is not None:
|
| 71 |
-
final_score = min(left_score, right_score)
|
| 72 |
-
delta = abs(left_score - right_score)
|
| 73 |
-
asymmetries.append({
|
| 74 |
-
"test": test_name,
|
| 75 |
-
"left_score": left_score,
|
| 76 |
-
"right_score": right_score,
|
| 77 |
-
"delta": delta,
|
| 78 |
-
})
|
| 79 |
-
elif left_score is not None:
|
| 80 |
-
final_score = left_score
|
| 81 |
-
elif right_score is not None:
|
| 82 |
-
final_score = right_score
|
| 83 |
-
else:
|
| 84 |
-
final_score = None
|
| 85 |
-
|
| 86 |
-
# Use the entry with the lower score for details
|
| 87 |
-
primary = (left["entry"] if left and (right is None or (left_score or 4) <= (right_score or 4))
|
| 88 |
-
else right["entry"] if right else entries[0])
|
| 89 |
-
|
| 90 |
-
per_test.append({
|
| 91 |
-
"test_name": test_name,
|
| 92 |
-
"score": final_score,
|
| 93 |
-
"judge": primary["judge"],
|
| 94 |
-
"features": primary["features"],
|
| 95 |
-
"needs_human": primary["judge"].needs_human,
|
| 96 |
-
})
|
| 97 |
-
|
| 98 |
-
self._check_flags(primary, low_confidence_flags, disagreement_flags)
|
| 99 |
-
|
| 100 |
-
# Process unilateral tests
|
| 101 |
-
for entry in unilateral:
|
| 102 |
-
judge = entry["judge"]
|
| 103 |
-
per_test.append({
|
| 104 |
-
"test_name": entry["movement"].test_name,
|
| 105 |
-
"score": judge.score,
|
| 106 |
-
"judge": judge,
|
| 107 |
-
"features": entry["features"],
|
| 108 |
-
"needs_human": judge.needs_human,
|
| 109 |
-
})
|
| 110 |
-
self._check_flags(entry, low_confidence_flags, disagreement_flags)
|
| 111 |
-
|
| 112 |
-
# Composite — null if any test unscored
|
| 113 |
-
all_scores = [t["score"] for t in per_test]
|
| 114 |
-
composite = sum(all_scores) if all(s is not None for s in all_scores) else None
|
| 115 |
-
|
| 116 |
-
return ReportResult(
|
| 117 |
-
per_test=per_test,
|
| 118 |
-
composite=composite,
|
| 119 |
-
asymmetries=asymmetries,
|
| 120 |
-
overlay_video_path=None, # Phase 4
|
| 121 |
-
pdf_path=None, # Phase 4
|
| 122 |
-
low_confidence_flags=low_confidence_flags,
|
| 123 |
-
disagreement_flags=disagreement_flags,
|
| 124 |
-
)
|
| 125 |
-
|
| 126 |
-
def _check_flags(self, entry: dict, low_conf: list, disagree: list):
|
| 127 |
-
"""Check quality gates and populate flag lists."""
|
| 128 |
-
judge = entry["judge"]
|
| 129 |
-
rubric = entry["rubric_score"]
|
| 130 |
-
test_name = entry["movement"].test_name
|
| 131 |
-
|
| 132 |
-
if judge.confidence < config.MIN_CONFIDENCE:
|
| 133 |
-
low_conf.append(f"{test_name}: judge confidence {judge.confidence:.2f}")
|
| 134 |
-
|
| 135 |
-
if (judge.score is not None and rubric.score is not None
|
| 136 |
-
and abs(judge.score - rubric.score) >= config.SCORE_DISAGREE_THRESH):
|
| 137 |
-
disagree.append(
|
| 138 |
-
f"{test_name}: rubric={rubric.score} vs judge={judge.score}"
|
| 139 |
-
)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ReportAgent — assembles per-test scorecard, composite, asymmetries.
|
| 3 |
+
|
| 4 |
+
Input: List of (MovementResult, BiomechFeatures, ScoreResult, JudgeResult) per test
|
| 5 |
+
Output: ReportResult(per_test, composite, asymmetries, overlay_video_path, pdf_path)
|
| 6 |
+
Failure: returns ReportResult with composite=None if any test unscored.
|
| 7 |
+
Params: 0 (pure assembly — no model).
|
| 8 |
+
License: n/a.
|
| 9 |
+
Gated: no.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from formscout.types import (
|
| 14 |
+
MovementResult, BiomechFeatures, ScoreResult, JudgeResult, ReportResult,
|
| 15 |
+
)
|
| 16 |
+
from formscout import config
|
| 17 |
+
|
| 18 |
+
# Bilateral tests that need L/R scoring
|
| 19 |
+
BILATERAL_TESTS = {"hurdle_step", "inline_lunge", "shoulder_mobility", "active_slr"}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class ReportAgent:
|
| 23 |
+
"""Assembles the final screening report from all test results."""
|
| 24 |
+
|
| 25 |
+
def run(self, test_results: list[dict]) -> ReportResult:
|
| 26 |
+
"""
|
| 27 |
+
Assemble the report.
|
| 28 |
+
|
| 29 |
+
Args:
|
| 30 |
+
test_results: list of dicts with keys:
|
| 31 |
+
- movement: MovementResult
|
| 32 |
+
- features: BiomechFeatures
|
| 33 |
+
- rubric_score: ScoreResult
|
| 34 |
+
- judge: JudgeResult
|
| 35 |
+
- side: str (for bilateral: "left" or "right")
|
| 36 |
+
"""
|
| 37 |
+
per_test = []
|
| 38 |
+
asymmetries = []
|
| 39 |
+
low_confidence_flags = []
|
| 40 |
+
disagreement_flags = []
|
| 41 |
+
|
| 42 |
+
# Group bilateral tests by test_name
|
| 43 |
+
bilateral_groups: dict[str, list[dict]] = {}
|
| 44 |
+
unilateral: list[dict] = []
|
| 45 |
+
|
| 46 |
+
for entry in test_results:
|
| 47 |
+
test_name = entry["movement"].test_name
|
| 48 |
+
if test_name in BILATERAL_TESTS:
|
| 49 |
+
bilateral_groups.setdefault(test_name, []).append(entry)
|
| 50 |
+
else:
|
| 51 |
+
unilateral.append(entry)
|
| 52 |
+
|
| 53 |
+
# Process bilateral tests — take the lower score, emit asymmetry
|
| 54 |
+
for test_name, entries in bilateral_groups.items():
|
| 55 |
+
scores = []
|
| 56 |
+
for entry in entries:
|
| 57 |
+
judge = entry["judge"]
|
| 58 |
+
side = entry.get("side", entry["movement"].side)
|
| 59 |
+
score = judge.score if judge.score is not None else None
|
| 60 |
+
scores.append({"side": side, "score": score, "entry": entry})
|
| 61 |
+
|
| 62 |
+
# Find best entry per side
|
| 63 |
+
left = next((s for s in scores if s["side"] == "left"), None)
|
| 64 |
+
right = next((s for s in scores if s["side"] == "right"), None)
|
| 65 |
+
|
| 66 |
+
left_score = left["score"] if left else None
|
| 67 |
+
right_score = right["score"] if right else None
|
| 68 |
+
|
| 69 |
+
# Report lower
|
| 70 |
+
if left_score is not None and right_score is not None:
|
| 71 |
+
final_score = min(left_score, right_score)
|
| 72 |
+
delta = abs(left_score - right_score)
|
| 73 |
+
asymmetries.append({
|
| 74 |
+
"test": test_name,
|
| 75 |
+
"left_score": left_score,
|
| 76 |
+
"right_score": right_score,
|
| 77 |
+
"delta": delta,
|
| 78 |
+
})
|
| 79 |
+
elif left_score is not None:
|
| 80 |
+
final_score = left_score
|
| 81 |
+
elif right_score is not None:
|
| 82 |
+
final_score = right_score
|
| 83 |
+
else:
|
| 84 |
+
final_score = None
|
| 85 |
+
|
| 86 |
+
# Use the entry with the lower score for details
|
| 87 |
+
primary = (left["entry"] if left and (right is None or (left_score or 4) <= (right_score or 4))
|
| 88 |
+
else right["entry"] if right else entries[0])
|
| 89 |
+
|
| 90 |
+
per_test.append({
|
| 91 |
+
"test_name": test_name,
|
| 92 |
+
"score": final_score,
|
| 93 |
+
"judge": primary["judge"],
|
| 94 |
+
"features": primary["features"],
|
| 95 |
+
"needs_human": primary["judge"].needs_human,
|
| 96 |
+
})
|
| 97 |
+
|
| 98 |
+
self._check_flags(primary, low_confidence_flags, disagreement_flags)
|
| 99 |
+
|
| 100 |
+
# Process unilateral tests
|
| 101 |
+
for entry in unilateral:
|
| 102 |
+
judge = entry["judge"]
|
| 103 |
+
per_test.append({
|
| 104 |
+
"test_name": entry["movement"].test_name,
|
| 105 |
+
"score": judge.score,
|
| 106 |
+
"judge": judge,
|
| 107 |
+
"features": entry["features"],
|
| 108 |
+
"needs_human": judge.needs_human,
|
| 109 |
+
})
|
| 110 |
+
self._check_flags(entry, low_confidence_flags, disagreement_flags)
|
| 111 |
+
|
| 112 |
+
# Composite — null if any test unscored
|
| 113 |
+
all_scores = [t["score"] for t in per_test]
|
| 114 |
+
composite = sum(all_scores) if all(s is not None for s in all_scores) else None
|
| 115 |
+
|
| 116 |
+
return ReportResult(
|
| 117 |
+
per_test=per_test,
|
| 118 |
+
composite=composite,
|
| 119 |
+
asymmetries=asymmetries,
|
| 120 |
+
overlay_video_path=None, # Phase 4
|
| 121 |
+
pdf_path=None, # Phase 4
|
| 122 |
+
low_confidence_flags=low_confidence_flags,
|
| 123 |
+
disagreement_flags=disagreement_flags,
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
def _check_flags(self, entry: dict, low_conf: list, disagree: list):
|
| 127 |
+
"""Check quality gates and populate flag lists."""
|
| 128 |
+
judge = entry["judge"]
|
| 129 |
+
rubric = entry["rubric_score"]
|
| 130 |
+
test_name = entry["movement"].test_name
|
| 131 |
+
|
| 132 |
+
if judge.confidence < config.MIN_CONFIDENCE:
|
| 133 |
+
low_conf.append(f"{test_name}: judge confidence {judge.confidence:.2f}")
|
| 134 |
+
|
| 135 |
+
if (judge.score is not None and rubric.score is not None
|
| 136 |
+
and abs(judge.score - rubric.score) >= config.SCORE_DISAGREE_THRESH):
|
| 137 |
+
disagree.append(
|
| 138 |
+
f"{test_name}: rubric={rubric.score} vs judge={judge.score}"
|
| 139 |
+
)
|
formscout/agents/visualizer.py
CHANGED
|
@@ -1,435 +1,418 @@
|
|
| 1 |
-
"""
|
| 2 |
-
PoseVisualizer — annotated overlay video with skeleton, trails, velocity arrows.
|
| 3 |
-
|
| 4 |
-
Input: IngestResult + Pose2DResult
|
| 5 |
-
Output: .mp4 path (or None on failure/empty layers)
|
| 6 |
-
Failure: returns None, never raises.
|
| 7 |
-
"""
|
| 8 |
-
from __future__ import annotations
|
| 9 |
-
|
| 10 |
-
import colorsys
|
| 11 |
-
import logging
|
| 12 |
-
import math
|
| 13 |
-
import tempfile
|
| 14 |
-
from collections import deque
|
| 15 |
-
|
| 16 |
-
import cv2
|
| 17 |
-
import numpy as np
|
| 18 |
-
|
| 19 |
-
logger = logging.getLogger(__name__)
|
| 20 |
-
|
| 21 |
-
# ── COCO constants ────────────────────────────────────────────────────────────
|
| 22 |
-
|
| 23 |
-
COCO_KEYPOINTS = [
|
| 24 |
-
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 25 |
-
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 26 |
-
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 27 |
-
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 28 |
-
]
|
| 29 |
-
|
| 30 |
-
COCO_SKELETON = [
|
| 31 |
-
(0, 1), (0, 2), (1, 3), (2, 4), # face
|
| 32 |
-
(5, 6), (5, 7), (7, 9), (6, 8), (8, 10), # arms
|
| 33 |
-
(5, 11), (6, 12), (11, 12), # torso
|
| 34 |
-
(11, 13), (13, 15), (12, 14), (14, 16), # legs
|
| 35 |
-
]
|
| 36 |
-
|
| 37 |
-
TRAIL_LENGTH = 10
|
| 38 |
-
MAX_ARROW_PX = 40
|
| 39 |
-
CONF_THRESHOLD = 0.3
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
# ── Kalman filter ─────────────────────────────────────────────────────────────
|
| 43 |
-
|
| 44 |
-
class SimpleKalmanFilter:
|
| 45 |
-
"""4-state Kalman filter (x, y, vx, vy) for joint tracking."""
|
| 46 |
-
|
| 47 |
-
def __init__(self, process_noise: float = 0.01, measurement_noise: float = 0.1):
|
| 48 |
-
self.is_initialized = False
|
| 49 |
-
self.state = np.zeros(4)
|
| 50 |
-
self.cov = np.eye(4) * 0.1
|
| 51 |
-
self.Q = np.eye(4) * process_noise
|
| 52 |
-
self.R = np.eye(2) * measurement_noise
|
| 53 |
-
self.H = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], dtype=float)
|
| 54 |
-
|
| 55 |
-
def predict(self, dt: float = 1.0):
|
| 56 |
-
F = np.array([[1, 0, dt, 0], [0, 1, 0, dt], [0, 0, 1, 0], [0, 0, 0, 1]], dtype=float)
|
| 57 |
-
self.state = F @ self.state
|
| 58 |
-
self.cov = F @ self.cov @ F.T + self.Q
|
| 59 |
-
|
| 60 |
-
def update(self, x: float, y: float):
|
| 61 |
-
z = np.array([x, y])
|
| 62 |
-
if not self.is_initialized:
|
| 63 |
-
self.state[:2] = z
|
| 64 |
-
self.is_initialized = True
|
| 65 |
-
return
|
| 66 |
-
S = self.H @ self.cov @ self.H.T + self.R
|
| 67 |
-
K = self.cov @ self.H.T @ np.linalg.inv(S)
|
| 68 |
-
self.state = self.state + K @ (z - self.H @ self.state)
|
| 69 |
-
self.cov = (np.eye(4) - K @ self.H) @ self.cov
|
| 70 |
-
|
| 71 |
-
def velocity_magnitude(self) -> float:
|
| 72 |
-
vx, vy = self.state[2], self.state[3]
|
| 73 |
-
return math.sqrt(vx * vx + vy * vy)
|
| 74 |
-
|
| 75 |
-
def velocity_vector(self) -> tuple[float, float]:
|
| 76 |
-
return float(self.state[2]), float(self.state[3])
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
# ── Velocity computation ──────────────────────────────────────────────────────
|
| 80 |
-
|
| 81 |
-
def compute_joint_velocity(
|
| 82 |
-
keypoints_per_frame: list[dict],
|
| 83 |
-
fps: float,
|
| 84 |
-
) -> dict[int, list[float]]:
|
| 85 |
-
"""
|
| 86 |
-
Compute Kalman-filtered per-joint speed (px/s) for each frame.
|
| 87 |
-
|
| 88 |
-
Returns dict[joint_idx, [speed_frame0, ...]] for all 17 COCO joints.
|
| 89 |
-
Missing/low-confidence keypoints yield speed=0.0 for that frame.
|
| 90 |
-
"""
|
| 91 |
-
dt = 1.0 / fps if fps > 0 else 1.0
|
| 92 |
-
filters: dict[int, SimpleKalmanFilter] = {j: SimpleKalmanFilter() for j in range(17)}
|
| 93 |
-
result: dict[int, list[float]] = {j: [] for j in range(17)}
|
| 94 |
-
|
| 95 |
-
for frame_kps in keypoints_per_frame:
|
| 96 |
-
for j in range(17):
|
| 97 |
-
kf = filters[j]
|
| 98 |
-
kp = frame_kps.get(j)
|
| 99 |
-
kf.predict(dt)
|
| 100 |
-
if kp and kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 101 |
-
kf.update(kp["x"], kp["y"])
|
| 102 |
-
speed = kf.velocity_magnitude()
|
| 103 |
-
else:
|
| 104 |
-
speed = 0.0
|
| 105 |
-
result[j].append(speed)
|
| 106 |
-
|
| 107 |
-
return result
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
# ── Helpers ───────────────────────────────────────────────────────────────────
|
| 111 |
-
|
| 112 |
-
def _conf_to_bgr(conf: float) -> tuple[int, int, int]:
|
| 113 |
-
"""Map confidence 0→1 to BGR color red→green via HSV."""
|
| 114 |
-
hue = conf * 120.0 / 360.0
|
| 115 |
-
r, g, b = colorsys.hsv_to_rgb(hue, 1.0, 1.0)
|
| 116 |
-
return (int(b * 255), int(g * 255), int(r * 255))
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
start
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
pose2d
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
#
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
if
|
| 300 |
-
out_frame =
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
if
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
)
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
continue
|
| 420 |
-
|
| 421 |
-
avg_speed = sum(speeds) / len(speeds)
|
| 422 |
-
peak_speed = max(speeds)
|
| 423 |
-
rows.append((COCO_KEYPOINTS[j], avg_speed, peak_speed))
|
| 424 |
-
|
| 425 |
-
if not rows:
|
| 426 |
-
return ""
|
| 427 |
-
|
| 428 |
-
rows.sort(key=lambda r: r[2], reverse=True)
|
| 429 |
-
lines = [
|
| 430 |
-
"| Joint | Avg (px/s) | Peak (px/s) |",
|
| 431 |
-
"|---|---|---|",
|
| 432 |
-
]
|
| 433 |
-
for name, avg, peak in rows:
|
| 434 |
-
lines.append(f"| {name} | {avg:.1f} | {peak:.1f} |")
|
| 435 |
-
return "\n".join(lines)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
PoseVisualizer — annotated overlay video with skeleton, trails, velocity arrows.
|
| 3 |
+
|
| 4 |
+
Input: IngestResult + Pose2DResult
|
| 5 |
+
Output: .mp4 path (or None on failure/empty layers)
|
| 6 |
+
Failure: returns None, never raises.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import colorsys
|
| 11 |
+
import logging
|
| 12 |
+
import math
|
| 13 |
+
import tempfile
|
| 14 |
+
from collections import deque
|
| 15 |
+
|
| 16 |
+
import cv2
|
| 17 |
+
import numpy as np
|
| 18 |
+
|
| 19 |
+
logger = logging.getLogger(__name__)
|
| 20 |
+
|
| 21 |
+
# ── COCO constants ────────────────────────────────────────────────────────────
|
| 22 |
+
|
| 23 |
+
COCO_KEYPOINTS = [
|
| 24 |
+
"nose", "left_eye", "right_eye", "left_ear", "right_ear",
|
| 25 |
+
"left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
|
| 26 |
+
"left_wrist", "right_wrist", "left_hip", "right_hip",
|
| 27 |
+
"left_knee", "right_knee", "left_ankle", "right_ankle",
|
| 28 |
+
]
|
| 29 |
+
|
| 30 |
+
COCO_SKELETON = [
|
| 31 |
+
(0, 1), (0, 2), (1, 3), (2, 4), # face
|
| 32 |
+
(5, 6), (5, 7), (7, 9), (6, 8), (8, 10), # arms
|
| 33 |
+
(5, 11), (6, 12), (11, 12), # torso
|
| 34 |
+
(11, 13), (13, 15), (12, 14), (14, 16), # legs
|
| 35 |
+
]
|
| 36 |
+
|
| 37 |
+
TRAIL_LENGTH = 10
|
| 38 |
+
MAX_ARROW_PX = 40
|
| 39 |
+
CONF_THRESHOLD = 0.3
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
# ── Kalman filter ─────────────────────────────────────────────────────────────
|
| 43 |
+
|
| 44 |
+
class SimpleKalmanFilter:
|
| 45 |
+
"""4-state Kalman filter (x, y, vx, vy) for joint tracking."""
|
| 46 |
+
|
| 47 |
+
def __init__(self, process_noise: float = 0.01, measurement_noise: float = 0.1):
|
| 48 |
+
self.is_initialized = False
|
| 49 |
+
self.state = np.zeros(4)
|
| 50 |
+
self.cov = np.eye(4) * 0.1
|
| 51 |
+
self.Q = np.eye(4) * process_noise
|
| 52 |
+
self.R = np.eye(2) * measurement_noise
|
| 53 |
+
self.H = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], dtype=float)
|
| 54 |
+
|
| 55 |
+
def predict(self, dt: float = 1.0):
|
| 56 |
+
F = np.array([[1, 0, dt, 0], [0, 1, 0, dt], [0, 0, 1, 0], [0, 0, 0, 1]], dtype=float)
|
| 57 |
+
self.state = F @ self.state
|
| 58 |
+
self.cov = F @ self.cov @ F.T + self.Q
|
| 59 |
+
|
| 60 |
+
def update(self, x: float, y: float):
|
| 61 |
+
z = np.array([x, y])
|
| 62 |
+
if not self.is_initialized:
|
| 63 |
+
self.state[:2] = z
|
| 64 |
+
self.is_initialized = True
|
| 65 |
+
return
|
| 66 |
+
S = self.H @ self.cov @ self.H.T + self.R
|
| 67 |
+
K = self.cov @ self.H.T @ np.linalg.inv(S)
|
| 68 |
+
self.state = self.state + K @ (z - self.H @ self.state)
|
| 69 |
+
self.cov = (np.eye(4) - K @ self.H) @ self.cov
|
| 70 |
+
|
| 71 |
+
def velocity_magnitude(self) -> float:
|
| 72 |
+
vx, vy = self.state[2], self.state[3]
|
| 73 |
+
return math.sqrt(vx * vx + vy * vy)
|
| 74 |
+
|
| 75 |
+
def velocity_vector(self) -> tuple[float, float]:
|
| 76 |
+
return float(self.state[2]), float(self.state[3])
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
# ── Velocity computation ──────────────────────────────────────────────────────
|
| 80 |
+
|
| 81 |
+
def compute_joint_velocity(
|
| 82 |
+
keypoints_per_frame: list[dict],
|
| 83 |
+
fps: float,
|
| 84 |
+
) -> dict[int, list[float]]:
|
| 85 |
+
"""
|
| 86 |
+
Compute Kalman-filtered per-joint speed (px/s) for each frame.
|
| 87 |
+
|
| 88 |
+
Returns dict[joint_idx, [speed_frame0, ...]] for all 17 COCO joints.
|
| 89 |
+
Missing/low-confidence keypoints yield speed=0.0 for that frame.
|
| 90 |
+
"""
|
| 91 |
+
dt = 1.0 / fps if fps > 0 else 1.0
|
| 92 |
+
filters: dict[int, SimpleKalmanFilter] = {j: SimpleKalmanFilter() for j in range(17)}
|
| 93 |
+
result: dict[int, list[float]] = {j: [] for j in range(17)}
|
| 94 |
+
|
| 95 |
+
for frame_kps in keypoints_per_frame:
|
| 96 |
+
for j in range(17):
|
| 97 |
+
kf = filters[j]
|
| 98 |
+
kp = frame_kps.get(j)
|
| 99 |
+
kf.predict(dt)
|
| 100 |
+
if kp and kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 101 |
+
kf.update(kp["x"], kp["y"])
|
| 102 |
+
speed = kf.velocity_magnitude()
|
| 103 |
+
else:
|
| 104 |
+
speed = 0.0
|
| 105 |
+
result[j].append(speed)
|
| 106 |
+
|
| 107 |
+
return result
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
# ── Helpers ───────────────────────────────────────────────────────────────────
|
| 111 |
+
|
| 112 |
+
def _conf_to_bgr(conf: float) -> tuple[int, int, int]:
|
| 113 |
+
"""Map confidence 0→1 to BGR color red→green via HSV."""
|
| 114 |
+
hue = conf * 120.0 / 360.0
|
| 115 |
+
r, g, b = colorsys.hsv_to_rgb(hue, 1.0, 1.0)
|
| 116 |
+
return (int(b * 255), int(g * 255), int(r * 255))
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
# ── PoseVisualizer ────────────────────────────────────────────────────────────
|
| 120 |
+
|
| 121 |
+
class PoseVisualizer:
|
| 122 |
+
"""Renders skeleton, trails, and velocity arrows onto video frames."""
|
| 123 |
+
|
| 124 |
+
def __init__(self):
|
| 125 |
+
self.last_velocities: dict[int, list[float]] = {}
|
| 126 |
+
|
| 127 |
+
# ── Skeleton ──────────────────────────────────────────────────────────────
|
| 128 |
+
|
| 129 |
+
def _draw_skeleton(self, frame: np.ndarray, kps: dict) -> np.ndarray:
|
| 130 |
+
"""Draw COCO-17 bones (white) and joints (confidence-colored) onto frame."""
|
| 131 |
+
visible = {j: kp for j, kp in kps.items() if kp.get("conf", 0.0) >= CONF_THRESHOLD}
|
| 132 |
+
|
| 133 |
+
# Bones
|
| 134 |
+
for j1, j2 in COCO_SKELETON:
|
| 135 |
+
if j1 in visible and j2 in visible:
|
| 136 |
+
p1 = (int(visible[j1]["x"]), int(visible[j1]["y"]))
|
| 137 |
+
p2 = (int(visible[j2]["x"]), int(visible[j2]["y"]))
|
| 138 |
+
cv2.line(frame, p1, p2, (255, 255, 255), 2)
|
| 139 |
+
|
| 140 |
+
# Joints
|
| 141 |
+
for j, kp in visible.items():
|
| 142 |
+
pt = (int(kp["x"]), int(kp["y"]))
|
| 143 |
+
color = _conf_to_bgr(kp["conf"])
|
| 144 |
+
cv2.circle(frame, pt, 4, color, -1)
|
| 145 |
+
cv2.circle(frame, pt, 5, (255, 255, 255), 1)
|
| 146 |
+
|
| 147 |
+
return frame
|
| 148 |
+
|
| 149 |
+
# ── Trails ───────────────────────────────────────────────────────────────
|
| 150 |
+
|
| 151 |
+
def _draw_trails(self, frame: np.ndarray, trail_history: dict) -> np.ndarray:
|
| 152 |
+
"""Draw fading motion trails for each joint."""
|
| 153 |
+
for joint_idx, trail in trail_history.items():
|
| 154 |
+
pts = list(trail)
|
| 155 |
+
if len(pts) < 2:
|
| 156 |
+
continue
|
| 157 |
+
for i in range(1, len(pts)):
|
| 158 |
+
alpha = i / len(pts)
|
| 159 |
+
brightness = int(255 * alpha)
|
| 160 |
+
color = (brightness, brightness, brightness)
|
| 161 |
+
thickness = max(1, int(3 * alpha))
|
| 162 |
+
p1 = (int(pts[i - 1][0]), int(pts[i - 1][1]))
|
| 163 |
+
p2 = (int(pts[i][0]), int(pts[i][1]))
|
| 164 |
+
cv2.line(frame, p1, p2, color, thickness)
|
| 165 |
+
return frame
|
| 166 |
+
|
| 167 |
+
# ── Velocity arrows ───────────────────────────────────────────────────────
|
| 168 |
+
|
| 169 |
+
def _draw_velocity_arrows(
|
| 170 |
+
self,
|
| 171 |
+
frame: np.ndarray,
|
| 172 |
+
kps: dict,
|
| 173 |
+
prev_kps: dict | None,
|
| 174 |
+
velocities: dict[int, list[float]],
|
| 175 |
+
frame_idx: int,
|
| 176 |
+
) -> np.ndarray:
|
| 177 |
+
"""Draw per-joint velocity arrows scaled by speed."""
|
| 178 |
+
if prev_kps is None:
|
| 179 |
+
return frame
|
| 180 |
+
|
| 181 |
+
all_speeds = [velocities[j][frame_idx] for j in range(17) if frame_idx < len(velocities.get(j, []))]
|
| 182 |
+
peak = max(all_speeds) if all_speeds else 1.0
|
| 183 |
+
if peak == 0.0:
|
| 184 |
+
return frame
|
| 185 |
+
|
| 186 |
+
for j in range(17):
|
| 187 |
+
kp = kps.get(j)
|
| 188 |
+
pk = prev_kps.get(j)
|
| 189 |
+
if not kp or not pk:
|
| 190 |
+
continue
|
| 191 |
+
if kp.get("conf", 0.0) < CONF_THRESHOLD:
|
| 192 |
+
continue
|
| 193 |
+
speeds = velocities.get(j, [])
|
| 194 |
+
if frame_idx >= len(speeds):
|
| 195 |
+
continue
|
| 196 |
+
speed = speeds[frame_idx]
|
| 197 |
+
if speed == 0.0:
|
| 198 |
+
continue
|
| 199 |
+
|
| 200 |
+
dx = kp["x"] - pk["x"]
|
| 201 |
+
dy = kp["y"] - pk["y"]
|
| 202 |
+
mag = math.sqrt(dx * dx + dy * dy)
|
| 203 |
+
if mag < 1e-6:
|
| 204 |
+
continue
|
| 205 |
+
|
| 206 |
+
length = min(speed / peak * MAX_ARROW_PX, MAX_ARROW_PX)
|
| 207 |
+
nx, ny = dx / mag, dy / mag
|
| 208 |
+
start = (int(kp["x"]), int(kp["y"]))
|
| 209 |
+
end = (int(kp["x"] + nx * length), int(kp["y"] + ny * length))
|
| 210 |
+
|
| 211 |
+
ratio = speed / peak
|
| 212 |
+
if ratio < 0.33:
|
| 213 |
+
color = (0, 200, 0) # green
|
| 214 |
+
elif ratio < 0.66:
|
| 215 |
+
color = (0, 140, 255) # orange
|
| 216 |
+
else:
|
| 217 |
+
color = (0, 0, 255) # red
|
| 218 |
+
|
| 219 |
+
cv2.arrowedLine(frame, start, end, color, 2, tipLength=0.35)
|
| 220 |
+
|
| 221 |
+
return frame
|
| 222 |
+
|
| 223 |
+
# ── Public ────────────────────────────────────────────────────────────────
|
| 224 |
+
|
| 225 |
+
def render_video(
|
| 226 |
+
self,
|
| 227 |
+
ingest,
|
| 228 |
+
pose2d,
|
| 229 |
+
layers: set[str],
|
| 230 |
+
output_path: str,
|
| 231 |
+
) -> str | None:
|
| 232 |
+
"""
|
| 233 |
+
Render annotated video. Returns output_path on success, None otherwise.
|
| 234 |
+
layers: subset of {"skeleton", "trails", "velocity_arrows"}
|
| 235 |
+
"""
|
| 236 |
+
if not layers:
|
| 237 |
+
return None
|
| 238 |
+
|
| 239 |
+
if not any(pose2d.keypoints):
|
| 240 |
+
return None
|
| 241 |
+
|
| 242 |
+
try:
|
| 243 |
+
velocities = compute_joint_velocity(pose2d.keypoints, ingest.fps)
|
| 244 |
+
self.last_velocities = velocities
|
| 245 |
+
|
| 246 |
+
frames = ingest.frames
|
| 247 |
+
orig_h, orig_w = frames[0].shape[:2]
|
| 248 |
+
fps = ingest.fps or 30.0
|
| 249 |
+
|
| 250 |
+
# Cap at 1280px wide — big frames are slow and don't need to be HQ
|
| 251 |
+
max_w = 1280
|
| 252 |
+
if orig_w > max_w:
|
| 253 |
+
scale = max_w / orig_w
|
| 254 |
+
out_w = max_w
|
| 255 |
+
out_h = int(orig_h * scale)
|
| 256 |
+
else:
|
| 257 |
+
scale = 1.0
|
| 258 |
+
out_w, out_h = orig_w, orig_h
|
| 259 |
+
|
| 260 |
+
# Scale keypoint coordinates to match resized frames
|
| 261 |
+
def _scale_kps(kps: dict) -> dict:
|
| 262 |
+
if scale == 1.0:
|
| 263 |
+
return kps
|
| 264 |
+
return {
|
| 265 |
+
j: {**kp, "x": kp["x"] * scale, "y": kp["y"] * scale}
|
| 266 |
+
for j, kp in kps.items()
|
| 267 |
+
}
|
| 268 |
+
|
| 269 |
+
scaled_keypoints = [_scale_kps(k) for k in pose2d.keypoints]
|
| 270 |
+
|
| 271 |
+
# Write raw mp4v to a temp file, then remux with ffmpeg faststart
|
| 272 |
+
import subprocess
|
| 273 |
+
import tempfile as _tf
|
| 274 |
+
tmp = _tf.NamedTemporaryFile(suffix="_raw.mp4", delete=False)
|
| 275 |
+
tmp_path = tmp.name
|
| 276 |
+
tmp.close()
|
| 277 |
+
|
| 278 |
+
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
|
| 279 |
+
writer = cv2.VideoWriter(tmp_path, fourcc, fps, (out_w, out_h))
|
| 280 |
+
if not writer.isOpened():
|
| 281 |
+
logger.warning("VideoWriter failed to open: %s", tmp_path)
|
| 282 |
+
return None
|
| 283 |
+
|
| 284 |
+
trail_history: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
|
| 285 |
+
prev_kps: dict | None = None
|
| 286 |
+
|
| 287 |
+
for frame_idx, (frame, kps) in enumerate(zip(frames, scaled_keypoints)):
|
| 288 |
+
if scale != 1.0:
|
| 289 |
+
out_frame = cv2.resize(frame, (out_w, out_h), interpolation=cv2.INTER_AREA)
|
| 290 |
+
else:
|
| 291 |
+
out_frame = frame.copy()
|
| 292 |
+
|
| 293 |
+
if "trails" in layers:
|
| 294 |
+
for j, kp in kps.items():
|
| 295 |
+
if kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 296 |
+
trail_history[j].append((kp["x"], kp["y"]))
|
| 297 |
+
out_frame = self._draw_trails(out_frame, trail_history)
|
| 298 |
+
|
| 299 |
+
if "skeleton" in layers:
|
| 300 |
+
out_frame = self._draw_skeleton(out_frame, kps)
|
| 301 |
+
|
| 302 |
+
if "velocity_arrows" in layers:
|
| 303 |
+
out_frame = self._draw_velocity_arrows(
|
| 304 |
+
out_frame, kps, prev_kps, velocities, frame_idx
|
| 305 |
+
)
|
| 306 |
+
|
| 307 |
+
writer.write(out_frame)
|
| 308 |
+
prev_kps = kps
|
| 309 |
+
|
| 310 |
+
writer.release()
|
| 311 |
+
|
| 312 |
+
# Remux with faststart so browsers can seek without downloading the whole file
|
| 313 |
+
try:
|
| 314 |
+
subprocess.run(
|
| 315 |
+
["ffmpeg", "-y", "-i", tmp_path, "-c", "copy",
|
| 316 |
+
"-movflags", "+faststart", output_path],
|
| 317 |
+
check=True, capture_output=True,
|
| 318 |
+
)
|
| 319 |
+
import os
|
| 320 |
+
os.unlink(tmp_path)
|
| 321 |
+
except Exception as ffmpeg_err:
|
| 322 |
+
logger.warning("ffmpeg remux failed (%s) — using raw mp4v", ffmpeg_err)
|
| 323 |
+
import shutil
|
| 324 |
+
shutil.move(tmp_path, output_path)
|
| 325 |
+
|
| 326 |
+
return output_path
|
| 327 |
+
|
| 328 |
+
except Exception as e:
|
| 329 |
+
logger.warning("render_video failed: %s", e)
|
| 330 |
+
return None
|
| 331 |
+
|
| 332 |
+
def render_frame(
|
| 333 |
+
self,
|
| 334 |
+
ingest,
|
| 335 |
+
pose2d,
|
| 336 |
+
frame_idx: int,
|
| 337 |
+
layers: set[str],
|
| 338 |
+
caption: str = "",
|
| 339 |
+
out_png: str | None = None,
|
| 340 |
+
) -> str | None:
|
| 341 |
+
"""Render a single annotated still (skeleton + optional trails + caption).
|
| 342 |
+
|
| 343 |
+
frame_idx is typically the governing frame from BiomechFeatures.timing.
|
| 344 |
+
Returns the PNG path on success, None on any failure. Never raises.
|
| 345 |
+
"""
|
| 346 |
+
try:
|
| 347 |
+
if not (0 <= frame_idx < len(ingest.frames)) or frame_idx >= len(pose2d.keypoints):
|
| 348 |
+
return None
|
| 349 |
+
|
| 350 |
+
frame = ingest.frames[frame_idx].copy()
|
| 351 |
+
kps = pose2d.keypoints[frame_idx]
|
| 352 |
+
|
| 353 |
+
if "trails" in layers:
|
| 354 |
+
trail: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
|
| 355 |
+
start = max(0, frame_idx - TRAIL_LENGTH)
|
| 356 |
+
for fi in range(start, frame_idx + 1):
|
| 357 |
+
for j, kp in pose2d.keypoints[fi].items():
|
| 358 |
+
if kp.get("conf", 0.0) >= CONF_THRESHOLD:
|
| 359 |
+
trail[j].append((kp["x"], kp["y"]))
|
| 360 |
+
frame = self._draw_trails(frame, trail)
|
| 361 |
+
|
| 362 |
+
if "skeleton" in layers:
|
| 363 |
+
frame = self._draw_skeleton(frame, kps)
|
| 364 |
+
|
| 365 |
+
if caption:
|
| 366 |
+
cv2.rectangle(frame, (0, 0), (frame.shape[1], 28), (0, 0, 0), -1)
|
| 367 |
+
cv2.putText(frame, caption[:80], (8, 20), cv2.FONT_HERSHEY_SIMPLEX,
|
| 368 |
+
0.55, (255, 255, 255), 1, cv2.LINE_AA)
|
| 369 |
+
|
| 370 |
+
if out_png is None:
|
| 371 |
+
out_png = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
|
| 372 |
+
|
| 373 |
+
ok = cv2.imwrite(out_png, frame)
|
| 374 |
+
return out_png if ok else None
|
| 375 |
+
except Exception as e:
|
| 376 |
+
logger.warning("render_frame failed: %s", e)
|
| 377 |
+
return None
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
# ── Velocity summary ──────────────────────────────────────────────────────────
|
| 381 |
+
|
| 382 |
+
def build_velocity_summary(
|
| 383 |
+
keypoints_per_frame: list[dict],
|
| 384 |
+
velocities: dict[int, list[float]],
|
| 385 |
+
) -> str:
|
| 386 |
+
"""Return markdown table of per-joint avg/peak velocity. Empty string if no valid joints."""
|
| 387 |
+
n_frames = len(keypoints_per_frame)
|
| 388 |
+
if n_frames == 0:
|
| 389 |
+
return ""
|
| 390 |
+
|
| 391 |
+
rows = []
|
| 392 |
+
for j in range(17):
|
| 393 |
+
detected = sum(
|
| 394 |
+
1 for kps in keypoints_per_frame
|
| 395 |
+
if kps.get(j, {}).get("conf", 0.0) >= CONF_THRESHOLD
|
| 396 |
+
)
|
| 397 |
+
if detected < n_frames * 0.5:
|
| 398 |
+
continue
|
| 399 |
+
|
| 400 |
+
speeds = velocities.get(j, [])
|
| 401 |
+
if not speeds:
|
| 402 |
+
continue
|
| 403 |
+
|
| 404 |
+
avg_speed = sum(speeds) / len(speeds)
|
| 405 |
+
peak_speed = max(speeds)
|
| 406 |
+
rows.append((COCO_KEYPOINTS[j], avg_speed, peak_speed))
|
| 407 |
+
|
| 408 |
+
if not rows:
|
| 409 |
+
return ""
|
| 410 |
+
|
| 411 |
+
rows.sort(key=lambda r: r[2], reverse=True)
|
| 412 |
+
lines = [
|
| 413 |
+
"| Joint | Avg (px/s) | Peak (px/s) |",
|
| 414 |
+
"|---|---|---|",
|
| 415 |
+
]
|
| 416 |
+
for name, avg, peak in rows:
|
| 417 |
+
lines.append(f"| {name} | {avg:.1f} | {peak:.1f} |")
|
| 418 |
+
return "\n".join(lines)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
formscout/analysis/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""FormScout movement-analysis engine: relevant joints, time series, Laban, charts."""
|
formscout/analysis/charts.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Matplotlib chart generators for the screening report.
|
| 3 |
+
|
| 4 |
+
Every function returns a PNG path on success or None on failure (never raises),
|
| 5 |
+
so a chart problem degrades the report but never blocks scoring. Charts use the
|
| 6 |
+
Silas palette and an Agg backend so they render headless on the Space.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import logging
|
| 11 |
+
|
| 12 |
+
import matplotlib
|
| 13 |
+
|
| 14 |
+
matplotlib.use("Agg")
|
| 15 |
+
import matplotlib.pyplot as plt # noqa: E402
|
| 16 |
+
import numpy as np # noqa: E402
|
| 17 |
+
|
| 18 |
+
from formscout.analysis.relevant_joints import COCO_NAMES # noqa: E402
|
| 19 |
+
|
| 20 |
+
logger = logging.getLogger(__name__)
|
| 21 |
+
|
| 22 |
+
TEAL = "#2b8a8a"
|
| 23 |
+
GOLD = "#e0a43b"
|
| 24 |
+
SAGE = "#9cbcad"
|
| 25 |
+
INK = "#243a34"
|
| 26 |
+
RED = "#d9534f"
|
| 27 |
+
_PALETTE = [TEAL, GOLD, SAGE, "#7a5ca0", "#c2683c", "#3c8dbc"]
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _save(fig, out_png: str) -> str | None:
|
| 31 |
+
try:
|
| 32 |
+
fig.savefig(out_png, dpi=110, bbox_inches="tight", facecolor="white")
|
| 33 |
+
return out_png
|
| 34 |
+
except Exception as e:
|
| 35 |
+
logger.warning("chart save failed: %s", e)
|
| 36 |
+
return None
|
| 37 |
+
finally:
|
| 38 |
+
plt.close(fig)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def angle_over_time(series: dict, primary: str | None, governing_idx: int | None,
|
| 42 |
+
out_png: str, title: str = "Joint angle over time") -> str | None:
|
| 43 |
+
"""Angle-vs-frame for the relevant angles; primary emphasised, key-frame marked."""
|
| 44 |
+
try:
|
| 45 |
+
if not series:
|
| 46 |
+
return None
|
| 47 |
+
fig, ax = plt.subplots(figsize=(6.4, 3.2))
|
| 48 |
+
for i, (name, vals) in enumerate(series.items()):
|
| 49 |
+
arr = np.array(vals, dtype=float)
|
| 50 |
+
is_primary = name == primary
|
| 51 |
+
ax.plot(np.arange(len(arr)), arr,
|
| 52 |
+
color=(TEAL if is_primary else _PALETTE[i % len(_PALETTE)]),
|
| 53 |
+
lw=2.4 if is_primary else 1.3,
|
| 54 |
+
alpha=1.0 if is_primary else 0.6,
|
| 55 |
+
label=name.replace("_", " ") + (" ★" if is_primary else ""))
|
| 56 |
+
if governing_idx is not None:
|
| 57 |
+
ax.axvline(governing_idx, color=GOLD, ls="--", lw=1.5, label="key frame")
|
| 58 |
+
ax.set_xlabel("frame")
|
| 59 |
+
ax.set_ylabel("degrees")
|
| 60 |
+
ax.set_title(title, color=INK)
|
| 61 |
+
ax.legend(fontsize=7, loc="best")
|
| 62 |
+
ax.grid(True, alpha=0.2)
|
| 63 |
+
return _save(fig, out_png)
|
| 64 |
+
except Exception as e:
|
| 65 |
+
logger.warning("angle_over_time failed: %s", e)
|
| 66 |
+
return None
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def velocity_profile(keypoints: list, fps: float, joints: list[int],
|
| 70 |
+
out_png: str, title: str = "Joint speed over time") -> str | None:
|
| 71 |
+
"""Per-frame speed (px/s) of the relevant joints across the clip."""
|
| 72 |
+
try:
|
| 73 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 74 |
+
vel = compute_joint_velocity(keypoints, fps or 30.0)
|
| 75 |
+
plot_joints = [j for j in joints if j in vel] or list(vel.keys())[:4]
|
| 76 |
+
if not plot_joints:
|
| 77 |
+
return None
|
| 78 |
+
fig, ax = plt.subplots(figsize=(6.4, 3.2))
|
| 79 |
+
for i, j in enumerate(plot_joints):
|
| 80 |
+
ax.plot(vel[j], color=_PALETTE[i % len(_PALETTE)], lw=1.6,
|
| 81 |
+
label=COCO_NAMES.get(j, str(j)).replace("_", " "))
|
| 82 |
+
ax.set_xlabel("frame")
|
| 83 |
+
ax.set_ylabel("speed (px/s)")
|
| 84 |
+
ax.set_title(title, color=INK)
|
| 85 |
+
ax.legend(fontsize=7, loc="best")
|
| 86 |
+
ax.grid(True, alpha=0.2)
|
| 87 |
+
return _save(fig, out_png)
|
| 88 |
+
except Exception as e:
|
| 89 |
+
logger.warning("velocity_profile failed: %s", e)
|
| 90 |
+
return None
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def laban_radar(effort: dict, out_png: str, title: str = "Laban Effort") -> str | None:
|
| 94 |
+
"""4-axis radar of the Effort factors (Space, Weight, Time, Flow)."""
|
| 95 |
+
try:
|
| 96 |
+
axes_order = ["space", "weight", "time", "flow"]
|
| 97 |
+
labels = ["Space\n(direct)", "Weight\n(strong)", "Time\n(sudden)", "Flow\n(free)"]
|
| 98 |
+
vals = [float(effort.get(k, 0.0)) for k in axes_order]
|
| 99 |
+
angles = np.linspace(0, 2 * np.pi, len(axes_order), endpoint=False).tolist()
|
| 100 |
+
vals_loop = vals + vals[:1]
|
| 101 |
+
angles_loop = angles + angles[:1]
|
| 102 |
+
|
| 103 |
+
fig, ax = plt.subplots(figsize=(4.2, 4.2), subplot_kw={"polar": True})
|
| 104 |
+
ax.plot(angles_loop, vals_loop, color=TEAL, lw=2)
|
| 105 |
+
ax.fill(angles_loop, vals_loop, color=TEAL, alpha=0.25)
|
| 106 |
+
ax.set_xticks(angles)
|
| 107 |
+
ax.set_xticklabels(labels, fontsize=8, color=INK)
|
| 108 |
+
ax.set_ylim(0, 1)
|
| 109 |
+
ax.set_yticks([0.25, 0.5, 0.75, 1.0])
|
| 110 |
+
ax.set_yticklabels(["", "0.5", "", "1.0"], fontsize=7)
|
| 111 |
+
ax.set_title(title, color=INK, pad=18)
|
| 112 |
+
return _save(fig, out_png)
|
| 113 |
+
except Exception as e:
|
| 114 |
+
logger.warning("laban_radar failed: %s", e)
|
| 115 |
+
return None
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def flexion_bars(flexion: dict, out_png: str,
|
| 119 |
+
title: str = "Relevant joint flexion") -> str | None:
|
| 120 |
+
"""Horizontal bars of relevant joint angles (deg) at the key frame."""
|
| 121 |
+
try:
|
| 122 |
+
if not flexion:
|
| 123 |
+
return None
|
| 124 |
+
names = [n.replace("_", " ") for n in flexion]
|
| 125 |
+
degs = [flexion[n]["deg"] for n in flexion]
|
| 126 |
+
colors = [TEAL if d >= 160 else GOLD if d >= 110 else RED for d in degs]
|
| 127 |
+
fig, ax = plt.subplots(figsize=(6.0, max(1.6, 0.5 * len(names) + 0.8)))
|
| 128 |
+
y = np.arange(len(names))
|
| 129 |
+
ax.barh(y, degs, color=colors)
|
| 130 |
+
ax.set_yticks(y)
|
| 131 |
+
ax.set_yticklabels(names, fontsize=8)
|
| 132 |
+
ax.set_xlim(0, 200)
|
| 133 |
+
ax.axvline(160, color=SAGE, ls=":", lw=1)
|
| 134 |
+
for yi, d in zip(y, degs):
|
| 135 |
+
ax.text(d + 3, yi, f"{d:.0f}°", va="center", fontsize=8, color=INK)
|
| 136 |
+
ax.set_xlabel("interior angle (°) · higher = more open")
|
| 137 |
+
ax.set_title(title, color=INK)
|
| 138 |
+
ax.invert_yaxis()
|
| 139 |
+
return _save(fig, out_png)
|
| 140 |
+
except Exception as e:
|
| 141 |
+
logger.warning("flexion_bars failed: %s", e)
|
| 142 |
+
return None
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def symmetry_bars(asymmetries: list, out_png: str,
|
| 146 |
+
title: str = "Left / right symmetry") -> str | None:
|
| 147 |
+
"""Grouped L vs R score bars for bilateral tests."""
|
| 148 |
+
try:
|
| 149 |
+
rows = [a for a in asymmetries
|
| 150 |
+
if a.get("left_score") is not None and a.get("right_score") is not None]
|
| 151 |
+
if not rows:
|
| 152 |
+
return None
|
| 153 |
+
names = [a["test"].replace("_", " ") for a in rows]
|
| 154 |
+
left = [a["left_score"] for a in rows]
|
| 155 |
+
right = [a["right_score"] for a in rows]
|
| 156 |
+
x = np.arange(len(names))
|
| 157 |
+
w = 0.36
|
| 158 |
+
fig, ax = plt.subplots(figsize=(6.4, 3.2))
|
| 159 |
+
ax.bar(x - w / 2, left, w, color=TEAL, label="left")
|
| 160 |
+
ax.bar(x + w / 2, right, w, color=GOLD, label="right")
|
| 161 |
+
ax.set_xticks(x)
|
| 162 |
+
ax.set_xticklabels(names, fontsize=8, rotation=15, ha="right")
|
| 163 |
+
ax.set_ylim(0, 3.4)
|
| 164 |
+
ax.set_ylabel("score (0–3)")
|
| 165 |
+
ax.set_title(title, color=INK)
|
| 166 |
+
ax.legend(fontsize=8)
|
| 167 |
+
ax.grid(True, axis="y", alpha=0.2)
|
| 168 |
+
return _save(fig, out_png)
|
| 169 |
+
except Exception as e:
|
| 170 |
+
logger.warning("symmetry_bars failed: %s", e)
|
| 171 |
+
return None
|
formscout/analysis/laban.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Laban Movement Analysis — Effort factors from pose kinematics.
|
| 3 |
+
|
| 4 |
+
Computes the four Effort factors over the joints relevant to a screening test:
|
| 5 |
+
|
| 6 |
+
- Space (indirect 0 … direct 1) — path directness of the leading joint
|
| 7 |
+
- Weight (light 0 … strong 1) — motion-energy of the relevant joints
|
| 8 |
+
- Time (sustained 0 … sudden 1) — impulsivity (peak vs mean speed)
|
| 9 |
+
- Flow (bound 0 … free 1) — smoothness (inverse normalised jerk)
|
| 10 |
+
|
| 11 |
+
These are reproducible kinematic heuristics, not clinical LMA notation — the
|
| 12 |
+
report labels them as such. Distances are normalised by torso length so the
|
| 13 |
+
factors are scale-invariant across cameras. Pure function — no model, no I/O.
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import math
|
| 18 |
+
|
| 19 |
+
from formscout.agents.biomechanics import _get_joint
|
| 20 |
+
from formscout.analysis.relevant_joints import (
|
| 21 |
+
COCO_NAMES, L_HIP, L_SHOULDER, R_HIP, R_SHOULDER, relevant_joints,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
# Heuristic calibration references (movement is normalised to torso-lengths/sec).
|
| 25 |
+
_WEIGHT_REF = 1.0 # energy (bl/s)^2 giving ~0.63 weight
|
| 26 |
+
_TIME_LO, _TIME_HI = 1.5, 4.0 # peak/mean speed ratio mapped to [0, 1]
|
| 27 |
+
_FLOW_JERK_REF = 6.0 # normalised jerk (bl/s^3) giving ~0.37 flow
|
| 28 |
+
|
| 29 |
+
_LABELS = {
|
| 30 |
+
"space": ("indirect", "direct"),
|
| 31 |
+
"weight": ("light", "strong"),
|
| 32 |
+
"time": ("sustained", "sudden"),
|
| 33 |
+
"flow": ("bound", "free"),
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _torso_scale(frames) -> float:
|
| 38 |
+
"""Median shoulder-hip distance across frames; 1.0 if unmeasurable."""
|
| 39 |
+
lengths = []
|
| 40 |
+
for kps in frames:
|
| 41 |
+
for sh, hip in ((L_SHOULDER, L_HIP), (R_SHOULDER, R_HIP)):
|
| 42 |
+
a, b = _get_joint(kps, sh), _get_joint(kps, hip)
|
| 43 |
+
if a and b:
|
| 44 |
+
lengths.append(math.hypot(a[0] - b[0], a[1] - b[1]))
|
| 45 |
+
if not lengths:
|
| 46 |
+
return 1.0
|
| 47 |
+
lengths.sort()
|
| 48 |
+
med = lengths[len(lengths) // 2]
|
| 49 |
+
return med if med > 1e-6 else 1.0
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _joint_kinematics(frames, joint_id: int, dt: float, scale: float) -> dict | None:
|
| 53 |
+
"""Speed/accel/jerk/directness for one joint trajectory (torso-length units)."""
|
| 54 |
+
pts = [_get_joint(kps, joint_id) for kps in frames]
|
| 55 |
+
valid = [(i, p) for i, p in enumerate(pts) if p is not None]
|
| 56 |
+
if len(valid) < 3:
|
| 57 |
+
return None
|
| 58 |
+
|
| 59 |
+
speeds, path_len = [], 0.0
|
| 60 |
+
for (i0, p0), (i1, p1) in zip(valid, valid[1:]):
|
| 61 |
+
d = math.hypot(p1[0] - p0[0], p1[1] - p0[1]) / scale
|
| 62 |
+
path_len += d
|
| 63 |
+
gap = max(1, i1 - i0)
|
| 64 |
+
speeds.append(d / (gap * dt))
|
| 65 |
+
if not speeds:
|
| 66 |
+
return None
|
| 67 |
+
|
| 68 |
+
net = math.hypot(valid[-1][1][0] - valid[0][1][0],
|
| 69 |
+
valid[-1][1][1] - valid[0][1][1]) / scale
|
| 70 |
+
directness = net / path_len if path_len > 1e-6 else 0.0
|
| 71 |
+
|
| 72 |
+
accels = [abs(speeds[i + 1] - speeds[i]) / dt for i in range(len(speeds) - 1)]
|
| 73 |
+
jerks = [abs(accels[i + 1] - accels[i]) / dt for i in range(len(accels) - 1)]
|
| 74 |
+
|
| 75 |
+
mean_speed = sum(speeds) / len(speeds)
|
| 76 |
+
peak_speed = max(speeds)
|
| 77 |
+
return {
|
| 78 |
+
"mean_speed": mean_speed,
|
| 79 |
+
"peak_speed": peak_speed,
|
| 80 |
+
"energy": sum(s * s for s in speeds) / len(speeds),
|
| 81 |
+
"ratio": peak_speed / (mean_speed + 1e-6),
|
| 82 |
+
"mean_jerk": (sum(jerks) / len(jerks)) if jerks else 0.0,
|
| 83 |
+
"directness": min(1.0, max(0.0, directness)),
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _clip01(x: float) -> float:
|
| 88 |
+
return min(1.0, max(0.0, x))
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def compute_laban(pose2d, test_name: str, fps: float) -> dict:
|
| 92 |
+
"""Return the four Effort factors, their labels, and body emphasis."""
|
| 93 |
+
frames = pose2d.keypoints
|
| 94 |
+
dt = 1.0 / fps if fps and fps > 0 else 1.0 / 30.0
|
| 95 |
+
scale = _torso_scale(frames)
|
| 96 |
+
joints = relevant_joints(test_name) or list(range(17))
|
| 97 |
+
|
| 98 |
+
kin = {j: k for j in joints if (k := _joint_kinematics(frames, j, dt, scale))}
|
| 99 |
+
if not kin:
|
| 100 |
+
return {
|
| 101 |
+
"effort": {"space": 0.0, "weight": 0.0, "time": 0.0, "flow": 0.0},
|
| 102 |
+
"labels": {k: v[0] for k, v in _LABELS.items()},
|
| 103 |
+
"body_emphasis": [],
|
| 104 |
+
"notes": "insufficient motion to estimate Effort",
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
leader = max(kin, key=lambda j: kin[j]["mean_speed"])
|
| 108 |
+
lead = kin[leader]
|
| 109 |
+
|
| 110 |
+
weight = _clip01(1.0 - math.exp(-(sum(k["energy"] for k in kin.values()) / len(kin)) / _WEIGHT_REF))
|
| 111 |
+
time = _clip01((lead["ratio"] - _TIME_LO) / (_TIME_HI - _TIME_LO))
|
| 112 |
+
flow = _clip01(math.exp(-lead["mean_jerk"] / _FLOW_JERK_REF))
|
| 113 |
+
space = _clip01(lead["directness"])
|
| 114 |
+
|
| 115 |
+
effort = {"space": space, "weight": weight, "time": time, "flow": flow}
|
| 116 |
+
labels = {k: _LABELS[k][1] if v >= 0.5 else _LABELS[k][0] for k, v in effort.items()}
|
| 117 |
+
|
| 118 |
+
emphasis = sorted(kin.items(), key=lambda kv: kv[1]["mean_speed"], reverse=True)[:3]
|
| 119 |
+
body_emphasis = [(COCO_NAMES.get(j, str(j)), round(k["mean_speed"], 3)) for j, k in emphasis]
|
| 120 |
+
|
| 121 |
+
return {
|
| 122 |
+
"effort": {k: round(v, 3) for k, v in effort.items()},
|
| 123 |
+
"labels": labels,
|
| 124 |
+
"body_emphasis": body_emphasis,
|
| 125 |
+
"leading_joint": COCO_NAMES.get(leader, str(leader)),
|
| 126 |
+
"notes": "kinematic Effort estimate (heuristic, not clinical LMA notation)",
|
| 127 |
+
}
|
formscout/analysis/relevant_joints.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Per-test relevant-joint and relevant-angle definitions.
|
| 3 |
+
|
| 4 |
+
Drives the "always describe just the joint relevant to the screening action"
|
| 5 |
+
requirement: each FMS test names the joints and the joint-angles that matter for
|
| 6 |
+
its rubric, plus the single primary angle used for the headline angle-over-time
|
| 7 |
+
graph. Angles are COCO (a, b, c) triplets — the angle is measured at joint b.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
# COCO-17 joint indices
|
| 12 |
+
NOSE = 0
|
| 13 |
+
L_SHOULDER, R_SHOULDER = 5, 6
|
| 14 |
+
L_ELBOW, R_ELBOW = 7, 8
|
| 15 |
+
L_WRIST, R_WRIST = 9, 10
|
| 16 |
+
L_HIP, R_HIP = 11, 12
|
| 17 |
+
L_KNEE, R_KNEE = 13, 14
|
| 18 |
+
L_ANKLE, R_ANKLE = 15, 16
|
| 19 |
+
|
| 20 |
+
COCO_NAMES = {
|
| 21 |
+
0: "nose", 1: "left_eye", 2: "right_eye", 3: "left_ear", 4: "right_ear",
|
| 22 |
+
5: "left_shoulder", 6: "right_shoulder", 7: "left_elbow", 8: "right_elbow",
|
| 23 |
+
9: "left_wrist", 10: "right_wrist", 11: "left_hip", 12: "right_hip",
|
| 24 |
+
13: "left_knee", 14: "right_knee", 15: "left_ankle", 16: "right_ankle",
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
# Each test: the joints that matter, the named angles (a, b, c) measured at b,
|
| 28 |
+
# and the primary angle for the headline graph.
|
| 29 |
+
RELEVANT: dict[str, dict] = {
|
| 30 |
+
"deep_squat": {
|
| 31 |
+
"joints": [L_SHOULDER, R_SHOULDER, L_HIP, R_HIP, L_KNEE, R_KNEE, L_ANKLE, R_ANKLE],
|
| 32 |
+
"angles": {
|
| 33 |
+
"left_knee_flexion": (L_HIP, L_KNEE, L_ANKLE),
|
| 34 |
+
"right_knee_flexion": (R_HIP, R_KNEE, R_ANKLE),
|
| 35 |
+
"left_hip_flexion": (L_SHOULDER, L_HIP, L_KNEE),
|
| 36 |
+
"right_hip_flexion": (R_SHOULDER, R_HIP, R_KNEE),
|
| 37 |
+
},
|
| 38 |
+
"primary_angle": "left_knee_flexion",
|
| 39 |
+
},
|
| 40 |
+
"hurdle_step": {
|
| 41 |
+
"joints": [L_HIP, R_HIP, L_KNEE, R_KNEE, L_ANKLE, R_ANKLE],
|
| 42 |
+
"angles": {
|
| 43 |
+
"left_knee_flexion": (L_HIP, L_KNEE, L_ANKLE),
|
| 44 |
+
"right_knee_flexion": (R_HIP, R_KNEE, R_ANKLE),
|
| 45 |
+
"left_hip_flexion": (L_SHOULDER, L_HIP, L_KNEE),
|
| 46 |
+
"right_hip_flexion": (R_SHOULDER, R_HIP, R_KNEE),
|
| 47 |
+
},
|
| 48 |
+
"primary_angle": "left_hip_flexion",
|
| 49 |
+
},
|
| 50 |
+
"inline_lunge": {
|
| 51 |
+
"joints": [L_HIP, R_HIP, L_KNEE, R_KNEE, L_ANKLE, R_ANKLE],
|
| 52 |
+
"angles": {
|
| 53 |
+
"left_knee_flexion": (L_HIP, L_KNEE, L_ANKLE),
|
| 54 |
+
"right_knee_flexion": (R_HIP, R_KNEE, R_ANKLE),
|
| 55 |
+
},
|
| 56 |
+
"primary_angle": "left_knee_flexion",
|
| 57 |
+
},
|
| 58 |
+
"shoulder_mobility": {
|
| 59 |
+
"joints": [L_SHOULDER, R_SHOULDER, L_ELBOW, R_ELBOW, L_WRIST, R_WRIST],
|
| 60 |
+
"angles": {
|
| 61 |
+
"left_shoulder_angle": (L_ELBOW, L_SHOULDER, L_HIP),
|
| 62 |
+
"right_shoulder_angle": (R_ELBOW, R_SHOULDER, R_HIP),
|
| 63 |
+
"left_elbow_flexion": (L_SHOULDER, L_ELBOW, L_WRIST),
|
| 64 |
+
"right_elbow_flexion": (R_SHOULDER, R_ELBOW, R_WRIST),
|
| 65 |
+
},
|
| 66 |
+
"primary_angle": "left_shoulder_angle",
|
| 67 |
+
},
|
| 68 |
+
"active_slr": {
|
| 69 |
+
"joints": [L_HIP, R_HIP, L_KNEE, R_KNEE, L_ANKLE, R_ANKLE],
|
| 70 |
+
"angles": {
|
| 71 |
+
"left_hip_flexion": (L_SHOULDER, L_HIP, L_KNEE),
|
| 72 |
+
"right_hip_flexion": (R_SHOULDER, R_HIP, R_KNEE),
|
| 73 |
+
"left_knee_flexion": (L_HIP, L_KNEE, L_ANKLE),
|
| 74 |
+
"right_knee_flexion": (R_HIP, R_KNEE, R_ANKLE),
|
| 75 |
+
},
|
| 76 |
+
"primary_angle": "left_hip_flexion",
|
| 77 |
+
},
|
| 78 |
+
"trunk_stability_pushup": {
|
| 79 |
+
"joints": [L_SHOULDER, R_SHOULDER, L_ELBOW, R_ELBOW, L_HIP, R_HIP, L_ANKLE, R_ANKLE],
|
| 80 |
+
"angles": {
|
| 81 |
+
"left_elbow_flexion": (L_SHOULDER, L_ELBOW, L_WRIST),
|
| 82 |
+
"right_elbow_flexion": (R_SHOULDER, R_ELBOW, R_WRIST),
|
| 83 |
+
"left_hip_line": (L_SHOULDER, L_HIP, L_ANKLE),
|
| 84 |
+
"right_hip_line": (R_SHOULDER, R_HIP, R_ANKLE),
|
| 85 |
+
},
|
| 86 |
+
"primary_angle": "left_hip_line",
|
| 87 |
+
},
|
| 88 |
+
"rotary_stability": {
|
| 89 |
+
"joints": [L_SHOULDER, R_SHOULDER, L_ELBOW, R_ELBOW, L_HIP, R_HIP, L_KNEE, R_KNEE],
|
| 90 |
+
"angles": {
|
| 91 |
+
"left_hip_line": (L_SHOULDER, L_HIP, L_KNEE),
|
| 92 |
+
"right_hip_line": (R_SHOULDER, R_HIP, R_KNEE),
|
| 93 |
+
},
|
| 94 |
+
"primary_angle": "left_hip_line",
|
| 95 |
+
},
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def relevant_joints(test_name: str) -> list[int]:
|
| 100 |
+
"""COCO joint indices relevant to this test (empty for unknown tests)."""
|
| 101 |
+
return list(RELEVANT.get(test_name, {}).get("joints", []))
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def relevant_angles(test_name: str) -> dict[str, tuple]:
|
| 105 |
+
"""Named (a, b, c) angle triplets relevant to this test."""
|
| 106 |
+
return dict(RELEVANT.get(test_name, {}).get("angles", {}))
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def primary_angle(test_name: str) -> str | None:
|
| 110 |
+
"""Name of the headline angle for the angle-over-time graph, or None."""
|
| 111 |
+
return RELEVANT.get(test_name, {}).get("primary_angle")
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def openness_label(angle_deg: float) -> str:
|
| 115 |
+
"""Describe how open/closed a joint is from its interior angle in degrees."""
|
| 116 |
+
if angle_deg >= 160:
|
| 117 |
+
return "open / extended"
|
| 118 |
+
if angle_deg >= 110:
|
| 119 |
+
return "mid-range"
|
| 120 |
+
if angle_deg >= 60:
|
| 121 |
+
return "flexed"
|
| 122 |
+
return "deeply flexed / closed"
|
formscout/analysis/timeseries.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Per-frame time series for the joints relevant to a screening test.
|
| 3 |
+
|
| 4 |
+
Builds angle-over-time series (used by the angle graph) and a per-joint flexion
|
| 5 |
+
summary at a chosen frame (degrees + open/closed label). Reuses the biomechanics
|
| 6 |
+
geometry helpers so angle definitions never diverge.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import math
|
| 11 |
+
|
| 12 |
+
from formscout.agents.biomechanics import _angle_between_points, _get_joint
|
| 13 |
+
from formscout.analysis.relevant_joints import openness_label, relevant_angles
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def angle_series(pose2d, test_name: str) -> dict[str, list[float]]:
|
| 17 |
+
"""Return {angle_name: [deg per frame]} for this test's relevant angles.
|
| 18 |
+
|
| 19 |
+
Frames where the angle cannot be measured hold NaN.
|
| 20 |
+
"""
|
| 21 |
+
angles = relevant_angles(test_name)
|
| 22 |
+
series: dict[str, list[float]] = {name: [] for name in angles}
|
| 23 |
+
for kps in pose2d.keypoints:
|
| 24 |
+
for name, (a, b, c) in angles.items():
|
| 25 |
+
pa, pb, pc = _get_joint(kps, a), _get_joint(kps, b), _get_joint(kps, c)
|
| 26 |
+
if pa and pb and pc:
|
| 27 |
+
series[name].append(_angle_between_points(pa, pb, pc))
|
| 28 |
+
else:
|
| 29 |
+
series[name].append(float("nan"))
|
| 30 |
+
return series
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def relevant_flexion_at(pose2d, test_name: str, frame_idx: int) -> dict[str, dict]:
|
| 34 |
+
"""At one frame, the relevant joint angles with degree value + openness label.
|
| 35 |
+
|
| 36 |
+
Returns {angle_name: {"deg": float, "openness": str}}; angles that cannot be
|
| 37 |
+
measured at that frame are omitted.
|
| 38 |
+
"""
|
| 39 |
+
out: dict[str, dict] = {}
|
| 40 |
+
if not (0 <= frame_idx < len(pose2d.keypoints)):
|
| 41 |
+
return out
|
| 42 |
+
kps = pose2d.keypoints[frame_idx]
|
| 43 |
+
for name, (a, b, c) in relevant_angles(test_name).items():
|
| 44 |
+
pa, pb, pc = _get_joint(kps, a), _get_joint(kps, b), _get_joint(kps, c)
|
| 45 |
+
if pa and pb and pc:
|
| 46 |
+
deg = _angle_between_points(pa, pb, pc)
|
| 47 |
+
if not math.isnan(deg):
|
| 48 |
+
out[name] = {"deg": deg, "openness": openness_label(deg)}
|
| 49 |
+
return out
|
formscout/config.py
CHANGED
|
@@ -143,6 +143,18 @@ DEEP_SQUAT_KNEE_TRACKING_MARGIN_PX = 20
|
|
| 143 |
LLAMA_CPP_HOST = "127.0.0.1"
|
| 144 |
LLAMA_CPP_PORT_VLM = 8080
|
| 145 |
LLAMA_CPP_PORT_EMBED = 8081
|
| 146 |
-
|
| 147 |
-
#
|
| 148 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 143 |
LLAMA_CPP_HOST = "127.0.0.1"
|
| 144 |
LLAMA_CPP_PORT_VLM = 8080
|
| 145 |
LLAMA_CPP_PORT_EMBED = 8081
|
| 146 |
+
|
| 147 |
+
# ─── Judge backend selection ────────────────────────────────────────────────
|
| 148 |
+
# "llama_cpp" — local llama-server (default for local dev; works perfectly)
|
| 149 |
+
# "transformers"— in-process Qwen3-VL via transformers, GPU on HF Spaces (ZeroGPU)
|
| 150 |
+
# "auto" — transformers on a Space (SPACE_ID set), llama_cpp locally
|
| 151 |
+
JUDGE_BACKEND = os.environ.get("FORMSCOUT_JUDGE_BACKEND", "auto")
|
| 152 |
+
JUDGE_HF_MODEL = os.environ.get("FORMSCOUT_JUDGE_HF_MODEL", "Qwen/Qwen3-VL-8B-Instruct")
|
| 153 |
+
ON_HF_SPACE = bool(os.environ.get("SPACE_ID"))
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def resolve_judge_backend() -> str:
|
| 157 |
+
"""Resolve the effective judge backend from JUDGE_BACKEND + environment."""
|
| 158 |
+
if JUDGE_BACKEND in ("llama_cpp", "transformers"):
|
| 159 |
+
return JUDGE_BACKEND
|
| 160 |
+
return "transformers" if ON_HF_SPACE else "llama_cpp"
|
formscout/pipeline.py
CHANGED
|
@@ -1,111 +1,111 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Director — deterministic state machine orchestrating the FormScout pipeline.
|
| 3 |
-
|
| 4 |
-
NOT an LLM. Runs each agent in sequence, applies quality gates, and assembles
|
| 5 |
-
the final PipelineState. Exposes run(video_path, config) -> PipelineState.
|
| 6 |
-
"""
|
| 7 |
-
from __future__ import annotations
|
| 8 |
-
|
| 9 |
-
from pathlib import Path
|
| 10 |
-
|
| 11 |
-
from formscout import config
|
| 12 |
-
from formscout.types import (
|
| 13 |
-
PipelineState, Body3DResult, MovementResult,
|
| 14 |
-
)
|
| 15 |
-
from formscout.agents.ingest import IngestAgent
|
| 16 |
-
from formscout.agents.pose2d import Pose2DAgent
|
| 17 |
-
from formscout.agents.body3d import Body3DAgent
|
| 18 |
-
from formscout.agents.biomechanics import BiomechanicsAgent
|
| 19 |
-
from formscout.agents.classifier import MovementClassifierAgent
|
| 20 |
-
from formscout.agents.judge import JudgeAgent
|
| 21 |
-
from formscout.agents.report import ReportAgent
|
| 22 |
-
from formscout.rubric import score_test
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
class Director:
|
| 26 |
-
"""
|
| 27 |
-
Orchestrates the FormScout agent pipeline as a deterministic state machine.
|
| 28 |
-
Quality gates are applied after each agent — never silently passes bad data.
|
| 29 |
-
"""
|
| 30 |
-
|
| 31 |
-
def __init__(self):
|
| 32 |
-
self._ingest = IngestAgent()
|
| 33 |
-
self._pose2d = Pose2DAgent()
|
| 34 |
-
self._body3d = Body3DAgent()
|
| 35 |
-
self._biomechanics = BiomechanicsAgent()
|
| 36 |
-
self._classifier = MovementClassifierAgent()
|
| 37 |
-
self._judge = JudgeAgent()
|
| 38 |
-
self._report = ReportAgent()
|
| 39 |
-
|
| 40 |
-
def run(self, video_path: str, test_name: str = "deep_squat", side: str = "na", model_key: str | None = None) -> PipelineState:
|
| 41 |
-
"""
|
| 42 |
-
Run the full pipeline on a single video.
|
| 43 |
-
test_name/side serve as manual override when provided (skips classifier).
|
| 44 |
-
model_key selects the pose backend (see config.POSE_MODELS).
|
| 45 |
-
"""
|
| 46 |
-
state = PipelineState(video_path=video_path)
|
| 47 |
-
|
| 48 |
-
# ─── Ingest ───
|
| 49 |
-
state.ingest = self._ingest.run(video_path)
|
| 50 |
-
if state.ingest.confidence < config.MIN_CONFIDENCE:
|
| 51 |
-
state.errors.append("ingest: low confidence — video may be corrupt")
|
| 52 |
-
return state
|
| 53 |
-
|
| 54 |
-
# ─── Pose 2D ───
|
| 55 |
-
state.pose2d = self._pose2d.run(state.ingest, model_key=model_key)
|
| 56 |
-
if state.pose2d.confidence < config.MIN_CONFIDENCE:
|
| 57 |
-
state.warnings.append("pose2d: low confidence — no clear person detected")
|
| 58 |
-
|
| 59 |
-
# ─── Body 3D (optional) ───
|
| 60 |
-
masks = state.segment.masks if state.segment else []
|
| 61 |
-
frames = state.ingest.frames if state.ingest else []
|
| 62 |
-
state.body3d = self._body3d.run(state.pose2d, masks, frames=frames)
|
| 63 |
-
|
| 64 |
-
# ─── Movement classification ───
|
| 65 |
-
if test_name and test_name != "unknown":
|
| 66 |
-
# Manual override
|
| 67 |
-
state.movement = MovementResult(
|
| 68 |
-
test_name=test_name, side=side,
|
| 69 |
-
confidence=1.0, notes="manually specified",
|
| 70 |
-
)
|
| 71 |
-
else:
|
| 72 |
-
state.movement = self._classifier.run(state.ingest, state.pose2d)
|
| 73 |
-
|
| 74 |
-
# Gate: unknown test → stop
|
| 75 |
-
if state.movement.test_name == "unknown":
|
| 76 |
-
state.errors.append("movement classifier returned 'unknown' — manual override required")
|
| 77 |
-
return state
|
| 78 |
-
|
| 79 |
-
# ─── Biomechanics ───
|
| 80 |
-
state.features = self._biomechanics.run(
|
| 81 |
-
state.pose2d,
|
| 82 |
-
state.body3d or Body3DResult(used=False, joints_3d=[]),
|
| 83 |
-
state.movement,
|
| 84 |
-
)
|
| 85 |
-
if state.features.confidence < config.MIN_CONFIDENCE:
|
| 86 |
-
state.warnings.append(
|
| 87 |
-
f"biomechanics: low confidence ({state.features.confidence:.2f}) — physio review recommended"
|
| 88 |
-
)
|
| 89 |
-
|
| 90 |
-
# ─── Rubric Score ───
|
| 91 |
-
rubric_result = score_test(state.features)
|
| 92 |
-
state.stgcn_score = rubric_result # Reusing field for rubric until ST-GCN is built
|
| 93 |
-
|
| 94 |
-
# ─── Judge ───
|
| 95 |
-
state.judge = self._judge.run(
|
| 96 |
-
state.features, rubric_result, state.movement, state.ingest,
|
| 97 |
-
)
|
| 98 |
-
|
| 99 |
-
# ─── Quality gates ───
|
| 100 |
-
# Gate: score disagreement
|
| 101 |
-
if (state.judge.score is not None and rubric_result.score is not None
|
| 102 |
-
and abs(state.judge.score - rubric_result.score) >= config.SCORE_DISAGREE_THRESH):
|
| 103 |
-
state.warnings.append(
|
| 104 |
-
f"score disagreement: rubric={rubric_result.score} vs judge={state.judge.score} — review recommended"
|
| 105 |
-
)
|
| 106 |
-
|
| 107 |
-
# Gate: needs_human
|
| 108 |
-
if state.judge.needs_human:
|
| 109 |
-
state.warnings.append("judge flagged needs_human — no auto-score emitted")
|
| 110 |
-
|
| 111 |
-
return state
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Director — deterministic state machine orchestrating the FormScout pipeline.
|
| 3 |
+
|
| 4 |
+
NOT an LLM. Runs each agent in sequence, applies quality gates, and assembles
|
| 5 |
+
the final PipelineState. Exposes run(video_path, config) -> PipelineState.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
from formscout import config
|
| 12 |
+
from formscout.types import (
|
| 13 |
+
PipelineState, Body3DResult, MovementResult,
|
| 14 |
+
)
|
| 15 |
+
from formscout.agents.ingest import IngestAgent
|
| 16 |
+
from formscout.agents.pose2d import Pose2DAgent
|
| 17 |
+
from formscout.agents.body3d import Body3DAgent
|
| 18 |
+
from formscout.agents.biomechanics import BiomechanicsAgent
|
| 19 |
+
from formscout.agents.classifier import MovementClassifierAgent
|
| 20 |
+
from formscout.agents.judge import JudgeAgent
|
| 21 |
+
from formscout.agents.report import ReportAgent
|
| 22 |
+
from formscout.rubric import score_test
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class Director:
|
| 26 |
+
"""
|
| 27 |
+
Orchestrates the FormScout agent pipeline as a deterministic state machine.
|
| 28 |
+
Quality gates are applied after each agent — never silently passes bad data.
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
def __init__(self):
|
| 32 |
+
self._ingest = IngestAgent()
|
| 33 |
+
self._pose2d = Pose2DAgent()
|
| 34 |
+
self._body3d = Body3DAgent()
|
| 35 |
+
self._biomechanics = BiomechanicsAgent()
|
| 36 |
+
self._classifier = MovementClassifierAgent()
|
| 37 |
+
self._judge = JudgeAgent()
|
| 38 |
+
self._report = ReportAgent()
|
| 39 |
+
|
| 40 |
+
def run(self, video_path: str, test_name: str = "deep_squat", side: str = "na", model_key: str | None = None) -> PipelineState:
|
| 41 |
+
"""
|
| 42 |
+
Run the full pipeline on a single video.
|
| 43 |
+
test_name/side serve as manual override when provided (skips classifier).
|
| 44 |
+
model_key selects the pose backend (see config.POSE_MODELS).
|
| 45 |
+
"""
|
| 46 |
+
state = PipelineState(video_path=video_path)
|
| 47 |
+
|
| 48 |
+
# ─── Ingest ───
|
| 49 |
+
state.ingest = self._ingest.run(video_path)
|
| 50 |
+
if state.ingest.confidence < config.MIN_CONFIDENCE:
|
| 51 |
+
state.errors.append("ingest: low confidence — video may be corrupt")
|
| 52 |
+
return state
|
| 53 |
+
|
| 54 |
+
# ─── Pose 2D ───
|
| 55 |
+
state.pose2d = self._pose2d.run(state.ingest, model_key=model_key)
|
| 56 |
+
if state.pose2d.confidence < config.MIN_CONFIDENCE:
|
| 57 |
+
state.warnings.append("pose2d: low confidence — no clear person detected")
|
| 58 |
+
|
| 59 |
+
# ─── Body 3D (optional) ───
|
| 60 |
+
masks = state.segment.masks if state.segment else []
|
| 61 |
+
frames = state.ingest.frames if state.ingest else []
|
| 62 |
+
state.body3d = self._body3d.run(state.pose2d, masks, frames=frames)
|
| 63 |
+
|
| 64 |
+
# ─── Movement classification ───
|
| 65 |
+
if test_name and test_name != "unknown":
|
| 66 |
+
# Manual override
|
| 67 |
+
state.movement = MovementResult(
|
| 68 |
+
test_name=test_name, side=side,
|
| 69 |
+
confidence=1.0, notes="manually specified",
|
| 70 |
+
)
|
| 71 |
+
else:
|
| 72 |
+
state.movement = self._classifier.run(state.ingest, state.pose2d)
|
| 73 |
+
|
| 74 |
+
# Gate: unknown test → stop
|
| 75 |
+
if state.movement.test_name == "unknown":
|
| 76 |
+
state.errors.append("movement classifier returned 'unknown' — manual override required")
|
| 77 |
+
return state
|
| 78 |
+
|
| 79 |
+
# ─── Biomechanics ───
|
| 80 |
+
state.features = self._biomechanics.run(
|
| 81 |
+
state.pose2d,
|
| 82 |
+
state.body3d or Body3DResult(used=False, joints_3d=[]),
|
| 83 |
+
state.movement,
|
| 84 |
+
)
|
| 85 |
+
if state.features.confidence < config.MIN_CONFIDENCE:
|
| 86 |
+
state.warnings.append(
|
| 87 |
+
f"biomechanics: low confidence ({state.features.confidence:.2f}) — physio review recommended"
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
# ─── Rubric Score ───
|
| 91 |
+
rubric_result = score_test(state.features)
|
| 92 |
+
state.stgcn_score = rubric_result # Reusing field for rubric until ST-GCN is built
|
| 93 |
+
|
| 94 |
+
# ─── Judge ───
|
| 95 |
+
state.judge = self._judge.run(
|
| 96 |
+
state.features, rubric_result, state.movement, state.ingest,
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
# ─── Quality gates ───
|
| 100 |
+
# Gate: score disagreement
|
| 101 |
+
if (state.judge.score is not None and rubric_result.score is not None
|
| 102 |
+
and abs(state.judge.score - rubric_result.score) >= config.SCORE_DISAGREE_THRESH):
|
| 103 |
+
state.warnings.append(
|
| 104 |
+
f"score disagreement: rubric={rubric_result.score} vs judge={state.judge.score} — review recommended"
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
# Gate: needs_human
|
| 108 |
+
if state.judge.needs_human:
|
| 109 |
+
state.warnings.append("judge flagged needs_human — no auto-score emitted")
|
| 110 |
+
|
| 111 |
+
return state
|
formscout/rubric/__init__.py
CHANGED
|
@@ -1,32 +1,32 @@
|
|
| 1 |
-
"""
|
| 2 |
-
FormScout rubric scorers — one pure-function scorer per FMS test.
|
| 3 |
-
"""
|
| 4 |
-
from formscout.rubric.deep_squat import score_deep_squat
|
| 5 |
-
from formscout.rubric.hurdle_step import score_hurdle_step
|
| 6 |
-
from formscout.rubric.inline_lunge import score_inline_lunge
|
| 7 |
-
from formscout.rubric.shoulder_mobility import score_shoulder_mobility
|
| 8 |
-
from formscout.rubric.active_slr import score_active_slr
|
| 9 |
-
from formscout.rubric.trunk_stability_pushup import score_trunk_stability_pushup
|
| 10 |
-
from formscout.rubric.rotary_stability import score_rotary_stability
|
| 11 |
-
from formscout.types import BiomechFeatures, ScoreResult
|
| 12 |
-
|
| 13 |
-
SCORERS = {
|
| 14 |
-
"deep_squat": score_deep_squat,
|
| 15 |
-
"hurdle_step": score_hurdle_step,
|
| 16 |
-
"inline_lunge": score_inline_lunge,
|
| 17 |
-
"shoulder_mobility": score_shoulder_mobility,
|
| 18 |
-
"active_slr": score_active_slr,
|
| 19 |
-
"trunk_stability_pushup": score_trunk_stability_pushup,
|
| 20 |
-
"rotary_stability": score_rotary_stability,
|
| 21 |
-
}
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
def score_test(features: BiomechFeatures) -> ScoreResult:
|
| 25 |
-
"""Dispatch to the appropriate rubric scorer by test name."""
|
| 26 |
-
fn = SCORERS.get(features.test_name)
|
| 27 |
-
if fn is None:
|
| 28 |
-
return ScoreResult(
|
| 29 |
-
score=1, rationale=f"No rubric for test '{features.test_name}'",
|
| 30 |
-
confidence=0.0, notes="unknown test",
|
| 31 |
-
)
|
| 32 |
-
return fn(features)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
FormScout rubric scorers — one pure-function scorer per FMS test.
|
| 3 |
+
"""
|
| 4 |
+
from formscout.rubric.deep_squat import score_deep_squat
|
| 5 |
+
from formscout.rubric.hurdle_step import score_hurdle_step
|
| 6 |
+
from formscout.rubric.inline_lunge import score_inline_lunge
|
| 7 |
+
from formscout.rubric.shoulder_mobility import score_shoulder_mobility
|
| 8 |
+
from formscout.rubric.active_slr import score_active_slr
|
| 9 |
+
from formscout.rubric.trunk_stability_pushup import score_trunk_stability_pushup
|
| 10 |
+
from formscout.rubric.rotary_stability import score_rotary_stability
|
| 11 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 12 |
+
|
| 13 |
+
SCORERS = {
|
| 14 |
+
"deep_squat": score_deep_squat,
|
| 15 |
+
"hurdle_step": score_hurdle_step,
|
| 16 |
+
"inline_lunge": score_inline_lunge,
|
| 17 |
+
"shoulder_mobility": score_shoulder_mobility,
|
| 18 |
+
"active_slr": score_active_slr,
|
| 19 |
+
"trunk_stability_pushup": score_trunk_stability_pushup,
|
| 20 |
+
"rotary_stability": score_rotary_stability,
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def score_test(features: BiomechFeatures) -> ScoreResult:
|
| 25 |
+
"""Dispatch to the appropriate rubric scorer by test name."""
|
| 26 |
+
fn = SCORERS.get(features.test_name)
|
| 27 |
+
if fn is None:
|
| 28 |
+
return ScoreResult(
|
| 29 |
+
score=1, rationale=f"No rubric for test '{features.test_name}'",
|
| 30 |
+
confidence=0.0, notes="unknown test",
|
| 31 |
+
)
|
| 32 |
+
return fn(features)
|
formscout/rubric/active_slr.py
CHANGED
|
@@ -1,51 +1,51 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Active Straight-Leg Raise rubric scorer — pure function, no model calls.
|
| 3 |
-
|
| 4 |
-
FMS ASLR Criteria (bilateral):
|
| 5 |
-
- Score 3: raised leg malleolus past contralateral knee (>70°), down leg flat.
|
| 6 |
-
- Score 2: malleolus between mid-thigh and knee (45-70°).
|
| 7 |
-
- Score 1: malleolus below mid-thigh (<45°).
|
| 8 |
-
- Score 0: PAIN — never auto-scored.
|
| 9 |
-
"""
|
| 10 |
-
from __future__ import annotations
|
| 11 |
-
|
| 12 |
-
from formscout.types import BiomechFeatures, ScoreResult
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
def score_active_slr(features: BiomechFeatures) -> ScoreResult:
|
| 16 |
-
"""Pure rubric scorer for active straight-leg raise."""
|
| 17 |
-
angles = features.angles
|
| 18 |
-
alignments = features.alignments
|
| 19 |
-
|
| 20 |
-
has_angle = "raised_leg_angle_deg" in angles
|
| 21 |
-
if not has_angle:
|
| 22 |
-
return ScoreResult(
|
| 23 |
-
score=1, rationale="Insufficient data: leg raise angle not measurable",
|
| 24 |
-
confidence=0.3, notes="missing key measurements",
|
| 25 |
-
)
|
| 26 |
-
|
| 27 |
-
angle = angles["raised_leg_angle_deg"]
|
| 28 |
-
past_knee = alignments.get("past_contralateral_knee", False)
|
| 29 |
-
past_mid = alignments.get("past_mid_thigh", False)
|
| 30 |
-
down_flat = alignments.get("down_leg_flat", True)
|
| 31 |
-
|
| 32 |
-
rationale_parts = []
|
| 33 |
-
|
| 34 |
-
if past_knee and down_flat:
|
| 35 |
-
score = 3
|
| 36 |
-
rationale_parts.append(f"Raised leg at {angle:.0f}° (past contralateral knee)")
|
| 37 |
-
elif past_mid:
|
| 38 |
-
score = 2
|
| 39 |
-
rationale_parts.append(f"Raised leg at {angle:.0f}° (between mid-thigh and knee)")
|
| 40 |
-
if not down_flat:
|
| 41 |
-
rationale_parts.append("down leg lifted off surface")
|
| 42 |
-
else:
|
| 43 |
-
score = 1
|
| 44 |
-
rationale_parts.append(f"Raised leg only {angle:.0f}° (below mid-thigh)")
|
| 45 |
-
|
| 46 |
-
confidence = features.confidence * 0.9
|
| 47 |
-
|
| 48 |
-
return ScoreResult(
|
| 49 |
-
score=score, rationale="; ".join(rationale_parts),
|
| 50 |
-
confidence=confidence, notes="",
|
| 51 |
-
)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Active Straight-Leg Raise rubric scorer — pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS ASLR Criteria (bilateral):
|
| 5 |
+
- Score 3: raised leg malleolus past contralateral knee (>70°), down leg flat.
|
| 6 |
+
- Score 2: malleolus between mid-thigh and knee (45-70°).
|
| 7 |
+
- Score 1: malleolus below mid-thigh (<45°).
|
| 8 |
+
- Score 0: PAIN — never auto-scored.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def score_active_slr(features: BiomechFeatures) -> ScoreResult:
|
| 16 |
+
"""Pure rubric scorer for active straight-leg raise."""
|
| 17 |
+
angles = features.angles
|
| 18 |
+
alignments = features.alignments
|
| 19 |
+
|
| 20 |
+
has_angle = "raised_leg_angle_deg" in angles
|
| 21 |
+
if not has_angle:
|
| 22 |
+
return ScoreResult(
|
| 23 |
+
score=1, rationale="Insufficient data: leg raise angle not measurable",
|
| 24 |
+
confidence=0.3, notes="missing key measurements",
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
angle = angles["raised_leg_angle_deg"]
|
| 28 |
+
past_knee = alignments.get("past_contralateral_knee", False)
|
| 29 |
+
past_mid = alignments.get("past_mid_thigh", False)
|
| 30 |
+
down_flat = alignments.get("down_leg_flat", True)
|
| 31 |
+
|
| 32 |
+
rationale_parts = []
|
| 33 |
+
|
| 34 |
+
if past_knee and down_flat:
|
| 35 |
+
score = 3
|
| 36 |
+
rationale_parts.append(f"Raised leg at {angle:.0f}° (past contralateral knee)")
|
| 37 |
+
elif past_mid:
|
| 38 |
+
score = 2
|
| 39 |
+
rationale_parts.append(f"Raised leg at {angle:.0f}° (between mid-thigh and knee)")
|
| 40 |
+
if not down_flat:
|
| 41 |
+
rationale_parts.append("down leg lifted off surface")
|
| 42 |
+
else:
|
| 43 |
+
score = 1
|
| 44 |
+
rationale_parts.append(f"Raised leg only {angle:.0f}° (below mid-thigh)")
|
| 45 |
+
|
| 46 |
+
confidence = features.confidence * 0.9
|
| 47 |
+
|
| 48 |
+
return ScoreResult(
|
| 49 |
+
score=score, rationale="; ".join(rationale_parts),
|
| 50 |
+
confidence=confidence, notes="",
|
| 51 |
+
)
|
formscout/rubric/hurdle_step.py
CHANGED
|
@@ -1,60 +1,60 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Hurdle Step rubric scorer — pure function, no model calls.
|
| 3 |
-
|
| 4 |
-
FMS Hurdle Step Criteria (bilateral — score each side, report lower):
|
| 5 |
-
- Score 3: hips/knees/ankles aligned, minimal trunk movement, dowel/posture stable,
|
| 6 |
-
no contact with hurdle.
|
| 7 |
-
- Score 2: movement completed with compensation (trunk lean, loss of alignment).
|
| 8 |
-
- Score 1: contact with hurdle, loss of balance, or inability to maintain alignment.
|
| 9 |
-
- Score 0: PAIN — never auto-scored.
|
| 10 |
-
"""
|
| 11 |
-
from __future__ import annotations
|
| 12 |
-
|
| 13 |
-
from formscout.types import BiomechFeatures, ScoreResult
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
def score_hurdle_step(features: BiomechFeatures) -> ScoreResult:
|
| 17 |
-
"""Pure rubric scorer for hurdle step."""
|
| 18 |
-
angles = features.angles
|
| 19 |
-
alignments = features.alignments
|
| 20 |
-
|
| 21 |
-
has_hip_flex = "step_hip_flexion_deg" in angles
|
| 22 |
-
if not has_hip_flex:
|
| 23 |
-
return ScoreResult(
|
| 24 |
-
score=1, rationale="Insufficient data: hip flexion not measurable",
|
| 25 |
-
confidence=0.3, notes="missing key measurements",
|
| 26 |
-
)
|
| 27 |
-
|
| 28 |
-
trunk_stable = alignments.get("trunk_stable", False)
|
| 29 |
-
stance_extended = alignments.get("stance_knee_extended", False)
|
| 30 |
-
hip_flex = angles.get("step_hip_flexion_deg", 0)
|
| 31 |
-
|
| 32 |
-
rationale_parts = []
|
| 33 |
-
|
| 34 |
-
# Score 3: good hip flexion, trunk stable, stance solid
|
| 35 |
-
if hip_flex > 90 and trunk_stable and stance_extended:
|
| 36 |
-
score = 3
|
| 37 |
-
rationale_parts.append("Hip flexion adequate, trunk stable, stance knee extended")
|
| 38 |
-
elif hip_flex > 70 or (trunk_stable and stance_extended):
|
| 39 |
-
score = 2
|
| 40 |
-
if not trunk_stable:
|
| 41 |
-
rationale_parts.append("trunk lean detected")
|
| 42 |
-
if not stance_extended:
|
| 43 |
-
rationale_parts.append("stance knee flexion")
|
| 44 |
-
if hip_flex <= 90:
|
| 45 |
-
rationale_parts.append(f"hip flexion {hip_flex:.0f}° (borderline)")
|
| 46 |
-
rationale_parts.insert(0, "Movement completed with compensation")
|
| 47 |
-
else:
|
| 48 |
-
score = 1
|
| 49 |
-
rationale_parts.append("Unable to maintain alignment")
|
| 50 |
-
if not trunk_stable:
|
| 51 |
-
rationale_parts.append("significant trunk lean")
|
| 52 |
-
if not stance_extended:
|
| 53 |
-
rationale_parts.append("stance knee collapse")
|
| 54 |
-
|
| 55 |
-
confidence = features.confidence * 0.85
|
| 56 |
-
|
| 57 |
-
return ScoreResult(
|
| 58 |
-
score=score, rationale="; ".join(rationale_parts),
|
| 59 |
-
confidence=confidence, notes="",
|
| 60 |
-
)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Hurdle Step rubric scorer — pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS Hurdle Step Criteria (bilateral — score each side, report lower):
|
| 5 |
+
- Score 3: hips/knees/ankles aligned, minimal trunk movement, dowel/posture stable,
|
| 6 |
+
no contact with hurdle.
|
| 7 |
+
- Score 2: movement completed with compensation (trunk lean, loss of alignment).
|
| 8 |
+
- Score 1: contact with hurdle, loss of balance, or inability to maintain alignment.
|
| 9 |
+
- Score 0: PAIN — never auto-scored.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def score_hurdle_step(features: BiomechFeatures) -> ScoreResult:
|
| 17 |
+
"""Pure rubric scorer for hurdle step."""
|
| 18 |
+
angles = features.angles
|
| 19 |
+
alignments = features.alignments
|
| 20 |
+
|
| 21 |
+
has_hip_flex = "step_hip_flexion_deg" in angles
|
| 22 |
+
if not has_hip_flex:
|
| 23 |
+
return ScoreResult(
|
| 24 |
+
score=1, rationale="Insufficient data: hip flexion not measurable",
|
| 25 |
+
confidence=0.3, notes="missing key measurements",
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
trunk_stable = alignments.get("trunk_stable", False)
|
| 29 |
+
stance_extended = alignments.get("stance_knee_extended", False)
|
| 30 |
+
hip_flex = angles.get("step_hip_flexion_deg", 0)
|
| 31 |
+
|
| 32 |
+
rationale_parts = []
|
| 33 |
+
|
| 34 |
+
# Score 3: good hip flexion, trunk stable, stance solid
|
| 35 |
+
if hip_flex > 90 and trunk_stable and stance_extended:
|
| 36 |
+
score = 3
|
| 37 |
+
rationale_parts.append("Hip flexion adequate, trunk stable, stance knee extended")
|
| 38 |
+
elif hip_flex > 70 or (trunk_stable and stance_extended):
|
| 39 |
+
score = 2
|
| 40 |
+
if not trunk_stable:
|
| 41 |
+
rationale_parts.append("trunk lean detected")
|
| 42 |
+
if not stance_extended:
|
| 43 |
+
rationale_parts.append("stance knee flexion")
|
| 44 |
+
if hip_flex <= 90:
|
| 45 |
+
rationale_parts.append(f"hip flexion {hip_flex:.0f}° (borderline)")
|
| 46 |
+
rationale_parts.insert(0, "Movement completed with compensation")
|
| 47 |
+
else:
|
| 48 |
+
score = 1
|
| 49 |
+
rationale_parts.append("Unable to maintain alignment")
|
| 50 |
+
if not trunk_stable:
|
| 51 |
+
rationale_parts.append("significant trunk lean")
|
| 52 |
+
if not stance_extended:
|
| 53 |
+
rationale_parts.append("stance knee collapse")
|
| 54 |
+
|
| 55 |
+
confidence = features.confidence * 0.85
|
| 56 |
+
|
| 57 |
+
return ScoreResult(
|
| 58 |
+
score=score, rationale="; ".join(rationale_parts),
|
| 59 |
+
confidence=confidence, notes="",
|
| 60 |
+
)
|
formscout/rubric/inline_lunge.py
CHANGED
|
@@ -1,58 +1,58 @@
|
|
| 1 |
-
"""
|
| 2 |
-
In-Line Lunge rubric scorer — pure function, no model calls.
|
| 3 |
-
|
| 4 |
-
FMS In-Line Lunge Criteria (bilateral):
|
| 5 |
-
- Score 3: dowel contacts maintained, no torso movement, knee touches behind heel.
|
| 6 |
-
- Score 2: movement completed with compensation (trunk lean, loss of balance).
|
| 7 |
-
- Score 1: loss of balance, inability to maintain foot contact or posture.
|
| 8 |
-
- Score 0: PAIN — never auto-scored.
|
| 9 |
-
"""
|
| 10 |
-
from __future__ import annotations
|
| 11 |
-
|
| 12 |
-
from formscout.types import BiomechFeatures, ScoreResult
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
def score_inline_lunge(features: BiomechFeatures) -> ScoreResult:
|
| 16 |
-
"""Pure rubric scorer for in-line lunge."""
|
| 17 |
-
angles = features.angles
|
| 18 |
-
alignments = features.alignments
|
| 19 |
-
|
| 20 |
-
has_knee = "front_knee_flexion_deg" in angles
|
| 21 |
-
if not has_knee:
|
| 22 |
-
return ScoreResult(
|
| 23 |
-
score=1, rationale="Insufficient data: knee flexion not measurable",
|
| 24 |
-
confidence=0.3, notes="missing key measurements",
|
| 25 |
-
)
|
| 26 |
-
|
| 27 |
-
knee_flex = angles.get("front_knee_flexion_deg", 180)
|
| 28 |
-
trunk_upright = alignments.get("trunk_upright", False)
|
| 29 |
-
knee_over_ankle = alignments.get("knee_over_ankle", False)
|
| 30 |
-
|
| 31 |
-
rationale_parts = []
|
| 32 |
-
|
| 33 |
-
# Good lunge: knee flexion < 90° (deep), trunk upright, knee aligned
|
| 34 |
-
deep_enough = knee_flex < 100
|
| 35 |
-
if deep_enough and trunk_upright and knee_over_ankle:
|
| 36 |
-
score = 3
|
| 37 |
-
rationale_parts.append("Deep lunge with trunk upright and knee aligned")
|
| 38 |
-
elif deep_enough or (trunk_upright and knee_over_ankle):
|
| 39 |
-
score = 2
|
| 40 |
-
if not trunk_upright:
|
| 41 |
-
rationale_parts.append(f"trunk lean {angles.get('trunk_lean_from_vertical_deg', '?')}°")
|
| 42 |
-
if not knee_over_ankle:
|
| 43 |
-
rationale_parts.append("knee drifts past ankle")
|
| 44 |
-
if not deep_enough:
|
| 45 |
-
rationale_parts.append(f"knee flexion {knee_flex:.0f}° (insufficient depth)")
|
| 46 |
-
rationale_parts.insert(0, "Completed with compensation")
|
| 47 |
-
else:
|
| 48 |
-
score = 1
|
| 49 |
-
rationale_parts.append("Unable to complete lunge pattern")
|
| 50 |
-
if not deep_enough:
|
| 51 |
-
rationale_parts.append(f"knee flexion only {knee_flex:.0f}°")
|
| 52 |
-
|
| 53 |
-
confidence = features.confidence * 0.85
|
| 54 |
-
|
| 55 |
-
return ScoreResult(
|
| 56 |
-
score=score, rationale="; ".join(rationale_parts),
|
| 57 |
-
confidence=confidence, notes="",
|
| 58 |
-
)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
In-Line Lunge rubric scorer — pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS In-Line Lunge Criteria (bilateral):
|
| 5 |
+
- Score 3: dowel contacts maintained, no torso movement, knee touches behind heel.
|
| 6 |
+
- Score 2: movement completed with compensation (trunk lean, loss of balance).
|
| 7 |
+
- Score 1: loss of balance, inability to maintain foot contact or posture.
|
| 8 |
+
- Score 0: PAIN — never auto-scored.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def score_inline_lunge(features: BiomechFeatures) -> ScoreResult:
|
| 16 |
+
"""Pure rubric scorer for in-line lunge."""
|
| 17 |
+
angles = features.angles
|
| 18 |
+
alignments = features.alignments
|
| 19 |
+
|
| 20 |
+
has_knee = "front_knee_flexion_deg" in angles
|
| 21 |
+
if not has_knee:
|
| 22 |
+
return ScoreResult(
|
| 23 |
+
score=1, rationale="Insufficient data: knee flexion not measurable",
|
| 24 |
+
confidence=0.3, notes="missing key measurements",
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
knee_flex = angles.get("front_knee_flexion_deg", 180)
|
| 28 |
+
trunk_upright = alignments.get("trunk_upright", False)
|
| 29 |
+
knee_over_ankle = alignments.get("knee_over_ankle", False)
|
| 30 |
+
|
| 31 |
+
rationale_parts = []
|
| 32 |
+
|
| 33 |
+
# Good lunge: knee flexion < 90° (deep), trunk upright, knee aligned
|
| 34 |
+
deep_enough = knee_flex < 100
|
| 35 |
+
if deep_enough and trunk_upright and knee_over_ankle:
|
| 36 |
+
score = 3
|
| 37 |
+
rationale_parts.append("Deep lunge with trunk upright and knee aligned")
|
| 38 |
+
elif deep_enough or (trunk_upright and knee_over_ankle):
|
| 39 |
+
score = 2
|
| 40 |
+
if not trunk_upright:
|
| 41 |
+
rationale_parts.append(f"trunk lean {angles.get('trunk_lean_from_vertical_deg', '?')}°")
|
| 42 |
+
if not knee_over_ankle:
|
| 43 |
+
rationale_parts.append("knee drifts past ankle")
|
| 44 |
+
if not deep_enough:
|
| 45 |
+
rationale_parts.append(f"knee flexion {knee_flex:.0f}° (insufficient depth)")
|
| 46 |
+
rationale_parts.insert(0, "Completed with compensation")
|
| 47 |
+
else:
|
| 48 |
+
score = 1
|
| 49 |
+
rationale_parts.append("Unable to complete lunge pattern")
|
| 50 |
+
if not deep_enough:
|
| 51 |
+
rationale_parts.append(f"knee flexion only {knee_flex:.0f}°")
|
| 52 |
+
|
| 53 |
+
confidence = features.confidence * 0.85
|
| 54 |
+
|
| 55 |
+
return ScoreResult(
|
| 56 |
+
score=score, rationale="; ".join(rationale_parts),
|
| 57 |
+
confidence=confidence, notes="",
|
| 58 |
+
)
|
formscout/rubric/rotary_stability.py
CHANGED
|
@@ -1,56 +1,56 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Rotary Stability rubric scorer — pure function, no model calls.
|
| 3 |
-
|
| 4 |
-
FMS Rotary Stability Criteria:
|
| 5 |
-
- Score 3: unilateral (same-side) arm/leg extension with trunk stable,
|
| 6 |
-
elbow/knee touch performed smoothly.
|
| 7 |
-
- Score 2: contralateral (opposite) arm/leg extension performed with trunk stable.
|
| 8 |
-
- Score 1: inability to maintain trunk stability during contralateral pattern.
|
| 9 |
-
- Score 0: PAIN (spinal flexion clearing test) — never auto-scored.
|
| 10 |
-
"""
|
| 11 |
-
from __future__ import annotations
|
| 12 |
-
|
| 13 |
-
from formscout.types import BiomechFeatures, ScoreResult
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
def score_rotary_stability(features: BiomechFeatures) -> ScoreResult:
|
| 17 |
-
"""Pure rubric scorer for rotary stability."""
|
| 18 |
-
angles = features.angles
|
| 19 |
-
alignments = features.alignments
|
| 20 |
-
|
| 21 |
-
has_data = "trunk_stability_std_px" in angles or "shoulder_level_diff_px" in angles
|
| 22 |
-
if not has_data:
|
| 23 |
-
return ScoreResult(
|
| 24 |
-
score=1, rationale="Insufficient data: trunk stability not measurable",
|
| 25 |
-
confidence=0.3, notes="missing key measurements",
|
| 26 |
-
)
|
| 27 |
-
|
| 28 |
-
trunk_stable = alignments.get("trunk_stable", False)
|
| 29 |
-
shoulders_level = alignments.get("shoulders_level", False)
|
| 30 |
-
hips_level = alignments.get("hips_level", False)
|
| 31 |
-
|
| 32 |
-
rationale_parts = []
|
| 33 |
-
|
| 34 |
-
# Without video classification of ipsi vs contra, assume contralateral (safer)
|
| 35 |
-
if trunk_stable and shoulders_level and hips_level:
|
| 36 |
-
score = 2 # Assume contralateral unless classifier says ipsilateral
|
| 37 |
-
rationale_parts.append("Trunk stable during extension, shoulders and hips level")
|
| 38 |
-
rationale_parts.append("scored as contralateral pattern (default)")
|
| 39 |
-
elif trunk_stable or (shoulders_level and hips_level):
|
| 40 |
-
score = 2
|
| 41 |
-
if not trunk_stable:
|
| 42 |
-
rationale_parts.append("minor trunk instability")
|
| 43 |
-
rationale_parts.insert(0, "Contralateral pattern with minor compensation")
|
| 44 |
-
else:
|
| 45 |
-
score = 1
|
| 46 |
-
std = angles.get("trunk_stability_std_px", 0)
|
| 47 |
-
rationale_parts.append(f"Trunk instability detected (std {std:.1f}px)")
|
| 48 |
-
if not shoulders_level:
|
| 49 |
-
rationale_parts.append("shoulder asymmetry during extension")
|
| 50 |
-
|
| 51 |
-
confidence = features.confidence * 0.75 # Lower confidence — hard to assess from 2D
|
| 52 |
-
|
| 53 |
-
return ScoreResult(
|
| 54 |
-
score=score, rationale="; ".join(rationale_parts),
|
| 55 |
-
confidence=confidence, notes="ipsi/contra distinction requires VLM classifier",
|
| 56 |
-
)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Rotary Stability rubric scorer — pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS Rotary Stability Criteria:
|
| 5 |
+
- Score 3: unilateral (same-side) arm/leg extension with trunk stable,
|
| 6 |
+
elbow/knee touch performed smoothly.
|
| 7 |
+
- Score 2: contralateral (opposite) arm/leg extension performed with trunk stable.
|
| 8 |
+
- Score 1: inability to maintain trunk stability during contralateral pattern.
|
| 9 |
+
- Score 0: PAIN (spinal flexion clearing test) — never auto-scored.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def score_rotary_stability(features: BiomechFeatures) -> ScoreResult:
|
| 17 |
+
"""Pure rubric scorer for rotary stability."""
|
| 18 |
+
angles = features.angles
|
| 19 |
+
alignments = features.alignments
|
| 20 |
+
|
| 21 |
+
has_data = "trunk_stability_std_px" in angles or "shoulder_level_diff_px" in angles
|
| 22 |
+
if not has_data:
|
| 23 |
+
return ScoreResult(
|
| 24 |
+
score=1, rationale="Insufficient data: trunk stability not measurable",
|
| 25 |
+
confidence=0.3, notes="missing key measurements",
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
trunk_stable = alignments.get("trunk_stable", False)
|
| 29 |
+
shoulders_level = alignments.get("shoulders_level", False)
|
| 30 |
+
hips_level = alignments.get("hips_level", False)
|
| 31 |
+
|
| 32 |
+
rationale_parts = []
|
| 33 |
+
|
| 34 |
+
# Without video classification of ipsi vs contra, assume contralateral (safer)
|
| 35 |
+
if trunk_stable and shoulders_level and hips_level:
|
| 36 |
+
score = 2 # Assume contralateral unless classifier says ipsilateral
|
| 37 |
+
rationale_parts.append("Trunk stable during extension, shoulders and hips level")
|
| 38 |
+
rationale_parts.append("scored as contralateral pattern (default)")
|
| 39 |
+
elif trunk_stable or (shoulders_level and hips_level):
|
| 40 |
+
score = 2
|
| 41 |
+
if not trunk_stable:
|
| 42 |
+
rationale_parts.append("minor trunk instability")
|
| 43 |
+
rationale_parts.insert(0, "Contralateral pattern with minor compensation")
|
| 44 |
+
else:
|
| 45 |
+
score = 1
|
| 46 |
+
std = angles.get("trunk_stability_std_px", 0)
|
| 47 |
+
rationale_parts.append(f"Trunk instability detected (std {std:.1f}px)")
|
| 48 |
+
if not shoulders_level:
|
| 49 |
+
rationale_parts.append("shoulder asymmetry during extension")
|
| 50 |
+
|
| 51 |
+
confidence = features.confidence * 0.75 # Lower confidence — hard to assess from 2D
|
| 52 |
+
|
| 53 |
+
return ScoreResult(
|
| 54 |
+
score=score, rationale="; ".join(rationale_parts),
|
| 55 |
+
confidence=confidence, notes="ipsi/contra distinction requires VLM classifier",
|
| 56 |
+
)
|
formscout/rubric/shoulder_mobility.py
CHANGED
|
@@ -1,46 +1,46 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Shoulder Mobility rubric scorer — pure function, no model calls.
|
| 3 |
-
|
| 4 |
-
FMS Shoulder Mobility Criteria (bilateral):
|
| 5 |
-
- Score 3: fists within one hand-length of each other.
|
| 6 |
-
- Score 2: fists within 1.5 hand-lengths.
|
| 7 |
-
- Score 1: fists more than 1.5 hand-lengths apart.
|
| 8 |
-
- Score 0: PAIN (clearing test) — never auto-scored.
|
| 9 |
-
"""
|
| 10 |
-
from __future__ import annotations
|
| 11 |
-
|
| 12 |
-
from formscout.types import BiomechFeatures, ScoreResult
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
def score_shoulder_mobility(features: BiomechFeatures) -> ScoreResult:
|
| 16 |
-
"""Pure rubric scorer for shoulder mobility."""
|
| 17 |
-
alignments = features.alignments
|
| 18 |
-
angles = features.angles
|
| 19 |
-
|
| 20 |
-
has_measure = "inter_fist_normalized" in angles
|
| 21 |
-
if not has_measure:
|
| 22 |
-
return ScoreResult(
|
| 23 |
-
score=1, rationale="Insufficient data: inter-fist distance not measurable",
|
| 24 |
-
confidence=0.3, notes="missing key measurements",
|
| 25 |
-
)
|
| 26 |
-
|
| 27 |
-
norm_dist = angles["inter_fist_normalized"]
|
| 28 |
-
within_one = alignments.get("fists_within_one_hand", False)
|
| 29 |
-
within_1_5 = alignments.get("fists_within_1_5_hand", False)
|
| 30 |
-
|
| 31 |
-
if within_one:
|
| 32 |
-
score = 3
|
| 33 |
-
rationale = f"Fists within one hand-length (normalized distance {norm_dist:.2f})"
|
| 34 |
-
elif within_1_5:
|
| 35 |
-
score = 2
|
| 36 |
-
rationale = f"Fists within 1.5 hand-lengths (normalized distance {norm_dist:.2f})"
|
| 37 |
-
else:
|
| 38 |
-
score = 1
|
| 39 |
-
rationale = f"Fists beyond 1.5 hand-lengths apart (normalized distance {norm_dist:.2f})"
|
| 40 |
-
|
| 41 |
-
confidence = features.confidence * 0.9
|
| 42 |
-
|
| 43 |
-
return ScoreResult(
|
| 44 |
-
score=score, rationale=rationale,
|
| 45 |
-
confidence=confidence, notes="",
|
| 46 |
-
)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Shoulder Mobility rubric scorer — pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS Shoulder Mobility Criteria (bilateral):
|
| 5 |
+
- Score 3: fists within one hand-length of each other.
|
| 6 |
+
- Score 2: fists within 1.5 hand-lengths.
|
| 7 |
+
- Score 1: fists more than 1.5 hand-lengths apart.
|
| 8 |
+
- Score 0: PAIN (clearing test) — never auto-scored.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def score_shoulder_mobility(features: BiomechFeatures) -> ScoreResult:
|
| 16 |
+
"""Pure rubric scorer for shoulder mobility."""
|
| 17 |
+
alignments = features.alignments
|
| 18 |
+
angles = features.angles
|
| 19 |
+
|
| 20 |
+
has_measure = "inter_fist_normalized" in angles
|
| 21 |
+
if not has_measure:
|
| 22 |
+
return ScoreResult(
|
| 23 |
+
score=1, rationale="Insufficient data: inter-fist distance not measurable",
|
| 24 |
+
confidence=0.3, notes="missing key measurements",
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
norm_dist = angles["inter_fist_normalized"]
|
| 28 |
+
within_one = alignments.get("fists_within_one_hand", False)
|
| 29 |
+
within_1_5 = alignments.get("fists_within_1_5_hand", False)
|
| 30 |
+
|
| 31 |
+
if within_one:
|
| 32 |
+
score = 3
|
| 33 |
+
rationale = f"Fists within one hand-length (normalized distance {norm_dist:.2f})"
|
| 34 |
+
elif within_1_5:
|
| 35 |
+
score = 2
|
| 36 |
+
rationale = f"Fists within 1.5 hand-lengths (normalized distance {norm_dist:.2f})"
|
| 37 |
+
else:
|
| 38 |
+
score = 1
|
| 39 |
+
rationale = f"Fists beyond 1.5 hand-lengths apart (normalized distance {norm_dist:.2f})"
|
| 40 |
+
|
| 41 |
+
confidence = features.confidence * 0.9
|
| 42 |
+
|
| 43 |
+
return ScoreResult(
|
| 44 |
+
score=score, rationale=rationale,
|
| 45 |
+
confidence=confidence, notes="",
|
| 46 |
+
)
|
formscout/rubric/trunk_stability_pushup.py
CHANGED
|
@@ -1,55 +1,55 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Trunk Stability Push-Up rubric scorer — pure function, no model calls.
|
| 3 |
-
|
| 4 |
-
FMS Trunk Stability Push-Up Criteria:
|
| 5 |
-
- Score 3: body moves as one unit (rigid) with hands at forehead level (men)
|
| 6 |
-
or chin level (women). No sag or segment lag.
|
| 7 |
-
- Score 2: body moves as one unit but with hands at chin (men) or clavicle (women).
|
| 8 |
-
- Score 1: unable to perform with hands lowered; body sags or segments.
|
| 9 |
-
- Score 0: PAIN (spinal extension clearing test) — never auto-scored.
|
| 10 |
-
"""
|
| 11 |
-
from __future__ import annotations
|
| 12 |
-
|
| 13 |
-
from formscout.types import BiomechFeatures, ScoreResult
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
def score_trunk_stability_pushup(features: BiomechFeatures) -> ScoreResult:
|
| 17 |
-
"""Pure rubric scorer for trunk stability push-up."""
|
| 18 |
-
angles = features.angles
|
| 19 |
-
alignments = features.alignments
|
| 20 |
-
|
| 21 |
-
has_data = "max_sag_px" in angles
|
| 22 |
-
if not has_data:
|
| 23 |
-
return ScoreResult(
|
| 24 |
-
score=1, rationale="Insufficient data: trunk rigidity not measurable",
|
| 25 |
-
confidence=0.3, notes="missing key measurements",
|
| 26 |
-
)
|
| 27 |
-
|
| 28 |
-
body_rigid = alignments.get("body_rigid", False)
|
| 29 |
-
no_sag = alignments.get("no_sag", False)
|
| 30 |
-
hands_high = alignments.get("hands_at_forehead", False)
|
| 31 |
-
|
| 32 |
-
rationale_parts = []
|
| 33 |
-
|
| 34 |
-
if body_rigid and hands_high:
|
| 35 |
-
score = 3
|
| 36 |
-
rationale_parts.append("Body rigid as one unit, hands at forehead position")
|
| 37 |
-
elif body_rigid or no_sag:
|
| 38 |
-
score = 2
|
| 39 |
-
if not hands_high:
|
| 40 |
-
rationale_parts.append("rigid body but hands in lower position")
|
| 41 |
-
else:
|
| 42 |
-
rationale_parts.append("minor trunk variance detected")
|
| 43 |
-
rationale_parts.insert(0, "Completed with regression")
|
| 44 |
-
else:
|
| 45 |
-
score = 1
|
| 46 |
-
sag = angles.get("max_sag_px", 0)
|
| 47 |
-
variance = angles.get("trunk_variance_px", 0)
|
| 48 |
-
rationale_parts.append(f"Body sag detected ({sag:.0f}px), variance {variance:.1f}px")
|
| 49 |
-
|
| 50 |
-
confidence = features.confidence * 0.8
|
| 51 |
-
|
| 52 |
-
return ScoreResult(
|
| 53 |
-
score=score, rationale="; ".join(rationale_parts),
|
| 54 |
-
confidence=confidence, notes="",
|
| 55 |
-
)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Trunk Stability Push-Up rubric scorer — pure function, no model calls.
|
| 3 |
+
|
| 4 |
+
FMS Trunk Stability Push-Up Criteria:
|
| 5 |
+
- Score 3: body moves as one unit (rigid) with hands at forehead level (men)
|
| 6 |
+
or chin level (women). No sag or segment lag.
|
| 7 |
+
- Score 2: body moves as one unit but with hands at chin (men) or clavicle (women).
|
| 8 |
+
- Score 1: unable to perform with hands lowered; body sags or segments.
|
| 9 |
+
- Score 0: PAIN (spinal extension clearing test) — never auto-scored.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from formscout.types import BiomechFeatures, ScoreResult
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def score_trunk_stability_pushup(features: BiomechFeatures) -> ScoreResult:
|
| 17 |
+
"""Pure rubric scorer for trunk stability push-up."""
|
| 18 |
+
angles = features.angles
|
| 19 |
+
alignments = features.alignments
|
| 20 |
+
|
| 21 |
+
has_data = "max_sag_px" in angles
|
| 22 |
+
if not has_data:
|
| 23 |
+
return ScoreResult(
|
| 24 |
+
score=1, rationale="Insufficient data: trunk rigidity not measurable",
|
| 25 |
+
confidence=0.3, notes="missing key measurements",
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
body_rigid = alignments.get("body_rigid", False)
|
| 29 |
+
no_sag = alignments.get("no_sag", False)
|
| 30 |
+
hands_high = alignments.get("hands_at_forehead", False)
|
| 31 |
+
|
| 32 |
+
rationale_parts = []
|
| 33 |
+
|
| 34 |
+
if body_rigid and hands_high:
|
| 35 |
+
score = 3
|
| 36 |
+
rationale_parts.append("Body rigid as one unit, hands at forehead position")
|
| 37 |
+
elif body_rigid or no_sag:
|
| 38 |
+
score = 2
|
| 39 |
+
if not hands_high:
|
| 40 |
+
rationale_parts.append("rigid body but hands in lower position")
|
| 41 |
+
else:
|
| 42 |
+
rationale_parts.append("minor trunk variance detected")
|
| 43 |
+
rationale_parts.insert(0, "Completed with regression")
|
| 44 |
+
else:
|
| 45 |
+
score = 1
|
| 46 |
+
sag = angles.get("max_sag_px", 0)
|
| 47 |
+
variance = angles.get("trunk_variance_px", 0)
|
| 48 |
+
rationale_parts.append(f"Body sag detected ({sag:.0f}px), variance {variance:.1f}px")
|
| 49 |
+
|
| 50 |
+
confidence = features.confidence * 0.8
|
| 51 |
+
|
| 52 |
+
return ScoreResult(
|
| 53 |
+
score=score, rationale="; ".join(rationale_parts),
|
| 54 |
+
confidence=confidence, notes="",
|
| 55 |
+
)
|
formscout/serving/__init__.py
CHANGED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Serving backends for the FormScout judge/classifier VLM."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def get_vlm_client():
|
| 6 |
+
"""Return the VLM client for the resolved judge backend.
|
| 7 |
+
|
| 8 |
+
transformers (in-process, ZeroGPU) on a Space; llama-server locally. Falls
|
| 9 |
+
back to the llama.cpp client if the transformers backend can't be imported.
|
| 10 |
+
"""
|
| 11 |
+
from formscout import config
|
| 12 |
+
|
| 13 |
+
if config.resolve_judge_backend() == "transformers":
|
| 14 |
+
try:
|
| 15 |
+
from formscout.serving.transformers_vlm import TransformersVLMClient
|
| 16 |
+
return TransformersVLMClient()
|
| 17 |
+
except Exception:
|
| 18 |
+
pass
|
| 19 |
+
from formscout.serving.llama_cpp import LlamaCppClient
|
| 20 |
+
return LlamaCppClient(port=config.LLAMA_CPP_PORT_VLM)
|
formscout/serving/llama_cpp.py
CHANGED
|
@@ -1,174 +1,148 @@
|
|
| 1 |
-
"""
|
| 2 |
-
llama.cpp HTTP client wrapper for FormScout.
|
| 3 |
-
|
| 4 |
-
Wraps the llama.cpp server's /completion and /embedding endpoints.
|
| 5 |
-
Falls back gracefully when the server is unavailable.
|
| 6 |
-
|
| 7 |
-
Model: Qwen3-VL-8B-Instruct (Q4_K_M GGUF) for VLM inference.
|
| 8 |
-
Model: Qwen3-VL-Embedding-8B (Q4_K_M GGUF) for embeddings.
|
| 9 |
-
Params: 8B each (shared backbone).
|
| 10 |
-
License: Apache-2.0.
|
| 11 |
-
"""
|
| 12 |
-
from __future__ import annotations
|
| 13 |
-
|
| 14 |
-
import base64
|
| 15 |
-
import json
|
| 16 |
-
import logging
|
| 17 |
-
from pathlib import Path
|
| 18 |
-
from typing import Any
|
| 19 |
-
|
| 20 |
-
import requests
|
| 21 |
-
|
| 22 |
-
from formscout import config
|
| 23 |
-
|
| 24 |
-
logger = logging.getLogger(__name__)
|
| 25 |
-
|
| 26 |
-
_TIMEOUT = 120 # seconds — VLM can be slow
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
class LlamaCppClient:
|
| 30 |
-
"""HTTP client for a llama.cpp server instance."""
|
| 31 |
-
|
| 32 |
-
def __init__(self, host: str | None = None, port: int | None = None):
|
| 33 |
-
self.host = host or config.LLAMA_CPP_HOST
|
| 34 |
-
self.port = port or config.LLAMA_CPP_PORT_VLM
|
| 35 |
-
self.base_url = f"http://{self.host}:{self.port}"
|
| 36 |
-
|
| 37 |
-
@property
|
| 38 |
-
def available(self) -> bool:
|
| 39 |
-
"""Check if the server is reachable."""
|
| 40 |
-
try:
|
| 41 |
-
r = requests.get(f"{self.base_url}/health", timeout=5)
|
| 42 |
-
return r.status_code == 200
|
| 43 |
-
except (requests.ConnectionError, requests.Timeout):
|
| 44 |
-
return False
|
| 45 |
-
|
| 46 |
-
def complete(
|
| 47 |
-
self,
|
| 48 |
-
prompt: str,
|
| 49 |
-
images: list[str] | None = None,
|
| 50 |
-
max_tokens: int = 512,
|
| 51 |
-
temperature: float = 0.1,
|
| 52 |
-
stop: list[str] | None = None,
|
| 53 |
-
) -> dict[str, Any]:
|
| 54 |
-
"""
|
| 55 |
-
Send a chat-completion request (OpenAI-compatible /v1/chat/completions —
|
| 56 |
-
required for multimodal: llama-server routes images through the mmproj
|
| 57 |
-
only on this endpoint). Returns parsed JSON if the response is JSON,
|
| 58 |
-
otherwise returns {"text": raw_text}.
|
| 59 |
-
|
| 60 |
-
Args:
|
| 61 |
-
prompt: The text prompt (system + user combined).
|
| 62 |
-
images: Optional list of base64-encoded JPEGs or file paths.
|
| 63 |
-
max_tokens: Max generation tokens.
|
| 64 |
-
temperature: Sampling temperature.
|
| 65 |
-
stop: Stop sequences (default: none — JSON output must not be truncated).
|
| 66 |
-
"""
|
| 67 |
-
content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
|
| 68 |
-
for img in images or []:
|
| 69 |
-
if len(img) < 4096 and Path(img).exists():
|
| 70 |
-
with open(img, "rb") as f:
|
| 71 |
-
b64 = base64.b64encode(f.read()).decode()
|
| 72 |
-
else:
|
| 73 |
-
b64 = img # already base64
|
| 74 |
-
content.append({
|
| 75 |
-
"type": "image_url",
|
| 76 |
-
"image_url": {"url": f"data:image/jpeg;base64,{b64}"},
|
| 77 |
-
})
|
| 78 |
-
|
| 79 |
-
payload: dict[str, Any] = {
|
| 80 |
-
"
|
| 81 |
-
"
|
| 82 |
-
"
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
)
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
def __init__(self, host: str | None = None, port: int | None = None):
|
| 150 |
-
self.host = host or config.LLAMA_CPP_HOST
|
| 151 |
-
self.port = port or config.LLAMA_CPP_PORT_EMBED
|
| 152 |
-
self.base_url = f"http://{self.host}:{self.port}"
|
| 153 |
-
|
| 154 |
-
@property
|
| 155 |
-
def available(self) -> bool:
|
| 156 |
-
try:
|
| 157 |
-
r = requests.get(f"{self.base_url}/health", timeout=5)
|
| 158 |
-
return r.status_code == 200
|
| 159 |
-
except (requests.ConnectionError, requests.Timeout):
|
| 160 |
-
return False
|
| 161 |
-
|
| 162 |
-
def embed(self, text: str) -> list[float] | None:
|
| 163 |
-
"""Get embedding vector for text. Returns None on failure."""
|
| 164 |
-
try:
|
| 165 |
-
r = requests.post(
|
| 166 |
-
f"{self.base_url}/embedding",
|
| 167 |
-
json={"content": text},
|
| 168 |
-
timeout=30,
|
| 169 |
-
)
|
| 170 |
-
r.raise_for_status()
|
| 171 |
-
data = r.json()
|
| 172 |
-
return data.get("embedding")
|
| 173 |
-
except Exception:
|
| 174 |
-
return None
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
llama.cpp HTTP client wrapper for FormScout.
|
| 3 |
+
|
| 4 |
+
Wraps the llama.cpp server's /completion and /embedding endpoints.
|
| 5 |
+
Falls back gracefully when the server is unavailable.
|
| 6 |
+
|
| 7 |
+
Model: Qwen3-VL-8B-Instruct (Q4_K_M GGUF) for VLM inference.
|
| 8 |
+
Model: Qwen3-VL-Embedding-8B (Q4_K_M GGUF) for embeddings.
|
| 9 |
+
Params: 8B each (shared backbone).
|
| 10 |
+
License: Apache-2.0.
|
| 11 |
+
"""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import base64
|
| 15 |
+
import json
|
| 16 |
+
import logging
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
from typing import Any
|
| 19 |
+
|
| 20 |
+
import requests
|
| 21 |
+
|
| 22 |
+
from formscout import config
|
| 23 |
+
|
| 24 |
+
logger = logging.getLogger(__name__)
|
| 25 |
+
|
| 26 |
+
_TIMEOUT = 120 # seconds — VLM can be slow
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class LlamaCppClient:
|
| 30 |
+
"""HTTP client for a llama.cpp server instance."""
|
| 31 |
+
|
| 32 |
+
def __init__(self, host: str | None = None, port: int | None = None):
|
| 33 |
+
self.host = host or config.LLAMA_CPP_HOST
|
| 34 |
+
self.port = port or config.LLAMA_CPP_PORT_VLM
|
| 35 |
+
self.base_url = f"http://{self.host}:{self.port}"
|
| 36 |
+
|
| 37 |
+
@property
|
| 38 |
+
def available(self) -> bool:
|
| 39 |
+
"""Check if the server is reachable."""
|
| 40 |
+
try:
|
| 41 |
+
r = requests.get(f"{self.base_url}/health", timeout=5)
|
| 42 |
+
return r.status_code == 200
|
| 43 |
+
except (requests.ConnectionError, requests.Timeout):
|
| 44 |
+
return False
|
| 45 |
+
|
| 46 |
+
def complete(
|
| 47 |
+
self,
|
| 48 |
+
prompt: str,
|
| 49 |
+
images: list[str] | None = None,
|
| 50 |
+
max_tokens: int = 512,
|
| 51 |
+
temperature: float = 0.1,
|
| 52 |
+
stop: list[str] | None = None,
|
| 53 |
+
) -> dict[str, Any]:
|
| 54 |
+
"""
|
| 55 |
+
Send a chat-completion request (OpenAI-compatible /v1/chat/completions —
|
| 56 |
+
required for multimodal: llama-server routes images through the mmproj
|
| 57 |
+
only on this endpoint). Returns parsed JSON if the response is JSON,
|
| 58 |
+
otherwise returns {"text": raw_text}.
|
| 59 |
+
|
| 60 |
+
Args:
|
| 61 |
+
prompt: The text prompt (system + user combined).
|
| 62 |
+
images: Optional list of base64-encoded JPEGs or file paths.
|
| 63 |
+
max_tokens: Max generation tokens.
|
| 64 |
+
temperature: Sampling temperature.
|
| 65 |
+
stop: Stop sequences (default: none — JSON output must not be truncated).
|
| 66 |
+
"""
|
| 67 |
+
content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
|
| 68 |
+
for img in images or []:
|
| 69 |
+
if len(img) < 4096 and Path(img).exists():
|
| 70 |
+
with open(img, "rb") as f:
|
| 71 |
+
b64 = base64.b64encode(f.read()).decode()
|
| 72 |
+
else:
|
| 73 |
+
b64 = img # already base64
|
| 74 |
+
content.append({
|
| 75 |
+
"type": "image_url",
|
| 76 |
+
"image_url": {"url": f"data:image/jpeg;base64,{b64}"},
|
| 77 |
+
})
|
| 78 |
+
|
| 79 |
+
payload: dict[str, Any] = {
|
| 80 |
+
"messages": [{"role": "user", "content": content}],
|
| 81 |
+
"max_tokens": max_tokens,
|
| 82 |
+
"temperature": temperature,
|
| 83 |
+
}
|
| 84 |
+
if stop:
|
| 85 |
+
payload["stop"] = stop
|
| 86 |
+
|
| 87 |
+
try:
|
| 88 |
+
r = requests.post(
|
| 89 |
+
f"{self.base_url}/v1/chat/completions",
|
| 90 |
+
json=payload,
|
| 91 |
+
timeout=_TIMEOUT,
|
| 92 |
+
)
|
| 93 |
+
r.raise_for_status()
|
| 94 |
+
result = r.json()
|
| 95 |
+
text = result["choices"][0]["message"]["content"] or ""
|
| 96 |
+
return self._parse_json_reply(text)
|
| 97 |
+
except requests.ConnectionError:
|
| 98 |
+
return {"error": "llama.cpp server not available", "text": ""}
|
| 99 |
+
except requests.Timeout:
|
| 100 |
+
return {"error": "llama.cpp server timeout", "text": ""}
|
| 101 |
+
except Exception as e:
|
| 102 |
+
return {"error": str(e), "text": ""}
|
| 103 |
+
|
| 104 |
+
@staticmethod
|
| 105 |
+
def _parse_json_reply(text: str) -> dict[str, Any]:
|
| 106 |
+
"""Parse model output as JSON, tolerating markdown fences."""
|
| 107 |
+
stripped = text.strip()
|
| 108 |
+
if stripped.startswith("```"):
|
| 109 |
+
stripped = stripped.split("\n", 1)[-1]
|
| 110 |
+
stripped = stripped.rsplit("```", 1)[0].strip()
|
| 111 |
+
try:
|
| 112 |
+
parsed = json.loads(stripped)
|
| 113 |
+
if isinstance(parsed, dict):
|
| 114 |
+
return parsed
|
| 115 |
+
except (json.JSONDecodeError, TypeError):
|
| 116 |
+
pass
|
| 117 |
+
return {"text": text}
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
class EmbeddingClient:
|
| 121 |
+
"""HTTP client for the llama.cpp embedding server."""
|
| 122 |
+
|
| 123 |
+
def __init__(self, host: str | None = None, port: int | None = None):
|
| 124 |
+
self.host = host or config.LLAMA_CPP_HOST
|
| 125 |
+
self.port = port or config.LLAMA_CPP_PORT_EMBED
|
| 126 |
+
self.base_url = f"http://{self.host}:{self.port}"
|
| 127 |
+
|
| 128 |
+
@property
|
| 129 |
+
def available(self) -> bool:
|
| 130 |
+
try:
|
| 131 |
+
r = requests.get(f"{self.base_url}/health", timeout=5)
|
| 132 |
+
return r.status_code == 200
|
| 133 |
+
except (requests.ConnectionError, requests.Timeout):
|
| 134 |
+
return False
|
| 135 |
+
|
| 136 |
+
def embed(self, text: str) -> list[float] | None:
|
| 137 |
+
"""Get embedding vector for text. Returns None on failure."""
|
| 138 |
+
try:
|
| 139 |
+
r = requests.post(
|
| 140 |
+
f"{self.base_url}/embedding",
|
| 141 |
+
json={"content": text},
|
| 142 |
+
timeout=30,
|
| 143 |
+
)
|
| 144 |
+
r.raise_for_status()
|
| 145 |
+
data = r.json()
|
| 146 |
+
return data.get("embedding")
|
| 147 |
+
except Exception:
|
| 148 |
+
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
formscout/serving/transformers_vlm.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
In-process Qwen3-VL backend via transformers — the HF Spaces (ZeroGPU) path.
|
| 3 |
+
|
| 4 |
+
Mirrors LlamaCppClient's interface (`available` + `complete(...) -> dict`) so it
|
| 5 |
+
is a drop-in alternative selected by config.resolve_judge_backend(). Inference is
|
| 6 |
+
wrapped in spaces.GPU when the `spaces` package is present (ZeroGPU); otherwise it
|
| 7 |
+
runs on whatever device transformers picks. The model is loaded lazily on first
|
| 8 |
+
use and cached for the process.
|
| 9 |
+
|
| 10 |
+
Any load/inference failure returns {"error": ..., "fallback": True} so JudgeAgent
|
| 11 |
+
falls back to the deterministic rubric score instead of flagging needs_human.
|
| 12 |
+
|
| 13 |
+
Model: Qwen3-VL-8B-Instruct (transformers weights, ~16 GB). License: Apache-2.0.
|
| 14 |
+
|
| 15 |
+
NOTE: requires validation on actual ZeroGPU hardware — it cannot be exercised in
|
| 16 |
+
the CPU test environment (would download 16 GB).
|
| 17 |
+
"""
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import base64
|
| 21 |
+
import importlib.util
|
| 22 |
+
import logging
|
| 23 |
+
|
| 24 |
+
from formscout import config
|
| 25 |
+
from formscout.serving.llama_cpp import LlamaCppClient
|
| 26 |
+
|
| 27 |
+
logger = logging.getLogger(__name__)
|
| 28 |
+
|
| 29 |
+
# Module-level model cache (loaded once per process).
|
| 30 |
+
_CACHE: dict = {}
|
| 31 |
+
|
| 32 |
+
# spaces.GPU decorator when on HF infra; a no-op decorator otherwise.
|
| 33 |
+
try: # pragma: no cover - depends on runtime env
|
| 34 |
+
import spaces
|
| 35 |
+
|
| 36 |
+
_gpu = spaces.GPU(duration=120)
|
| 37 |
+
except Exception: # pragma: no cover
|
| 38 |
+
def _gpu(fn):
|
| 39 |
+
return fn
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@_gpu
|
| 43 |
+
def _generate(model_id: str, prompt: str, pil_images: list, max_tokens: int,
|
| 44 |
+
temperature: float) -> str: # pragma: no cover - needs GPU + model
|
| 45 |
+
"""Load (cached) and run Qwen3-VL; returns the raw decoded string."""
|
| 46 |
+
import torch
|
| 47 |
+
from transformers import AutoModelForImageTextToText, AutoProcessor
|
| 48 |
+
|
| 49 |
+
if "model" not in _CACHE:
|
| 50 |
+
_CACHE["processor"] = AutoProcessor.from_pretrained(model_id)
|
| 51 |
+
_CACHE["model"] = AutoModelForImageTextToText.from_pretrained(
|
| 52 |
+
model_id, torch_dtype="auto", device_map="auto",
|
| 53 |
+
)
|
| 54 |
+
processor, model = _CACHE["processor"], _CACHE["model"]
|
| 55 |
+
|
| 56 |
+
content = [{"type": "image", "image": im} for im in pil_images]
|
| 57 |
+
content.append({"type": "text", "text": prompt})
|
| 58 |
+
messages = [{"role": "user", "content": content}]
|
| 59 |
+
|
| 60 |
+
inputs = processor.apply_chat_template(
|
| 61 |
+
messages, tokenize=True, add_generation_prompt=True,
|
| 62 |
+
return_tensors="pt", return_dict=True,
|
| 63 |
+
).to(model.device)
|
| 64 |
+
|
| 65 |
+
with torch.no_grad():
|
| 66 |
+
out = model.generate(
|
| 67 |
+
**inputs, max_new_tokens=max_tokens,
|
| 68 |
+
do_sample=temperature > 0, temperature=max(temperature, 1e-2),
|
| 69 |
+
)
|
| 70 |
+
gen = out[:, inputs["input_ids"].shape[1]:]
|
| 71 |
+
return processor.batch_decode(gen, skip_special_tokens=True)[0]
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
class TransformersVLMClient:
|
| 75 |
+
"""In-process Qwen3-VL client (ZeroGPU on Spaces)."""
|
| 76 |
+
|
| 77 |
+
def __init__(self, model_id: str | None = None):
|
| 78 |
+
self.model_id = model_id or config.JUDGE_HF_MODEL
|
| 79 |
+
self._failed = False
|
| 80 |
+
|
| 81 |
+
@property
|
| 82 |
+
def available(self) -> bool:
|
| 83 |
+
"""Cheap check — does NOT load the model (so tests stay download-free)."""
|
| 84 |
+
if self._failed:
|
| 85 |
+
return False
|
| 86 |
+
return importlib.util.find_spec("transformers") is not None
|
| 87 |
+
|
| 88 |
+
def complete(self, prompt: str, images: list[str] | None = None,
|
| 89 |
+
max_tokens: int = 512, temperature: float = 0.1,
|
| 90 |
+
stop: list[str] | None = None) -> dict:
|
| 91 |
+
try:
|
| 92 |
+
pil_images = self._decode_images(images)
|
| 93 |
+
text = _generate(self.model_id, prompt, pil_images, max_tokens, temperature)
|
| 94 |
+
return LlamaCppClient._parse_json_reply(text)
|
| 95 |
+
except Exception as e: # pragma: no cover - needs GPU + model
|
| 96 |
+
logger.warning("transformers VLM failed (%s) — falling back to rubric", e)
|
| 97 |
+
self._failed = True
|
| 98 |
+
return {"error": str(e), "fallback": True, "text": ""}
|
| 99 |
+
|
| 100 |
+
@staticmethod
|
| 101 |
+
def _decode_images(images: list[str] | None) -> list:
|
| 102 |
+
"""Decode base64 JPEGs (as the JudgeAgent encodes them) into PIL images."""
|
| 103 |
+
if not images:
|
| 104 |
+
return []
|
| 105 |
+
import io
|
| 106 |
+
|
| 107 |
+
from PIL import Image
|
| 108 |
+
|
| 109 |
+
out = []
|
| 110 |
+
for img in images:
|
| 111 |
+
try:
|
| 112 |
+
raw = base64.b64decode(img)
|
| 113 |
+
out.append(Image.open(io.BytesIO(raw)).convert("RGB"))
|
| 114 |
+
except Exception:
|
| 115 |
+
continue
|
| 116 |
+
return out
|
formscout/session.py
CHANGED
|
@@ -1,194 +1,283 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Screening-session accumulator.
|
| 3 |
-
|
| 4 |
-
Accumulates one SessionEntry per analyzed clip, persists each to a temp session
|
| 5 |
-
dir (session.json + analysis.md + key-frame PNGs), and on finish builds a
|
| 6 |
-
ReportResult (via ReportAgent) + a PDF (via PdfReportAgent).
|
| 7 |
-
|
| 8 |
-
Pure orchestration — no Gradio imports. Disk writes tolerate failure with a
|
| 9 |
-
logged warning and never block scoring.
|
| 10 |
-
"""
|
| 11 |
-
from __future__ import annotations
|
| 12 |
-
|
| 13 |
-
import json
|
| 14 |
-
import logging
|
| 15 |
-
import os
|
| 16 |
-
import tempfile
|
| 17 |
-
import uuid
|
| 18 |
-
from dataclasses import dataclass, replace
|
| 19 |
-
|
| 20 |
-
from formscout.rubric import score_test
|
| 21 |
-
from formscout.types import MovementResult, ReportResult, SessionEntry
|
| 22 |
-
|
| 23 |
-
logger = logging.getLogger(__name__)
|
| 24 |
-
|
| 25 |
-
# Maps each test to the BiomechFeatures.timing key holding its governing frame.
|
| 26 |
-
TIMING_KEY = {
|
| 27 |
-
"deep_squat": "deepest_frame",
|
| 28 |
-
"hurdle_step": "peak_step_frame",
|
| 29 |
-
"inline_lunge": "deepest_lunge_frame",
|
| 30 |
-
"shoulder_mobility": "measure_frame",
|
| 31 |
-
"active_slr": "peak_raise_frame",
|
| 32 |
-
"trunk_stability_pushup": "max_sag_frame",
|
| 33 |
-
"rotary_stability": "peak_extension_frame",
|
| 34 |
-
}
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
@dataclass
|
| 38 |
-
class Session:
|
| 39 |
-
"""Mutable session: an id, its temp dir, and accumulated entries."""
|
| 40 |
-
session_id: str
|
| 41 |
-
session_dir: str
|
| 42 |
-
entries: list # list[SessionEntry]
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
def new_session() -> Session:
|
| 46 |
-
sid = uuid.uuid4().hex[:12]
|
| 47 |
-
base = os.path.join(tempfile.gettempdir(), "formscout_sessions", sid)
|
| 48 |
-
try:
|
| 49 |
-
os.makedirs(os.path.join(base, "keyframes"), exist_ok=True)
|
| 50 |
-
except Exception as e:
|
| 51 |
-
logger.warning("session dir create failed: %s", e)
|
| 52 |
-
return Session(session_id=sid, session_dir=base, entries=[])
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
def governing_frame_index(features) -> int | None:
|
| 56 |
-
"""Return the governing frame index for this test, or None."""
|
| 57 |
-
key = TIMING_KEY.get(features.test_name)
|
| 58 |
-
if key is None:
|
| 59 |
-
return None
|
| 60 |
-
idx = features.timing.get(key)
|
| 61 |
-
return int(idx) if isinstance(idx, (int, float)) else None
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
def worst_compensation_caption(judge, features) -> str:
|
| 65 |
-
"""Short caption naming the worst compensation for the key-frame still."""
|
| 66 |
-
if judge and getattr(judge, "compensation_tags", None):
|
| 67 |
-
return ", ".join(judge.compensation_tags)
|
| 68 |
-
failed = [k.replace("_", " ") for k, v in features.alignments.items() if v is False]
|
| 69 |
-
return ("compensation: " + ", ".join(failed)) if failed else "key position"
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
def add_analysis(session, *, ingest, pose2d, features, judge, test_name, side,
|
| 73 |
-
draw_trails: bool = False) -> SessionEntry:
|
| 74 |
-
"""Build a SessionEntry from a completed analysis, render its key-frame,
|
| 75 |
-
persist the session, append, and return the entry."""
|
| 76 |
-
movement = MovementResult(test_name=test_name, side=side, confidence=1.0)
|
| 77 |
-
rubric = score_test(features)
|
| 78 |
-
|
| 79 |
-
needs_human = bool((judge and judge.needs_human) or rubric.needs_human)
|
| 80 |
-
if needs_human:
|
| 81 |
-
score = None
|
| 82 |
-
elif judge and judge.score is not None:
|
| 83 |
-
score = judge.score
|
| 84 |
-
else:
|
| 85 |
-
score = rubric.score
|
| 86 |
-
|
| 87 |
-
keyframe_path = None
|
| 88 |
-
idx = governing_frame_index(features)
|
| 89 |
-
if idx is not None and 0 <= idx < len(pose2d.keypoints):
|
| 90 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 91 |
-
caption = (f"{test_name.replace('_', ' ').title()} "
|
| 92 |
-
f"({side}) — {worst_compensation_caption(judge, features)}")
|
| 93 |
-
layers = {"skeleton", "trails"} if draw_trails else {"skeleton"}
|
| 94 |
-
out_png = os.path.join(session.session_dir, "keyframes", f"{test_name}_{side}.png")
|
| 95 |
-
try:
|
| 96 |
-
keyframe_path = PoseVisualizer().render_frame(ingest, pose2d, idx, layers, caption, out_png)
|
| 97 |
-
except Exception as e:
|
| 98 |
-
logger.warning("keyframe render failed: %s", e)
|
| 99 |
-
|
| 100 |
-
measurements = {}
|
| 101 |
-
measurements.update(features.angles)
|
| 102 |
-
measurements.update(features.alignments)
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Screening-session accumulator.
|
| 3 |
+
|
| 4 |
+
Accumulates one SessionEntry per analyzed clip, persists each to a temp session
|
| 5 |
+
dir (session.json + analysis.md + key-frame PNGs), and on finish builds a
|
| 6 |
+
ReportResult (via ReportAgent) + a PDF (via PdfReportAgent).
|
| 7 |
+
|
| 8 |
+
Pure orchestration — no Gradio imports. Disk writes tolerate failure with a
|
| 9 |
+
logged warning and never block scoring.
|
| 10 |
+
"""
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import json
|
| 14 |
+
import logging
|
| 15 |
+
import os
|
| 16 |
+
import tempfile
|
| 17 |
+
import uuid
|
| 18 |
+
from dataclasses import dataclass, replace
|
| 19 |
+
|
| 20 |
+
from formscout.rubric import score_test
|
| 21 |
+
from formscout.types import MovementResult, ReportResult, SessionEntry
|
| 22 |
+
|
| 23 |
+
logger = logging.getLogger(__name__)
|
| 24 |
+
|
| 25 |
+
# Maps each test to the BiomechFeatures.timing key holding its governing frame.
|
| 26 |
+
TIMING_KEY = {
|
| 27 |
+
"deep_squat": "deepest_frame",
|
| 28 |
+
"hurdle_step": "peak_step_frame",
|
| 29 |
+
"inline_lunge": "deepest_lunge_frame",
|
| 30 |
+
"shoulder_mobility": "measure_frame",
|
| 31 |
+
"active_slr": "peak_raise_frame",
|
| 32 |
+
"trunk_stability_pushup": "max_sag_frame",
|
| 33 |
+
"rotary_stability": "peak_extension_frame",
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
@dataclass
|
| 38 |
+
class Session:
|
| 39 |
+
"""Mutable session: an id, its temp dir, and accumulated entries."""
|
| 40 |
+
session_id: str
|
| 41 |
+
session_dir: str
|
| 42 |
+
entries: list # list[SessionEntry]
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def new_session() -> Session:
|
| 46 |
+
sid = uuid.uuid4().hex[:12]
|
| 47 |
+
base = os.path.join(tempfile.gettempdir(), "formscout_sessions", sid)
|
| 48 |
+
try:
|
| 49 |
+
os.makedirs(os.path.join(base, "keyframes"), exist_ok=True)
|
| 50 |
+
except Exception as e:
|
| 51 |
+
logger.warning("session dir create failed: %s", e)
|
| 52 |
+
return Session(session_id=sid, session_dir=base, entries=[])
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def governing_frame_index(features) -> int | None:
|
| 56 |
+
"""Return the governing frame index for this test, or None."""
|
| 57 |
+
key = TIMING_KEY.get(features.test_name)
|
| 58 |
+
if key is None:
|
| 59 |
+
return None
|
| 60 |
+
idx = features.timing.get(key)
|
| 61 |
+
return int(idx) if isinstance(idx, (int, float)) else None
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def worst_compensation_caption(judge, features) -> str:
|
| 65 |
+
"""Short caption naming the worst compensation for the key-frame still."""
|
| 66 |
+
if judge and getattr(judge, "compensation_tags", None):
|
| 67 |
+
return ", ".join(judge.compensation_tags)
|
| 68 |
+
failed = [k.replace("_", " ") for k, v in features.alignments.items() if v is False]
|
| 69 |
+
return ("compensation: " + ", ".join(failed)) if failed else "key position"
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def add_analysis(session, *, ingest, pose2d, features, judge, test_name, side,
|
| 73 |
+
draw_trails: bool = False) -> SessionEntry:
|
| 74 |
+
"""Build a SessionEntry from a completed analysis, render its key-frame,
|
| 75 |
+
persist the session, append, and return the entry."""
|
| 76 |
+
movement = MovementResult(test_name=test_name, side=side, confidence=1.0)
|
| 77 |
+
rubric = score_test(features)
|
| 78 |
+
|
| 79 |
+
needs_human = bool((judge and judge.needs_human) or rubric.needs_human)
|
| 80 |
+
if needs_human:
|
| 81 |
+
score = None
|
| 82 |
+
elif judge and judge.score is not None:
|
| 83 |
+
score = judge.score
|
| 84 |
+
else:
|
| 85 |
+
score = rubric.score
|
| 86 |
+
|
| 87 |
+
keyframe_path = None
|
| 88 |
+
idx = governing_frame_index(features)
|
| 89 |
+
if idx is not None and 0 <= idx < len(pose2d.keypoints):
|
| 90 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 91 |
+
caption = (f"{test_name.replace('_', ' ').title()} "
|
| 92 |
+
f"({side}) — {worst_compensation_caption(judge, features)}")
|
| 93 |
+
layers = {"skeleton", "trails"} if draw_trails else {"skeleton"}
|
| 94 |
+
out_png = os.path.join(session.session_dir, "keyframes", f"{test_name}_{side}.png")
|
| 95 |
+
try:
|
| 96 |
+
keyframe_path = PoseVisualizer().render_frame(ingest, pose2d, idx, layers, caption, out_png)
|
| 97 |
+
except Exception as e:
|
| 98 |
+
logger.warning("keyframe render failed: %s", e)
|
| 99 |
+
|
| 100 |
+
measurements = {}
|
| 101 |
+
measurements.update(features.angles)
|
| 102 |
+
measurements.update(features.alignments)
|
| 103 |
+
|
| 104 |
+
laban, flexion, chart_paths = _run_analysis(session, pose2d, test_name, side, idx)
|
| 105 |
+
|
| 106 |
+
entry = SessionEntry(
|
| 107 |
+
test_name=test_name, side=side, score=score, needs_human=needs_human,
|
| 108 |
+
rationale=(judge.rationale if judge else rubric.rationale),
|
| 109 |
+
compensation_tags=list(judge.compensation_tags) if judge else [],
|
| 110 |
+
corrective_hint=(judge.corrective_hint if judge else ""),
|
| 111 |
+
measurements=measurements,
|
| 112 |
+
confidence=(judge.confidence if judge else rubric.confidence),
|
| 113 |
+
view=features.view,
|
| 114 |
+
keyframe_path=keyframe_path,
|
| 115 |
+
movement=movement, features=features, rubric_score=rubric, judge=judge,
|
| 116 |
+
laban=laban, flexion=flexion, chart_paths=chart_paths,
|
| 117 |
+
)
|
| 118 |
+
session.entries.append(entry)
|
| 119 |
+
_persist(session)
|
| 120 |
+
return entry
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def _run_analysis(session, pose2d, test_name, side, governing_idx):
|
| 124 |
+
"""Compute Laban Effort, relevant-joint flexion, and per-test charts.
|
| 125 |
+
|
| 126 |
+
Returns (laban, flexion, chart_paths); any failure degrades to (None, None, {})
|
| 127 |
+
without blocking the entry."""
|
| 128 |
+
try:
|
| 129 |
+
from formscout.analysis import charts as C
|
| 130 |
+
from formscout.analysis.laban import compute_laban
|
| 131 |
+
from formscout.analysis.relevant_joints import primary_angle, relevant_joints
|
| 132 |
+
from formscout.analysis.timeseries import angle_series, relevant_flexion_at
|
| 133 |
+
except Exception as e:
|
| 134 |
+
logger.warning("analysis engine unavailable: %s", e)
|
| 135 |
+
return None, None, {}
|
| 136 |
+
|
| 137 |
+
try:
|
| 138 |
+
laban = compute_laban(pose2d, test_name, pose2d.fps)
|
| 139 |
+
gidx = governing_idx if governing_idx is not None else 0
|
| 140 |
+
flexion = relevant_flexion_at(pose2d, test_name, gidx)
|
| 141 |
+
series = angle_series(pose2d, test_name)
|
| 142 |
+
|
| 143 |
+
cdir = os.path.join(session.session_dir, "charts")
|
| 144 |
+
os.makedirs(cdir, exist_ok=True)
|
| 145 |
+
tag = f"{test_name}_{side}"
|
| 146 |
+
produced = {
|
| 147 |
+
"angle": C.angle_over_time(series, primary_angle(test_name), governing_idx,
|
| 148 |
+
os.path.join(cdir, f"{tag}_angle.png"),
|
| 149 |
+
title=f"{test_name.replace('_', ' ').title()} — angle over time"),
|
| 150 |
+
"velocity": C.velocity_profile(pose2d.keypoints, pose2d.fps, relevant_joints(test_name),
|
| 151 |
+
os.path.join(cdir, f"{tag}_vel.png")),
|
| 152 |
+
"radar": C.laban_radar(laban["effort"], os.path.join(cdir, f"{tag}_radar.png")),
|
| 153 |
+
"flexion": C.flexion_bars(flexion, os.path.join(cdir, f"{tag}_flex.png")),
|
| 154 |
+
}
|
| 155 |
+
chart_paths = {k: p for k, p in produced.items() if p}
|
| 156 |
+
return laban, flexion, chart_paths
|
| 157 |
+
except Exception as e:
|
| 158 |
+
logger.warning("analysis/charts failed: %s", e)
|
| 159 |
+
return None, None, {}
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def finish_session(session) -> tuple[ReportResult | None, str | None]:
|
| 163 |
+
"""Build the composite report + PDF. Returns (report, pdf_path).
|
| 164 |
+
Returns (None, None) for an empty session."""
|
| 165 |
+
if not session.entries:
|
| 166 |
+
return None, None
|
| 167 |
+
|
| 168 |
+
from formscout.agents.report import ReportAgent
|
| 169 |
+
report_inputs = [{
|
| 170 |
+
"movement": e.movement, "features": e.features,
|
| 171 |
+
"rubric_score": e.rubric_score, "judge": e.judge, "side": e.side,
|
| 172 |
+
} for e in session.entries]
|
| 173 |
+
report = ReportAgent().run(report_inputs)
|
| 174 |
+
|
| 175 |
+
pdf_path = None
|
| 176 |
+
try:
|
| 177 |
+
from formscout.agents.pdf_report import PdfReportAgent
|
| 178 |
+
pdf_path = PdfReportAgent().run(report, session.entries, session.session_dir)
|
| 179 |
+
except Exception as e:
|
| 180 |
+
logger.warning("pdf generation failed: %s", e)
|
| 181 |
+
|
| 182 |
+
report = replace(report, pdf_path=pdf_path)
|
| 183 |
+
return report, pdf_path
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
# ���─ Persistence ───────────────────────────────────────────────────────────────
|
| 187 |
+
|
| 188 |
+
def _jsonable(d: dict) -> dict:
|
| 189 |
+
out = {}
|
| 190 |
+
for k, v in d.items():
|
| 191 |
+
if isinstance(v, float):
|
| 192 |
+
out[k] = round(v, 2)
|
| 193 |
+
elif isinstance(v, (int, str, bool)) or v is None:
|
| 194 |
+
out[k] = v
|
| 195 |
+
else:
|
| 196 |
+
out[k] = str(v)
|
| 197 |
+
return out
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _entry_display(e: SessionEntry) -> dict:
|
| 201 |
+
return {
|
| 202 |
+
"test_name": e.test_name, "side": e.side, "score": e.score,
|
| 203 |
+
"needs_human": e.needs_human, "rationale": e.rationale,
|
| 204 |
+
"compensation_tags": list(e.compensation_tags), "corrective_hint": e.corrective_hint,
|
| 205 |
+
"measurements": _jsonable(e.measurements), "confidence": round(e.confidence, 2),
|
| 206 |
+
"view": e.view, "keyframe_path": e.keyframe_path,
|
| 207 |
+
"laban": e.laban, "flexion": _jsonable_flexion(e.flexion), "chart_paths": e.chart_paths,
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def _jsonable_flexion(flexion: dict | None) -> dict:
|
| 212 |
+
if not flexion:
|
| 213 |
+
return {}
|
| 214 |
+
return {k: {"deg": round(v["deg"], 1), "openness": v["openness"]} for k, v in flexion.items()}
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def _render_markdown(session: Session) -> str:
|
| 218 |
+
lines = ["# FormScout — Session Log", ""]
|
| 219 |
+
for e in session.entries:
|
| 220 |
+
title = e.test_name.replace("_", " ").title()
|
| 221 |
+
if e.side in ("left", "right"):
|
| 222 |
+
title += f" ({e.side})"
|
| 223 |
+
score = "Clinician review required" if e.needs_human else f"{e.score}/3"
|
| 224 |
+
lines.append(f"## {title} — {score}")
|
| 225 |
+
lines.append(f"*view: {e.view} · confidence: {e.confidence:.0%}*")
|
| 226 |
+
lines.append("")
|
| 227 |
+
lines.append(e.rationale or "")
|
| 228 |
+
if e.compensation_tags:
|
| 229 |
+
lines.append(f"- **Compensations:** {', '.join(e.compensation_tags)}")
|
| 230 |
+
if e.corrective_hint:
|
| 231 |
+
lines.append(f"- **Corrective:** {e.corrective_hint}")
|
| 232 |
+
|
| 233 |
+
# Relevant-joint flexion (degrees + open/closed)
|
| 234 |
+
if e.flexion:
|
| 235 |
+
lines.append("\n### Relevant joint flexion (key frame)")
|
| 236 |
+
lines.append("| Joint angle | Degrees | State |")
|
| 237 |
+
lines.append("|---|---|---|")
|
| 238 |
+
for name, v in e.flexion.items():
|
| 239 |
+
lines.append(f"| {name.replace('_', ' ')} | {v['deg']:.1f}° | {v['openness']} |")
|
| 240 |
+
|
| 241 |
+
# Laban Effort
|
| 242 |
+
if e.laban:
|
| 243 |
+
eff, lab = e.laban.get("effort", {}), e.laban.get("labels", {})
|
| 244 |
+
lines.append("\n### Laban Effort (kinematic estimate)")
|
| 245 |
+
lines.append("| Factor | Value | Quality |")
|
| 246 |
+
lines.append("|---|---|---|")
|
| 247 |
+
for k in ("space", "weight", "time", "flow"):
|
| 248 |
+
lines.append(f"| {k.title()} | {eff.get(k, 0):.2f} | {lab.get(k, '')} |")
|
| 249 |
+
if e.laban.get("body_emphasis"):
|
| 250 |
+
emph = ", ".join(f"{n} ({s})" for n, s in e.laban["body_emphasis"])
|
| 251 |
+
lines.append(f"\n- **Body emphasis:** {emph}")
|
| 252 |
+
if e.laban.get("leading_joint"):
|
| 253 |
+
lines.append(f"- **Leading joint:** {e.laban['leading_joint']}")
|
| 254 |
+
lines.append(f"- *{e.laban.get('notes', '')}*")
|
| 255 |
+
|
| 256 |
+
# Full measurement dump
|
| 257 |
+
if e.measurements:
|
| 258 |
+
lines.append("\n### All measurements")
|
| 259 |
+
lines.append("| Measurement | Value |")
|
| 260 |
+
lines.append("|---|---|")
|
| 261 |
+
for k, v in e.measurements.items():
|
| 262 |
+
val = f"{v:.2f}" if isinstance(v, float) else str(v)
|
| 263 |
+
lines.append(f"| {k.replace('_', ' ')} | {val} |")
|
| 264 |
+
if e.features and e.features.symmetry_delta is not None:
|
| 265 |
+
lines.append(f"\n- **L/R symmetry delta:** {e.features.symmetry_delta:.1f}")
|
| 266 |
+
|
| 267 |
+
# Artifacts
|
| 268 |
+
if e.keyframe_path:
|
| 269 |
+
lines.append(f"\n- Key frame: `{e.keyframe_path}`")
|
| 270 |
+
for kind, path in (e.chart_paths or {}).items():
|
| 271 |
+
lines.append(f"- Chart ({kind}): `{path}`")
|
| 272 |
+
lines.append("")
|
| 273 |
+
return "\n".join(lines)
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def _persist(session: Session) -> None:
|
| 277 |
+
try:
|
| 278 |
+
with open(os.path.join(session.session_dir, "session.json"), "w") as f:
|
| 279 |
+
json.dump([_entry_display(e) for e in session.entries], f, indent=2)
|
| 280 |
+
with open(os.path.join(session.session_dir, "analysis.md"), "w") as f:
|
| 281 |
+
f.write(_render_markdown(session))
|
| 282 |
+
except Exception as e:
|
| 283 |
+
logger.warning("session persist failed: %s", e)
|
formscout/startup.py
CHANGED
|
@@ -1,47 +1,47 @@
|
|
| 1 |
-
"""
|
| 2 |
-
Checkpoint bootstrap — downloads missing model files from HF model repo on first run.
|
| 3 |
-
Called once at app startup before build_app(); no-ops if files already present.
|
| 4 |
-
"""
|
| 5 |
-
from __future__ import annotations
|
| 6 |
-
|
| 7 |
-
import logging
|
| 8 |
-
from pathlib import Path
|
| 9 |
-
|
| 10 |
-
logger = logging.getLogger(__name__)
|
| 11 |
-
|
| 12 |
-
CHECKPOINT_REPO = "silas-therapy/formscout-checkpoints"
|
| 13 |
-
ROOT = Path(__file__).parent.parent
|
| 14 |
-
|
| 15 |
-
_CHECKPOINTS = [
|
| 16 |
-
"checkpoints/yolo26/yolo26n-pose.pt",
|
| 17 |
-
"checkpoints/yolo26/yolo26s-pose.pt",
|
| 18 |
-
"checkpoints/yolo26/yolo26m-pose.pt",
|
| 19 |
-
"checkpoints/yolo26/yolo26l-pose.pt",
|
| 20 |
-
"checkpoints/yolo26/yolo26x-pose.pt",
|
| 21 |
-
"checkpoints/mediapipe/pose_landmarker_full.task",
|
| 22 |
-
]
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
def ensure_checkpoints() -> None:
|
| 26 |
-
"""Download any missing checkpoints from silas-therapy/formscout-checkpoints."""
|
| 27 |
-
try:
|
| 28 |
-
from huggingface_hub import hf_hub_download
|
| 29 |
-
except ImportError:
|
| 30 |
-
logger.warning("huggingface_hub not installed — skipping checkpoint download")
|
| 31 |
-
return
|
| 32 |
-
|
| 33 |
-
for rel_path in _CHECKPOINTS:
|
| 34 |
-
local = ROOT / rel_path
|
| 35 |
-
if local.exists():
|
| 36 |
-
continue
|
| 37 |
-
logger.info("Downloading %s ...", rel_path)
|
| 38 |
-
try:
|
| 39 |
-
local.parent.mkdir(parents=True, exist_ok=True)
|
| 40 |
-
hf_hub_download(
|
| 41 |
-
repo_id=CHECKPOINT_REPO,
|
| 42 |
-
filename=rel_path,
|
| 43 |
-
local_dir=str(ROOT),
|
| 44 |
-
)
|
| 45 |
-
logger.info("Downloaded %s", rel_path)
|
| 46 |
-
except Exception as e:
|
| 47 |
-
logger.warning("Failed to download %s: %s", rel_path, e)
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Checkpoint bootstrap — downloads missing model files from HF model repo on first run.
|
| 3 |
+
Called once at app startup before build_app(); no-ops if files already present.
|
| 4 |
+
"""
|
| 5 |
+
from __future__ import annotations
|
| 6 |
+
|
| 7 |
+
import logging
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
logger = logging.getLogger(__name__)
|
| 11 |
+
|
| 12 |
+
CHECKPOINT_REPO = "silas-therapy/formscout-checkpoints"
|
| 13 |
+
ROOT = Path(__file__).parent.parent
|
| 14 |
+
|
| 15 |
+
_CHECKPOINTS = [
|
| 16 |
+
"checkpoints/yolo26/yolo26n-pose.pt",
|
| 17 |
+
"checkpoints/yolo26/yolo26s-pose.pt",
|
| 18 |
+
"checkpoints/yolo26/yolo26m-pose.pt",
|
| 19 |
+
"checkpoints/yolo26/yolo26l-pose.pt",
|
| 20 |
+
"checkpoints/yolo26/yolo26x-pose.pt",
|
| 21 |
+
"checkpoints/mediapipe/pose_landmarker_full.task",
|
| 22 |
+
]
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def ensure_checkpoints() -> None:
|
| 26 |
+
"""Download any missing checkpoints from silas-therapy/formscout-checkpoints."""
|
| 27 |
+
try:
|
| 28 |
+
from huggingface_hub import hf_hub_download
|
| 29 |
+
except ImportError:
|
| 30 |
+
logger.warning("huggingface_hub not installed — skipping checkpoint download")
|
| 31 |
+
return
|
| 32 |
+
|
| 33 |
+
for rel_path in _CHECKPOINTS:
|
| 34 |
+
local = ROOT / rel_path
|
| 35 |
+
if local.exists():
|
| 36 |
+
continue
|
| 37 |
+
logger.info("Downloading %s ...", rel_path)
|
| 38 |
+
try:
|
| 39 |
+
local.parent.mkdir(parents=True, exist_ok=True)
|
| 40 |
+
hf_hub_download(
|
| 41 |
+
repo_id=CHECKPOINT_REPO,
|
| 42 |
+
filename=rel_path,
|
| 43 |
+
local_dir=str(ROOT),
|
| 44 |
+
)
|
| 45 |
+
logger.info("Downloaded %s", rel_path)
|
| 46 |
+
except Exception as e:
|
| 47 |
+
logger.warning("Failed to download %s: %s", rel_path, e)
|
formscout/types.py
CHANGED
|
@@ -164,6 +164,9 @@ class SessionEntry:
|
|
| 164 |
features: BiomechFeatures
|
| 165 |
rubric_score: ScoreResult
|
| 166 |
judge: JudgeResult | None
|
|
|
|
|
|
|
|
|
|
| 167 |
|
| 168 |
|
| 169 |
@dataclass
|
|
|
|
| 164 |
features: BiomechFeatures
|
| 165 |
rubric_score: ScoreResult
|
| 166 |
judge: JudgeResult | None
|
| 167 |
+
laban: dict | None = None # Laban Effort factors + labels + body emphasis
|
| 168 |
+
flexion: dict | None = None # relevant joint angles at key frame: {name: {deg, openness}}
|
| 169 |
+
chart_paths: dict | None = None # {"angle"|"velocity"|"radar"|"flexion": png path}
|
| 170 |
|
| 171 |
|
| 172 |
@dataclass
|
formscout/ui/theme.py
CHANGED
|
@@ -1,250 +1,272 @@
|
|
| 1 |
-
"""
|
| 2 |
-
FormScout custom Gradio theme —
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
}
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
background:
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
}
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
background: #
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
}
|
| 172 |
-
|
| 173 |
-
.
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
}
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
}
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
FormScout custom Gradio theme — Silas Therapy palette.
|
| 3 |
+
|
| 4 |
+
Warm cream + sage green ground, petrol-teal primary, golden-orange accent,
|
| 5 |
+
dark slate-green text. Matches https://silastherapy.sk branding.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import gradio as gr
|
| 10 |
+
|
| 11 |
+
# ── Silas palette ──────────────────────────────────────────────────────────────
|
| 12 |
+
CREAM = "#f7eedd"
|
| 13 |
+
CREAM_DEEP = "#f1e4cc"
|
| 14 |
+
SAGE = "#bcd3c8"
|
| 15 |
+
SAGE_DEEP = "#9cbcad"
|
| 16 |
+
TEAL = "#2b8a8a"
|
| 17 |
+
TEAL_HOVER = "#1f6e6e"
|
| 18 |
+
TEAL_DEEP = "#175757"
|
| 19 |
+
GOLD = "#e0a43b"
|
| 20 |
+
GOLD_DEEP = "#cf922a"
|
| 21 |
+
INK = "#243a34"
|
| 22 |
+
INK_MUTED = "#4a5f57"
|
| 23 |
+
INK_FAINT = "#6b7d75"
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def formscout_theme() -> gr.Theme:
|
| 27 |
+
"""Create the FormScout theme in the Silas Therapy palette."""
|
| 28 |
+
return gr.themes.Soft(
|
| 29 |
+
primary_hue=gr.themes.colors.teal,
|
| 30 |
+
secondary_hue=gr.themes.colors.amber,
|
| 31 |
+
neutral_hue=gr.themes.colors.stone,
|
| 32 |
+
font=[
|
| 33 |
+
gr.themes.GoogleFont("Inter"),
|
| 34 |
+
"ui-sans-serif",
|
| 35 |
+
"system-ui",
|
| 36 |
+
"sans-serif",
|
| 37 |
+
],
|
| 38 |
+
font_mono=[
|
| 39 |
+
gr.themes.GoogleFont("JetBrains Mono"),
|
| 40 |
+
"ui-monospace",
|
| 41 |
+
"monospace",
|
| 42 |
+
],
|
| 43 |
+
).set(
|
| 44 |
+
# Background — warm cream into soft sage
|
| 45 |
+
body_background_fill=f"linear-gradient(135deg, {CREAM} 0%, {CREAM_DEEP} 45%, {SAGE} 100%)",
|
| 46 |
+
body_background_fill_dark=f"linear-gradient(135deg, {CREAM} 0%, {CREAM_DEEP} 45%, {SAGE} 100%)",
|
| 47 |
+
# Blocks — soft white cards over the sage ground
|
| 48 |
+
block_background_fill="rgba(255, 255, 255, 0.72)",
|
| 49 |
+
block_background_fill_dark="rgba(255, 255, 255, 0.72)",
|
| 50 |
+
block_border_width="1px",
|
| 51 |
+
block_border_color="rgba(43, 138, 138, 0.22)",
|
| 52 |
+
block_shadow="0 6px 22px rgba(36, 58, 52, 0.10)",
|
| 53 |
+
block_radius="14px",
|
| 54 |
+
# Buttons — petrol teal primary, sage secondary
|
| 55 |
+
button_primary_background_fill=f"linear-gradient(135deg, {TEAL} 0%, {TEAL_HOVER} 100%)",
|
| 56 |
+
button_primary_background_fill_hover=f"linear-gradient(135deg, {TEAL_HOVER} 0%, {TEAL_DEEP} 100%)",
|
| 57 |
+
button_primary_text_color="white",
|
| 58 |
+
button_primary_border_color="rgba(43, 138, 138, 0.45)",
|
| 59 |
+
button_secondary_background_fill="rgba(156, 188, 173, 0.55)",
|
| 60 |
+
button_secondary_text_color=INK,
|
| 61 |
+
# Inputs
|
| 62 |
+
input_background_fill="rgba(255, 255, 255, 0.92)",
|
| 63 |
+
input_background_fill_dark="rgba(255, 255, 255, 0.92)",
|
| 64 |
+
input_background_fill_focus="rgba(255, 255, 255, 1.0)",
|
| 65 |
+
input_border_color="rgba(43, 138, 138, 0.30)",
|
| 66 |
+
input_border_color_focus="rgba(43, 138, 138, 0.75)",
|
| 67 |
+
# Labels — pin light in both modes so no dark dropdown header appears
|
| 68 |
+
block_label_background_fill="rgba(188, 211, 200, 0.55)",
|
| 69 |
+
block_label_background_fill_dark="rgba(188, 211, 200, 0.55)",
|
| 70 |
+
block_label_text_color=INK,
|
| 71 |
+
block_label_text_color_dark=INK,
|
| 72 |
+
# Text
|
| 73 |
+
body_text_color=INK,
|
| 74 |
+
body_text_color_dark=INK,
|
| 75 |
+
block_title_text_color=TEAL_DEEP,
|
| 76 |
+
# Spacing
|
| 77 |
+
block_padding="16px",
|
| 78 |
+
layout_gap="16px",
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
FORMSCOUT_CSS = f"""
|
| 83 |
+
/* FormScout — Silas Therapy theme */
|
| 84 |
+
|
| 85 |
+
.gradio-container {{
|
| 86 |
+
max-width: 1400px !important;
|
| 87 |
+
margin: 0 auto;
|
| 88 |
+
}}
|
| 89 |
+
|
| 90 |
+
/* Header styling */
|
| 91 |
+
.formscout-header {{
|
| 92 |
+
text-align: center;
|
| 93 |
+
padding: 20px 0;
|
| 94 |
+
border-bottom: 2px solid rgba(43, 138, 138, 0.30);
|
| 95 |
+
margin-bottom: 20px;
|
| 96 |
+
}}
|
| 97 |
+
|
| 98 |
+
.formscout-header h1 {{
|
| 99 |
+
font-size: 2.2em;
|
| 100 |
+
background: linear-gradient(135deg, {TEAL} 0%, {GOLD} 100%);
|
| 101 |
+
-webkit-background-clip: text;
|
| 102 |
+
-webkit-text-fill-color: transparent;
|
| 103 |
+
background-clip: text;
|
| 104 |
+
margin-bottom: 8px;
|
| 105 |
+
}}
|
| 106 |
+
|
| 107 |
+
/* Safety banner — golden */
|
| 108 |
+
.safety-banner {{
|
| 109 |
+
background: linear-gradient(90deg, rgba(224, 164, 59, 0.20), rgba(224, 164, 59, 0.08));
|
| 110 |
+
border: 1px solid rgba(224, 164, 59, 0.55);
|
| 111 |
+
border-radius: 8px;
|
| 112 |
+
padding: 12px 16px;
|
| 113 |
+
margin: 12px 0;
|
| 114 |
+
font-size: 0.9em;
|
| 115 |
+
text-align: center;
|
| 116 |
+
color: {GOLD_DEEP};
|
| 117 |
+
}}
|
| 118 |
+
|
| 119 |
+
/* Score display */
|
| 120 |
+
.score-card {{
|
| 121 |
+
background: rgba(43, 138, 138, 0.08);
|
| 122 |
+
border: 2px solid rgba(43, 138, 138, 0.35);
|
| 123 |
+
border-radius: 16px;
|
| 124 |
+
padding: 24px;
|
| 125 |
+
text-align: center;
|
| 126 |
+
}}
|
| 127 |
+
|
| 128 |
+
.score-value {{
|
| 129 |
+
font-size: 4em;
|
| 130 |
+
font-weight: 800;
|
| 131 |
+
background: linear-gradient(135deg, {TEAL} 0%, {GOLD} 100%);
|
| 132 |
+
-webkit-background-clip: text;
|
| 133 |
+
-webkit-text-fill-color: transparent;
|
| 134 |
+
background-clip: text;
|
| 135 |
+
}}
|
| 136 |
+
|
| 137 |
+
/* Confidence meter */
|
| 138 |
+
.confidence-bar {{
|
| 139 |
+
height: 8px;
|
| 140 |
+
border-radius: 4px;
|
| 141 |
+
background: rgba(43, 138, 138, 0.18);
|
| 142 |
+
overflow: hidden;
|
| 143 |
+
margin-top: 8px;
|
| 144 |
+
}}
|
| 145 |
+
|
| 146 |
+
.confidence-fill {{
|
| 147 |
+
height: 100%;
|
| 148 |
+
border-radius: 4px;
|
| 149 |
+
background: linear-gradient(90deg, #d9534f, {GOLD}, {TEAL});
|
| 150 |
+
transition: width 0.5s ease;
|
| 151 |
+
}}
|
| 152 |
+
|
| 153 |
+
/* Pipeline steps indicator */
|
| 154 |
+
.pipeline-steps {{
|
| 155 |
+
display: flex;
|
| 156 |
+
gap: 4px;
|
| 157 |
+
align-items: center;
|
| 158 |
+
padding: 8px 0;
|
| 159 |
+
}}
|
| 160 |
+
|
| 161 |
+
.pipeline-step {{
|
| 162 |
+
flex: 1;
|
| 163 |
+
height: 4px;
|
| 164 |
+
border-radius: 2px;
|
| 165 |
+
background: rgba(43, 138, 138, 0.18);
|
| 166 |
+
transition: background 0.3s ease;
|
| 167 |
+
}}
|
| 168 |
+
|
| 169 |
+
.pipeline-step.active {{
|
| 170 |
+
background: {TEAL};
|
| 171 |
+
}}
|
| 172 |
+
|
| 173 |
+
.pipeline-step.complete {{
|
| 174 |
+
background: {GOLD};
|
| 175 |
+
}}
|
| 176 |
+
|
| 177 |
+
/* Asymmetry indicator */
|
| 178 |
+
.asymmetry-bar {{
|
| 179 |
+
display: flex;
|
| 180 |
+
align-items: center;
|
| 181 |
+
gap: 8px;
|
| 182 |
+
padding: 8px 12px;
|
| 183 |
+
background: rgba(156, 188, 173, 0.30);
|
| 184 |
+
border-radius: 8px;
|
| 185 |
+
margin: 4px 0;
|
| 186 |
+
}}
|
| 187 |
+
|
| 188 |
+
.asymmetry-label {{
|
| 189 |
+
min-width: 60px;
|
| 190 |
+
font-size: 0.85em;
|
| 191 |
+
color: {INK_MUTED};
|
| 192 |
+
}}
|
| 193 |
+
|
| 194 |
+
.asymmetry-track {{
|
| 195 |
+
flex: 1;
|
| 196 |
+
height: 6px;
|
| 197 |
+
background: rgba(43, 138, 138, 0.12);
|
| 198 |
+
border-radius: 3px;
|
| 199 |
+
position: relative;
|
| 200 |
+
}}
|
| 201 |
+
|
| 202 |
+
.asymmetry-marker {{
|
| 203 |
+
position: absolute;
|
| 204 |
+
top: -3px;
|
| 205 |
+
width: 12px;
|
| 206 |
+
height: 12px;
|
| 207 |
+
border-radius: 50%;
|
| 208 |
+
background: {TEAL};
|
| 209 |
+
border: 2px solid {GOLD};
|
| 210 |
+
}}
|
| 211 |
+
|
| 212 |
+
/* Topographic pattern accent */
|
| 213 |
+
.topo-accent {{
|
| 214 |
+
color: {INK_FAINT};
|
| 215 |
+
background-image:
|
| 216 |
+
repeating-linear-gradient(
|
| 217 |
+
0deg,
|
| 218 |
+
transparent,
|
| 219 |
+
transparent 40px,
|
| 220 |
+
rgba(43, 138, 138, 0.04) 40px,
|
| 221 |
+
rgba(43, 138, 138, 0.04) 41px
|
| 222 |
+
),
|
| 223 |
+
repeating-linear-gradient(
|
| 224 |
+
90deg,
|
| 225 |
+
transparent,
|
| 226 |
+
transparent 40px,
|
| 227 |
+
rgba(43, 138, 138, 0.03) 40px,
|
| 228 |
+
rgba(43, 138, 138, 0.03) 41px
|
| 229 |
+
);
|
| 230 |
+
}}
|
| 231 |
+
|
| 232 |
+
/* Warning/error states */
|
| 233 |
+
.needs-review {{
|
| 234 |
+
border-color: rgba(224, 164, 59, 0.65) !important;
|
| 235 |
+
background: rgba(224, 164, 59, 0.10) !important;
|
| 236 |
+
}}
|
| 237 |
+
|
| 238 |
+
.low-confidence {{
|
| 239 |
+
opacity: 0.7;
|
| 240 |
+
border-style: dashed !important;
|
| 241 |
+
}}
|
| 242 |
+
|
| 243 |
+
/* Rubric drawer */
|
| 244 |
+
.rubric-item {{
|
| 245 |
+
display: flex;
|
| 246 |
+
align-items: center;
|
| 247 |
+
gap: 8px;
|
| 248 |
+
padding: 6px 10px;
|
| 249 |
+
border-radius: 6px;
|
| 250 |
+
margin: 2px 0;
|
| 251 |
+
}}
|
| 252 |
+
|
| 253 |
+
.rubric-met {{
|
| 254 |
+
background: rgba(43, 138, 138, 0.10);
|
| 255 |
+
border-left: 3px solid {TEAL};
|
| 256 |
+
}}
|
| 257 |
+
|
| 258 |
+
.rubric-unmet {{
|
| 259 |
+
background: rgba(217, 83, 79, 0.10);
|
| 260 |
+
border-left: 3px solid #d9534f;
|
| 261 |
+
}}
|
| 262 |
+
|
| 263 |
+
/* Responsive */
|
| 264 |
+
@media (max-width: 768px) {{
|
| 265 |
+
.gradio-container {{
|
| 266 |
+
padding: 8px !important;
|
| 267 |
+
}}
|
| 268 |
+
.score-value {{
|
| 269 |
+
font-size: 3em;
|
| 270 |
+
}}
|
| 271 |
+
}}
|
| 272 |
+
"""
|
requirements.txt
CHANGED
|
@@ -2,16 +2,18 @@ gradio>=5.0
|
|
| 2 |
ultralytics>=8.3
|
| 3 |
torch>=2.3
|
| 4 |
opencv-python>=4.10
|
| 5 |
-
imageio-ffmpeg>=0.5
|
| 6 |
numpy>=1.26
|
| 7 |
scipy>=1.13
|
| 8 |
pillow>=10.3
|
| 9 |
reportlab>=4.0
|
|
|
|
| 10 |
requests>=2.31
|
| 11 |
pytest>=8.2
|
| 12 |
ruff>=0.4
|
| 13 |
black>=24.4
|
| 14 |
huggingface_hub>=0.23
|
| 15 |
transformers>=4.44
|
|
|
|
|
|
|
| 16 |
onnxruntime>=1.18
|
| 17 |
mediapipe>=0.10
|
|
|
|
| 2 |
ultralytics>=8.3
|
| 3 |
torch>=2.3
|
| 4 |
opencv-python>=4.10
|
|
|
|
| 5 |
numpy>=1.26
|
| 6 |
scipy>=1.13
|
| 7 |
pillow>=10.3
|
| 8 |
reportlab>=4.0
|
| 9 |
+
matplotlib>=3.8
|
| 10 |
requests>=2.31
|
| 11 |
pytest>=8.2
|
| 12 |
ruff>=0.4
|
| 13 |
black>=24.4
|
| 14 |
huggingface_hub>=0.23
|
| 15 |
transformers>=4.44
|
| 16 |
+
accelerate>=0.30
|
| 17 |
+
spaces>=0.30
|
| 18 |
onnxruntime>=1.18
|
| 19 |
mediapipe>=0.10
|
scripts/hf_upload.sh
CHANGED
|
@@ -1,97 +1,97 @@
|
|
| 1 |
-
#!/usr/bin/env bash
|
| 2 |
-
# Upload the FormScout source tree to both the model repo and the Space.
|
| 3 |
-
#
|
| 4 |
-
# Usage:
|
| 5 |
-
# ./scripts/hf_upload.sh # message from last git commit
|
| 6 |
-
# ./scripts/hf_upload.sh "feat: my change" # custom message
|
| 7 |
-
#
|
| 8 |
-
# Pushes to:
|
| 9 |
-
# silas-therapy/small-functional-movement-screening (model repo)
|
| 10 |
-
# spaces/silas-therapy/small-functional-movement-screening (Gradio Space)
|
| 11 |
-
#
|
| 12 |
-
# `hf upload` does NOT read .hfignore — it only honors .gitignore, and only at
|
| 13 |
-
# commit time (after hashing and pre-uploading everything). So we parse
|
| 14 |
-
# .hfignore ourselves into --exclude globs and pass them explicitly.
|
| 15 |
-
#
|
| 16 |
-
# If the filtered file count still exceeds LARGE_THRESHOLD, we fall back to
|
| 17 |
-
# `hf upload-large-folder` (resumable, multi-threaded). Caveats of that mode:
|
| 18 |
-
# no --create-pr and no custom commit message — it commits directly to main
|
| 19 |
-
# in multiple commits.
|
| 20 |
-
set -euo pipefail
|
| 21 |
-
|
| 22 |
-
cd "$(dirname "$0")/.."
|
| 23 |
-
|
| 24 |
-
MODEL_REPO="silas-therapy/small-functional-movement-screening"
|
| 25 |
-
SPACE_REPO="spaces/silas-therapy/small-functional-movement-screening"
|
| 26 |
-
MSG="${1:-$(git log -1 --pretty=%s)}"
|
| 27 |
-
LARGE_THRESHOLD="${FORMSCOUT_HF_LARGE_THRESHOLD:-500}"
|
| 28 |
-
|
| 29 |
-
# Belt-and-suspenders extras on top of .hfignore. `.cache/` is the resume
|
| 30 |
-
# state upload-large-folder writes into the folder being uploaded.
|
| 31 |
-
PATTERNS=(
|
| 32 |
-
"*.pdf"
|
| 33 |
-
"**/node_modules/**"
|
| 34 |
-
".cache/**"
|
| 35 |
-
)
|
| 36 |
-
|
| 37 |
-
# Parse .hfignore into fnmatch-style globs. fnmatch's `*` crosses `/`, but a
|
| 38 |
-
# bare name like `.DS_Store` or `dir/` only matches at the root, so emit both
|
| 39 |
-
# the rooted and `**/`-prefixed forms.
|
| 40 |
-
while IFS= read -r line; do
|
| 41 |
-
line="${line%%#*}"
|
| 42 |
-
line="${line#"${line%%[![:space:]]*}"}"
|
| 43 |
-
line="${line%"${line##*[![:space:]]}"}"
|
| 44 |
-
[[ -z "$line" ]] && continue
|
| 45 |
-
if [[ "$line" == */ ]]; then
|
| 46 |
-
PATTERNS+=("${line}**" "**/${line}**")
|
| 47 |
-
else
|
| 48 |
-
PATTERNS+=("$line" "**/$line")
|
| 49 |
-
fi
|
| 50 |
-
done < .hfignore
|
| 51 |
-
|
| 52 |
-
EXCLUDES=()
|
| 53 |
-
for p in "${PATTERNS[@]}"; do
|
| 54 |
-
EXCLUDES+=(--exclude="$p")
|
| 55 |
-
done
|
| 56 |
-
|
| 57 |
-
# Count what would actually be uploaded, using the same filter the hub client
|
| 58 |
-
# applies, so the mode decision matches reality.
|
| 59 |
-
N_FILES=$(python3 - "${PATTERNS[@]}" <<'EOF'
|
| 60 |
-
import sys
|
| 61 |
-
from pathlib import Path
|
| 62 |
-
from huggingface_hub.utils import filter_repo_objects
|
| 63 |
-
|
| 64 |
-
patterns = sys.argv[1:]
|
| 65 |
-
files = (
|
| 66 |
-
str(p) for p in Path(".").rglob("*")
|
| 67 |
-
if p.is_file() and p.parts[0] != ".git"
|
| 68 |
-
)
|
| 69 |
-
print(len(list(filter_repo_objects(files, ignore_patterns=patterns))))
|
| 70 |
-
EOF
|
| 71 |
-
)
|
| 72 |
-
echo "── $N_FILES files to upload after .hfignore filtering"
|
| 73 |
-
|
| 74 |
-
if (( N_FILES == 0 )); then
|
| 75 |
-
echo "✗ nothing to upload — check .hfignore" >&2
|
| 76 |
-
exit 1
|
| 77 |
-
fi
|
| 78 |
-
|
| 79 |
-
upload_repo() {
|
| 80 |
-
local repo="$1"
|
| 81 |
-
if (( N_FILES > LARGE_THRESHOLD )); then
|
| 82 |
-
echo "── $repo: $N_FILES files > $LARGE_THRESHOLD, using upload-large-folder"
|
| 83 |
-
echo " (resumable; commits directly to main — no PR, no custom message)"
|
| 84 |
-
hf upload-large-folder "$repo" . "${EXCLUDES[@]}"
|
| 85 |
-
else
|
| 86 |
-
echo "── uploading to: $repo"
|
| 87 |
-
hf upload "$repo" . . \
|
| 88 |
-
"${EXCLUDES[@]}" \
|
| 89 |
-
--create-pr \
|
| 90 |
-
--commit-message="$MSG"
|
| 91 |
-
fi
|
| 92 |
-
}
|
| 93 |
-
|
| 94 |
-
upload_repo "$MODEL_REPO"
|
| 95 |
-
upload_repo "$SPACE_REPO"
|
| 96 |
-
|
| 97 |
-
echo "✓ done"
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# Upload the FormScout source tree to both the model repo and the Space.
|
| 3 |
+
#
|
| 4 |
+
# Usage:
|
| 5 |
+
# ./scripts/hf_upload.sh # message from last git commit
|
| 6 |
+
# ./scripts/hf_upload.sh "feat: my change" # custom message
|
| 7 |
+
#
|
| 8 |
+
# Pushes to:
|
| 9 |
+
# silas-therapy/small-functional-movement-screening (model repo)
|
| 10 |
+
# spaces/silas-therapy/small-functional-movement-screening (Gradio Space)
|
| 11 |
+
#
|
| 12 |
+
# `hf upload` does NOT read .hfignore — it only honors .gitignore, and only at
|
| 13 |
+
# commit time (after hashing and pre-uploading everything). So we parse
|
| 14 |
+
# .hfignore ourselves into --exclude globs and pass them explicitly.
|
| 15 |
+
#
|
| 16 |
+
# If the filtered file count still exceeds LARGE_THRESHOLD, we fall back to
|
| 17 |
+
# `hf upload-large-folder` (resumable, multi-threaded). Caveats of that mode:
|
| 18 |
+
# no --create-pr and no custom commit message — it commits directly to main
|
| 19 |
+
# in multiple commits.
|
| 20 |
+
set -euo pipefail
|
| 21 |
+
|
| 22 |
+
cd "$(dirname "$0")/.."
|
| 23 |
+
|
| 24 |
+
MODEL_REPO="silas-therapy/small-functional-movement-screening"
|
| 25 |
+
SPACE_REPO="spaces/silas-therapy/small-functional-movement-screening"
|
| 26 |
+
MSG="${1:-$(git log -1 --pretty=%s)}"
|
| 27 |
+
LARGE_THRESHOLD="${FORMSCOUT_HF_LARGE_THRESHOLD:-500}"
|
| 28 |
+
|
| 29 |
+
# Belt-and-suspenders extras on top of .hfignore. `.cache/` is the resume
|
| 30 |
+
# state upload-large-folder writes into the folder being uploaded.
|
| 31 |
+
PATTERNS=(
|
| 32 |
+
"*.pdf"
|
| 33 |
+
"**/node_modules/**"
|
| 34 |
+
".cache/**"
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
# Parse .hfignore into fnmatch-style globs. fnmatch's `*` crosses `/`, but a
|
| 38 |
+
# bare name like `.DS_Store` or `dir/` only matches at the root, so emit both
|
| 39 |
+
# the rooted and `**/`-prefixed forms.
|
| 40 |
+
while IFS= read -r line; do
|
| 41 |
+
line="${line%%#*}"
|
| 42 |
+
line="${line#"${line%%[![:space:]]*}"}"
|
| 43 |
+
line="${line%"${line##*[![:space:]]}"}"
|
| 44 |
+
[[ -z "$line" ]] && continue
|
| 45 |
+
if [[ "$line" == */ ]]; then
|
| 46 |
+
PATTERNS+=("${line}**" "**/${line}**")
|
| 47 |
+
else
|
| 48 |
+
PATTERNS+=("$line" "**/$line")
|
| 49 |
+
fi
|
| 50 |
+
done < .hfignore
|
| 51 |
+
|
| 52 |
+
EXCLUDES=()
|
| 53 |
+
for p in "${PATTERNS[@]}"; do
|
| 54 |
+
EXCLUDES+=(--exclude="$p")
|
| 55 |
+
done
|
| 56 |
+
|
| 57 |
+
# Count what would actually be uploaded, using the same filter the hub client
|
| 58 |
+
# applies, so the mode decision matches reality.
|
| 59 |
+
N_FILES=$(python3 - "${PATTERNS[@]}" <<'EOF'
|
| 60 |
+
import sys
|
| 61 |
+
from pathlib import Path
|
| 62 |
+
from huggingface_hub.utils import filter_repo_objects
|
| 63 |
+
|
| 64 |
+
patterns = sys.argv[1:]
|
| 65 |
+
files = (
|
| 66 |
+
str(p) for p in Path(".").rglob("*")
|
| 67 |
+
if p.is_file() and p.parts[0] != ".git"
|
| 68 |
+
)
|
| 69 |
+
print(len(list(filter_repo_objects(files, ignore_patterns=patterns))))
|
| 70 |
+
EOF
|
| 71 |
+
)
|
| 72 |
+
echo "── $N_FILES files to upload after .hfignore filtering"
|
| 73 |
+
|
| 74 |
+
if (( N_FILES == 0 )); then
|
| 75 |
+
echo "✗ nothing to upload — check .hfignore" >&2
|
| 76 |
+
exit 1
|
| 77 |
+
fi
|
| 78 |
+
|
| 79 |
+
upload_repo() {
|
| 80 |
+
local repo="$1"
|
| 81 |
+
if (( N_FILES > LARGE_THRESHOLD )); then
|
| 82 |
+
echo "── $repo: $N_FILES files > $LARGE_THRESHOLD, using upload-large-folder"
|
| 83 |
+
echo " (resumable; commits directly to main — no PR, no custom message)"
|
| 84 |
+
hf upload-large-folder "$repo" . "${EXCLUDES[@]}"
|
| 85 |
+
else
|
| 86 |
+
echo "── uploading to: $repo"
|
| 87 |
+
hf upload "$repo" . . \
|
| 88 |
+
"${EXCLUDES[@]}" \
|
| 89 |
+
--create-pr \
|
| 90 |
+
--commit-message="$MSG"
|
| 91 |
+
fi
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
upload_repo "$MODEL_REPO"
|
| 95 |
+
upload_repo "$SPACE_REPO"
|
| 96 |
+
|
| 97 |
+
echo "✓ done"
|
scripts/serve_judge.sh
CHANGED
|
@@ -1,35 +1,35 @@
|
|
| 1 |
-
#!/usr/bin/env bash
|
| 2 |
-
# Launch llama-server with the FormScout Judge/Classifier VLM.
|
| 3 |
-
#
|
| 4 |
-
# Default model: Qwen3-VL-8B-Instruct Q4_K_M (checkpoints/qwen3-vl/).
|
| 5 |
-
# To serve a fine-tuned GGUF instead, set:
|
| 6 |
-
# FORMSCOUT_JUDGE_GGUF=/path/to/finetuned.gguf
|
| 7 |
-
# FORMSCOUT_JUDGE_MMPROJ=/path/to/mmproj.gguf (only if it ships its own)
|
| 8 |
-
#
|
| 9 |
-
# Requires: brew install llama.cpp
|
| 10 |
-
set -euo pipefail
|
| 11 |
-
|
| 12 |
-
# Homebrew bin may be missing from non-interactive shells
|
| 13 |
-
export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
|
| 14 |
-
|
| 15 |
-
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
| 16 |
-
GGUF="${FORMSCOUT_JUDGE_GGUF:-$ROOT/checkpoints/qwen3-vl/Qwen3VL-8B-Instruct-Q4_K_M.gguf}"
|
| 17 |
-
MMPROJ="${FORMSCOUT_JUDGE_MMPROJ:-$ROOT/checkpoints/qwen3-vl/mmproj-Qwen3VL-8B-Instruct-F16.gguf}"
|
| 18 |
-
HOST="${FORMSCOUT_LLAMA_HOST:-127.0.0.1}"
|
| 19 |
-
PORT="${FORMSCOUT_LLAMA_PORT:-8080}"
|
| 20 |
-
|
| 21 |
-
if [[ ! -f "$GGUF" ]]; then
|
| 22 |
-
echo "Model not found: $GGUF" >&2
|
| 23 |
-
echo "Download it with:" >&2
|
| 24 |
-
echo " python3 -c \"from huggingface_hub import hf_hub_download; [hf_hub_download('Qwen/Qwen3-VL-8B-Instruct-GGUF', f, local_dir='$ROOT/checkpoints/qwen3-vl') for f in ['Qwen3VL-8B-Instruct-Q4_K_M.gguf', 'mmproj-Qwen3VL-8B-Instruct-F16.gguf']]\"" >&2
|
| 25 |
-
exit 1
|
| 26 |
-
fi
|
| 27 |
-
|
| 28 |
-
exec llama-server \
|
| 29 |
-
--model "$GGUF" \
|
| 30 |
-
--mmproj "$MMPROJ" \
|
| 31 |
-
--host "$HOST" \
|
| 32 |
-
--port "$PORT" \
|
| 33 |
-
--ctx-size 16384 \
|
| 34 |
-
--n-gpu-layers 99 \
|
| 35 |
-
--no-warmup
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# Launch llama-server with the FormScout Judge/Classifier VLM.
|
| 3 |
+
#
|
| 4 |
+
# Default model: Qwen3-VL-8B-Instruct Q4_K_M (checkpoints/qwen3-vl/).
|
| 5 |
+
# To serve a fine-tuned GGUF instead, set:
|
| 6 |
+
# FORMSCOUT_JUDGE_GGUF=/path/to/finetuned.gguf
|
| 7 |
+
# FORMSCOUT_JUDGE_MMPROJ=/path/to/mmproj.gguf (only if it ships its own)
|
| 8 |
+
#
|
| 9 |
+
# Requires: brew install llama.cpp
|
| 10 |
+
set -euo pipefail
|
| 11 |
+
|
| 12 |
+
# Homebrew bin may be missing from non-interactive shells
|
| 13 |
+
export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
|
| 14 |
+
|
| 15 |
+
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
| 16 |
+
GGUF="${FORMSCOUT_JUDGE_GGUF:-$ROOT/checkpoints/qwen3-vl/Qwen3VL-8B-Instruct-Q4_K_M.gguf}"
|
| 17 |
+
MMPROJ="${FORMSCOUT_JUDGE_MMPROJ:-$ROOT/checkpoints/qwen3-vl/mmproj-Qwen3VL-8B-Instruct-F16.gguf}"
|
| 18 |
+
HOST="${FORMSCOUT_LLAMA_HOST:-127.0.0.1}"
|
| 19 |
+
PORT="${FORMSCOUT_LLAMA_PORT:-8080}"
|
| 20 |
+
|
| 21 |
+
if [[ ! -f "$GGUF" ]]; then
|
| 22 |
+
echo "Model not found: $GGUF" >&2
|
| 23 |
+
echo "Download it with:" >&2
|
| 24 |
+
echo " python3 -c \"from huggingface_hub import hf_hub_download; [hf_hub_download('Qwen/Qwen3-VL-8B-Instruct-GGUF', f, local_dir='$ROOT/checkpoints/qwen3-vl') for f in ['Qwen3VL-8B-Instruct-Q4_K_M.gguf', 'mmproj-Qwen3VL-8B-Instruct-F16.gguf']]\"" >&2
|
| 25 |
+
exit 1
|
| 26 |
+
fi
|
| 27 |
+
|
| 28 |
+
exec llama-server \
|
| 29 |
+
--model "$GGUF" \
|
| 30 |
+
--mmproj "$MMPROJ" \
|
| 31 |
+
--host "$HOST" \
|
| 32 |
+
--port "$PORT" \
|
| 33 |
+
--ctx-size 16384 \
|
| 34 |
+
--n-gpu-layers 99 \
|
| 35 |
+
--no-warmup
|
tests/test_analysis.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for the movement-analysis engine — no GPU, no model downloads."""
|
| 2 |
+
import math
|
| 3 |
+
import numpy as np
|
| 4 |
+
|
| 5 |
+
from formscout.types import Pose2DResult
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _pose_from(traj_by_joint, n, base_conf=0.9):
|
| 9 |
+
"""Build a Pose2DResult; traj_by_joint maps joint->callable(i)->(x,y)."""
|
| 10 |
+
kps = []
|
| 11 |
+
for i in range(n):
|
| 12 |
+
frame = {}
|
| 13 |
+
for j in range(17):
|
| 14 |
+
if j in traj_by_joint:
|
| 15 |
+
x, y = traj_by_joint[j](i)
|
| 16 |
+
else:
|
| 17 |
+
x, y = float(100 + j * 5), float(100 + j * 5)
|
| 18 |
+
frame[j] = {"x": float(x), "y": float(y), "conf": base_conf}
|
| 19 |
+
kps.append(frame)
|
| 20 |
+
return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
# ── relevant_joints ────────────────────────────────────────────────────────────
|
| 24 |
+
|
| 25 |
+
def test_relevant_joints_and_primary_consistent():
|
| 26 |
+
from formscout.analysis import relevant_joints as RJ
|
| 27 |
+
for test in RJ.RELEVANT:
|
| 28 |
+
joints = RJ.relevant_joints(test)
|
| 29 |
+
angles = RJ.relevant_angles(test)
|
| 30 |
+
prim = RJ.primary_angle(test)
|
| 31 |
+
assert joints, f"{test} has no relevant joints"
|
| 32 |
+
assert prim in angles, f"{test} primary angle not in angles"
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def test_openness_label_monotonic():
|
| 36 |
+
from formscout.analysis.relevant_joints import openness_label
|
| 37 |
+
assert "open" in openness_label(175)
|
| 38 |
+
assert openness_label(30).endswith("closed")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# ── timeseries ──────────────────────────────────────────────────────────────────
|
| 42 |
+
|
| 43 |
+
def test_angle_series_length_matches_frames():
|
| 44 |
+
from formscout.analysis.timeseries import angle_series
|
| 45 |
+
pose = _pose_from({}, n=8)
|
| 46 |
+
series = angle_series(pose, "deep_squat")
|
| 47 |
+
assert "left_knee_flexion" in series
|
| 48 |
+
for name, vals in series.items():
|
| 49 |
+
assert len(vals) == 8
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def test_relevant_flexion_reports_degrees_and_openness():
|
| 53 |
+
from formscout.analysis.timeseries import relevant_flexion_at
|
| 54 |
+
# Straight leg: hip, knee, ankle collinear vertical → ~180°
|
| 55 |
+
straight = {
|
| 56 |
+
11: lambda i: (200, 100), 13: lambda i: (200, 200), 15: lambda i: (200, 300),
|
| 57 |
+
12: lambda i: (260, 100), 14: lambda i: (260, 200), 16: lambda i: (260, 300),
|
| 58 |
+
5: lambda i: (200, 40), 6: lambda i: (260, 40),
|
| 59 |
+
}
|
| 60 |
+
pose = _pose_from(straight, n=5)
|
| 61 |
+
flex = relevant_flexion_at(pose, "deep_squat", 2)
|
| 62 |
+
assert "left_knee_flexion" in flex
|
| 63 |
+
assert flex["left_knee_flexion"]["deg"] > 160
|
| 64 |
+
assert "open" in flex["left_knee_flexion"]["openness"]
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
# ── laban ───────────────────────────────────────────────────────────────────────
|
| 68 |
+
|
| 69 |
+
def test_laban_factors_in_unit_range():
|
| 70 |
+
from formscout.analysis.laban import compute_laban
|
| 71 |
+
pose = _pose_from({13: lambda i: (100 + i * 8, 200)}, n=20)
|
| 72 |
+
res = compute_laban(pose, "deep_squat", fps=30.0)
|
| 73 |
+
for k, v in res["effort"].items():
|
| 74 |
+
assert 0.0 <= v <= 1.0, f"{k}={v} out of range"
|
| 75 |
+
assert set(res["labels"]) == {"space", "weight", "time", "flow"}
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def test_laban_straight_line_is_direct():
|
| 79 |
+
from formscout.analysis.laban import compute_laban
|
| 80 |
+
# Knee travels in a straight horizontal line → high directness (Space)
|
| 81 |
+
pose = _pose_from({13: lambda i: (100 + i * 10, 200)}, n=20)
|
| 82 |
+
res = compute_laban(pose, "deep_squat", fps=30.0)
|
| 83 |
+
assert res["effort"]["space"] > 0.8
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def test_laban_static_clip_low_energy():
|
| 87 |
+
from formscout.analysis.laban import compute_laban
|
| 88 |
+
pose = _pose_from({13: lambda i: (200, 200)}, n=20) # no motion
|
| 89 |
+
res = compute_laban(pose, "deep_squat", fps=30.0)
|
| 90 |
+
assert res["effort"]["weight"] < 0.2
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# ── charts ──────────────────────────────────────────────────────────────────────
|
| 94 |
+
|
| 95 |
+
def test_angle_over_time_chart(tmp_path):
|
| 96 |
+
from formscout.analysis import charts
|
| 97 |
+
out = str(tmp_path / "angle.png")
|
| 98 |
+
series = {"left_knee_flexion": [90, 100, 110, 95], "right_knee_flexion": [88, 99, 108, 94]}
|
| 99 |
+
p = charts.angle_over_time(series, "left_knee_flexion", 2, out)
|
| 100 |
+
assert p == out
|
| 101 |
+
import os
|
| 102 |
+
assert os.path.getsize(out) > 0
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def test_velocity_profile_chart(tmp_path):
|
| 106 |
+
from formscout.analysis import charts
|
| 107 |
+
out = str(tmp_path / "vel.png")
|
| 108 |
+
kps = [{j: {"x": 100 + j + i * 3, "y": 100 + j, "conf": 0.9} for j in range(17)} for i in range(10)]
|
| 109 |
+
p = charts.velocity_profile(kps, 30.0, [13, 14, 11, 12], out)
|
| 110 |
+
import os
|
| 111 |
+
assert p == out and os.path.getsize(out) > 0
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def test_laban_radar_chart(tmp_path):
|
| 115 |
+
from formscout.analysis import charts
|
| 116 |
+
out = str(tmp_path / "radar.png")
|
| 117 |
+
p = charts.laban_radar({"space": 0.8, "weight": 0.4, "time": 0.6, "flow": 0.3}, out)
|
| 118 |
+
import os
|
| 119 |
+
assert p == out and os.path.getsize(out) > 0
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def test_flexion_bars_chart(tmp_path):
|
| 123 |
+
from formscout.analysis import charts
|
| 124 |
+
out = str(tmp_path / "flex.png")
|
| 125 |
+
flex = {"left_knee_flexion": {"deg": 95.0, "openness": "flexed"},
|
| 126 |
+
"left_hip_flexion": {"deg": 120.0, "openness": "mid-range"}}
|
| 127 |
+
p = charts.flexion_bars(flex, out)
|
| 128 |
+
import os
|
| 129 |
+
assert p == out and os.path.getsize(out) > 0
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def test_symmetry_bars_chart(tmp_path):
|
| 133 |
+
from formscout.analysis import charts
|
| 134 |
+
out = str(tmp_path / "sym.png")
|
| 135 |
+
asym = [{"test": "hurdle_step", "left_score": 2, "right_score": 3, "delta": 1}]
|
| 136 |
+
p = charts.symmetry_bars(asym, out)
|
| 137 |
+
import os
|
| 138 |
+
assert p == out and os.path.getsize(out) > 0
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def test_charts_return_none_on_empty(tmp_path):
|
| 142 |
+
from formscout.analysis import charts
|
| 143 |
+
assert charts.angle_over_time({}, None, None, str(tmp_path / "a.png")) is None
|
| 144 |
+
assert charts.flexion_bars({}, str(tmp_path / "f.png")) is None
|
| 145 |
+
assert charts.symmetry_bars([], str(tmp_path / "s.png")) is None
|
tests/test_judge_backend.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Judge backend selection + transformers-fallback — no GPU, no downloads."""
|
| 2 |
+
import importlib
|
| 3 |
+
|
| 4 |
+
import formscout.config as config
|
| 5 |
+
from formscout.serving import get_vlm_client
|
| 6 |
+
from formscout.serving.llama_cpp import LlamaCppClient
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def _reload_config(monkeypatch, **env):
|
| 10 |
+
for k, v in env.items():
|
| 11 |
+
if v is None:
|
| 12 |
+
monkeypatch.delenv(k, raising=False)
|
| 13 |
+
else:
|
| 14 |
+
monkeypatch.setenv(k, v)
|
| 15 |
+
importlib.reload(config)
|
| 16 |
+
return config
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def test_resolve_backend_default_local(monkeypatch):
|
| 20 |
+
cfg = _reload_config(monkeypatch, FORMSCOUT_JUDGE_BACKEND=None, SPACE_ID=None)
|
| 21 |
+
assert cfg.resolve_judge_backend() == "llama_cpp"
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def test_resolve_backend_auto_on_space(monkeypatch):
|
| 25 |
+
cfg = _reload_config(monkeypatch, FORMSCOUT_JUDGE_BACKEND="auto", SPACE_ID="me/space")
|
| 26 |
+
assert cfg.resolve_judge_backend() == "transformers"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def test_resolve_backend_explicit(monkeypatch):
|
| 30 |
+
cfg = _reload_config(monkeypatch, FORMSCOUT_JUDGE_BACKEND="llama_cpp", SPACE_ID="me/space")
|
| 31 |
+
assert cfg.resolve_judge_backend() == "llama_cpp"
|
| 32 |
+
importlib.reload(config) # restore
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def test_factory_returns_llama_cpp_locally(monkeypatch):
|
| 36 |
+
_reload_config(monkeypatch, FORMSCOUT_JUDGE_BACKEND="llama_cpp", SPACE_ID=None)
|
| 37 |
+
client = get_vlm_client()
|
| 38 |
+
assert isinstance(client, LlamaCppClient)
|
| 39 |
+
importlib.reload(config)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def test_transformers_client_available_is_cheap_bool():
|
| 43 |
+
# available must not load/download the model — just report a bool
|
| 44 |
+
from formscout.serving.transformers_vlm import TransformersVLMClient
|
| 45 |
+
c = TransformersVLMClient()
|
| 46 |
+
assert isinstance(c.available, bool)
|
| 47 |
+
c._failed = True
|
| 48 |
+
assert c.available is False
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def test_judge_uses_rubric_on_fallback_sentinel():
|
| 52 |
+
from formscout.agents.judge import JudgeAgent
|
| 53 |
+
from formscout.types import BiomechFeatures, ScoreResult, MovementResult, IngestResult
|
| 54 |
+
import numpy as np
|
| 55 |
+
|
| 56 |
+
agent = JudgeAgent()
|
| 57 |
+
|
| 58 |
+
class _FakeClient:
|
| 59 |
+
available = True
|
| 60 |
+
|
| 61 |
+
def complete(self, *a, **k):
|
| 62 |
+
return {"error": "no gpu", "fallback": True, "text": ""}
|
| 63 |
+
|
| 64 |
+
agent._client = _FakeClient()
|
| 65 |
+
features = BiomechFeatures(test_name="deep_squat", view="2d", side="na",
|
| 66 |
+
angles={}, alignments={}, symmetry_delta=None,
|
| 67 |
+
timing={}, confidence=0.9)
|
| 68 |
+
rubric = ScoreResult(score=2, rationale="rubric says 2", confidence=0.8)
|
| 69 |
+
movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
|
| 70 |
+
ingest = IngestResult(frames=[np.zeros((10, 10, 3), np.uint8)], fps=30.0,
|
| 71 |
+
duration=0.03, n_people=1, width=10, height=10)
|
| 72 |
+
|
| 73 |
+
res = agent.run(features, rubric, movement, ingest)
|
| 74 |
+
assert res.score == 2
|
| 75 |
+
assert "rubric-only" in res.rationale
|
tests/test_keyframe.py
CHANGED
|
@@ -1,37 +1,37 @@
|
|
| 1 |
-
"""Tests for PoseVisualizer.render_frame — single annotated still."""
|
| 2 |
-
import os
|
| 3 |
-
import numpy as np
|
| 4 |
-
|
| 5 |
-
from formscout.types import IngestResult, Pose2DResult
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
def _ingest(n=5, h=480, w=640):
|
| 9 |
-
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 10 |
-
return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
def _pose(n=5):
|
| 14 |
-
kps = []
|
| 15 |
-
for i in range(n):
|
| 16 |
-
kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
|
| 17 |
-
for j in range(17)})
|
| 18 |
-
return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
def test_render_frame_writes_png(tmp_path):
|
| 22 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 23 |
-
out = str(tmp_path / "key.png")
|
| 24 |
-
path = PoseVisualizer().render_frame(_ingest(), _pose(), frame_idx=2,
|
| 25 |
-
layers={"skeleton"}, caption="Deep Squat — heels elevated",
|
| 26 |
-
out_png=out)
|
| 27 |
-
assert path == out
|
| 28 |
-
assert os.path.exists(out)
|
| 29 |
-
assert os.path.getsize(out) > 0
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
def test_render_frame_bad_index_returns_none(tmp_path):
|
| 33 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 34 |
-
out = str(tmp_path / "key.png")
|
| 35 |
-
path = PoseVisualizer().render_frame(_ingest(n=3), _pose(n=3), frame_idx=99,
|
| 36 |
-
layers={"skeleton"}, caption="", out_png=out)
|
| 37 |
-
assert path is None
|
|
|
|
| 1 |
+
"""Tests for PoseVisualizer.render_frame — single annotated still."""
|
| 2 |
+
import os
|
| 3 |
+
import numpy as np
|
| 4 |
+
|
| 5 |
+
from formscout.types import IngestResult, Pose2DResult
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _ingest(n=5, h=480, w=640):
|
| 9 |
+
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 10 |
+
return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _pose(n=5):
|
| 14 |
+
kps = []
|
| 15 |
+
for i in range(n):
|
| 16 |
+
kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
|
| 17 |
+
for j in range(17)})
|
| 18 |
+
return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def test_render_frame_writes_png(tmp_path):
|
| 22 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 23 |
+
out = str(tmp_path / "key.png")
|
| 24 |
+
path = PoseVisualizer().render_frame(_ingest(), _pose(), frame_idx=2,
|
| 25 |
+
layers={"skeleton"}, caption="Deep Squat — heels elevated",
|
| 26 |
+
out_png=out)
|
| 27 |
+
assert path == out
|
| 28 |
+
assert os.path.exists(out)
|
| 29 |
+
assert os.path.getsize(out) > 0
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def test_render_frame_bad_index_returns_none(tmp_path):
|
| 33 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 34 |
+
out = str(tmp_path / "key.png")
|
| 35 |
+
path = PoseVisualizer().render_frame(_ingest(n=3), _pose(n=3), frame_idx=99,
|
| 36 |
+
layers={"skeleton"}, caption="", out_png=out)
|
| 37 |
+
assert path is None
|
tests/test_pdf_report.py
CHANGED
|
@@ -1,51 +1,51 @@
|
|
| 1 |
-
"""Tests for PdfReportAgent — no GPU, no model downloads."""
|
| 2 |
-
import os
|
| 3 |
-
|
| 4 |
-
from formscout.types import (
|
| 5 |
-
ReportResult, SessionEntry, MovementResult, BiomechFeatures, ScoreResult, JudgeResult,
|
| 6 |
-
)
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
def _entry(test_name="deep_squat", score=2, needs_human=False):
|
| 10 |
-
movement = MovementResult(test_name=test_name, side="na", confidence=1.0)
|
| 11 |
-
features = BiomechFeatures(
|
| 12 |
-
test_name=test_name, view="2d", side="na",
|
| 13 |
-
angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": False},
|
| 14 |
-
symmetry_delta=None, timing={"deepest_frame": 1}, confidence=0.9,
|
| 15 |
-
)
|
| 16 |
-
rubric = ScoreResult(score=2, rationale="rubric ok", confidence=0.8)
|
| 17 |
-
judge = JudgeResult(score=None if needs_human else score, rationale="judge rationale",
|
| 18 |
-
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 19 |
-
confidence=0.85, needs_human=needs_human)
|
| 20 |
-
return SessionEntry(
|
| 21 |
-
test_name=test_name, side="na", score=None if needs_human else score,
|
| 22 |
-
needs_human=needs_human, rationale="judge rationale",
|
| 23 |
-
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 24 |
-
measurements={"left_knee_flexion_deg": 95.0, "knees_tracking_over_feet": False},
|
| 25 |
-
confidence=0.85, view="2d", keyframe_path=None,
|
| 26 |
-
movement=movement, features=features, rubric_score=rubric, judge=judge,
|
| 27 |
-
)
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
def _report(composite=2):
|
| 31 |
-
return ReportResult(
|
| 32 |
-
per_test=[], composite=composite, asymmetries=[],
|
| 33 |
-
overlay_video_path=None, pdf_path=None,
|
| 34 |
-
low_confidence_flags=[], disagreement_flags=[],
|
| 35 |
-
)
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
def test_pdf_is_created(tmp_path):
|
| 39 |
-
from formscout.agents.pdf_report import PdfReportAgent
|
| 40 |
-
path = PdfReportAgent().run(_report(2), [_entry()], str(tmp_path))
|
| 41 |
-
assert path is not None
|
| 42 |
-
assert os.path.exists(path)
|
| 43 |
-
assert os.path.getsize(path) > 1000 # a real PDF, not an empty file
|
| 44 |
-
with open(path, "rb") as f:
|
| 45 |
-
assert f.read(5) == b"%PDF-"
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
def test_pdf_handles_incomplete_composite(tmp_path):
|
| 49 |
-
from formscout.agents.pdf_report import PdfReportAgent
|
| 50 |
-
path = PdfReportAgent().run(_report(None), [_entry(needs_human=True)], str(tmp_path))
|
| 51 |
-
assert path is not None and os.path.exists(path)
|
|
|
|
| 1 |
+
"""Tests for PdfReportAgent — no GPU, no model downloads."""
|
| 2 |
+
import os
|
| 3 |
+
|
| 4 |
+
from formscout.types import (
|
| 5 |
+
ReportResult, SessionEntry, MovementResult, BiomechFeatures, ScoreResult, JudgeResult,
|
| 6 |
+
)
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def _entry(test_name="deep_squat", score=2, needs_human=False):
|
| 10 |
+
movement = MovementResult(test_name=test_name, side="na", confidence=1.0)
|
| 11 |
+
features = BiomechFeatures(
|
| 12 |
+
test_name=test_name, view="2d", side="na",
|
| 13 |
+
angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": False},
|
| 14 |
+
symmetry_delta=None, timing={"deepest_frame": 1}, confidence=0.9,
|
| 15 |
+
)
|
| 16 |
+
rubric = ScoreResult(score=2, rationale="rubric ok", confidence=0.8)
|
| 17 |
+
judge = JudgeResult(score=None if needs_human else score, rationale="judge rationale",
|
| 18 |
+
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 19 |
+
confidence=0.85, needs_human=needs_human)
|
| 20 |
+
return SessionEntry(
|
| 21 |
+
test_name=test_name, side="na", score=None if needs_human else score,
|
| 22 |
+
needs_human=needs_human, rationale="judge rationale",
|
| 23 |
+
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 24 |
+
measurements={"left_knee_flexion_deg": 95.0, "knees_tracking_over_feet": False},
|
| 25 |
+
confidence=0.85, view="2d", keyframe_path=None,
|
| 26 |
+
movement=movement, features=features, rubric_score=rubric, judge=judge,
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _report(composite=2):
|
| 31 |
+
return ReportResult(
|
| 32 |
+
per_test=[], composite=composite, asymmetries=[],
|
| 33 |
+
overlay_video_path=None, pdf_path=None,
|
| 34 |
+
low_confidence_flags=[], disagreement_flags=[],
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def test_pdf_is_created(tmp_path):
|
| 39 |
+
from formscout.agents.pdf_report import PdfReportAgent
|
| 40 |
+
path = PdfReportAgent().run(_report(2), [_entry()], str(tmp_path))
|
| 41 |
+
assert path is not None
|
| 42 |
+
assert os.path.exists(path)
|
| 43 |
+
assert os.path.getsize(path) > 1000 # a real PDF, not an empty file
|
| 44 |
+
with open(path, "rb") as f:
|
| 45 |
+
assert f.read(5) == b"%PDF-"
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def test_pdf_handles_incomplete_composite(tmp_path):
|
| 49 |
+
from formscout.agents.pdf_report import PdfReportAgent
|
| 50 |
+
path = PdfReportAgent().run(_report(None), [_entry(needs_human=True)], str(tmp_path))
|
| 51 |
+
assert path is not None and os.path.exists(path)
|
tests/test_phase2.py
CHANGED
|
@@ -1,354 +1,354 @@
|
|
| 1 |
-
"""Tests for all rubric scorers and Phase 2 agents."""
|
| 2 |
-
|
| 3 |
-
from formscout.types import (
|
| 4 |
-
BiomechFeatures, ScoreResult, MovementResult, JudgeResult, ReportResult,
|
| 5 |
-
)
|
| 6 |
-
from formscout.rubric import score_test, SCORERS
|
| 7 |
-
from formscout.rubric.hurdle_step import score_hurdle_step
|
| 8 |
-
from formscout.rubric.inline_lunge import score_inline_lunge
|
| 9 |
-
from formscout.rubric.shoulder_mobility import score_shoulder_mobility
|
| 10 |
-
from formscout.rubric.active_slr import score_active_slr
|
| 11 |
-
from formscout.rubric.trunk_stability_pushup import score_trunk_stability_pushup
|
| 12 |
-
from formscout.rubric.rotary_stability import score_rotary_stability
|
| 13 |
-
from formscout.agents.judge import JudgeAgent
|
| 14 |
-
from formscout.agents.report import ReportAgent
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
def _make_features(test_name, angles=None, alignments=None, side="na", sym_delta=None):
|
| 18 |
-
return BiomechFeatures(
|
| 19 |
-
test_name=test_name, view="2d", side=side,
|
| 20 |
-
angles=angles or {}, alignments=alignments or {},
|
| 21 |
-
symmetry_delta=sym_delta, timing={}, confidence=0.8,
|
| 22 |
-
)
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
# ─── Rubric dispatch ─────────────────────────────────────────────────────────
|
| 26 |
-
|
| 27 |
-
class TestRubricDispatch:
|
| 28 |
-
def test_all_tests_have_scorers(self):
|
| 29 |
-
expected = {"deep_squat", "hurdle_step", "inline_lunge", "shoulder_mobility",
|
| 30 |
-
"active_slr", "trunk_stability_pushup", "rotary_stability"}
|
| 31 |
-
assert set(SCORERS.keys()) == expected
|
| 32 |
-
|
| 33 |
-
def test_dispatch_unknown_test(self):
|
| 34 |
-
f = _make_features("unknown_test")
|
| 35 |
-
r = score_test(f)
|
| 36 |
-
assert r.confidence == 0.0
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
# ─── Hurdle Step ──────────────────────────────────────────────────────────────
|
| 40 |
-
|
| 41 |
-
class TestHurdleStep:
|
| 42 |
-
def test_score_3_good_form(self):
|
| 43 |
-
f = _make_features("hurdle_step", angles={
|
| 44 |
-
"step_hip_flexion_deg": 100.0, "stance_knee_angle_deg": 175.0,
|
| 45 |
-
"shoulder_tilt_deg": 5.0,
|
| 46 |
-
}, alignments={"trunk_stable": True, "stance_knee_extended": True})
|
| 47 |
-
r = score_hurdle_step(f)
|
| 48 |
-
assert r.score == 3
|
| 49 |
-
|
| 50 |
-
def test_score_2_compensation(self):
|
| 51 |
-
f = _make_features("hurdle_step", angles={
|
| 52 |
-
"step_hip_flexion_deg": 80.0, "stance_knee_angle_deg": 170.0,
|
| 53 |
-
}, alignments={"trunk_stable": True, "stance_knee_extended": True})
|
| 54 |
-
r = score_hurdle_step(f)
|
| 55 |
-
assert r.score == 2
|
| 56 |
-
|
| 57 |
-
def test_score_1_poor(self):
|
| 58 |
-
f = _make_features("hurdle_step", angles={
|
| 59 |
-
"step_hip_flexion_deg": 50.0, "stance_knee_angle_deg": 140.0,
|
| 60 |
-
}, alignments={"trunk_stable": False, "stance_knee_extended": False})
|
| 61 |
-
r = score_hurdle_step(f)
|
| 62 |
-
assert r.score == 1
|
| 63 |
-
|
| 64 |
-
def test_never_scores_zero(self):
|
| 65 |
-
f = _make_features("hurdle_step", angles={
|
| 66 |
-
"step_hip_flexion_deg": 30.0,
|
| 67 |
-
}, alignments={"trunk_stable": False, "stance_knee_extended": False})
|
| 68 |
-
r = score_hurdle_step(f)
|
| 69 |
-
assert r.score >= 1
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
# ─── Inline Lunge ─────────────────────────────────────────────────────────────
|
| 73 |
-
|
| 74 |
-
class TestInlineLunge:
|
| 75 |
-
def test_score_3_deep_and_aligned(self):
|
| 76 |
-
f = _make_features("inline_lunge", angles={
|
| 77 |
-
"front_knee_flexion_deg": 85.0, "trunk_lean_from_vertical_deg": 5.0,
|
| 78 |
-
}, alignments={"trunk_upright": True, "knee_over_ankle": True})
|
| 79 |
-
r = score_inline_lunge(f)
|
| 80 |
-
assert r.score == 3
|
| 81 |
-
|
| 82 |
-
def test_score_1_shallow(self):
|
| 83 |
-
f = _make_features("inline_lunge", angles={
|
| 84 |
-
"front_knee_flexion_deg": 140.0,
|
| 85 |
-
}, alignments={"trunk_upright": False, "knee_over_ankle": False})
|
| 86 |
-
r = score_inline_lunge(f)
|
| 87 |
-
assert r.score == 1
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
# ─── Shoulder Mobility ────────────────────────────────────────────────────────
|
| 91 |
-
|
| 92 |
-
class TestShoulderMobility:
|
| 93 |
-
def test_score_3_close(self):
|
| 94 |
-
f = _make_features("shoulder_mobility", angles={
|
| 95 |
-
"inter_fist_normalized": 0.25,
|
| 96 |
-
}, alignments={"fists_within_one_hand": True, "fists_within_1_5_hand": True})
|
| 97 |
-
r = score_shoulder_mobility(f)
|
| 98 |
-
assert r.score == 3
|
| 99 |
-
|
| 100 |
-
def test_score_2_moderate(self):
|
| 101 |
-
f = _make_features("shoulder_mobility", angles={
|
| 102 |
-
"inter_fist_normalized": 0.45,
|
| 103 |
-
}, alignments={"fists_within_one_hand": False, "fists_within_1_5_hand": True})
|
| 104 |
-
r = score_shoulder_mobility(f)
|
| 105 |
-
assert r.score == 2
|
| 106 |
-
|
| 107 |
-
def test_score_1_far(self):
|
| 108 |
-
f = _make_features("shoulder_mobility", angles={
|
| 109 |
-
"inter_fist_normalized": 0.8,
|
| 110 |
-
}, alignments={"fists_within_one_hand": False, "fists_within_1_5_hand": False})
|
| 111 |
-
r = score_shoulder_mobility(f)
|
| 112 |
-
assert r.score == 1
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
# ─── Active SLR ───────────────────────────────────────────────────────────────
|
| 116 |
-
|
| 117 |
-
class TestActiveSLR:
|
| 118 |
-
def test_score_3_high_raise(self):
|
| 119 |
-
f = _make_features("active_slr", angles={
|
| 120 |
-
"raised_leg_angle_deg": 80.0,
|
| 121 |
-
}, alignments={"past_contralateral_knee": True, "past_mid_thigh": True, "down_leg_flat": True})
|
| 122 |
-
r = score_active_slr(f)
|
| 123 |
-
assert r.score == 3
|
| 124 |
-
|
| 125 |
-
def test_score_2_moderate_raise(self):
|
| 126 |
-
f = _make_features("active_slr", angles={
|
| 127 |
-
"raised_leg_angle_deg": 55.0,
|
| 128 |
-
}, alignments={"past_contralateral_knee": False, "past_mid_thigh": True, "down_leg_flat": True})
|
| 129 |
-
r = score_active_slr(f)
|
| 130 |
-
assert r.score == 2
|
| 131 |
-
|
| 132 |
-
def test_score_1_low_raise(self):
|
| 133 |
-
f = _make_features("active_slr", angles={
|
| 134 |
-
"raised_leg_angle_deg": 30.0,
|
| 135 |
-
}, alignments={"past_contralateral_knee": False, "past_mid_thigh": False, "down_leg_flat": True})
|
| 136 |
-
r = score_active_slr(f)
|
| 137 |
-
assert r.score == 1
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
# ─── Trunk Stability Push-Up ─────────────────────────────────────────────────
|
| 141 |
-
|
| 142 |
-
class TestTrunkStabilityPushup:
|
| 143 |
-
def test_score_3_rigid_hands_high(self):
|
| 144 |
-
f = _make_features("trunk_stability_pushup", angles={
|
| 145 |
-
"max_sag_px": 10.0, "trunk_variance_px": 5.0,
|
| 146 |
-
}, alignments={"body_rigid": True, "no_sag": True, "hands_at_forehead": True})
|
| 147 |
-
r = score_trunk_stability_pushup(f)
|
| 148 |
-
assert r.score == 3
|
| 149 |
-
|
| 150 |
-
def test_score_1_sag(self):
|
| 151 |
-
f = _make_features("trunk_stability_pushup", angles={
|
| 152 |
-
"max_sag_px": 50.0, "trunk_variance_px": 25.0,
|
| 153 |
-
}, alignments={"body_rigid": False, "no_sag": False, "hands_at_forehead": True})
|
| 154 |
-
r = score_trunk_stability_pushup(f)
|
| 155 |
-
assert r.score == 1
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
# ─── Rotary Stability ────────────────────────────────────────────────────────
|
| 159 |
-
|
| 160 |
-
class TestRotaryStability:
|
| 161 |
-
def test_score_2_stable(self):
|
| 162 |
-
f = _make_features("rotary_stability", angles={
|
| 163 |
-
"trunk_stability_std_px": 8.0, "shoulder_level_diff_px": 10.0, "hip_level_diff_px": 12.0,
|
| 164 |
-
}, alignments={"trunk_stable": True, "shoulders_level": True, "hips_level": True})
|
| 165 |
-
r = score_rotary_stability(f)
|
| 166 |
-
assert r.score == 2 # Default to 2 (contralateral assumption)
|
| 167 |
-
|
| 168 |
-
def test_score_1_unstable(self):
|
| 169 |
-
f = _make_features("rotary_stability", angles={
|
| 170 |
-
"trunk_stability_std_px": 30.0, "shoulder_level_diff_px": 35.0, "hip_level_diff_px": 30.0,
|
| 171 |
-
}, alignments={"trunk_stable": False, "shoulders_level": False, "hips_level": False})
|
| 172 |
-
r = score_rotary_stability(f)
|
| 173 |
-
assert r.score == 1
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
# ─── JudgeAgent fallback ─────────────────────────────────────────────────────
|
| 177 |
-
|
| 178 |
-
class TestJudgeAgent:
|
| 179 |
-
def test_fallback_when_judge_disabled(self, monkeypatch):
|
| 180 |
-
"""When ENABLE_JUDGE=False, judge promotes rubric score."""
|
| 181 |
-
from formscout import config
|
| 182 |
-
monkeypatch.setattr(config, "ENABLE_JUDGE", False)
|
| 183 |
-
agent = JudgeAgent()
|
| 184 |
-
features = _make_features("deep_squat", angles={"left_femur_from_horizontal_deg": 70.0})
|
| 185 |
-
rubric = ScoreResult(score=3, rationale="all good", confidence=0.9)
|
| 186 |
-
movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
|
| 187 |
-
result = agent.run(features, rubric, movement)
|
| 188 |
-
assert isinstance(result, JudgeResult)
|
| 189 |
-
assert result.score == 3
|
| 190 |
-
assert "[rubric-only]" in result.rationale
|
| 191 |
-
|
| 192 |
-
def test_fallback_when_server_unavailable(self, monkeypatch):
|
| 193 |
-
"""ENABLE_JUDGE=True but llama-server down → rubric fallback, never a crash."""
|
| 194 |
-
from unittest.mock import PropertyMock, patch
|
| 195 |
-
from formscout import config
|
| 196 |
-
monkeypatch.setattr(config, "ENABLE_JUDGE", True)
|
| 197 |
-
agent = JudgeAgent()
|
| 198 |
-
with patch.object(type(agent._client), "available", new_callable=PropertyMock, return_value=False):
|
| 199 |
-
features = _make_features("deep_squat")
|
| 200 |
-
rubric = ScoreResult(score=2, rationale="heels up", confidence=0.8)
|
| 201 |
-
movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
|
| 202 |
-
result = agent.run(features, rubric, movement)
|
| 203 |
-
assert result.score == 2
|
| 204 |
-
assert "[rubric-only]" in result.rationale
|
| 205 |
-
|
| 206 |
-
def test_vlm_response_parsed_into_judge_result(self, monkeypatch):
|
| 207 |
-
"""ENABLE_JUDGE=True with live client → VLM JSON becomes JudgeResult."""
|
| 208 |
-
from unittest.mock import PropertyMock, patch
|
| 209 |
-
from formscout import config
|
| 210 |
-
monkeypatch.setattr(config, "ENABLE_JUDGE", True)
|
| 211 |
-
agent = JudgeAgent()
|
| 212 |
-
vlm_json = {
|
| 213 |
-
"test": "deep_squat", "side": "na", "score": 2, "needs_human": False,
|
| 214 |
-
"rationale": "Femur 5° above horizontal; 2D estimate.",
|
| 215 |
-
"compensation_tags": ["forward_lean"], "corrective_hint": "Sit back into heels.",
|
| 216 |
-
"confidence": 0.78,
|
| 217 |
-
}
|
| 218 |
-
with patch.object(type(agent._client), "available", new_callable=PropertyMock, return_value=True), \
|
| 219 |
-
patch.object(agent._client, "complete", return_value=vlm_json):
|
| 220 |
-
features = _make_features("deep_squat")
|
| 221 |
-
rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
|
| 222 |
-
movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
|
| 223 |
-
result = agent.run(features, rubric, movement)
|
| 224 |
-
assert result.score == 2
|
| 225 |
-
assert result.compensation_tags == ["forward_lean"]
|
| 226 |
-
assert result.needs_human is False
|
| 227 |
-
|
| 228 |
-
def test_vlm_needs_human_yields_no_score(self, monkeypatch):
|
| 229 |
-
"""needs_human=True from the VLM must produce score=None."""
|
| 230 |
-
from unittest.mock import PropertyMock, patch
|
| 231 |
-
from formscout import config
|
| 232 |
-
monkeypatch.setattr(config, "ENABLE_JUDGE", True)
|
| 233 |
-
agent = JudgeAgent()
|
| 234 |
-
vlm_json = {"score": 1, "needs_human": True, "rationale": "Possible pain.", "confidence": 0.9}
|
| 235 |
-
with patch.object(type(agent._client), "available", new_callable=PropertyMock, return_value=True), \
|
| 236 |
-
patch.object(agent._client, "complete", return_value=vlm_json):
|
| 237 |
-
result = agent.run(
|
| 238 |
-
_make_features("deep_squat"),
|
| 239 |
-
ScoreResult(score=1, rationale="x", confidence=0.5),
|
| 240 |
-
MovementResult(test_name="deep_squat", side="na", confidence=1.0),
|
| 241 |
-
)
|
| 242 |
-
assert result.needs_human is True
|
| 243 |
-
assert result.score is None
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
# ─── LlamaCppClient (chat-completions endpoint) ──────────────────────────────
|
| 247 |
-
|
| 248 |
-
class TestLlamaCppClient:
|
| 249 |
-
def test_parse_plain_json(self):
|
| 250 |
-
from formscout.serving.llama_cpp import LlamaCppClient
|
| 251 |
-
assert LlamaCppClient._parse_json_reply('{"score": 3}') == {"score": 3}
|
| 252 |
-
|
| 253 |
-
def test_parse_fenced_json(self):
|
| 254 |
-
from formscout.serving.llama_cpp import LlamaCppClient
|
| 255 |
-
fenced = '```json\n{"score": 2, "needs_human": false}\n```'
|
| 256 |
-
assert LlamaCppClient._parse_json_reply(fenced) == {"score": 2, "needs_human": False}
|
| 257 |
-
|
| 258 |
-
def test_parse_non_json_returns_text(self):
|
| 259 |
-
from formscout.serving.llama_cpp import LlamaCppClient
|
| 260 |
-
assert LlamaCppClient._parse_json_reply("not json") == {"text": "not json"}
|
| 261 |
-
|
| 262 |
-
def test_complete_posts_chat_endpoint_with_images(self):
|
| 263 |
-
from unittest.mock import MagicMock, patch
|
| 264 |
-
from formscout.serving.llama_cpp import LlamaCppClient
|
| 265 |
-
|
| 266 |
-
client = LlamaCppClient(port=8080)
|
| 267 |
-
resp = MagicMock()
|
| 268 |
-
resp.json.return_value = {"choices": [{"message": {"content": '{"ok": true}'}}]}
|
| 269 |
-
resp.raise_for_status.return_value = None
|
| 270 |
-
with patch("formscout.serving.llama_cpp.requests.post", return_value=resp) as mock_post:
|
| 271 |
-
result = client.complete("score this", images=["aGVsbG8=" * 600])
|
| 272 |
-
assert result == {"ok": True}
|
| 273 |
-
url = mock_post.call_args.args[0] if mock_post.call_args.args else mock_post.call_args.kwargs.get("url")
|
| 274 |
-
assert url.endswith("/v1/chat/completions")
|
| 275 |
-
payload = mock_post.call_args.kwargs["json"]
|
| 276 |
-
content = payload["messages"][0]["content"]
|
| 277 |
-
assert content[0] == {"type": "text", "text": "score this"}
|
| 278 |
-
assert content[1]["type"] == "image_url"
|
| 279 |
-
assert content[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
| 280 |
-
|
| 281 |
-
def test_complete_connection_error_returns_safe_dict(self):
|
| 282 |
-
from unittest.mock import patch
|
| 283 |
-
import requests as _requests
|
| 284 |
-
from formscout.serving.llama_cpp import LlamaCppClient
|
| 285 |
-
|
| 286 |
-
client = LlamaCppClient(port=8080)
|
| 287 |
-
with patch("formscout.serving.llama_cpp.requests.post", side_effect=_requests.ConnectionError):
|
| 288 |
-
result = client.complete("hello")
|
| 289 |
-
assert "error" in result
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
# ─── ReportAgent ──────────────────────────────────────────────────────────────
|
| 293 |
-
|
| 294 |
-
class TestReportAgent:
|
| 295 |
-
def test_single_test_report(self):
|
| 296 |
-
agent = ReportAgent()
|
| 297 |
-
entries = [{
|
| 298 |
-
"movement": MovementResult(test_name="deep_squat", side="na", confidence=1.0),
|
| 299 |
-
"features": _make_features("deep_squat"),
|
| 300 |
-
"rubric_score": ScoreResult(score=3, rationale="ok", confidence=0.9),
|
| 301 |
-
"judge": JudgeResult(
|
| 302 |
-
score=3, rationale="good", compensation_tags=[], corrective_hint="",
|
| 303 |
-
confidence=0.9,
|
| 304 |
-
),
|
| 305 |
-
"side": "na",
|
| 306 |
-
}]
|
| 307 |
-
result = agent.run(entries)
|
| 308 |
-
assert isinstance(result, ReportResult)
|
| 309 |
-
assert len(result.per_test) == 1
|
| 310 |
-
assert result.per_test[0]["score"] == 3
|
| 311 |
-
|
| 312 |
-
def test_bilateral_reports_lower_score(self):
|
| 313 |
-
agent = ReportAgent()
|
| 314 |
-
entries = [
|
| 315 |
-
{
|
| 316 |
-
"movement": MovementResult(test_name="hurdle_step", side="left", confidence=1.0),
|
| 317 |
-
"features": _make_features("hurdle_step", side="left"),
|
| 318 |
-
"rubric_score": ScoreResult(score=3, rationale="ok", confidence=0.9),
|
| 319 |
-
"judge": JudgeResult(
|
| 320 |
-
score=3, rationale="", compensation_tags=[], corrective_hint="",
|
| 321 |
-
confidence=0.9,
|
| 322 |
-
),
|
| 323 |
-
"side": "left",
|
| 324 |
-
},
|
| 325 |
-
{
|
| 326 |
-
"movement": MovementResult(test_name="hurdle_step", side="right", confidence=1.0),
|
| 327 |
-
"features": _make_features("hurdle_step", side="right"),
|
| 328 |
-
"rubric_score": ScoreResult(score=2, rationale="comp", confidence=0.8),
|
| 329 |
-
"judge": JudgeResult(
|
| 330 |
-
score=2, rationale="", compensation_tags=[], corrective_hint="",
|
| 331 |
-
confidence=0.8,
|
| 332 |
-
),
|
| 333 |
-
"side": "right",
|
| 334 |
-
},
|
| 335 |
-
]
|
| 336 |
-
result = agent.run(entries)
|
| 337 |
-
assert result.per_test[0]["score"] == 2 # lower of 3 and 2
|
| 338 |
-
assert len(result.asymmetries) == 1
|
| 339 |
-
assert result.asymmetries[0]["delta"] == 1
|
| 340 |
-
|
| 341 |
-
def test_composite_none_when_unscored(self):
|
| 342 |
-
agent = ReportAgent()
|
| 343 |
-
entries = [{
|
| 344 |
-
"movement": MovementResult(test_name="deep_squat", side="na", confidence=1.0),
|
| 345 |
-
"features": _make_features("deep_squat"),
|
| 346 |
-
"rubric_score": ScoreResult(score=1, rationale="", confidence=0.5),
|
| 347 |
-
"judge": JudgeResult(
|
| 348 |
-
score=None, rationale="pain", compensation_tags=[], corrective_hint="",
|
| 349 |
-
confidence=0.0, needs_human=True,
|
| 350 |
-
),
|
| 351 |
-
"side": "na",
|
| 352 |
-
}]
|
| 353 |
-
result = agent.run(entries)
|
| 354 |
-
assert result.composite is None
|
|
|
|
| 1 |
+
"""Tests for all rubric scorers and Phase 2 agents."""
|
| 2 |
+
|
| 3 |
+
from formscout.types import (
|
| 4 |
+
BiomechFeatures, ScoreResult, MovementResult, JudgeResult, ReportResult,
|
| 5 |
+
)
|
| 6 |
+
from formscout.rubric import score_test, SCORERS
|
| 7 |
+
from formscout.rubric.hurdle_step import score_hurdle_step
|
| 8 |
+
from formscout.rubric.inline_lunge import score_inline_lunge
|
| 9 |
+
from formscout.rubric.shoulder_mobility import score_shoulder_mobility
|
| 10 |
+
from formscout.rubric.active_slr import score_active_slr
|
| 11 |
+
from formscout.rubric.trunk_stability_pushup import score_trunk_stability_pushup
|
| 12 |
+
from formscout.rubric.rotary_stability import score_rotary_stability
|
| 13 |
+
from formscout.agents.judge import JudgeAgent
|
| 14 |
+
from formscout.agents.report import ReportAgent
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _make_features(test_name, angles=None, alignments=None, side="na", sym_delta=None):
|
| 18 |
+
return BiomechFeatures(
|
| 19 |
+
test_name=test_name, view="2d", side=side,
|
| 20 |
+
angles=angles or {}, alignments=alignments or {},
|
| 21 |
+
symmetry_delta=sym_delta, timing={}, confidence=0.8,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
# ─── Rubric dispatch ─────────────────────────────────────────────────────────
|
| 26 |
+
|
| 27 |
+
class TestRubricDispatch:
|
| 28 |
+
def test_all_tests_have_scorers(self):
|
| 29 |
+
expected = {"deep_squat", "hurdle_step", "inline_lunge", "shoulder_mobility",
|
| 30 |
+
"active_slr", "trunk_stability_pushup", "rotary_stability"}
|
| 31 |
+
assert set(SCORERS.keys()) == expected
|
| 32 |
+
|
| 33 |
+
def test_dispatch_unknown_test(self):
|
| 34 |
+
f = _make_features("unknown_test")
|
| 35 |
+
r = score_test(f)
|
| 36 |
+
assert r.confidence == 0.0
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# ─── Hurdle Step ──────────────────────────────────────────────────────────────
|
| 40 |
+
|
| 41 |
+
class TestHurdleStep:
|
| 42 |
+
def test_score_3_good_form(self):
|
| 43 |
+
f = _make_features("hurdle_step", angles={
|
| 44 |
+
"step_hip_flexion_deg": 100.0, "stance_knee_angle_deg": 175.0,
|
| 45 |
+
"shoulder_tilt_deg": 5.0,
|
| 46 |
+
}, alignments={"trunk_stable": True, "stance_knee_extended": True})
|
| 47 |
+
r = score_hurdle_step(f)
|
| 48 |
+
assert r.score == 3
|
| 49 |
+
|
| 50 |
+
def test_score_2_compensation(self):
|
| 51 |
+
f = _make_features("hurdle_step", angles={
|
| 52 |
+
"step_hip_flexion_deg": 80.0, "stance_knee_angle_deg": 170.0,
|
| 53 |
+
}, alignments={"trunk_stable": True, "stance_knee_extended": True})
|
| 54 |
+
r = score_hurdle_step(f)
|
| 55 |
+
assert r.score == 2
|
| 56 |
+
|
| 57 |
+
def test_score_1_poor(self):
|
| 58 |
+
f = _make_features("hurdle_step", angles={
|
| 59 |
+
"step_hip_flexion_deg": 50.0, "stance_knee_angle_deg": 140.0,
|
| 60 |
+
}, alignments={"trunk_stable": False, "stance_knee_extended": False})
|
| 61 |
+
r = score_hurdle_step(f)
|
| 62 |
+
assert r.score == 1
|
| 63 |
+
|
| 64 |
+
def test_never_scores_zero(self):
|
| 65 |
+
f = _make_features("hurdle_step", angles={
|
| 66 |
+
"step_hip_flexion_deg": 30.0,
|
| 67 |
+
}, alignments={"trunk_stable": False, "stance_knee_extended": False})
|
| 68 |
+
r = score_hurdle_step(f)
|
| 69 |
+
assert r.score >= 1
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
# ─── Inline Lunge ─────────────────────────────────────────────────────────────
|
| 73 |
+
|
| 74 |
+
class TestInlineLunge:
|
| 75 |
+
def test_score_3_deep_and_aligned(self):
|
| 76 |
+
f = _make_features("inline_lunge", angles={
|
| 77 |
+
"front_knee_flexion_deg": 85.0, "trunk_lean_from_vertical_deg": 5.0,
|
| 78 |
+
}, alignments={"trunk_upright": True, "knee_over_ankle": True})
|
| 79 |
+
r = score_inline_lunge(f)
|
| 80 |
+
assert r.score == 3
|
| 81 |
+
|
| 82 |
+
def test_score_1_shallow(self):
|
| 83 |
+
f = _make_features("inline_lunge", angles={
|
| 84 |
+
"front_knee_flexion_deg": 140.0,
|
| 85 |
+
}, alignments={"trunk_upright": False, "knee_over_ankle": False})
|
| 86 |
+
r = score_inline_lunge(f)
|
| 87 |
+
assert r.score == 1
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# ─── Shoulder Mobility ────────────────────────────────────────────────────────
|
| 91 |
+
|
| 92 |
+
class TestShoulderMobility:
|
| 93 |
+
def test_score_3_close(self):
|
| 94 |
+
f = _make_features("shoulder_mobility", angles={
|
| 95 |
+
"inter_fist_normalized": 0.25,
|
| 96 |
+
}, alignments={"fists_within_one_hand": True, "fists_within_1_5_hand": True})
|
| 97 |
+
r = score_shoulder_mobility(f)
|
| 98 |
+
assert r.score == 3
|
| 99 |
+
|
| 100 |
+
def test_score_2_moderate(self):
|
| 101 |
+
f = _make_features("shoulder_mobility", angles={
|
| 102 |
+
"inter_fist_normalized": 0.45,
|
| 103 |
+
}, alignments={"fists_within_one_hand": False, "fists_within_1_5_hand": True})
|
| 104 |
+
r = score_shoulder_mobility(f)
|
| 105 |
+
assert r.score == 2
|
| 106 |
+
|
| 107 |
+
def test_score_1_far(self):
|
| 108 |
+
f = _make_features("shoulder_mobility", angles={
|
| 109 |
+
"inter_fist_normalized": 0.8,
|
| 110 |
+
}, alignments={"fists_within_one_hand": False, "fists_within_1_5_hand": False})
|
| 111 |
+
r = score_shoulder_mobility(f)
|
| 112 |
+
assert r.score == 1
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
# ─── Active SLR ───────────────────────────────────────────────────────────────
|
| 116 |
+
|
| 117 |
+
class TestActiveSLR:
|
| 118 |
+
def test_score_3_high_raise(self):
|
| 119 |
+
f = _make_features("active_slr", angles={
|
| 120 |
+
"raised_leg_angle_deg": 80.0,
|
| 121 |
+
}, alignments={"past_contralateral_knee": True, "past_mid_thigh": True, "down_leg_flat": True})
|
| 122 |
+
r = score_active_slr(f)
|
| 123 |
+
assert r.score == 3
|
| 124 |
+
|
| 125 |
+
def test_score_2_moderate_raise(self):
|
| 126 |
+
f = _make_features("active_slr", angles={
|
| 127 |
+
"raised_leg_angle_deg": 55.0,
|
| 128 |
+
}, alignments={"past_contralateral_knee": False, "past_mid_thigh": True, "down_leg_flat": True})
|
| 129 |
+
r = score_active_slr(f)
|
| 130 |
+
assert r.score == 2
|
| 131 |
+
|
| 132 |
+
def test_score_1_low_raise(self):
|
| 133 |
+
f = _make_features("active_slr", angles={
|
| 134 |
+
"raised_leg_angle_deg": 30.0,
|
| 135 |
+
}, alignments={"past_contralateral_knee": False, "past_mid_thigh": False, "down_leg_flat": True})
|
| 136 |
+
r = score_active_slr(f)
|
| 137 |
+
assert r.score == 1
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
# ─── Trunk Stability Push-Up ─────────────────────────────────────────────────
|
| 141 |
+
|
| 142 |
+
class TestTrunkStabilityPushup:
|
| 143 |
+
def test_score_3_rigid_hands_high(self):
|
| 144 |
+
f = _make_features("trunk_stability_pushup", angles={
|
| 145 |
+
"max_sag_px": 10.0, "trunk_variance_px": 5.0,
|
| 146 |
+
}, alignments={"body_rigid": True, "no_sag": True, "hands_at_forehead": True})
|
| 147 |
+
r = score_trunk_stability_pushup(f)
|
| 148 |
+
assert r.score == 3
|
| 149 |
+
|
| 150 |
+
def test_score_1_sag(self):
|
| 151 |
+
f = _make_features("trunk_stability_pushup", angles={
|
| 152 |
+
"max_sag_px": 50.0, "trunk_variance_px": 25.0,
|
| 153 |
+
}, alignments={"body_rigid": False, "no_sag": False, "hands_at_forehead": True})
|
| 154 |
+
r = score_trunk_stability_pushup(f)
|
| 155 |
+
assert r.score == 1
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
# ─── Rotary Stability ────────────────────────────────────────────────────────
|
| 159 |
+
|
| 160 |
+
class TestRotaryStability:
|
| 161 |
+
def test_score_2_stable(self):
|
| 162 |
+
f = _make_features("rotary_stability", angles={
|
| 163 |
+
"trunk_stability_std_px": 8.0, "shoulder_level_diff_px": 10.0, "hip_level_diff_px": 12.0,
|
| 164 |
+
}, alignments={"trunk_stable": True, "shoulders_level": True, "hips_level": True})
|
| 165 |
+
r = score_rotary_stability(f)
|
| 166 |
+
assert r.score == 2 # Default to 2 (contralateral assumption)
|
| 167 |
+
|
| 168 |
+
def test_score_1_unstable(self):
|
| 169 |
+
f = _make_features("rotary_stability", angles={
|
| 170 |
+
"trunk_stability_std_px": 30.0, "shoulder_level_diff_px": 35.0, "hip_level_diff_px": 30.0,
|
| 171 |
+
}, alignments={"trunk_stable": False, "shoulders_level": False, "hips_level": False})
|
| 172 |
+
r = score_rotary_stability(f)
|
| 173 |
+
assert r.score == 1
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
# ─── JudgeAgent fallback ─────────────────────────────────────────────────────
|
| 177 |
+
|
| 178 |
+
class TestJudgeAgent:
|
| 179 |
+
def test_fallback_when_judge_disabled(self, monkeypatch):
|
| 180 |
+
"""When ENABLE_JUDGE=False, judge promotes rubric score."""
|
| 181 |
+
from formscout import config
|
| 182 |
+
monkeypatch.setattr(config, "ENABLE_JUDGE", False)
|
| 183 |
+
agent = JudgeAgent()
|
| 184 |
+
features = _make_features("deep_squat", angles={"left_femur_from_horizontal_deg": 70.0})
|
| 185 |
+
rubric = ScoreResult(score=3, rationale="all good", confidence=0.9)
|
| 186 |
+
movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
|
| 187 |
+
result = agent.run(features, rubric, movement)
|
| 188 |
+
assert isinstance(result, JudgeResult)
|
| 189 |
+
assert result.score == 3
|
| 190 |
+
assert "[rubric-only]" in result.rationale
|
| 191 |
+
|
| 192 |
+
def test_fallback_when_server_unavailable(self, monkeypatch):
|
| 193 |
+
"""ENABLE_JUDGE=True but llama-server down → rubric fallback, never a crash."""
|
| 194 |
+
from unittest.mock import PropertyMock, patch
|
| 195 |
+
from formscout import config
|
| 196 |
+
monkeypatch.setattr(config, "ENABLE_JUDGE", True)
|
| 197 |
+
agent = JudgeAgent()
|
| 198 |
+
with patch.object(type(agent._client), "available", new_callable=PropertyMock, return_value=False):
|
| 199 |
+
features = _make_features("deep_squat")
|
| 200 |
+
rubric = ScoreResult(score=2, rationale="heels up", confidence=0.8)
|
| 201 |
+
movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
|
| 202 |
+
result = agent.run(features, rubric, movement)
|
| 203 |
+
assert result.score == 2
|
| 204 |
+
assert "[rubric-only]" in result.rationale
|
| 205 |
+
|
| 206 |
+
def test_vlm_response_parsed_into_judge_result(self, monkeypatch):
|
| 207 |
+
"""ENABLE_JUDGE=True with live client → VLM JSON becomes JudgeResult."""
|
| 208 |
+
from unittest.mock import PropertyMock, patch
|
| 209 |
+
from formscout import config
|
| 210 |
+
monkeypatch.setattr(config, "ENABLE_JUDGE", True)
|
| 211 |
+
agent = JudgeAgent()
|
| 212 |
+
vlm_json = {
|
| 213 |
+
"test": "deep_squat", "side": "na", "score": 2, "needs_human": False,
|
| 214 |
+
"rationale": "Femur 5° above horizontal; 2D estimate.",
|
| 215 |
+
"compensation_tags": ["forward_lean"], "corrective_hint": "Sit back into heels.",
|
| 216 |
+
"confidence": 0.78,
|
| 217 |
+
}
|
| 218 |
+
with patch.object(type(agent._client), "available", new_callable=PropertyMock, return_value=True), \
|
| 219 |
+
patch.object(agent._client, "complete", return_value=vlm_json):
|
| 220 |
+
features = _make_features("deep_squat")
|
| 221 |
+
rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
|
| 222 |
+
movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
|
| 223 |
+
result = agent.run(features, rubric, movement)
|
| 224 |
+
assert result.score == 2
|
| 225 |
+
assert result.compensation_tags == ["forward_lean"]
|
| 226 |
+
assert result.needs_human is False
|
| 227 |
+
|
| 228 |
+
def test_vlm_needs_human_yields_no_score(self, monkeypatch):
|
| 229 |
+
"""needs_human=True from the VLM must produce score=None."""
|
| 230 |
+
from unittest.mock import PropertyMock, patch
|
| 231 |
+
from formscout import config
|
| 232 |
+
monkeypatch.setattr(config, "ENABLE_JUDGE", True)
|
| 233 |
+
agent = JudgeAgent()
|
| 234 |
+
vlm_json = {"score": 1, "needs_human": True, "rationale": "Possible pain.", "confidence": 0.9}
|
| 235 |
+
with patch.object(type(agent._client), "available", new_callable=PropertyMock, return_value=True), \
|
| 236 |
+
patch.object(agent._client, "complete", return_value=vlm_json):
|
| 237 |
+
result = agent.run(
|
| 238 |
+
_make_features("deep_squat"),
|
| 239 |
+
ScoreResult(score=1, rationale="x", confidence=0.5),
|
| 240 |
+
MovementResult(test_name="deep_squat", side="na", confidence=1.0),
|
| 241 |
+
)
|
| 242 |
+
assert result.needs_human is True
|
| 243 |
+
assert result.score is None
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
# ─── LlamaCppClient (chat-completions endpoint) ──────────────────────────────
|
| 247 |
+
|
| 248 |
+
class TestLlamaCppClient:
|
| 249 |
+
def test_parse_plain_json(self):
|
| 250 |
+
from formscout.serving.llama_cpp import LlamaCppClient
|
| 251 |
+
assert LlamaCppClient._parse_json_reply('{"score": 3}') == {"score": 3}
|
| 252 |
+
|
| 253 |
+
def test_parse_fenced_json(self):
|
| 254 |
+
from formscout.serving.llama_cpp import LlamaCppClient
|
| 255 |
+
fenced = '```json\n{"score": 2, "needs_human": false}\n```'
|
| 256 |
+
assert LlamaCppClient._parse_json_reply(fenced) == {"score": 2, "needs_human": False}
|
| 257 |
+
|
| 258 |
+
def test_parse_non_json_returns_text(self):
|
| 259 |
+
from formscout.serving.llama_cpp import LlamaCppClient
|
| 260 |
+
assert LlamaCppClient._parse_json_reply("not json") == {"text": "not json"}
|
| 261 |
+
|
| 262 |
+
def test_complete_posts_chat_endpoint_with_images(self):
|
| 263 |
+
from unittest.mock import MagicMock, patch
|
| 264 |
+
from formscout.serving.llama_cpp import LlamaCppClient
|
| 265 |
+
|
| 266 |
+
client = LlamaCppClient(port=8080)
|
| 267 |
+
resp = MagicMock()
|
| 268 |
+
resp.json.return_value = {"choices": [{"message": {"content": '{"ok": true}'}}]}
|
| 269 |
+
resp.raise_for_status.return_value = None
|
| 270 |
+
with patch("formscout.serving.llama_cpp.requests.post", return_value=resp) as mock_post:
|
| 271 |
+
result = client.complete("score this", images=["aGVsbG8=" * 600])
|
| 272 |
+
assert result == {"ok": True}
|
| 273 |
+
url = mock_post.call_args.args[0] if mock_post.call_args.args else mock_post.call_args.kwargs.get("url")
|
| 274 |
+
assert url.endswith("/v1/chat/completions")
|
| 275 |
+
payload = mock_post.call_args.kwargs["json"]
|
| 276 |
+
content = payload["messages"][0]["content"]
|
| 277 |
+
assert content[0] == {"type": "text", "text": "score this"}
|
| 278 |
+
assert content[1]["type"] == "image_url"
|
| 279 |
+
assert content[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
| 280 |
+
|
| 281 |
+
def test_complete_connection_error_returns_safe_dict(self):
|
| 282 |
+
from unittest.mock import patch
|
| 283 |
+
import requests as _requests
|
| 284 |
+
from formscout.serving.llama_cpp import LlamaCppClient
|
| 285 |
+
|
| 286 |
+
client = LlamaCppClient(port=8080)
|
| 287 |
+
with patch("formscout.serving.llama_cpp.requests.post", side_effect=_requests.ConnectionError):
|
| 288 |
+
result = client.complete("hello")
|
| 289 |
+
assert "error" in result
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
# ─── ReportAgent ──────────────────────────────────────────────────────────────
|
| 293 |
+
|
| 294 |
+
class TestReportAgent:
|
| 295 |
+
def test_single_test_report(self):
|
| 296 |
+
agent = ReportAgent()
|
| 297 |
+
entries = [{
|
| 298 |
+
"movement": MovementResult(test_name="deep_squat", side="na", confidence=1.0),
|
| 299 |
+
"features": _make_features("deep_squat"),
|
| 300 |
+
"rubric_score": ScoreResult(score=3, rationale="ok", confidence=0.9),
|
| 301 |
+
"judge": JudgeResult(
|
| 302 |
+
score=3, rationale="good", compensation_tags=[], corrective_hint="",
|
| 303 |
+
confidence=0.9,
|
| 304 |
+
),
|
| 305 |
+
"side": "na",
|
| 306 |
+
}]
|
| 307 |
+
result = agent.run(entries)
|
| 308 |
+
assert isinstance(result, ReportResult)
|
| 309 |
+
assert len(result.per_test) == 1
|
| 310 |
+
assert result.per_test[0]["score"] == 3
|
| 311 |
+
|
| 312 |
+
def test_bilateral_reports_lower_score(self):
|
| 313 |
+
agent = ReportAgent()
|
| 314 |
+
entries = [
|
| 315 |
+
{
|
| 316 |
+
"movement": MovementResult(test_name="hurdle_step", side="left", confidence=1.0),
|
| 317 |
+
"features": _make_features("hurdle_step", side="left"),
|
| 318 |
+
"rubric_score": ScoreResult(score=3, rationale="ok", confidence=0.9),
|
| 319 |
+
"judge": JudgeResult(
|
| 320 |
+
score=3, rationale="", compensation_tags=[], corrective_hint="",
|
| 321 |
+
confidence=0.9,
|
| 322 |
+
),
|
| 323 |
+
"side": "left",
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"movement": MovementResult(test_name="hurdle_step", side="right", confidence=1.0),
|
| 327 |
+
"features": _make_features("hurdle_step", side="right"),
|
| 328 |
+
"rubric_score": ScoreResult(score=2, rationale="comp", confidence=0.8),
|
| 329 |
+
"judge": JudgeResult(
|
| 330 |
+
score=2, rationale="", compensation_tags=[], corrective_hint="",
|
| 331 |
+
confidence=0.8,
|
| 332 |
+
),
|
| 333 |
+
"side": "right",
|
| 334 |
+
},
|
| 335 |
+
]
|
| 336 |
+
result = agent.run(entries)
|
| 337 |
+
assert result.per_test[0]["score"] == 2 # lower of 3 and 2
|
| 338 |
+
assert len(result.asymmetries) == 1
|
| 339 |
+
assert result.asymmetries[0]["delta"] == 1
|
| 340 |
+
|
| 341 |
+
def test_composite_none_when_unscored(self):
|
| 342 |
+
agent = ReportAgent()
|
| 343 |
+
entries = [{
|
| 344 |
+
"movement": MovementResult(test_name="deep_squat", side="na", confidence=1.0),
|
| 345 |
+
"features": _make_features("deep_squat"),
|
| 346 |
+
"rubric_score": ScoreResult(score=1, rationale="", confidence=0.5),
|
| 347 |
+
"judge": JudgeResult(
|
| 348 |
+
score=None, rationale="pain", compensation_tags=[], corrective_hint="",
|
| 349 |
+
confidence=0.0, needs_human=True,
|
| 350 |
+
),
|
| 351 |
+
"side": "na",
|
| 352 |
+
}]
|
| 353 |
+
result = agent.run(entries)
|
| 354 |
+
assert result.composite is None
|
tests/test_session.py
CHANGED
|
@@ -1,94 +1,94 @@
|
|
| 1 |
-
"""Tests for the FMS session accumulator — no GPU, no model downloads."""
|
| 2 |
-
import numpy as np
|
| 3 |
-
|
| 4 |
-
from formscout.types import (
|
| 5 |
-
IngestResult, Pose2DResult, BiomechFeatures, ScoreResult, JudgeResult,
|
| 6 |
-
MovementResult, SessionEntry,
|
| 7 |
-
)
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
def test_session_entry_holds_typed_objects():
|
| 11 |
-
movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
|
| 12 |
-
features = BiomechFeatures(
|
| 13 |
-
test_name="deep_squat", view="2d", side="na",
|
| 14 |
-
angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": True},
|
| 15 |
-
symmetry_delta=None, timing={"deepest_frame": 2}, confidence=0.9,
|
| 16 |
-
)
|
| 17 |
-
rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
|
| 18 |
-
judge = JudgeResult(score=2, rationale="ok", compensation_tags=["heels elevated"],
|
| 19 |
-
corrective_hint="ankle mobility", confidence=0.85)
|
| 20 |
-
entry = SessionEntry(
|
| 21 |
-
test_name="deep_squat", side="na", score=2, needs_human=False,
|
| 22 |
-
rationale="ok", compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 23 |
-
measurements={"left_knee_flexion_deg": 95.0}, confidence=0.85, view="2d",
|
| 24 |
-
keyframe_path=None, movement=movement, features=features,
|
| 25 |
-
rubric_score=rubric, judge=judge,
|
| 26 |
-
)
|
| 27 |
-
assert entry.score == 2
|
| 28 |
-
assert entry.movement.test_name == "deep_squat"
|
| 29 |
-
assert entry.rubric_score.score == 2
|
| 30 |
-
assert entry.judge.compensation_tags == ["heels elevated"]
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
def _ingest(n=5, h=480, w=640):
|
| 34 |
-
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 35 |
-
return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
def _pose(n=5):
|
| 39 |
-
kps = []
|
| 40 |
-
for i in range(n):
|
| 41 |
-
kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
|
| 42 |
-
for j in range(17)})
|
| 43 |
-
return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
def _features(test_name="deep_squat", side="na", frame_key="deepest_frame"):
|
| 47 |
-
return BiomechFeatures(
|
| 48 |
-
test_name=test_name, view="2d", side=side,
|
| 49 |
-
angles={"left_knee_flexion_deg": 95.0},
|
| 50 |
-
alignments={"knees_tracking_over_feet": False},
|
| 51 |
-
symmetry_delta=None, timing={frame_key: 2}, confidence=0.9,
|
| 52 |
-
)
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
def _judge(score=2, needs_human=False):
|
| 56 |
-
return JudgeResult(
|
| 57 |
-
score=None if needs_human else score, rationale="r",
|
| 58 |
-
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 59 |
-
confidence=0.85, needs_human=needs_human,
|
| 60 |
-
)
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
def test_add_analysis_appends_entry_and_writes_files():
|
| 64 |
-
import os
|
| 65 |
-
from formscout import session as S
|
| 66 |
-
sess = S.new_session()
|
| 67 |
-
entry = S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
|
| 68 |
-
features=_features(), judge=_judge(), test_name="deep_squat", side="na")
|
| 69 |
-
assert len(sess.entries) == 1
|
| 70 |
-
assert entry.score == 2
|
| 71 |
-
assert os.path.exists(os.path.join(sess.session_dir, "session.json"))
|
| 72 |
-
assert os.path.exists(os.path.join(sess.session_dir, "analysis.md"))
|
| 73 |
-
# key-frame still written (deepest_frame=2 is valid)
|
| 74 |
-
assert entry.keyframe_path and os.path.exists(entry.keyframe_path)
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
def test_finish_composite_null_when_needs_human():
|
| 78 |
-
from formscout import session as S
|
| 79 |
-
sess = S.new_session()
|
| 80 |
-
S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(), features=_features(),
|
| 81 |
-
judge=_judge(score=3), test_name="deep_squat", side="na")
|
| 82 |
-
S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
|
| 83 |
-
features=_features("trunk_stability_pushup", frame_key="max_sag_frame"),
|
| 84 |
-
judge=_judge(needs_human=True), test_name="trunk_stability_pushup", side="na")
|
| 85 |
-
report, pdf_path = S.finish_session(sess)
|
| 86 |
-
assert report is not None
|
| 87 |
-
assert report.composite is None # one test needs_human
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
def test_finish_empty_session_returns_none():
|
| 91 |
-
from formscout import session as S
|
| 92 |
-
sess = S.new_session()
|
| 93 |
-
report, pdf_path = S.finish_session(sess)
|
| 94 |
-
assert report is None and pdf_path is None
|
|
|
|
| 1 |
+
"""Tests for the FMS session accumulator — no GPU, no model downloads."""
|
| 2 |
+
import numpy as np
|
| 3 |
+
|
| 4 |
+
from formscout.types import (
|
| 5 |
+
IngestResult, Pose2DResult, BiomechFeatures, ScoreResult, JudgeResult,
|
| 6 |
+
MovementResult, SessionEntry,
|
| 7 |
+
)
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def test_session_entry_holds_typed_objects():
|
| 11 |
+
movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
|
| 12 |
+
features = BiomechFeatures(
|
| 13 |
+
test_name="deep_squat", view="2d", side="na",
|
| 14 |
+
angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": True},
|
| 15 |
+
symmetry_delta=None, timing={"deepest_frame": 2}, confidence=0.9,
|
| 16 |
+
)
|
| 17 |
+
rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
|
| 18 |
+
judge = JudgeResult(score=2, rationale="ok", compensation_tags=["heels elevated"],
|
| 19 |
+
corrective_hint="ankle mobility", confidence=0.85)
|
| 20 |
+
entry = SessionEntry(
|
| 21 |
+
test_name="deep_squat", side="na", score=2, needs_human=False,
|
| 22 |
+
rationale="ok", compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 23 |
+
measurements={"left_knee_flexion_deg": 95.0}, confidence=0.85, view="2d",
|
| 24 |
+
keyframe_path=None, movement=movement, features=features,
|
| 25 |
+
rubric_score=rubric, judge=judge,
|
| 26 |
+
)
|
| 27 |
+
assert entry.score == 2
|
| 28 |
+
assert entry.movement.test_name == "deep_squat"
|
| 29 |
+
assert entry.rubric_score.score == 2
|
| 30 |
+
assert entry.judge.compensation_tags == ["heels elevated"]
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _ingest(n=5, h=480, w=640):
|
| 34 |
+
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 35 |
+
return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _pose(n=5):
|
| 39 |
+
kps = []
|
| 40 |
+
for i in range(n):
|
| 41 |
+
kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
|
| 42 |
+
for j in range(17)})
|
| 43 |
+
return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _features(test_name="deep_squat", side="na", frame_key="deepest_frame"):
|
| 47 |
+
return BiomechFeatures(
|
| 48 |
+
test_name=test_name, view="2d", side=side,
|
| 49 |
+
angles={"left_knee_flexion_deg": 95.0},
|
| 50 |
+
alignments={"knees_tracking_over_feet": False},
|
| 51 |
+
symmetry_delta=None, timing={frame_key: 2}, confidence=0.9,
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _judge(score=2, needs_human=False):
|
| 56 |
+
return JudgeResult(
|
| 57 |
+
score=None if needs_human else score, rationale="r",
|
| 58 |
+
compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
|
| 59 |
+
confidence=0.85, needs_human=needs_human,
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def test_add_analysis_appends_entry_and_writes_files():
|
| 64 |
+
import os
|
| 65 |
+
from formscout import session as S
|
| 66 |
+
sess = S.new_session()
|
| 67 |
+
entry = S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
|
| 68 |
+
features=_features(), judge=_judge(), test_name="deep_squat", side="na")
|
| 69 |
+
assert len(sess.entries) == 1
|
| 70 |
+
assert entry.score == 2
|
| 71 |
+
assert os.path.exists(os.path.join(sess.session_dir, "session.json"))
|
| 72 |
+
assert os.path.exists(os.path.join(sess.session_dir, "analysis.md"))
|
| 73 |
+
# key-frame still written (deepest_frame=2 is valid)
|
| 74 |
+
assert entry.keyframe_path and os.path.exists(entry.keyframe_path)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def test_finish_composite_null_when_needs_human():
|
| 78 |
+
from formscout import session as S
|
| 79 |
+
sess = S.new_session()
|
| 80 |
+
S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(), features=_features(),
|
| 81 |
+
judge=_judge(score=3), test_name="deep_squat", side="na")
|
| 82 |
+
S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
|
| 83 |
+
features=_features("trunk_stability_pushup", frame_key="max_sag_frame"),
|
| 84 |
+
judge=_judge(needs_human=True), test_name="trunk_stability_pushup", side="na")
|
| 85 |
+
report, pdf_path = S.finish_session(sess)
|
| 86 |
+
assert report is not None
|
| 87 |
+
assert report.composite is None # one test needs_human
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def test_finish_empty_session_returns_none():
|
| 91 |
+
from formscout import session as S
|
| 92 |
+
sess = S.new_session()
|
| 93 |
+
report, pdf_path = S.finish_session(sess)
|
| 94 |
+
assert report is None and pdf_path is None
|
tests/test_visualizer.py
CHANGED
|
@@ -1,176 +1,176 @@
|
|
| 1 |
-
"""Tests for PoseVisualizer — no GPU, no model downloads."""
|
| 2 |
-
import numpy as np
|
| 3 |
-
import pytest
|
| 4 |
-
from formscout.types import IngestResult, Pose2DResult
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
def _make_ingest(n=5, h=480, w=640, fps=30.0):
|
| 8 |
-
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 9 |
-
return IngestResult(frames=frames, fps=fps, duration=n / fps, n_people=1, width=w, height=h)
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
def _make_pose(n=5, w=640, h=480):
|
| 13 |
-
"""Synthetic Pose2DResult: 17 joints at fixed pixel positions, conf=0.9."""
|
| 14 |
-
kps_per_frame = []
|
| 15 |
-
for i in range(n):
|
| 16 |
-
frame_kps = {}
|
| 17 |
-
for j in range(17):
|
| 18 |
-
frame_kps[j] = {
|
| 19 |
-
"x": float(50 + j * 30 + i * 2),
|
| 20 |
-
"y": float(100 + j * 20),
|
| 21 |
-
"conf": 0.9,
|
| 22 |
-
}
|
| 23 |
-
kps_per_frame.append(frame_kps)
|
| 24 |
-
return Pose2DResult(keypoints=kps_per_frame, fps=30.0, confidence=0.9, notes="")
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
class TestComputeJointVelocity:
|
| 28 |
-
def test_returns_17_joints(self):
|
| 29 |
-
from formscout.agents.visualizer import compute_joint_velocity
|
| 30 |
-
pose = _make_pose(n=5)
|
| 31 |
-
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 32 |
-
assert len(result) == 17
|
| 33 |
-
|
| 34 |
-
def test_each_list_has_n_frames(self):
|
| 35 |
-
from formscout.agents.visualizer import compute_joint_velocity
|
| 36 |
-
pose = _make_pose(n=5)
|
| 37 |
-
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 38 |
-
for joint_idx, speeds in result.items():
|
| 39 |
-
assert len(speeds) == 5, f"joint {joint_idx} has {len(speeds)} speeds, expected 5"
|
| 40 |
-
|
| 41 |
-
def test_speeds_are_non_negative(self):
|
| 42 |
-
from formscout.agents.visualizer import compute_joint_velocity
|
| 43 |
-
pose = _make_pose(n=5)
|
| 44 |
-
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 45 |
-
for speeds in result.values():
|
| 46 |
-
assert all(s >= 0.0 for s in speeds)
|
| 47 |
-
|
| 48 |
-
def test_missing_keypoints_give_zero_speed(self):
|
| 49 |
-
from formscout.agents.visualizer import compute_joint_velocity
|
| 50 |
-
empty_kps = [{} for _ in range(5)]
|
| 51 |
-
result = compute_joint_velocity(empty_kps, fps=30.0)
|
| 52 |
-
for speeds in result.values():
|
| 53 |
-
assert all(s == 0.0 for s in speeds)
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
class TestDrawSkeleton:
|
| 57 |
-
def test_skeleton_draws_without_error(self):
|
| 58 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 59 |
-
vis = PoseVisualizer()
|
| 60 |
-
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 61 |
-
kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
|
| 62 |
-
for j in range(17)}
|
| 63 |
-
result = vis._draw_skeleton(frame.copy(), kps)
|
| 64 |
-
assert result.shape == frame.shape
|
| 65 |
-
assert not np.array_equal(result, frame)
|
| 66 |
-
|
| 67 |
-
def test_low_confidence_keypoints_not_drawn(self):
|
| 68 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 69 |
-
vis = PoseVisualizer()
|
| 70 |
-
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 71 |
-
kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.1} for j in range(17)}
|
| 72 |
-
result = vis._draw_skeleton(frame.copy(), kps)
|
| 73 |
-
assert np.array_equal(result, frame)
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
class TestDrawTrails:
|
| 77 |
-
def test_trails_draw_without_error(self):
|
| 78 |
-
from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
|
| 79 |
-
from collections import deque
|
| 80 |
-
vis = PoseVisualizer()
|
| 81 |
-
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 82 |
-
trail_history = {
|
| 83 |
-
0: deque([(100 + i * 5, 200 + i * 3) for i in range(5)], maxlen=TRAIL_LENGTH)
|
| 84 |
-
}
|
| 85 |
-
result = vis._draw_trails(frame.copy(), trail_history)
|
| 86 |
-
assert result.shape == frame.shape
|
| 87 |
-
assert not np.array_equal(result, frame)
|
| 88 |
-
|
| 89 |
-
def test_short_trail_no_crash(self):
|
| 90 |
-
from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
|
| 91 |
-
from collections import deque
|
| 92 |
-
vis = PoseVisualizer()
|
| 93 |
-
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 94 |
-
trail_history = {0: deque([(100, 200)], maxlen=TRAIL_LENGTH)}
|
| 95 |
-
result = vis._draw_trails(frame.copy(), trail_history)
|
| 96 |
-
assert np.array_equal(result, frame)
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
class TestDrawVelocityArrows:
|
| 100 |
-
def test_arrows_draw_without_error(self):
|
| 101 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 102 |
-
vis = PoseVisualizer()
|
| 103 |
-
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 104 |
-
kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
|
| 105 |
-
for j in range(17)}
|
| 106 |
-
prev_kps = {j: {"x": float(48 + j * 30), "y": float(98 + j * 20), "conf": 0.9}
|
| 107 |
-
for j in range(17)}
|
| 108 |
-
velocities = {j: [0.0] * 5 for j in range(17)}
|
| 109 |
-
velocities[5] = [0.0, 10.0, 50.0, 80.0, 120.0]
|
| 110 |
-
result = vis._draw_velocity_arrows(frame.copy(), kps, prev_kps, velocities, frame_idx=4)
|
| 111 |
-
assert result.shape == frame.shape
|
| 112 |
-
|
| 113 |
-
def test_no_prev_kps_no_crash(self):
|
| 114 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 115 |
-
vis = PoseVisualizer()
|
| 116 |
-
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 117 |
-
kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.9} for j in range(17)}
|
| 118 |
-
velocities = {j: [50.0] * 5 for j in range(17)}
|
| 119 |
-
result = vis._draw_velocity_arrows(frame.copy(), kps, None, velocities, frame_idx=0)
|
| 120 |
-
assert result.shape == frame.shape
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
class TestRenderVideo:
|
| 124 |
-
def test_creates_mp4_file(self, tmp_path):
|
| 125 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 126 |
-
vis = PoseVisualizer()
|
| 127 |
-
ingest = _make_ingest(n=5)
|
| 128 |
-
pose = _make_pose(n=5)
|
| 129 |
-
out = str(tmp_path / "out.mp4")
|
| 130 |
-
result = vis.render_video(ingest, pose, {"skeleton"}, out)
|
| 131 |
-
assert result is not None
|
| 132 |
-
import os
|
| 133 |
-
assert os.path.exists(result)
|
| 134 |
-
assert os.path.getsize(result) > 0
|
| 135 |
-
|
| 136 |
-
def test_empty_layers_returns_none(self, tmp_path):
|
| 137 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 138 |
-
vis = PoseVisualizer()
|
| 139 |
-
out = str(tmp_path / "out.mp4")
|
| 140 |
-
result = vis.render_video(_make_ingest(), _make_pose(), set(), out)
|
| 141 |
-
assert result is None
|
| 142 |
-
|
| 143 |
-
def test_no_detections_returns_none(self, tmp_path):
|
| 144 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 145 |
-
vis = PoseVisualizer()
|
| 146 |
-
ingest = _make_ingest(n=5)
|
| 147 |
-
empty_pose = Pose2DResult(
|
| 148 |
-
keypoints=[{} for _ in range(5)], fps=30.0, confidence=0.0, notes=""
|
| 149 |
-
)
|
| 150 |
-
out = str(tmp_path / "out.mp4")
|
| 151 |
-
result = vis.render_video(ingest, empty_pose, {"skeleton"}, out)
|
| 152 |
-
assert result is None
|
| 153 |
-
|
| 154 |
-
def test_last_velocities_set_after_render(self, tmp_path):
|
| 155 |
-
from formscout.agents.visualizer import PoseVisualizer
|
| 156 |
-
vis = PoseVisualizer()
|
| 157 |
-
out = str(tmp_path / "out.mp4")
|
| 158 |
-
vis.render_video(_make_ingest(n=5), _make_pose(n=5), {"skeleton"}, out)
|
| 159 |
-
assert len(vis.last_velocities) == 17
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
class TestBuildVelocitySummary:
|
| 163 |
-
def test_returns_markdown_table(self):
|
| 164 |
-
from formscout.agents.visualizer import build_velocity_summary, compute_joint_velocity
|
| 165 |
-
pose = _make_pose(n=10)
|
| 166 |
-
vels = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 167 |
-
result = build_velocity_summary(pose.keypoints, vels)
|
| 168 |
-
assert "|" in result
|
| 169 |
-
assert any(name in result for name in ["knee", "shoulder", "hip", "ankle"])
|
| 170 |
-
|
| 171 |
-
def test_empty_keypoints_returns_empty_string(self):
|
| 172 |
-
from formscout.agents.visualizer import build_velocity_summary
|
| 173 |
-
empty_kps = [{} for _ in range(5)]
|
| 174 |
-
vels = {j: [0.0] * 5 for j in range(17)}
|
| 175 |
-
result = build_velocity_summary(empty_kps, vels)
|
| 176 |
-
assert result == ""
|
|
|
|
| 1 |
+
"""Tests for PoseVisualizer — no GPU, no model downloads."""
|
| 2 |
+
import numpy as np
|
| 3 |
+
import pytest
|
| 4 |
+
from formscout.types import IngestResult, Pose2DResult
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def _make_ingest(n=5, h=480, w=640, fps=30.0):
|
| 8 |
+
frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
|
| 9 |
+
return IngestResult(frames=frames, fps=fps, duration=n / fps, n_people=1, width=w, height=h)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def _make_pose(n=5, w=640, h=480):
|
| 13 |
+
"""Synthetic Pose2DResult: 17 joints at fixed pixel positions, conf=0.9."""
|
| 14 |
+
kps_per_frame = []
|
| 15 |
+
for i in range(n):
|
| 16 |
+
frame_kps = {}
|
| 17 |
+
for j in range(17):
|
| 18 |
+
frame_kps[j] = {
|
| 19 |
+
"x": float(50 + j * 30 + i * 2),
|
| 20 |
+
"y": float(100 + j * 20),
|
| 21 |
+
"conf": 0.9,
|
| 22 |
+
}
|
| 23 |
+
kps_per_frame.append(frame_kps)
|
| 24 |
+
return Pose2DResult(keypoints=kps_per_frame, fps=30.0, confidence=0.9, notes="")
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class TestComputeJointVelocity:
|
| 28 |
+
def test_returns_17_joints(self):
|
| 29 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 30 |
+
pose = _make_pose(n=5)
|
| 31 |
+
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 32 |
+
assert len(result) == 17
|
| 33 |
+
|
| 34 |
+
def test_each_list_has_n_frames(self):
|
| 35 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 36 |
+
pose = _make_pose(n=5)
|
| 37 |
+
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 38 |
+
for joint_idx, speeds in result.items():
|
| 39 |
+
assert len(speeds) == 5, f"joint {joint_idx} has {len(speeds)} speeds, expected 5"
|
| 40 |
+
|
| 41 |
+
def test_speeds_are_non_negative(self):
|
| 42 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 43 |
+
pose = _make_pose(n=5)
|
| 44 |
+
result = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 45 |
+
for speeds in result.values():
|
| 46 |
+
assert all(s >= 0.0 for s in speeds)
|
| 47 |
+
|
| 48 |
+
def test_missing_keypoints_give_zero_speed(self):
|
| 49 |
+
from formscout.agents.visualizer import compute_joint_velocity
|
| 50 |
+
empty_kps = [{} for _ in range(5)]
|
| 51 |
+
result = compute_joint_velocity(empty_kps, fps=30.0)
|
| 52 |
+
for speeds in result.values():
|
| 53 |
+
assert all(s == 0.0 for s in speeds)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class TestDrawSkeleton:
|
| 57 |
+
def test_skeleton_draws_without_error(self):
|
| 58 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 59 |
+
vis = PoseVisualizer()
|
| 60 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 61 |
+
kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
|
| 62 |
+
for j in range(17)}
|
| 63 |
+
result = vis._draw_skeleton(frame.copy(), kps)
|
| 64 |
+
assert result.shape == frame.shape
|
| 65 |
+
assert not np.array_equal(result, frame)
|
| 66 |
+
|
| 67 |
+
def test_low_confidence_keypoints_not_drawn(self):
|
| 68 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 69 |
+
vis = PoseVisualizer()
|
| 70 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 71 |
+
kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.1} for j in range(17)}
|
| 72 |
+
result = vis._draw_skeleton(frame.copy(), kps)
|
| 73 |
+
assert np.array_equal(result, frame)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
class TestDrawTrails:
|
| 77 |
+
def test_trails_draw_without_error(self):
|
| 78 |
+
from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
|
| 79 |
+
from collections import deque
|
| 80 |
+
vis = PoseVisualizer()
|
| 81 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 82 |
+
trail_history = {
|
| 83 |
+
0: deque([(100 + i * 5, 200 + i * 3) for i in range(5)], maxlen=TRAIL_LENGTH)
|
| 84 |
+
}
|
| 85 |
+
result = vis._draw_trails(frame.copy(), trail_history)
|
| 86 |
+
assert result.shape == frame.shape
|
| 87 |
+
assert not np.array_equal(result, frame)
|
| 88 |
+
|
| 89 |
+
def test_short_trail_no_crash(self):
|
| 90 |
+
from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
|
| 91 |
+
from collections import deque
|
| 92 |
+
vis = PoseVisualizer()
|
| 93 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 94 |
+
trail_history = {0: deque([(100, 200)], maxlen=TRAIL_LENGTH)}
|
| 95 |
+
result = vis._draw_trails(frame.copy(), trail_history)
|
| 96 |
+
assert np.array_equal(result, frame)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
class TestDrawVelocityArrows:
|
| 100 |
+
def test_arrows_draw_without_error(self):
|
| 101 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 102 |
+
vis = PoseVisualizer()
|
| 103 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 104 |
+
kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
|
| 105 |
+
for j in range(17)}
|
| 106 |
+
prev_kps = {j: {"x": float(48 + j * 30), "y": float(98 + j * 20), "conf": 0.9}
|
| 107 |
+
for j in range(17)}
|
| 108 |
+
velocities = {j: [0.0] * 5 for j in range(17)}
|
| 109 |
+
velocities[5] = [0.0, 10.0, 50.0, 80.0, 120.0]
|
| 110 |
+
result = vis._draw_velocity_arrows(frame.copy(), kps, prev_kps, velocities, frame_idx=4)
|
| 111 |
+
assert result.shape == frame.shape
|
| 112 |
+
|
| 113 |
+
def test_no_prev_kps_no_crash(self):
|
| 114 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 115 |
+
vis = PoseVisualizer()
|
| 116 |
+
frame = np.zeros((480, 640, 3), dtype=np.uint8)
|
| 117 |
+
kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.9} for j in range(17)}
|
| 118 |
+
velocities = {j: [50.0] * 5 for j in range(17)}
|
| 119 |
+
result = vis._draw_velocity_arrows(frame.copy(), kps, None, velocities, frame_idx=0)
|
| 120 |
+
assert result.shape == frame.shape
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
class TestRenderVideo:
|
| 124 |
+
def test_creates_mp4_file(self, tmp_path):
|
| 125 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 126 |
+
vis = PoseVisualizer()
|
| 127 |
+
ingest = _make_ingest(n=5)
|
| 128 |
+
pose = _make_pose(n=5)
|
| 129 |
+
out = str(tmp_path / "out.mp4")
|
| 130 |
+
result = vis.render_video(ingest, pose, {"skeleton"}, out)
|
| 131 |
+
assert result is not None
|
| 132 |
+
import os
|
| 133 |
+
assert os.path.exists(result)
|
| 134 |
+
assert os.path.getsize(result) > 0
|
| 135 |
+
|
| 136 |
+
def test_empty_layers_returns_none(self, tmp_path):
|
| 137 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 138 |
+
vis = PoseVisualizer()
|
| 139 |
+
out = str(tmp_path / "out.mp4")
|
| 140 |
+
result = vis.render_video(_make_ingest(), _make_pose(), set(), out)
|
| 141 |
+
assert result is None
|
| 142 |
+
|
| 143 |
+
def test_no_detections_returns_none(self, tmp_path):
|
| 144 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 145 |
+
vis = PoseVisualizer()
|
| 146 |
+
ingest = _make_ingest(n=5)
|
| 147 |
+
empty_pose = Pose2DResult(
|
| 148 |
+
keypoints=[{} for _ in range(5)], fps=30.0, confidence=0.0, notes=""
|
| 149 |
+
)
|
| 150 |
+
out = str(tmp_path / "out.mp4")
|
| 151 |
+
result = vis.render_video(ingest, empty_pose, {"skeleton"}, out)
|
| 152 |
+
assert result is None
|
| 153 |
+
|
| 154 |
+
def test_last_velocities_set_after_render(self, tmp_path):
|
| 155 |
+
from formscout.agents.visualizer import PoseVisualizer
|
| 156 |
+
vis = PoseVisualizer()
|
| 157 |
+
out = str(tmp_path / "out.mp4")
|
| 158 |
+
vis.render_video(_make_ingest(n=5), _make_pose(n=5), {"skeleton"}, out)
|
| 159 |
+
assert len(vis.last_velocities) == 17
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
class TestBuildVelocitySummary:
|
| 163 |
+
def test_returns_markdown_table(self):
|
| 164 |
+
from formscout.agents.visualizer import build_velocity_summary, compute_joint_velocity
|
| 165 |
+
pose = _make_pose(n=10)
|
| 166 |
+
vels = compute_joint_velocity(pose.keypoints, fps=30.0)
|
| 167 |
+
result = build_velocity_summary(pose.keypoints, vels)
|
| 168 |
+
assert "|" in result
|
| 169 |
+
assert any(name in result for name in ["knee", "shoulder", "hip", "ankle"])
|
| 170 |
+
|
| 171 |
+
def test_empty_keypoints_returns_empty_string(self):
|
| 172 |
+
from formscout.agents.visualizer import build_velocity_summary
|
| 173 |
+
empty_kps = [{} for _ in range(5)]
|
| 174 |
+
vels = {j: [0.0] * 5 for j in range(17)}
|
| 175 |
+
result = build_velocity_summary(empty_kps, vels)
|
| 176 |
+
assert result == ""
|