fix: pin dropdown label backgrounds light (no dark FMS Test header)

#8
Files changed (48) hide show
  1. .hfignore +37 -37
  2. CLAUDE.md +199 -192
  3. README.md +118 -118
  4. app.py +11 -11
  5. docs/superpowers/plans/2026-06-09-pose-model-selector.md +734 -734
  6. docs/superpowers/plans/2026-06-09-pose-visualizer.md +914 -914
  7. docs/superpowers/plans/2026-06-13-full-fms-session-pdf.md +1209 -1209
  8. docs/superpowers/specs/2026-06-09-pose-model-selector-design.md +171 -171
  9. docs/superpowers/specs/2026-06-09-pose-visualizer-design.md +197 -197
  10. docs/superpowers/specs/2026-06-13-full-fms-session-pdf-design.md +154 -154
  11. formscout/agents/classifier.py +102 -102
  12. formscout/agents/ingest.py +7 -28
  13. formscout/agents/judge.py +125 -136
  14. formscout/agents/pdf_report.py +175 -115
  15. formscout/agents/pose2d.py +232 -232
  16. formscout/agents/report.py +139 -139
  17. formscout/agents/visualizer.py +418 -435
  18. formscout/analysis/__init__.py +1 -0
  19. formscout/analysis/charts.py +171 -0
  20. formscout/analysis/laban.py +127 -0
  21. formscout/analysis/relevant_joints.py +122 -0
  22. formscout/analysis/timeseries.py +49 -0
  23. formscout/config.py +15 -3
  24. formscout/pipeline.py +111 -111
  25. formscout/rubric/__init__.py +32 -32
  26. formscout/rubric/active_slr.py +51 -51
  27. formscout/rubric/hurdle_step.py +60 -60
  28. formscout/rubric/inline_lunge.py +58 -58
  29. formscout/rubric/rotary_stability.py +56 -56
  30. formscout/rubric/shoulder_mobility.py +46 -46
  31. formscout/rubric/trunk_stability_pushup.py +55 -55
  32. formscout/serving/__init__.py +20 -0
  33. formscout/serving/llama_cpp.py +148 -174
  34. formscout/serving/transformers_vlm.py +116 -0
  35. formscout/session.py +283 -194
  36. formscout/startup.py +47 -47
  37. formscout/types.py +3 -0
  38. formscout/ui/theme.py +272 -250
  39. requirements.txt +3 -1
  40. scripts/hf_upload.sh +97 -97
  41. scripts/serve_judge.sh +35 -35
  42. tests/test_analysis.py +145 -0
  43. tests/test_judge_backend.py +75 -0
  44. tests/test_keyframe.py +37 -37
  45. tests/test_pdf_report.py +51 -51
  46. tests/test_phase2.py +354 -354
  47. tests/test_session.py +94 -94
  48. tests/test_visualizer.py +176 -176
.hfignore CHANGED
@@ -1,37 +1,37 @@
1
- # Python
2
- __pycache__/
3
- *.py[cod]
4
- *.egg-info/
5
- dist/
6
- build/
7
- .eggs/
8
- *.egg
9
-
10
- # Virtual environments
11
- .venv/
12
- venv/
13
- env/
14
-
15
- # Secrets / local config
16
- .env
17
- .env.*
18
-
19
- # Model weights (managed separately)
20
- checkpoints/
21
- *.pt
22
- *.pth
23
- *.gguf
24
- *.bin
25
-
26
- # Run artifacts
27
- traces/
28
- *.mp4
29
-
30
- # Dev tooling
31
- .pytest_cache/
32
- .ruff_cache/
33
- .DS_Store
34
- .claude/
35
-
36
- # Git
37
- .git/
 
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.egg-info/
5
+ dist/
6
+ build/
7
+ .eggs/
8
+ *.egg
9
+
10
+ # Virtual environments
11
+ .venv/
12
+ venv/
13
+ env/
14
+
15
+ # Secrets / local config
16
+ .env
17
+ .env.*
18
+
19
+ # Model weights (managed separately)
20
+ checkpoints/
21
+ *.pt
22
+ *.pth
23
+ *.gguf
24
+ *.bin
25
+
26
+ # Run artifacts
27
+ traces/
28
+ *.mp4
29
+
30
+ # Dev tooling
31
+ .pytest_cache/
32
+ .ruff_cache/
33
+ .DS_Store
34
+ .claude/
35
+
36
+ # Git
37
+ .git/
CLAUDE.md CHANGED
@@ -1,192 +1,199 @@
1
- # CLAUDE.md
2
-
3
- This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
4
-
5
- ## Project overview
6
-
7
- FormScout is a Gradio app (Hugging Face Space) that scores Functional Movement Screen (FMS) videos 0–3 per test with a written rationale and an annotated overlay. It is a **screening aid** — not a diagnosis, not an injury predictor. Built for the Build Small Hackathon (Backyard AI track). Full product spec is in `docs/FormScout-FMS-Spec.md`; the engineering contract is in `docs/plans/FormScout-Build-Prompt.md`.
8
-
9
- **Current status:** Phase 2 complete. All 7 FMS test rubric scorers, JudgeAgent, MovementClassifierAgent, ReportAgent, PoseVisualizer (overlay video), and a user-selectable pose-model registry are implemented and tested (86/87 passing). Phase 3 is next (ST-GCN fine-tune + RAG retrieval).
10
-
11
- ## Common commands
12
-
13
- ```bash
14
- # Run the Gradio app locally
15
- python3 app.py
16
-
17
- # Headless pipeline test (no Gradio)
18
- python3 -m formscout.run sample.mp4
19
-
20
- # Run all tests
21
- pytest tests/
22
-
23
- # Run a single test file or test
24
- pytest tests/test_phase2.py
25
- pytest tests/test_biomechanics.py::TestBiomechanicsAgent::test_deep_squat_score
26
-
27
- # Lint / format
28
- ruff check . && ruff format .
29
-
30
- # Start the local VLM judge server (llama.cpp, port 8080)
31
- ./scripts/serve_judge.sh
32
-
33
- # Push source tree to the HF model repo + Space (PRs; message from last commit)
34
- ./scripts/hf_upload.sh
35
-
36
- # Run Svelte component tests (when frontend work is added)
37
- npx vitest run
38
- ```
39
-
40
- ## Architecture
41
-
42
- The pipeline is a sequence of **typed specialist agents**. Each agent accepts and returns a frozen dataclass from `formscout/types.py`. The Director in `formscout/pipeline.py` orchestrates them as a deterministic state machine (not an LLM).
43
-
44
- ### Agent pipeline
45
-
46
- ```
47
- IngestAgent → Pose2DAgent → [Body3DAgent — optional]
48
- → MovementClassifierAgent → BiomechanicsAgent
49
- → rubric/score_test() → JudgeAgent → ReportAgent
50
- ```
51
-
52
- The **Director** (`pipeline.py`) owns the flow. `app.py` creates one `Director()` instance and calls `director.run(video_path, test_name, side, model_key)` per submission. The Gradio UI passes `test_name` directly (from dropdown), bypassing the classifier; `model_key` selects the pose backend from `config.POSE_MODELS`.
53
-
54
- `PoseVisualizer` (`formscout/agents/visualizer.py`) renders the annotated overlay video (skeleton, trails, velocity arrows) from `IngestResult` + `Pose2DResult`. It is called from `app.py` after the pipeline run — it is a UI-layer component, not a Director stage. It returns `None` on failure, never raises.
55
-
56
- ### The tiering rule (most important invariant)
57
-
58
- **The 2D path is the default and must stand alone as a complete, functional pipeline.** `Body3DAgent` is only activated when `config.ENABLE_3D == True` AND the checkpoint loads successfully. If 3D is off or fails, `Body3DResult(used=False, ...)` is returned — this is a normal success path, not an error. `BiomechFeatures.view` is `"2d"` or `"3d"` so the `JudgeAgent` can caveat its rationale appropriately. Never put `Body3DAgent` on the critical path.
59
-
60
- ### Feature flags in `config.py` and their current state
61
-
62
- | Flag | Default | Meaning |
63
- |------|---------|---------|
64
- | `ENABLE_JUDGE` | `True` | Judge/Classifier call Qwen3-VL via llama-server; graceful rubric fallback when the server is down |
65
- | `ENABLE_3D` | `False` | When False, Body3DAgent returns `used=False` immediately |
66
- | `ENABLE_STGCN` | `False` | Phase 3 — ST-GCN learned scoring head |
67
- | `ENABLE_RAG` | `False` | Phase 3 — RetrievalAgent exemplar lookup |
68
-
69
- All model IDs, thresholds, k-values, and feature flags live in `config.py` — never scattered literals.
70
-
71
- ### Fallback chain (important for local dev and Spaces)
72
-
73
- 1. `ENABLE_JUDGE=False` JudgeAgent returns rubric score wrapped as JudgeResult (no VLM needed)
74
- 2. `ENABLE_JUDGE=True` + llama.cpp server unreachable → same fallback, logs a warning
75
- 3. `ENABLE_JUDGE=True` + server available calls Qwen3-VL-8B-Instruct at `127.0.0.1:8080`
76
-
77
- Start the VLM server with `scripts/serve_judge.sh` (downloads live in `checkpoints/qwen3-vl/`, gitignored). To use a fine-tuned GGUF, set `FORMSCOUT_JUDGE_GGUF` (and `FORMSCOUT_JUDGE_MMPROJ` if it ships its own projector) — no code change needed. Multimodal requests go through the OpenAI-compatible `/v1/chat/completions` endpoint (the legacy `/completion` + `image_data` path does not work with modern llama-server).
78
-
79
- This means the app is **fully functional without any GPU or llama.cpp** — rubric scoring is pure Python.
80
-
81
- ### Rubric scorers
82
-
83
- Each FMS test has a pure-function scorer in `formscout/rubric/`:
84
-
85
- ```
86
- score_deep_squat / score_hurdle_step / score_inline_lunge /
87
- score_shoulder_mobility / score_active_slr /
88
- score_trunk_stability_pushup / score_rotary_stability
89
- ```
90
-
91
- All accept `BiomechFeatures` and return `ScoreResult`. Dispatch via `rubric.score_test(features)`. **Rubric functions must remain pure** — no model calls, no I/O.
92
-
93
- ### Bilateral tests
94
-
95
- `hurdle_step`, `inline_lunge`, `shoulder_mobility`, `active_slr` are bilateral. `ReportAgent` groups them by test name, takes the **lower** score, and always emits the asymmetry delta even when scores are equal. `composite` is `None` when any test is unscored.
96
-
97
- ### Types contract
98
-
99
- Every agent I/O is a frozen dataclass from `formscout/types.py`. Key types:
100
-
101
- - `IngestResult` — decoded frames (np.ndarray list), fps, duration, dimensions
102
- - `Pose2DResult` per-frame keypoints as `dict[int, {x, y, conf}]` (COCO 17 joints)
103
- - `Body3DResult` — optional 3D joints, always has `used: bool`
104
- - `MovementResult` — `test_name` (validated enum), `side` ("left"|"right"|"na")
105
- - `BiomechFeatures` — `angles: dict`, `alignments: dict`, `view: "2d"|"3d"`, `symmetry_delta`
106
- - `ScoreResult` `score: int` (0–3), `rationale`, `needs_human`
107
- - `JudgeResult` — same as ScoreResult + `compensation_tags`, `corrective_hint`; `score=None` when `needs_human=True`
108
- - `PipelineState` — mutable accumulator threaded through the Director
109
-
110
- `MovementResult` and `JudgeResult` validate their fields in `__post_init__` — passing invalid values raises immediately.
111
-
112
- ### Pose model selection and checkpoints
113
-
114
- `config.POSE_MODELS` is a registry of pose backends: MediaPipe (CPU-friendly), five YOLO26 sizes (n/s/m/l/x), and Sapiens2 variants (Phase 3, need the custom `sapiens` repo installed). `config.DEFAULT_POSE_MODEL` is YOLO26n. The Gradio UI exposes a dropdown built from `config.available_pose_models()` (filters to checkpoints actually present) and passes the chosen `model_key` through `Director.run` to `Pose2DAgent`. `config.YOLO_POSE_MODEL` is a backward-compat alias only.
115
-
116
- Checkpoints are **not** committed (`checkpoints/` is gitignored). `formscout/startup.py:ensure_checkpoints()` downloads missing YOLO26/MediaPipe files from the `silas-therapy/formscout-checkpoints` HF repo once at app startup. Models load once per process and are cached — never inside the inference hot path.
117
-
118
- ### llama.cpp serving
119
-
120
- `formscout/serving/llama_cpp.py` provides `LlamaCppClient` (VLM, port 8080) and `EmbeddingClient` (embeddings, port 8081). Both check `/health` before use and return safe error dicts when unavailable. Only active when the corresponding `ENABLE_*` flag is True.
121
-
122
- ### Deploying to Hugging Face
123
-
124
- The repo deploys to both `silas-therapy/small-functional-movement-screening` (model repo) and the Space of the same name (README frontmatter is the Space config). Use `./scripts/hf_upload.sh` — never raw `hf upload .`: the `hf` CLI does **not** read `.hfignore`, so a raw upload hashes the entire `.venv` (~44k files) and pushes torch binaries. The script parses `.hfignore` into `--exclude` globs, preflights the file count, creates PRs on both repos, and auto-switches to `hf upload-large-folder` (resumable, but no PR / no commit message) above 500 files.
125
-
126
- ## Key constraints and invariants
127
-
128
- - **No cloud model APIs.** All inference runs on-Space (ZeroGPU). No OpenAI/Anthropic/Gemini calls.
129
- - **Pain is never auto-scored.** Any clearing test or visible distress sets `needs_human=True` — enforced in rubric functions and JudgeAgent. `JudgeResult.score` must be `None` when `needs_human=True`.
130
- - **Quality gates (Director, never silently skip):**
131
- - Any agent `confidence < config.MIN_CONFIDENCE` (0.6) warn or stop
132
- - `|rubric.score - judge.score| >= 1` → flag disagreement
133
- - `MovementResult.test_name == "unknown"` → stop pipeline, surface manual override
134
- - `JudgeAgent.needs_human == True` → no numeric score emitted
135
- - **Composite is null** when any test is unscored. Never show a partial 0–21 as complete.
136
- - **Pipeline runs headless.** No Gradio imports in any agent file.
137
- - **Safety banner** ("Screening aid not a diagnosis…") must always be visible in the UI — appears at top and bottom of `app.py`.
138
-
139
- ## Engineering standards
140
-
141
- - Every agent: one public entrypoint, typed dataclass I/O from `types.py`, `confidence: float` and `notes: str` on every result.
142
- - Models load once at module/instance init never inside the inference hot path.
143
- - Every agent module docstring states: purpose, inputs, outputs, failure behavior, model param count, license, and gated status.
144
- - `tracing.py` records structured per-agent I/O for any run; one full run gets exported to the Hub.
145
- - Every agent ships with a pytest in `tests/` that runs without model downloads and asserts the typed contract.
146
-
147
- ## Model stack (~17.6B total — stay under 32B)
148
-
149
- | Component | Model | Params | Status |
150
- |---|---|---|---|
151
- | 2D pose (primary) | YOLO26-Pose n/s/m/l/x (default: n) | 0.0007–0.058B | Ready (auto-downloaded at startup) |
152
- | 2D pose (CPU alt) | MediaPipe Pose Landmarker (full) | ~0.004B | Ready (auto-downloaded at startup) |
153
- | 2D pose (HQ alt) | `facebook/sapiens2-pose-0.4b/0.8b/1b/5b` | 0.4–5B | Phase 3 — needs custom `sapiens` repo |
154
- | Segmentation | SAM 3.1 base | ~0.85B | Access accepted |
155
- | 3D biomechanics | `facebook/sam-3d-body-dinov3` | ~0.84B | **Access ACCEPTED Jun 4 2026** |
156
- | Learned scoring | ST-GCN (pyskl) | ~0.03B | Phase 3 |
157
- | Judge + Classifier | Qwen3-VL-8B-Instruct (llama.cpp) | 8B | **Online** — `scripts/serve_judge.sh`, ENABLE_JUDGE=True |
158
- | Retrieval | Qwen3-VL-Embedding-8B (llama.cpp) | 8B | Phase 3 |
159
-
160
- Track the running sum in `MODEL_BUDGET.md`. The two Qwen3-VL-8B models share a backbone.
161
-
162
- ## Gradio + Svelte UI guidance
163
-
164
- The UI uses **Gradio `gr.Blocks`** with custom CSS/theme (`formscout/ui/theme.py`). Custom Svelte components for score dial, asymmetry bars, rubric drawer are planned for Phase 4. Use `gradio-svelte-expert` agent for Svelte component work.
165
-
166
- - ZeroGPU: wrap heavy inference (`Pose2DAgent.run`, `Body3DAgent.run`) in `@spaces.GPU` before deploying to Spaces.
167
- - Verify Gradio APIs against current docs before use pin exact versions in `requirements.txt`.
168
-
169
- ## Build phases
170
-
171
- 1. **Phase 0 — Recon:** Complete. See `RECON.md`.
172
- 2. **Phase 1 — Spine:** ✅ Complete. Deep Squat end-to-end.
173
- 3. **Phase 2 All 7 tests:** ✅ Complete. Classifier, Judge, Report agents; all rubric scorers; Gradio UI.
174
- 4. **Phase 3 Learned scoring + retrieval:** ST-GCN fine-tune on physio clips, publish to Hub. RetrievalAgent with embedding index.
175
- 5. **Phase 4 — Polish + ship:** Custom Svelte UI components, agent trace to Hub, blog post. (Overlay video done via `PoseVisualizer`; full 7-test session + PDF export done via `formscout/session.py` + `PdfReportAgent`.)
176
-
177
- ## Known issues
178
-
179
- - `tests/test_biomechanics.py::TestBiomechanicsAgent::test_unimplemented_test_returns_low_confidence` fails: expects `"not yet implemented"` in `result.notes` but biomechanics returns empty string. Minor — low priority.
180
-
181
- ## Badge checklist (definition of done)
182
-
183
- - [ ] Space runs green; upload → scorecard works on real clips
184
- - [ ] Param sum verified ≤ 32B in `MODEL_BUDGET.md`
185
- - [ ] 🔌 **Off the Grid** — no cloud model APIs anywhere in the pipeline
186
- - [ ] 🎯 **Well-Tuned** fine-tuned ST-GCN head published to Hub with honest model card
187
- - [ ] 🎨 **Off-Brand** — custom, non-default Gradio UI (scout/trail theme)
188
- - [ ] 🦙 **Llama Champion** — VLM + embedder served via llama.cpp (GGUF)
189
- - [ ] 📡 **Sharing is Caring** — one full agent trace (all I/O) published to Hub
190
- - [ ] 📓 **Field Notes** blog post written, honesty section (FMS limitations) front-and-center
191
- - [ ] Demo video + social post recorded
192
- - [ ] Safety banner present; pain/clearing never auto-scored; low-confidence flagged
 
 
 
 
 
 
 
 
1
+ # CLAUDE.md
2
+
3
+ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
4
+
5
+ ## Project overview
6
+
7
+ FormScout is a Gradio app (Hugging Face Space) that scores Functional Movement Screen (FMS) videos 0–3 per test with a written rationale and an annotated overlay. It is a **screening aid** — not a diagnosis, not an injury predictor. Built for the Build Small Hackathon (Backyard AI track). Full product spec is in `docs/FormScout-FMS-Spec.md`; the engineering contract is in `docs/plans/FormScout-Build-Prompt.md`.
8
+
9
+ **Current status:** Phase 2 complete. All 7 FMS test rubric scorers, JudgeAgent, MovementClassifierAgent, ReportAgent, PoseVisualizer (overlay video), and a user-selectable pose-model registry are implemented and tested (86/87 passing). Phase 3 is next (ST-GCN fine-tune + RAG retrieval).
10
+
11
+ ## Common commands
12
+
13
+ ```bash
14
+ # Run the Gradio app locally
15
+ python3 app.py
16
+
17
+ # Headless pipeline test (no Gradio)
18
+ python3 -m formscout.run sample.mp4
19
+
20
+ # Run all tests
21
+ pytest tests/
22
+
23
+ # Run a single test file or test
24
+ pytest tests/test_phase2.py
25
+ pytest tests/test_biomechanics.py::TestBiomechanicsAgent::test_deep_squat_score
26
+
27
+ # Lint / format
28
+ ruff check . && ruff format .
29
+
30
+ # Start the local VLM judge server (llama.cpp, port 8080)
31
+ ./scripts/serve_judge.sh
32
+
33
+ # Push source tree to the HF model repo + Space (PRs; message from last commit)
34
+ ./scripts/hf_upload.sh
35
+
36
+ # Run Svelte component tests (when frontend work is added)
37
+ npx vitest run
38
+ ```
39
+
40
+ ## Architecture
41
+
42
+ The pipeline is a sequence of **typed specialist agents**. Each agent accepts and returns a frozen dataclass from `formscout/types.py`. The Director in `formscout/pipeline.py` orchestrates them as a deterministic state machine (not an LLM).
43
+
44
+ ### Agent pipeline
45
+
46
+ ```
47
+ IngestAgent → Pose2DAgent → [Body3DAgent — optional]
48
+ → MovementClassifierAgent → BiomechanicsAgent
49
+ → rubric/score_test() → JudgeAgent → ReportAgent
50
+ ```
51
+
52
+ The **Director** (`pipeline.py`) owns the flow. `app.py` creates one `Director()` instance and calls `director.run(video_path, test_name, side, model_key)` per submission. The Gradio UI passes `test_name` directly (from dropdown), bypassing the classifier; `model_key` selects the pose backend from `config.POSE_MODELS`.
53
+
54
+ `PoseVisualizer` (`formscout/agents/visualizer.py`) renders the annotated overlay video (skeleton, trails, velocity arrows) from `IngestResult` + `Pose2DResult`. It is called from `app.py` after the pipeline run — it is a UI-layer component, not a Director stage. It returns `None` on failure, never raises.
55
+
56
+ ### The tiering rule (most important invariant)
57
+
58
+ **The 2D path is the default and must stand alone as a complete, functional pipeline.** `Body3DAgent` is only activated when `config.ENABLE_3D == True` AND the checkpoint loads successfully. If 3D is off or fails, `Body3DResult(used=False, ...)` is returned — this is a normal success path, not an error. `BiomechFeatures.view` is `"2d"` or `"3d"` so the `JudgeAgent` can caveat its rationale appropriately. Never put `Body3DAgent` on the critical path.
59
+
60
+ ### Feature flags in `config.py` and their current state
61
+
62
+ | Flag | Default | Meaning |
63
+ |------|---------|---------|
64
+ | `ENABLE_JUDGE` | `True` | Judge/Classifier call Qwen3-VL via llama-server; graceful rubric fallback when the server is down |
65
+ | `ENABLE_3D` | `False` | When False, Body3DAgent returns `used=False` immediately |
66
+ | `ENABLE_STGCN` | `False` | Phase 3 — ST-GCN learned scoring head |
67
+ | `ENABLE_RAG` | `False` | Phase 3 — RetrievalAgent exemplar lookup |
68
+
69
+ All model IDs, thresholds, k-values, and feature flags live in `config.py` — never scattered literals.
70
+
71
+ ### Judge backend selection (local vs Space)
72
+
73
+ `config.resolve_judge_backend()` picks the VLM backend via `FORMSCOUT_JUDGE_BACKEND` (`llama_cpp` | `transformers` | `auto`). `auto` (default) uses **llama-server locally** and the **in-process transformers backend on a Space** (detected via `SPACE_ID`). `JudgeAgent` gets its client from `serving.get_vlm_client()`.
74
+
75
+ - **`llama_cpp`** `LlamaCppClient`llama-server at `127.0.0.1:8080` (start with `scripts/serve_judge.sh`). The local path; works perfectly.
76
+ - **`transformers`** — `TransformersVLMClient` loads Qwen3-VL-8B via transformers, GPU-wrapped with `spaces.GPU` (ZeroGPU). Lazy model load, cached per process. On any load/inference failure it returns `{"fallback": True}` and the Judge falls back to the rubric. **Needs validation on real ZeroGPU hardware** — not exercised in CPU tests.
77
+
78
+ ### Fallback chain (important for local dev and Spaces)
79
+
80
+ 1. `ENABLE_JUDGE=False` → JudgeAgent returns rubric score wrapped as JudgeResult (no VLM needed)
81
+ 2. `ENABLE_JUDGE=True` + selected backend unavailable / transformers load fails → same rubric fallback, logs a warning
82
+ 3. `ENABLE_JUDGE=True` + backend available → calls Qwen3-VL-8B-Instruct (llama-server locally, transformers/ZeroGPU on a Space)
83
+
84
+ Start the VLM server with `scripts/serve_judge.sh` (downloads live in `checkpoints/qwen3-vl/`, gitignored). To use a fine-tuned GGUF, set `FORMSCOUT_JUDGE_GGUF` (and `FORMSCOUT_JUDGE_MMPROJ` if it ships its own projector) — no code change needed. Multimodal requests go through the OpenAI-compatible `/v1/chat/completions` endpoint (the legacy `/completion` + `image_data` path does not work with modern llama-server).
85
+
86
+ This means the app is **fully functional without any GPU or llama.cpp** — rubric scoring is pure Python.
87
+
88
+ ### Rubric scorers
89
+
90
+ Each FMS test has a pure-function scorer in `formscout/rubric/`:
91
+
92
+ ```
93
+ score_deep_squat / score_hurdle_step / score_inline_lunge /
94
+ score_shoulder_mobility / score_active_slr /
95
+ score_trunk_stability_pushup / score_rotary_stability
96
+ ```
97
+
98
+ All accept `BiomechFeatures` and return `ScoreResult`. Dispatch via `rubric.score_test(features)`. **Rubric functions must remain pure** — no model calls, no I/O.
99
+
100
+ ### Bilateral tests
101
+
102
+ `hurdle_step`, `inline_lunge`, `shoulder_mobility`, `active_slr` are bilateral. `ReportAgent` groups them by test name, takes the **lower** score, and always emits the asymmetry delta even when scores are equal. `composite` is `None` when any test is unscored.
103
+
104
+ ### Types contract
105
+
106
+ Every agent I/O is a frozen dataclass from `formscout/types.py`. Key types:
107
+
108
+ - `IngestResult` — decoded frames (np.ndarray list), fps, duration, dimensions
109
+ - `Pose2DResult` — per-frame keypoints as `dict[int, {x, y, conf}]` (COCO 17 joints)
110
+ - `Body3DResult` optional 3D joints, always has `used: bool`
111
+ - `MovementResult` — `test_name` (validated enum), `side` ("left"|"right"|"na")
112
+ - `BiomechFeatures` `angles: dict`, `alignments: dict`, `view: "2d"|"3d"`, `symmetry_delta`
113
+ - `ScoreResult` — `score: int` (0–3), `rationale`, `needs_human`
114
+ - `JudgeResult` same as ScoreResult + `compensation_tags`, `corrective_hint`; `score=None` when `needs_human=True`
115
+ - `PipelineState` — mutable accumulator threaded through the Director
116
+
117
+ `MovementResult` and `JudgeResult` validate their fields in `__post_init__` — passing invalid values raises immediately.
118
+
119
+ ### Pose model selection and checkpoints
120
+
121
+ `config.POSE_MODELS` is a registry of pose backends: MediaPipe (CPU-friendly), five YOLO26 sizes (n/s/m/l/x), and Sapiens2 variants (Phase 3, need the custom `sapiens` repo installed). `config.DEFAULT_POSE_MODEL` is YOLO26n. The Gradio UI exposes a dropdown built from `config.available_pose_models()` (filters to checkpoints actually present) and passes the chosen `model_key` through `Director.run` to `Pose2DAgent`. `config.YOLO_POSE_MODEL` is a backward-compat alias only.
122
+
123
+ Checkpoints are **not** committed (`checkpoints/` is gitignored). `formscout/startup.py:ensure_checkpoints()` downloads missing YOLO26/MediaPipe files from the `silas-therapy/formscout-checkpoints` HF repo once at app startup. Models load once per process and are cached — never inside the inference hot path.
124
+
125
+ ### llama.cpp serving
126
+
127
+ `formscout/serving/llama_cpp.py` provides `LlamaCppClient` (VLM, port 8080) and `EmbeddingClient` (embeddings, port 8081). Both check `/health` before use and return safe error dicts when unavailable. Only active when the corresponding `ENABLE_*` flag is True.
128
+
129
+ ### Deploying to Hugging Face
130
+
131
+ The repo deploys to both `silas-therapy/small-functional-movement-screening` (model repo) and the Space of the same name (README frontmatter is the Space config). Use `./scripts/hf_upload.sh` — never raw `hf upload .`: the `hf` CLI does **not** read `.hfignore`, so a raw upload hashes the entire `.venv` (~44k files) and pushes torch binaries. The script parses `.hfignore` into `--exclude` globs, preflights the file count, creates PRs on both repos, and auto-switches to `hf upload-large-folder` (resumable, but no PR / no commit message) above 500 files.
132
+
133
+ ## Key constraints and invariants
134
+
135
+ - **No cloud model APIs.** All inference runs on-Space (ZeroGPU). No OpenAI/Anthropic/Gemini calls.
136
+ - **Pain is never auto-scored.** Any clearing test or visible distress sets `needs_human=True` — enforced in rubric functions and JudgeAgent. `JudgeResult.score` must be `None` when `needs_human=True`.
137
+ - **Quality gates (Director, never silently skip):**
138
+ - Any agent `confidence < config.MIN_CONFIDENCE` (0.6) → warn or stop
139
+ - `|rubric.score - judge.score| >= 1` → flag disagreement
140
+ - `MovementResult.test_name == "unknown"` → stop pipeline, surface manual override
141
+ - `JudgeAgent.needs_human == True` no numeric score emitted
142
+ - **Composite is null** when any test is unscored. Never show a partial 0–21 as complete.
143
+ - **Pipeline runs headless.** No Gradio imports in any agent file.
144
+ - **Safety banner** ("Screening aid not a diagnosis…") must always be visible in the UI — appears at top and bottom of `app.py`.
145
+
146
+ ## Engineering standards
147
+
148
+ - Every agent: one public entrypoint, typed dataclass I/O from `types.py`, `confidence: float` and `notes: str` on every result.
149
+ - Models load once at module/instance init never inside the inference hot path.
150
+ - Every agent module docstring states: purpose, inputs, outputs, failure behavior, model param count, license, and gated status.
151
+ - `tracing.py` records structured per-agent I/O for any run; one full run gets exported to the Hub.
152
+ - Every agent ships with a pytest in `tests/` that runs without model downloads and asserts the typed contract.
153
+
154
+ ## Model stack (~17.6B total stay under 32B)
155
+
156
+ | Component | Model | Params | Status |
157
+ |---|---|---|---|
158
+ | 2D pose (primary) | YOLO26-Pose n/s/m/l/x (default: n) | 0.0007–0.058B | Ready (auto-downloaded at startup) |
159
+ | 2D pose (CPU alt) | MediaPipe Pose Landmarker (full) | ~0.004B | Ready (auto-downloaded at startup) |
160
+ | 2D pose (HQ alt) | `facebook/sapiens2-pose-0.4b/0.8b/1b/5b` | 0.4–5B | Phase 3 needs custom `sapiens` repo |
161
+ | Segmentation | SAM 3.1 base | ~0.85B | Access accepted |
162
+ | 3D biomechanics | `facebook/sam-3d-body-dinov3` | ~0.84B | **Access ACCEPTED Jun 4 2026** |
163
+ | Learned scoring | ST-GCN (pyskl) | ~0.03B | Phase 3 |
164
+ | Judge + Classifier | Qwen3-VL-8B-Instruct (llama.cpp) | 8B | **Online** `scripts/serve_judge.sh`, ENABLE_JUDGE=True |
165
+ | Retrieval | Qwen3-VL-Embedding-8B (llama.cpp) | 8B | Phase 3 |
166
+
167
+ Track the running sum in `MODEL_BUDGET.md`. The two Qwen3-VL-8B models share a backbone.
168
+
169
+ ## Gradio + Svelte UI guidance
170
+
171
+ The UI uses **Gradio `gr.Blocks`** with custom CSS/theme (`formscout/ui/theme.py`). Custom Svelte components for score dial, asymmetry bars, rubric drawer are planned for Phase 4. Use `gradio-svelte-expert` agent for Svelte component work.
172
+
173
+ - ZeroGPU: wrap heavy inference (`Pose2DAgent.run`, `Body3DAgent.run`) in `@spaces.GPU` before deploying to Spaces.
174
+ - Verify Gradio APIs against current docs before use pin exact versions in `requirements.txt`.
175
+
176
+ ## Build phases
177
+
178
+ 1. **Phase 0 — Recon:** ✅ Complete. See `RECON.md`.
179
+ 2. **Phase 1 Spine:** Complete. Deep Squat end-to-end.
180
+ 3. **Phase 2 — All 7 tests:** ✅ Complete. Classifier, Judge, Report agents; all rubric scorers; Gradio UI.
181
+ 4. **Phase 3 Learned scoring + retrieval:** ST-GCN fine-tune on physio clips, publish to Hub. RetrievalAgent with embedding index.
182
+ 5. **Phase 4 — Polish + ship:** Custom Svelte UI components, agent trace to Hub, blog post. (Overlay video done via `PoseVisualizer`; full 7-test session + PDF export done via `formscout/session.py` + `PdfReportAgent`.)
183
+
184
+ ## Known issues
185
+
186
+ - `tests/test_biomechanics.py::TestBiomechanicsAgent::test_unimplemented_test_returns_low_confidence` fails: expects `"not yet implemented"` in `result.notes` but biomechanics returns empty string. Minor �� low priority.
187
+
188
+ ## Badge checklist (definition of done)
189
+
190
+ - [ ] Space runs green; upload scorecard works on real clips
191
+ - [ ] Param sum verified 32B in `MODEL_BUDGET.md`
192
+ - [ ] 🔌 **Off the Grid** no cloud model APIs anywhere in the pipeline
193
+ - [ ] 🎯 **Well-Tuned** — fine-tuned ST-GCN head published to Hub with honest model card
194
+ - [ ] 🎨 **Off-Brand** — custom, non-default Gradio UI (scout/trail theme)
195
+ - [ ] 🦙 **Llama Champion** — VLM + embedder served via llama.cpp (GGUF)
196
+ - [ ] 📡 **Sharing is Caring** — one full agent trace (all I/O) published to Hub
197
+ - [ ] 📓 **Field Notes** — blog post written, honesty section (FMS limitations) front-and-center
198
+ - [ ] Demo video + social post recorded
199
+ - [ ] Safety banner present; pain/clearing never auto-scored; low-confidence flagged
README.md CHANGED
@@ -1,118 +1,118 @@
1
- ---
2
- title: FormScout
3
- emoji: 🏔️
4
- colorFrom: green
5
- colorTo: green
6
- sdk: gradio
7
- app_file: app.py
8
- pinned: false
9
- license: apache-2.0
10
- short_description: FMS video scoring — movement screen aid
11
- ---
12
-
13
- # FormScout
14
-
15
- FMS (Functional Movement Screen) scoring pipeline — a screening aid that scores movement videos 0–3 per test with a written rationale and annotated overlay.
16
-
17
- **⚠️ Screening aid — not a diagnosis. Pain or clearing tests require a clinician.**
18
-
19
- ## Running locally
20
-
21
- ### 1. Clone and install
22
-
23
- ```bash
24
- git clone https://huggingface.co/silas-therapy/small-functional-movement-screening
25
- cd small-functional-movement-screening
26
- python3 -m venv .venv && source .venv/bin/activate
27
- pip install -r requirements.txt
28
- ```
29
-
30
- ### 2. Start the VLM judge (optional but recommended)
31
-
32
- The judge uses Qwen3-VL-8B-Instruct via llama.cpp. Without it the app falls back to the deterministic rubric score — fully functional, no GPU needed.
33
-
34
- ```bash
35
- # Install llama.cpp once
36
- brew install llama.cpp
37
-
38
- # Download the model (one-time, ~6 GB)
39
- python3 -c "
40
- from huggingface_hub import hf_hub_download
41
- for f in ['Qwen3VL-8B-Instruct-Q4_K_M.gguf', 'mmproj-Qwen3VL-8B-Instruct-F16.gguf']:
42
- hf_hub_download('Qwen/Qwen3-VL-8B-Instruct-GGUF', f, local_dir='checkpoints/qwen3-vl')
43
- "
44
-
45
- # Start the server (keep this terminal open)
46
- ./scripts/serve_judge.sh
47
- ```
48
-
49
- To use a fine-tuned GGUF instead of the default:
50
- ```bash
51
- FORMSCOUT_JUDGE_GGUF=/path/to/finetuned.gguf ./scripts/serve_judge.sh
52
- ```
53
-
54
- ### 3. Launch the Gradio app
55
-
56
- ```bash
57
- python3 app.py
58
- # → http://127.0.0.1:7860
59
- ```
60
-
61
- Upload a video, select the FMS test from the dropdown, and click **Analyze**.
62
-
63
- ### 4. Headless pipeline (no Gradio)
64
-
65
- ```bash
66
- python3 -m formscout.run sample.mp4
67
- ```
68
-
69
- ### 5. Tests
70
-
71
- ```bash
72
- pytest tests/ -v
73
- ```
74
-
75
- ### 6. Upload to Hugging Face
76
-
77
- ```bash
78
- # Pushes source to both model repo and Space, opens a PR on each
79
- ./scripts/hf_upload.sh
80
-
81
- # Or with a custom commit message
82
- ./scripts/hf_upload.sh "feat: my change"
83
- ```
84
-
85
- ## Architecture
86
-
87
- Typed specialist agents orchestrated by a deterministic Director:
88
-
89
- ```
90
- Ingest → Pose2D → [Body3D optional] → Biomechanics → Rubric Score → [Judge] → Report
91
- ```
92
-
93
- | Agent | Model | Status |
94
- |---|---|---|
95
- | Pose2D | YOLO26l-Pose (0.026B) + MediaPipe fallback | ✅ |
96
- | Body3D | SAM 3D Body DINOv3 (0.84B) | gated, off by default |
97
- | Judge + Classifier | Qwen3-VL-8B-Instruct via llama.cpp (8B) | ✅ |
98
- | Scoring Head | ST-GCN (0.03B) | Phase 3 |
99
- | Retrieval | Qwen3-VL-Embedding-8B (8B) | Phase 3 |
100
-
101
- See [CLAUDE.md](CLAUDE.md) for full architecture and invariants.
102
-
103
- ## Feature flags (`formscout/config.py`)
104
-
105
- | Flag | Default | Meaning |
106
- |---|---|---|
107
- | `ENABLE_JUDGE` | `True` | VLM judge via llama-server; rubric fallback when server is down |
108
- | `ENABLE_3D` | `False` | SAM 3D Body — off until integrated |
109
- | `ENABLE_STGCN` | `False` | Phase 3 |
110
- | `ENABLE_RAG` | `False` | Phase 3 |
111
-
112
- ## Model budget
113
-
114
- ~18B params total (under 32B cap). See [MODEL_BUDGET.md](MODEL_BUDGET.md).
115
-
116
- ## License
117
-
118
- Apache-2.0. Built for the Build Small Hackathon (Backyard AI track).
 
1
+ ---
2
+ title: FormScout
3
+ emoji: 🏔️
4
+ colorFrom: green
5
+ colorTo: yellow
6
+ sdk: gradio
7
+ app_file: app.py
8
+ pinned: false
9
+ license: apache-2.0
10
+ short_description: FMS video scoring — movement screen aid
11
+ ---
12
+
13
+ # FormScout
14
+
15
+ FMS (Functional Movement Screen) scoring pipeline — a screening aid that scores movement videos 0–3 per test with a written rationale and annotated overlay.
16
+
17
+ **⚠️ Screening aid — not a diagnosis. Pain or clearing tests require a clinician.**
18
+
19
+ ## Running locally
20
+
21
+ ### 1. Clone and install
22
+
23
+ ```bash
24
+ git clone https://huggingface.co/silas-therapy/small-functional-movement-screening
25
+ cd small-functional-movement-screening
26
+ python3 -m venv .venv && source .venv/bin/activate
27
+ pip install -r requirements.txt
28
+ ```
29
+
30
+ ### 2. Start the VLM judge (optional but recommended)
31
+
32
+ The judge uses Qwen3-VL-8B-Instruct via llama.cpp. Without it the app falls back to the deterministic rubric score — fully functional, no GPU needed.
33
+
34
+ ```bash
35
+ # Install llama.cpp once
36
+ brew install llama.cpp
37
+
38
+ # Download the model (one-time, ~6 GB)
39
+ python3 -c "
40
+ from huggingface_hub import hf_hub_download
41
+ for f in ['Qwen3VL-8B-Instruct-Q4_K_M.gguf', 'mmproj-Qwen3VL-8B-Instruct-F16.gguf']:
42
+ hf_hub_download('Qwen/Qwen3-VL-8B-Instruct-GGUF', f, local_dir='checkpoints/qwen3-vl')
43
+ "
44
+
45
+ # Start the server (keep this terminal open)
46
+ ./scripts/serve_judge.sh
47
+ ```
48
+
49
+ To use a fine-tuned GGUF instead of the default:
50
+ ```bash
51
+ FORMSCOUT_JUDGE_GGUF=/path/to/finetuned.gguf ./scripts/serve_judge.sh
52
+ ```
53
+
54
+ ### 3. Launch the Gradio app
55
+
56
+ ```bash
57
+ python3 app.py
58
+ # → http://127.0.0.1:7860
59
+ ```
60
+
61
+ Upload a video, select the FMS test from the dropdown, and click **Analyze**.
62
+
63
+ ### 4. Headless pipeline (no Gradio)
64
+
65
+ ```bash
66
+ python3 -m formscout.run sample.mp4
67
+ ```
68
+
69
+ ### 5. Tests
70
+
71
+ ```bash
72
+ pytest tests/ -v
73
+ ```
74
+
75
+ ### 6. Upload to Hugging Face
76
+
77
+ ```bash
78
+ # Pushes source to both model repo and Space, opens a PR on each
79
+ ./scripts/hf_upload.sh
80
+
81
+ # Or with a custom commit message
82
+ ./scripts/hf_upload.sh "feat: my change"
83
+ ```
84
+
85
+ ## Architecture
86
+
87
+ Typed specialist agents orchestrated by a deterministic Director:
88
+
89
+ ```
90
+ Ingest → Pose2D → [Body3D optional] → Biomechanics → Rubric Score → [Judge] → Report
91
+ ```
92
+
93
+ | Agent | Model | Status |
94
+ |---|---|---|
95
+ | Pose2D | YOLO26l-Pose (0.026B) + MediaPipe fallback | ✅ |
96
+ | Body3D | SAM 3D Body DINOv3 (0.84B) | gated, off by default |
97
+ | Judge + Classifier | Qwen3-VL-8B-Instruct via llama.cpp (8B) | ✅ |
98
+ | Scoring Head | ST-GCN (0.03B) | Phase 3 |
99
+ | Retrieval | Qwen3-VL-Embedding-8B (8B) | Phase 3 |
100
+
101
+ See [CLAUDE.md](CLAUDE.md) for full architecture and invariants.
102
+
103
+ ## Feature flags (`formscout/config.py`)
104
+
105
+ | Flag | Default | Meaning |
106
+ |---|---|---|
107
+ | `ENABLE_JUDGE` | `True` | VLM judge via llama-server; rubric fallback when server is down |
108
+ | `ENABLE_3D` | `False` | SAM 3D Body — off until integrated |
109
+ | `ENABLE_STGCN` | `False` | Phase 3 |
110
+ | `ENABLE_RAG` | `False` | Phase 3 |
111
+
112
+ ## Model budget
113
+
114
+ ~18B params total (under 32B cap). See [MODEL_BUDGET.md](MODEL_BUDGET.md).
115
+
116
+ ## License
117
+
118
+ Apache-2.0. Built for the Build Small Hackathon (Backyard AI track).
app.py CHANGED
@@ -125,22 +125,22 @@ def _render_score_card(score: int, confidence: float, needs_human: bool) -> str:
125
  if needs_human:
126
  return """
127
  <div class="score-card needs-review">
128
- <div style="font-size: 1.2em; color: #fbbf24; margin-bottom: 8px;">⚠️ Needs Clinician Review</div>
129
- <div style="font-size: 0.9em; color: #94a3b8;">Pain or clearing test detected — cannot auto-score</div>
130
  </div>
131
  """
132
 
133
  conf_pct = int(confidence * 100)
134
- conf_color = "#059669" if confidence >= 0.7 else "#f59e0b" if confidence >= 0.4 else "#ef4444"
135
 
136
  return f"""
137
  <div class="score-card">
138
  <div class="score-value">{score}/3</div>
139
- <div style="font-size: 0.95em; color: #94a3b8; margin-top: 4px;">
140
  {SCORE_DESCRIPTIONS.get(score, '')}
141
  </div>
142
  <div style="margin-top: 12px;">
143
- <div style="display: flex; justify-content: space-between; font-size: 0.8em; color: #64748b;">
144
  <span>Confidence</span>
145
  <span style="color: {conf_color};">{conf_pct}%</span>
146
  </div>
@@ -155,9 +155,9 @@ def _render_score_card(score: int, confidence: float, needs_human: bool) -> str:
155
  def _render_empty_state() -> str:
156
  """Render placeholder when no video processed yet."""
157
  return """
158
- <div class="score-card" style="opacity: 0.5;">
159
  <div style="font-size: 2em; margin-bottom: 8px;">🏔️</div>
160
- <div style="color: #64748b;">Upload a video to begin</div>
161
  </div>
162
  """
163
 
@@ -319,7 +319,7 @@ def build_app() -> gr.Blocks:
319
  gr.HTML("""
320
  <div class="formscout-header">
321
  <h1>🏔️ FormScout</h1>
322
- <p style="color: #94a3b8; font-size: 0.95em;">
323
  Functional Movement Screen · Automated Scoring Aid
324
  </p>
325
  </div>
@@ -362,7 +362,7 @@ def build_app() -> gr.Blocks:
362
 
363
  overlay_layers = gr.CheckboxGroup(
364
  choices=["Skeleton", "Trails", "Velocity arrows"],
365
- value=["Skeleton", "Trails"],
366
  label="Overlay Layers",
367
  )
368
 
@@ -414,9 +414,9 @@ def build_app() -> gr.Blocks:
414
  gr.HTML(f'<div class="safety-banner" style="margin-top: 20px;">{DISCLAIMER}</div>')
415
 
416
  gr.Markdown(
417
- "<center style='color: #64748b; font-size: 0.8em; margin-top: 12px;'>"
418
  "FormScout · ~18B params · Off the Grid · "
419
- "<a href='https://github.com/' style='color: #86efac;'>Built for Build Small Hackathon</a>"
420
  "</center>"
421
  )
422
 
 
125
  if needs_human:
126
  return """
127
  <div class="score-card needs-review">
128
+ <div style="font-size: 1.2em; color: #cf922a; margin-bottom: 8px;">⚠️ Needs Clinician Review</div>
129
+ <div style="font-size: 0.9em; color: #4a5f57;">Pain or clearing test detected — cannot auto-score</div>
130
  </div>
131
  """
132
 
133
  conf_pct = int(confidence * 100)
134
+ conf_color = "#2b8a8a" if confidence >= 0.7 else "#cf922a" if confidence >= 0.4 else "#d9534f"
135
 
136
  return f"""
137
  <div class="score-card">
138
  <div class="score-value">{score}/3</div>
139
+ <div style="font-size: 0.95em; color: #4a5f57; margin-top: 4px;">
140
  {SCORE_DESCRIPTIONS.get(score, '')}
141
  </div>
142
  <div style="margin-top: 12px;">
143
+ <div style="display: flex; justify-content: space-between; font-size: 0.8em; color: #6b7d75;">
144
  <span>Confidence</span>
145
  <span style="color: {conf_color};">{conf_pct}%</span>
146
  </div>
 
155
  def _render_empty_state() -> str:
156
  """Render placeholder when no video processed yet."""
157
  return """
158
+ <div class="score-card" style="opacity: 0.6;">
159
  <div style="font-size: 2em; margin-bottom: 8px;">🏔️</div>
160
+ <div style="color: #6b7d75;">Upload a video to begin</div>
161
  </div>
162
  """
163
 
 
319
  gr.HTML("""
320
  <div class="formscout-header">
321
  <h1>🏔️ FormScout</h1>
322
+ <p style="color: #4a5f57; font-size: 0.95em;">
323
  Functional Movement Screen · Automated Scoring Aid
324
  </p>
325
  </div>
 
362
 
363
  overlay_layers = gr.CheckboxGroup(
364
  choices=["Skeleton", "Trails", "Velocity arrows"],
365
+ value=["Skeleton", "Trails", "Velocity arrows"],
366
  label="Overlay Layers",
367
  )
368
 
 
414
  gr.HTML(f'<div class="safety-banner" style="margin-top: 20px;">{DISCLAIMER}</div>')
415
 
416
  gr.Markdown(
417
+ "<center style='color: #6b7d75; font-size: 0.8em; margin-top: 12px;'>"
418
  "FormScout · ~18B params · Off the Grid · "
419
+ "<a href='https://silastherapy.sk' style='color: #1f6e6e;'>Silas Therapy · Build Small Hackathon</a>"
420
  "</center>"
421
  )
422
 
docs/superpowers/plans/2026-06-09-pose-model-selector.md CHANGED
@@ -1,734 +1,734 @@
1
- # Pose Model Selector Implementation Plan
2
-
3
- > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
4
-
5
- **Goal:** Replace the hard-coded YOLO26l default with a 10-model dropdown (MediaPipe, YOLO26 n→x, Sapiens2 0.4B→5B) wired end-to-end from UI through the Director to `Pose2DAgent`.
6
-
7
- **Architecture:** Unified `POSE_MODELS` registry in `config.py` drives a `gr.Dropdown` in `app.py`; the selected key flows through `Director.run()` into `Pose2DAgent.run(model_key)`, which dispatches to one of three private sub-runners (`_run_yolo`, `_run_mediapipe`, `_run_sapiens2`), all producing the same COCO-17 `list[dict]` contract.
8
-
9
- **Tech Stack:** `ultralytics` (YOLO), `onnxruntime` + `huggingface_hub` (MediaPipe), `transformers` (Sapiens2), `gradio` (UI).
10
-
11
- ---
12
-
13
- ## File map
14
-
15
- | File | Change |
16
- |---|---|
17
- | `formscout/config.py` | Replace `YOLO_POSE_MODELS` with `POSE_MODELS` dict + `DEFAULT_POSE_MODEL` |
18
- | `formscout/agents/pose2d.py` | Add `_run_yolo`, `_run_mediapipe`, `_run_sapiens2`; update `run()` signature |
19
- | `formscout/pipeline.py` | Change `pose_model_path` param to `model_key` |
20
- | `app.py` | Add `pose_model_dropdown`, fix `_map_inputs` + `process_video` |
21
- | `requirements.txt` | Add `onnxruntime>=1.18` |
22
- | `tests/test_pose2d.py` | Add mocked tests for each backend |
23
-
24
- ---
25
-
26
- ## Task 1: Add unified `POSE_MODELS` registry to `config.py`
27
-
28
- **Files:**
29
- - Modify: `formscout/config.py`
30
-
31
- - [ ] **Step 1: Open `formscout/config.py` and replace the `YOLO_POSE_MODELS` block**
32
-
33
- Replace lines 12–20 (the `YOLO_POSE_MODELS` dict and `YOLO_POSE_MODEL` / `YOLO_POSE_MODEL_HQ` lines) with:
34
-
35
- ```python
36
- _YOLO_DIR = ROOT / "checkpoints" / "yolo26"
37
-
38
- POSE_MODELS: dict[str, dict] = {
39
- # ── MediaPipe (Qualcomm HF, ONNX Runtime) ──────────────────────────────
40
- "MediaPipe-Pose ⬇ ~16 MB, CPU-friendly": {
41
- "backend": "mediapipe",
42
- "hf_id": "qualcomm/MediaPipe-Pose-Estimation",
43
- "params_m": 4.2,
44
- },
45
- # ── YOLO26 (local checkpoints) ─────────────────────────────────────────
46
- "YOLO26n — nano (0.7M, fastest)": {
47
- "backend": "yolo",
48
- "path": str(_YOLO_DIR / "yolo26n-pose.pt"),
49
- "params_m": 0.7,
50
- },
51
- "YOLO26s — small (3.5M)": {
52
- "backend": "yolo",
53
- "path": str(_YOLO_DIR / "yolo26s-pose.pt"),
54
- "params_m": 3.5,
55
- },
56
- "YOLO26m — medium (9M)": {
57
- "backend": "yolo",
58
- "path": str(_YOLO_DIR / "yolo26m-pose.pt"),
59
- "params_m": 9.0,
60
- },
61
- "YOLO26l — large (25.9M)": {
62
- "backend": "yolo",
63
- "path": str(_YOLO_DIR / "yolo26l-pose.pt"),
64
- "params_m": 25.9,
65
- },
66
- "YOLO26x — extra-large (57.6M)": {
67
- "backend": "yolo",
68
- "path": str(_YOLO_DIR / "yolo26x-pose.pt"),
69
- "params_m": 57.6,
70
- },
71
- # ── Sapiens2 (HF download, transformers) ──────────────────────────────
72
- "Sapiens2-0.4B ⬇ ~1.6 GB": {
73
- "backend": "sapiens2",
74
- "hf_id": "facebook/sapiens2-pose-0.4b",
75
- "params_m": 400,
76
- },
77
- "Sapiens2-0.8B ⬇ ~3.2 GB": {
78
- "backend": "sapiens2",
79
- "hf_id": "facebook/sapiens2-pose-0.8b",
80
- "params_m": 800,
81
- },
82
- "Sapiens2-1B ⬇ ~4 GB": {
83
- "backend": "sapiens2",
84
- "hf_id": "facebook/sapiens2-pose-1b",
85
- "params_m": 1000,
86
- },
87
- "Sapiens2-5B ⬇ ~20 GB, large GPU": {
88
- "backend": "sapiens2",
89
- "hf_id": "facebook/sapiens2-pose-5b",
90
- "params_m": 5000,
91
- },
92
- }
93
-
94
- DEFAULT_POSE_MODEL = "YOLO26n — nano (0.7M, fastest)"
95
-
96
- # Backward-compat aliases — kept for any direct references outside the agent
97
- YOLO_POSE_MODEL = str(_YOLO_DIR / "yolo26l-pose.pt")
98
- YOLO_POSE_MODEL_HQ = str(_YOLO_DIR / "yolo26x-pose.pt")
99
- ```
100
-
101
- - [ ] **Step 2: Verify import is clean**
102
-
103
- ```bash
104
- python3 -c "from formscout import config; print(list(config.POSE_MODELS.keys()))"
105
- ```
106
-
107
- Expected: list of 10 model labels, starting with `MediaPipe-Pose...`
108
-
109
- - [ ] **Step 3: Commit**
110
-
111
- ```bash
112
- git add formscout/config.py
113
- git commit -m "feat: unified POSE_MODELS registry with MediaPipe, YOLO26 n-x, Sapiens2 0.4-5B"
114
- git push
115
- ```
116
-
117
- ---
118
-
119
- ## Task 2: Refactor `Pose2DAgent` — YOLO sub-runner + new `run()` signature
120
-
121
- **Files:**
122
- - Modify: `formscout/agents/pose2d.py`
123
- - Modify: `tests/test_pose2d.py`
124
-
125
- - [ ] **Step 1: Write failing test for the new `model_key` signature**
126
-
127
- Add to `tests/test_pose2d.py`:
128
-
129
- ```python
130
- def test_run_accepts_model_key(pose2d_agent):
131
- """run() must accept model_key kwarg, not model_path."""
132
- import inspect
133
- sig = inspect.signature(pose2d_agent.run)
134
- assert "model_key" in sig.parameters
135
- assert "model_path" not in sig.parameters
136
- ```
137
-
138
- - [ ] **Step 2: Run to confirm it fails**
139
-
140
- ```bash
141
- pytest tests/test_pose2d.py::TestPose2DAgent::test_run_accepts_model_key -v
142
- ```
143
-
144
- Expected: FAIL — `model_path` still present in signature.
145
-
146
- - [ ] **Step 3: Rewrite `formscout/agents/pose2d.py`**
147
-
148
- Replace the entire file with:
149
-
150
- ```python
151
- """
152
- Pose2DAgent — 2D per-frame keypoint extraction.
153
-
154
- Backends: yolo (local ONNX), mediapipe (Qualcomm HF/ONNX Runtime),
155
- sapiens2 (Meta HF/transformers).
156
- All backends output COCO-17 keypoints: dict[int, {x, y, conf}] per frame.
157
-
158
- Input: IngestResult
159
- Output: Pose2DResult(keypoints per frame, fps, confidence)
160
- Failure: Pose2DResult(confidence=0.0, notes=<reason>) — never raises.
161
- """
162
- from __future__ import annotations
163
-
164
- import logging
165
- import numpy as np
166
-
167
- from formscout import config
168
- from formscout.types import IngestResult, Pose2DResult
169
-
170
- logger = logging.getLogger(__name__)
171
-
172
- COCO_KEYPOINTS = [
173
- "nose", "left_eye", "right_eye", "left_ear", "right_ear",
174
- "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
175
- "left_wrist", "right_wrist", "left_hip", "right_hip",
176
- "left_knee", "right_knee", "left_ankle", "right_ankle",
177
- ]
178
-
179
- # BlazePose-33 → COCO-17 index mapping
180
- _BLAZEPOSE_TO_COCO: dict[int, int] = {
181
- 0: 0, # nose
182
- 1: 2, # left_eye (inner → left_eye)
183
- 2: 1, # right_eye (inner → right_eye) — swapped: BlazePose 1=left_eye_inner
184
- 3: 3, # left_ear
185
- 4: 4, # right_ear
186
- 5: 5, # left_shoulder → COCO left_shoulder... wait
187
- # Correct BlazePose-33 COCO mapping (canonical):
188
- # BlazePose idx : COCO idx
189
- # 0 nose → COCO 0
190
- # 2 left_eye → COCO 1
191
- # 5 right_eye → COCO 2
192
- # 7 left_ear → COCO 3
193
- # 8 right_ear → COCO 4
194
- # 11 left_shoulder → COCO 5
195
- # 12 right_shoulder → COCO 6
196
- # 13 left_elbow → COCO 7
197
- # 14 right_elbow → COCO 8
198
- # 15 left_wrist → COCO 9
199
- # 16 right_wrist → COCO 10
200
- # 23 left_hip → COCO 11
201
- # 24 right_hip → COCO 12
202
- # 25 left_knee → COCO 13
203
- # 26 right_knee → COCO 14
204
- # 27 left_ankle → COCO 15
205
- # 28 right_ankle → COCO 16
206
- }
207
-
208
- # BlazePose source index → COCO target index (correct mapping, no duplicates)
209
- _BP_SRC = [0, 2, 5, 7, 8, 11, 12, 13, 14, 15, 16, 23, 24, 25, 26, 27, 28]
210
- _BP_DST = list(range(17)) # COCO 0..16
211
-
212
- _model_cache: dict[str, object] = {}
213
-
214
-
215
- # ── YOLO backend ─────────────────────────────────────────────────────────────
216
-
217
- def _get_yolo(path: str) -> object:
218
- if path not in _model_cache:
219
- from ultralytics import YOLO
220
- _model_cache[path] = YOLO(path)
221
- return _model_cache[path]
222
-
223
-
224
- def _run_yolo(frames: list, path: str) -> list[dict]:
225
- model = _get_yolo(path)
226
- out = []
227
- for frame in frames:
228
- try:
229
- results = model(frame, verbose=False)
230
- kps: dict[int, dict] = {}
231
- if results and results[0].keypoints is not None:
232
- kp = results[0].keypoints
233
- if kp.xy is not None and len(kp.xy) > 0:
234
- xy = kp.xy[0].cpu().numpy()
235
- conf = kp.conf[0].cpu().numpy()
236
- for j in range(min(len(xy), 17)):
237
- kps[j] = {"x": float(xy[j, 0]), "y": float(xy[j, 1]), "conf": float(conf[j])}
238
- out.append(kps)
239
- except Exception:
240
- out.append({})
241
- return out
242
-
243
-
244
- # ── MediaPipe backend ────────────────────────────────────────────────────────
245
-
246
- def _get_mediapipe_sessions(hf_id: str):
247
- """Return (detector_session, landmark_session) cached by hf_id."""
248
- cache_key = f"mp:{hf_id}"
249
- if cache_key not in _model_cache:
250
- from huggingface_hub import snapshot_download
251
- import onnxruntime as ort
252
- from pathlib import Path
253
-
254
- snap = Path(snapshot_download(hf_id))
255
- onnx_files = sorted(snap.glob("**/*.onnx"), key=lambda p: p.stat().st_size)
256
- if len(onnx_files) < 2:
257
- raise RuntimeError(f"Expected 2 ONNX files in {snap}, found {len(onnx_files)}")
258
- # Smaller file = pose detector; larger = pose landmark detector
259
- det_sess = ort.InferenceSession(str(onnx_files[0]))
260
- lmk_sess = ort.InferenceSession(str(onnx_files[-1]))
261
- _model_cache[cache_key] = (det_sess, lmk_sess)
262
- return _model_cache[cache_key]
263
-
264
-
265
- def _preprocess_mediapipe(frame: np.ndarray, size: int = 256) -> np.ndarray:
266
- """Resize to size×size, normalize to [0,1], add batch dim → (1,3,H,W)."""
267
- import cv2
268
- img = cv2.resize(frame, (size, size)).astype(np.float32) / 255.0
269
- return img.transpose(2, 0, 1)[None] # (1, 3, 256, 256)
270
-
271
-
272
- def _run_mediapipe(frames: list, hf_id: str) -> list[dict]:
273
- try:
274
- det_sess, lmk_sess = _get_mediapipe_sessions(hf_id)
275
- except Exception as e:
276
- logger.warning("mediapipe load failed: %s", e)
277
- return [{} for _ in frames]
278
-
279
- import cv2
280
- h_orig, w_orig = frames[0].shape[:2] if frames else (480, 640)
281
- out = []
282
-
283
- for frame in frames:
284
- try:
285
- h, w = frame.shape[:2]
286
- inp = _preprocess_mediapipe(frame)
287
-
288
- # Run landmark detector directly on full frame (single-person FMS use-case)
289
- lmk_input_name = lmk_sess.get_inputs()[0].name
290
- lmk_out = lmk_sess.run(None, {lmk_input_name: inp})
291
-
292
- # lmk_out[0] shape: (1, 33, 3) — [x, y, visibility] normalized 0..1
293
- landmarks = lmk_out[0][0] # (33, 3)
294
-
295
- kps: dict[int, dict] = {}
296
- for coco_idx, bp_idx in zip(_BP_DST, _BP_SRC):
297
- if bp_idx < len(landmarks):
298
- lm = landmarks[bp_idx]
299
- kps[coco_idx] = {
300
- "x": float(lm[0] * w),
301
- "y": float(lm[1] * h),
302
- "conf": float(lm[2]), # visibility score
303
- }
304
- out.append(kps)
305
- except Exception:
306
- out.append({})
307
-
308
- return out
309
-
310
-
311
- # ── Sapiens2 backend ─────────────────────────────────────────────────────────
312
-
313
- # COCO-17 keypoint names in order (used to map Sapiens2 named output → COCO index)
314
- _COCO_NAMES = [
315
- "nose", "left_eye", "right_eye", "left_ear", "right_ear",
316
- "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
317
- "left_wrist", "right_wrist", "left_hip", "right_hip",
318
- "left_knee", "right_knee", "left_ankle", "right_ankle",
319
- ]
320
-
321
-
322
- def _get_sapiens2(hf_id: str) -> object:
323
- if hf_id not in _model_cache:
324
- from transformers import pipeline as hf_pipeline
325
- _model_cache[hf_id] = hf_pipeline("pose-estimation", model=hf_id)
326
- return _model_cache[hf_id]
327
-
328
-
329
- def _run_sapiens2(frames: list, hf_id: str) -> list[dict]:
330
- try:
331
- pipe = _get_sapiens2(hf_id)
332
- except Exception as e:
333
- logger.warning("sapiens2 load failed: %s", e)
334
- return [{} for _ in frames]
335
-
336
- from PIL import Image
337
- out = []
338
-
339
- for frame in frames:
340
- try:
341
- pil_img = Image.fromarray(frame)
342
- result = pipe(pil_img)
343
-
344
- # result is a list of person dicts; take the first (highest confidence)
345
- if not result:
346
- out.append({})
347
- continue
348
-
349
- person = result[0]
350
- keypoints = person.get("keypoints", [])
351
- scores = person.get("keypoint_scores", [])
352
-
353
- # Build name→(x,y,score) lookup from pipeline output
354
- kp_lookup: dict[str, tuple] = {}
355
- for i, kp in enumerate(keypoints):
356
- name = kp.get("label", "") if isinstance(kp, dict) else ""
357
- x = kp.get("x", 0.0) if isinstance(kp, dict) else float(kp[0])
358
- y = kp.get("y", 0.0) if isinstance(kp, dict) else float(kp[1])
359
- score = scores[i] if i < len(scores) else 0.0
360
- if name:
361
- kp_lookup[name] = (x, y, float(score))
362
-
363
- kps: dict[int, dict] = {}
364
- for coco_idx, name in enumerate(_COCO_NAMES):
365
- if name in kp_lookup:
366
- x, y, s = kp_lookup[name]
367
- kps[coco_idx] = {"x": x, "y": y, "conf": s}
368
-
369
- out.append(kps)
370
- except Exception:
371
- out.append({})
372
-
373
- return out
374
-
375
-
376
- # ── Agent ────────────────────────────────────────────────────────────────────
377
-
378
- class Pose2DAgent:
379
- """Extracts COCO-17 keypoints per frame; dispatches to YOLO, MediaPipe, or Sapiens2."""
380
-
381
- def run(self, ingest: IngestResult, model_key: str | None = None) -> Pose2DResult:
382
- if not ingest.frames:
383
- return Pose2DResult(keypoints=[], fps=ingest.fps, confidence=0.0, notes="no frames in ingest")
384
-
385
- key = model_key or config.DEFAULT_POSE_MODEL
386
- spec = config.POSE_MODELS.get(key)
387
- if spec is None:
388
- logger.warning("Unknown model_key %r — falling back to %s", key, config.DEFAULT_POSE_MODEL)
389
- spec = config.POSE_MODELS[config.DEFAULT_POSE_MODEL]
390
-
391
- backend = spec["backend"]
392
- try:
393
- if backend == "yolo":
394
- kps_per_frame = _run_yolo(ingest.frames, spec["path"])
395
- elif backend == "mediapipe":
396
- kps_per_frame = _run_mediapipe(ingest.frames, spec["hf_id"])
397
- elif backend == "sapiens2":
398
- kps_per_frame = _run_sapiens2(ingest.frames, spec["hf_id"])
399
- else:
400
- return Pose2DResult(
401
- keypoints=[{} for _ in ingest.frames],
402
- fps=ingest.fps, confidence=0.0,
403
- notes=f"unknown backend: {backend}",
404
- )
405
- except Exception as e:
406
- return Pose2DResult(
407
- keypoints=[{} for _ in ingest.frames],
408
- fps=ingest.fps, confidence=0.0,
409
- notes=str(e),
410
- )
411
-
412
- n_detected = sum(1 for f in kps_per_frame if f)
413
- total_conf = sum(
414
- sum(kp["conf"] for kp in f.values()) / len(f)
415
- for f in kps_per_frame if f
416
- )
417
- overall_conf = (total_conf / n_detected) if n_detected > 0 else 0.0
418
- notes = "" if n_detected > 0 else "no person detected in any frame"
419
-
420
- return Pose2DResult(
421
- keypoints=kps_per_frame,
422
- fps=ingest.fps,
423
- confidence=overall_conf,
424
- notes=notes,
425
- )
426
- ```
427
-
428
- - [ ] **Step 4: Run the new signature test**
429
-
430
- ```bash
431
- pytest tests/test_pose2d.py::TestPose2DAgent::test_run_accepts_model_key -v
432
- ```
433
-
434
- Expected: PASS
435
-
436
- - [ ] **Step 5: Run full existing pose2d test suite**
437
-
438
- ```bash
439
- pytest tests/test_pose2d.py -v
440
- ```
441
-
442
- Expected: all existing tests pass (they will skip if YOLO model unavailable in env — that's OK).
443
-
444
- - [ ] **Step 6: Commit and push**
445
-
446
- ```bash
447
- git add formscout/agents/pose2d.py tests/test_pose2d.py
448
- git commit -m "feat: Pose2DAgent — three backends (yolo/mediapipe/sapiens2), model_key dispatch"
449
- git push
450
- ```
451
-
452
- ---
453
-
454
- ## Task 3: Add `onnxruntime` to requirements
455
-
456
- **Files:**
457
- - Modify: `requirements.txt`
458
-
459
- - [ ] **Step 1: Add onnxruntime**
460
-
461
- Open `requirements.txt` and add after the existing `transformers` line:
462
-
463
- ```
464
- onnxruntime>=1.18
465
- ```
466
-
467
- - [ ] **Step 2: Verify it installs**
468
-
469
- ```bash
470
- pip install onnxruntime --quiet && python3 -c "import onnxruntime; print(onnxruntime.__version__)"
471
- ```
472
-
473
- Expected: version string printed, no errors.
474
-
475
- - [ ] **Step 3: Commit and push**
476
-
477
- ```bash
478
- git add requirements.txt
479
- git commit -m "chore: add onnxruntime for MediaPipe ONNX backend"
480
- git push
481
- ```
482
-
483
- ---
484
-
485
- ## Task 4: Update `Director.run()` — `pose_model_path` → `model_key`
486
-
487
- **Files:**
488
- - Modify: `formscout/pipeline.py`
489
-
490
- - [ ] **Step 1: Update the signature and the `pose2d` call**
491
-
492
- In `formscout/pipeline.py`, change `Director.run()`:
493
-
494
- ```python
495
- def run(self, video_path: str, test_name: str = "deep_squat", side: str = "na", model_key: str | None = None) -> PipelineState:
496
- """
497
- Run the full pipeline on a single video.
498
- test_name/side serve as manual override when provided (skips classifier).
499
- model_key selects the pose backend (see config.POSE_MODELS).
500
- """
501
- state = PipelineState(video_path=video_path)
502
-
503
- # ─── Ingest ───
504
- state.ingest = self._ingest.run(video_path)
505
- if state.ingest.confidence < config.MIN_CONFIDENCE:
506
- state.errors.append("ingest: low confidence — video may be corrupt")
507
- return state
508
-
509
- # ─── Pose 2D ───
510
- state.pose2d = self._pose2d.run(state.ingest, model_key=model_key)
511
- # ... rest of method unchanged
512
- ```
513
-
514
- (Only the signature line and the `self._pose2d.run(...)` call change — everything else stays the same.)
515
-
516
- - [ ] **Step 2: Verify import is clean**
517
-
518
- ```bash
519
- python3 -c "from formscout.pipeline import Director; d = Director(); print('ok')"
520
- ```
521
-
522
- Expected: `ok` (models load lazily so no crash here).
523
-
524
- - [ ] **Step 3: Commit and push**
525
-
526
- ```bash
527
- git add formscout/pipeline.py
528
- git commit -m "feat: Director.run() accepts model_key, threads to Pose2DAgent"
529
- git push
530
- ```
531
-
532
- ---
533
-
534
- ## Task 5: Wire the UI — pose model dropdown in `app.py`
535
-
536
- **Files:**
537
- - Modify: `app.py`
538
-
539
- - [ ] **Step 1: Update `process_video` to use `model_key` and the unified registry**
540
-
541
- Replace the existing `process_video` function signature and the old `YOLO_POSE_MODELS.get()` lookup:
542
-
543
- ```python
544
- def process_video(video_path: str, test_name: str, side: str, model_key: str):
545
- """Process an uploaded video through the FormScout pipeline."""
546
- if not video_path:
547
- return (
548
- _render_empty_state(),
549
- "Upload a video to begin analysis.",
550
- "",
551
- "",
552
- )
553
-
554
- director = Director()
555
- state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
556
- ```
557
-
558
- (Remove the `pose_model_path = config.YOLO_POSE_MODELS.get(...)` line entirely.)
559
-
560
- - [ ] **Step 2: Add the `pose_model_dropdown` in `build_app()`**
561
-
562
- Inside `build_app()`, after the `side_dropdown` block (around line 265) and before `submit_btn`, add:
563
-
564
- ```python
565
- pose_model_dropdown = gr.Dropdown(
566
- choices=list(config.POSE_MODELS.keys()),
567
- value=config.DEFAULT_POSE_MODEL,
568
- label="Pose Model",
569
- )
570
- ```
571
-
572
- - [ ] **Step 3: Update `_map_inputs` to pass the model key**
573
-
574
- Replace the existing `_map_inputs` closure:
575
-
576
- ```python
577
- def _map_inputs(video, test_display_name, side_display, pose_model_key):
578
- """Map UI display values to internal values."""
579
- test_map = {name: val for name, val in FMS_TESTS}
580
- test_name = test_map.get(test_display_name, "deep_squat")
581
- side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
582
- return process_video(video, test_name, side, pose_model_key)
583
- ```
584
-
585
- - [ ] **Step 4: Update `submit_btn.click` to include `pose_model_dropdown`**
586
-
587
- Replace the existing `.click(...)` call:
588
-
589
- ```python
590
- submit_btn.click(
591
- fn=_map_inputs,
592
- inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown],
593
- outputs=[score_html, pipeline_md, score_details, alerts_md],
594
- )
595
- ```
596
-
597
- - [ ] **Step 5: Smoke-test the app starts**
598
-
599
- ```bash
600
- python3 -c "from app import build_app; app = build_app(); print('app built ok')"
601
- ```
602
-
603
- Expected: `app built ok` — no import or config errors.
604
-
605
- - [ ] **Step 6: Commit and push**
606
-
607
- ```bash
608
- git add app.py
609
- git commit -m "feat: pose model dropdown in UI, wired through process_video → Director"
610
- git push
611
- ```
612
-
613
- ---
614
-
615
- ## Task 6: Add mocked backend tests
616
-
617
- **Files:**
618
- - Modify: `tests/test_pose2d.py`
619
-
620
- - [ ] **Step 1: Add mocked YOLO test**
621
-
622
- Append to `tests/test_pose2d.py`:
623
-
624
- ```python
625
- import unittest.mock as mock
626
- import numpy as np
627
- from formscout.types import IngestResult, Pose2DResult
628
-
629
-
630
- def _blank_ingest_3():
631
- frames = [np.zeros((480, 640, 3), dtype=np.uint8) for _ in range(3)]
632
- return IngestResult(frames=frames, fps=30.0, duration=0.1, n_people=1, width=640, height=480)
633
-
634
-
635
- class TestPose2DBackendsMocked:
636
- """Backend dispatch tests — no real model downloads."""
637
-
638
- def test_yolo_backend_dispatches(self):
639
- from formscout.agents.pose2d import Pose2DAgent, _run_yolo
640
- fake_kps = [{0: {"x": 10.0, "y": 20.0, "conf": 0.9}} for _ in range(3)]
641
- with mock.patch("formscout.agents.pose2d._run_yolo", return_value=fake_kps) as m:
642
- agent = Pose2DAgent()
643
- result = agent.run(_blank_ingest_3(), model_key="YOLO26n — nano (0.7M, fastest)")
644
- m.assert_called_once()
645
- assert isinstance(result, Pose2DResult)
646
- assert len(result.keypoints) == 3
647
- assert result.confidence > 0.0
648
-
649
- def test_mediapipe_backend_dispatches(self):
650
- from formscout.agents.pose2d import Pose2DAgent
651
- fake_kps = [{i: {"x": float(i), "y": float(i), "conf": 0.8} for i in range(17)} for _ in range(3)]
652
- with mock.patch("formscout.agents.pose2d._run_mediapipe", return_value=fake_kps) as m:
653
- agent = Pose2DAgent()
654
- result = agent.run(_blank_ingest_3(), model_key="MediaPipe-Pose ⬇ ~16 MB, CPU-friendly")
655
- m.assert_called_once()
656
- assert isinstance(result, Pose2DResult)
657
- assert len(result.keypoints) == 3
658
- assert all(len(f) == 17 for f in result.keypoints)
659
-
660
- def test_sapiens2_backend_dispatches(self):
661
- from formscout.agents.pose2d import Pose2DAgent
662
- fake_kps = [{i: {"x": float(i), "y": float(i), "conf": 0.85} for i in range(17)} for _ in range(3)]
663
- with mock.patch("formscout.agents.pose2d._run_sapiens2", return_value=fake_kps) as m:
664
- agent = Pose2DAgent()
665
- result = agent.run(_blank_ingest_3(), model_key="Sapiens2-0.4B ⬇ ~1.6 GB")
666
- m.assert_called_once()
667
- assert isinstance(result, Pose2DResult)
668
- assert len(result.keypoints) == 3
669
-
670
- def test_unknown_model_key_falls_back(self):
671
- from formscout.agents.pose2d import Pose2DAgent
672
- fake_kps = [{0: {"x": 1.0, "y": 2.0, "conf": 0.7}} for _ in range(3)]
673
- with mock.patch("formscout.agents.pose2d._run_yolo", return_value=fake_kps):
674
- agent = Pose2DAgent()
675
- result = agent.run(_blank_ingest_3(), model_key="nonexistent-model-xyz")
676
- assert isinstance(result, Pose2DResult) # graceful fallback, no crash
677
-
678
- def test_confidence_zero_on_empty_keypoints(self):
679
- from formscout.agents.pose2d import Pose2DAgent
680
- with mock.patch("formscout.agents.pose2d._run_yolo", return_value=[{}, {}, {}]):
681
- agent = Pose2DAgent()
682
- result = agent.run(_blank_ingest_3(), model_key="YOLO26n — nano (0.7M, fastest)")
683
- assert result.confidence == 0.0
684
- assert "no person" in result.notes.lower()
685
- ```
686
-
687
- - [ ] **Step 2: Run the new tests**
688
-
689
- ```bash
690
- pytest tests/test_pose2d.py::TestPose2DBackendsMocked -v
691
- ```
692
-
693
- Expected: all 5 tests PASS.
694
-
695
- - [ ] **Step 3: Run the full test suite to check for regressions**
696
-
697
- ```bash
698
- pytest tests/ -v --tb=short 2>&1 | tail -30
699
- ```
700
-
701
- Expected: same pass/fail ratio as before (45/46 known passing). The one known failure (`test_unimplemented_test_returns_low_confidence`) is pre-existing — ignore it.
702
-
703
- - [ ] **Step 4: Commit and push**
704
-
705
- ```bash
706
- git add tests/test_pose2d.py
707
- git commit -m "test: mocked backend dispatch tests for YOLO, MediaPipe, Sapiens2"
708
- git push
709
- ```
710
-
711
- ---
712
-
713
- ## Self-review
714
-
715
- **Spec coverage:**
716
- - ✅ Unified `POSE_MODELS` registry (Task 1)
717
- - ✅ `DEFAULT_POSE_MODEL = YOLO26n` (Task 1)
718
- - ✅ Backward-compat `YOLO_POSE_MODEL` / `YOLO_POSE_MODEL_HQ` aliases (Task 1)
719
- - ✅ `_run_yolo` sub-runner (Task 2)
720
- - ✅ `_run_mediapipe` with ONNX Runtime + BlazePose→COCO-17 mapping (Task 2)
721
- - ✅ `_run_sapiens2` with transformers pipeline + named-keypoint→COCO-17 mapping (Task 2)
722
- - ✅ `Pose2DAgent.run(model_key)` dispatch + fallback on unknown key (Task 2)
723
- - ✅ `onnxruntime` added to requirements (Task 3)
724
- - ✅ `Director.run(model_key)` threads key to agent (Task 4)
725
- - ✅ `pose_model_dropdown` in UI (Task 5)
726
- - ✅ `_map_inputs` + `submit_btn.click` wired (Task 5)
727
- - ✅ Error handling: unknown key → warning + fallback; download failure → confidence=0 (Task 2)
728
- - ✅ Mocked tests for all three backends (Task 6)
729
-
730
- **Placeholder scan:** None found.
731
-
732
- **Type consistency:** `model_key: str | None` used consistently across `Pose2DAgent.run`, `Director.run`, `process_video`. `config.POSE_MODELS` and `config.DEFAULT_POSE_MODEL` referenced consistently.
733
-
734
- **Note on Sapiens2 keypoint format:** The `_run_sapiens2` implementation uses **named keypoint lookup** (by label string) rather than assuming fixed indices 0–16 = COCO. This is the safe approach — the transformers pipeline returns labeled keypoints and the code maps by name. If the pipeline returns unnamed keypoints (index-only), the `kp_lookup` will be empty and the frame will gracefully return `{}`.
 
1
+ # Pose Model Selector Implementation Plan
2
+
3
+ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
4
+
5
+ **Goal:** Replace the hard-coded YOLO26l default with a 10-model dropdown (MediaPipe, YOLO26 n→x, Sapiens2 0.4B→5B) wired end-to-end from UI through the Director to `Pose2DAgent`.
6
+
7
+ **Architecture:** Unified `POSE_MODELS` registry in `config.py` drives a `gr.Dropdown` in `app.py`; the selected key flows through `Director.run()` into `Pose2DAgent.run(model_key)`, which dispatches to one of three private sub-runners (`_run_yolo`, `_run_mediapipe`, `_run_sapiens2`), all producing the same COCO-17 `list[dict]` contract.
8
+
9
+ **Tech Stack:** `ultralytics` (YOLO), `onnxruntime` + `huggingface_hub` (MediaPipe), `transformers` (Sapiens2), `gradio` (UI).
10
+
11
+ ---
12
+
13
+ ## File map
14
+
15
+ | File | Change |
16
+ |---|---|
17
+ | `formscout/config.py` | Replace `YOLO_POSE_MODELS` with `POSE_MODELS` dict + `DEFAULT_POSE_MODEL` |
18
+ | `formscout/agents/pose2d.py` | Add `_run_yolo`, `_run_mediapipe`, `_run_sapiens2`; update `run()` signature |
19
+ | `formscout/pipeline.py` | Change `pose_model_path` param to `model_key` |
20
+ | `app.py` | Add `pose_model_dropdown`, fix `_map_inputs` + `process_video` |
21
+ | `requirements.txt` | Add `onnxruntime>=1.18` |
22
+ | `tests/test_pose2d.py` | Add mocked tests for each backend |
23
+
24
+ ---
25
+
26
+ ## Task 1: Add unified `POSE_MODELS` registry to `config.py`
27
+
28
+ **Files:**
29
+ - Modify: `formscout/config.py`
30
+
31
+ - [ ] **Step 1: Open `formscout/config.py` and replace the `YOLO_POSE_MODELS` block**
32
+
33
+ Replace lines 12–20 (the `YOLO_POSE_MODELS` dict and `YOLO_POSE_MODEL` / `YOLO_POSE_MODEL_HQ` lines) with:
34
+
35
+ ```python
36
+ _YOLO_DIR = ROOT / "checkpoints" / "yolo26"
37
+
38
+ POSE_MODELS: dict[str, dict] = {
39
+ # ── MediaPipe (Qualcomm HF, ONNX Runtime) ──────────────────────────────
40
+ "MediaPipe-Pose ⬇ ~16 MB, CPU-friendly": {
41
+ "backend": "mediapipe",
42
+ "hf_id": "qualcomm/MediaPipe-Pose-Estimation",
43
+ "params_m": 4.2,
44
+ },
45
+ # ── YOLO26 (local checkpoints) ─────────────────────────────────────────
46
+ "YOLO26n — nano (0.7M, fastest)": {
47
+ "backend": "yolo",
48
+ "path": str(_YOLO_DIR / "yolo26n-pose.pt"),
49
+ "params_m": 0.7,
50
+ },
51
+ "YOLO26s — small (3.5M)": {
52
+ "backend": "yolo",
53
+ "path": str(_YOLO_DIR / "yolo26s-pose.pt"),
54
+ "params_m": 3.5,
55
+ },
56
+ "YOLO26m — medium (9M)": {
57
+ "backend": "yolo",
58
+ "path": str(_YOLO_DIR / "yolo26m-pose.pt"),
59
+ "params_m": 9.0,
60
+ },
61
+ "YOLO26l — large (25.9M)": {
62
+ "backend": "yolo",
63
+ "path": str(_YOLO_DIR / "yolo26l-pose.pt"),
64
+ "params_m": 25.9,
65
+ },
66
+ "YOLO26x — extra-large (57.6M)": {
67
+ "backend": "yolo",
68
+ "path": str(_YOLO_DIR / "yolo26x-pose.pt"),
69
+ "params_m": 57.6,
70
+ },
71
+ # ── Sapiens2 (HF download, transformers) ─────────────────────────���─────
72
+ "Sapiens2-0.4B ⬇ ~1.6 GB": {
73
+ "backend": "sapiens2",
74
+ "hf_id": "facebook/sapiens2-pose-0.4b",
75
+ "params_m": 400,
76
+ },
77
+ "Sapiens2-0.8B ⬇ ~3.2 GB": {
78
+ "backend": "sapiens2",
79
+ "hf_id": "facebook/sapiens2-pose-0.8b",
80
+ "params_m": 800,
81
+ },
82
+ "Sapiens2-1B ⬇ ~4 GB": {
83
+ "backend": "sapiens2",
84
+ "hf_id": "facebook/sapiens2-pose-1b",
85
+ "params_m": 1000,
86
+ },
87
+ "Sapiens2-5B ⬇ ~20 GB, large GPU": {
88
+ "backend": "sapiens2",
89
+ "hf_id": "facebook/sapiens2-pose-5b",
90
+ "params_m": 5000,
91
+ },
92
+ }
93
+
94
+ DEFAULT_POSE_MODEL = "YOLO26n — nano (0.7M, fastest)"
95
+
96
+ # Backward-compat aliases — kept for any direct references outside the agent
97
+ YOLO_POSE_MODEL = str(_YOLO_DIR / "yolo26l-pose.pt")
98
+ YOLO_POSE_MODEL_HQ = str(_YOLO_DIR / "yolo26x-pose.pt")
99
+ ```
100
+
101
+ - [ ] **Step 2: Verify import is clean**
102
+
103
+ ```bash
104
+ python3 -c "from formscout import config; print(list(config.POSE_MODELS.keys()))"
105
+ ```
106
+
107
+ Expected: list of 10 model labels, starting with `MediaPipe-Pose...`
108
+
109
+ - [ ] **Step 3: Commit**
110
+
111
+ ```bash
112
+ git add formscout/config.py
113
+ git commit -m "feat: unified POSE_MODELS registry with MediaPipe, YOLO26 n-x, Sapiens2 0.4-5B"
114
+ git push
115
+ ```
116
+
117
+ ---
118
+
119
+ ## Task 2: Refactor `Pose2DAgent` — YOLO sub-runner + new `run()` signature
120
+
121
+ **Files:**
122
+ - Modify: `formscout/agents/pose2d.py`
123
+ - Modify: `tests/test_pose2d.py`
124
+
125
+ - [ ] **Step 1: Write failing test for the new `model_key` signature**
126
+
127
+ Add to `tests/test_pose2d.py`:
128
+
129
+ ```python
130
+ def test_run_accepts_model_key(pose2d_agent):
131
+ """run() must accept model_key kwarg, not model_path."""
132
+ import inspect
133
+ sig = inspect.signature(pose2d_agent.run)
134
+ assert "model_key" in sig.parameters
135
+ assert "model_path" not in sig.parameters
136
+ ```
137
+
138
+ - [ ] **Step 2: Run to confirm it fails**
139
+
140
+ ```bash
141
+ pytest tests/test_pose2d.py::TestPose2DAgent::test_run_accepts_model_key -v
142
+ ```
143
+
144
+ Expected: FAIL — `model_path` still present in signature.
145
+
146
+ - [ ] **Step 3: Rewrite `formscout/agents/pose2d.py`**
147
+
148
+ Replace the entire file with:
149
+
150
+ ```python
151
+ """
152
+ Pose2DAgent — 2D per-frame keypoint extraction.
153
+
154
+ Backends: yolo (local ONNX), mediapipe (Qualcomm HF/ONNX Runtime),
155
+ sapiens2 (Meta HF/transformers).
156
+ All backends output COCO-17 keypoints: dict[int, {x, y, conf}] per frame.
157
+
158
+ Input: IngestResult
159
+ Output: Pose2DResult(keypoints per frame, fps, confidence)
160
+ Failure: Pose2DResult(confidence=0.0, notes=<reason>) — never raises.
161
+ """
162
+ from __future__ import annotations
163
+
164
+ import logging
165
+ import numpy as np
166
+
167
+ from formscout import config
168
+ from formscout.types import IngestResult, Pose2DResult
169
+
170
+ logger = logging.getLogger(__name__)
171
+
172
+ COCO_KEYPOINTS = [
173
+ "nose", "left_eye", "right_eye", "left_ear", "right_ear",
174
+ "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
175
+ "left_wrist", "right_wrist", "left_hip", "right_hip",
176
+ "left_knee", "right_knee", "left_ankle", "right_ankle",
177
+ ]
178
+
179
+ # BlazePose-33 → COCO-17 index mapping
180
+ _BLAZEPOSE_TO_COCO: dict[int, int] = {
181
+ 0: 0, # nose
182
+ 1: 2, # left_eye (inner → left_eye)
183
+ 2: 1, # right_eye (inner → right_eye) — swapped: BlazePose 1=left_eye_inner
184
+ 3: 3, # left_ear
185
+ 4: 4, # right_ear
186
+ 5: 5, # left_shoulder → COCO left_shoulder... wait
187
+ # Correct BlazePose-33 COCO mapping (canonical):
188
+ # BlazePose idx : COCO idx
189
+ # 0 nose → COCO 0
190
+ # 2 left_eye → COCO 1
191
+ # 5 right_eye → COCO 2
192
+ # 7 left_ear → COCO 3
193
+ # 8 right_ear → COCO 4
194
+ # 11 left_shoulder → COCO 5
195
+ # 12 right_shoulder → COCO 6
196
+ # 13 left_elbow → COCO 7
197
+ # 14 right_elbow → COCO 8
198
+ # 15 left_wrist → COCO 9
199
+ # 16 right_wrist → COCO 10
200
+ # 23 left_hip → COCO 11
201
+ # 24 right_hip → COCO 12
202
+ # 25 left_knee → COCO 13
203
+ # 26 right_knee → COCO 14
204
+ # 27 left_ankle → COCO 15
205
+ # 28 right_ankle → COCO 16
206
+ }
207
+
208
+ # BlazePose source index → COCO target index (correct mapping, no duplicates)
209
+ _BP_SRC = [0, 2, 5, 7, 8, 11, 12, 13, 14, 15, 16, 23, 24, 25, 26, 27, 28]
210
+ _BP_DST = list(range(17)) # COCO 0..16
211
+
212
+ _model_cache: dict[str, object] = {}
213
+
214
+
215
+ # ── YOLO backend ─────────────────────────────────────────────────────────────
216
+
217
+ def _get_yolo(path: str) -> object:
218
+ if path not in _model_cache:
219
+ from ultralytics import YOLO
220
+ _model_cache[path] = YOLO(path)
221
+ return _model_cache[path]
222
+
223
+
224
+ def _run_yolo(frames: list, path: str) -> list[dict]:
225
+ model = _get_yolo(path)
226
+ out = []
227
+ for frame in frames:
228
+ try:
229
+ results = model(frame, verbose=False)
230
+ kps: dict[int, dict] = {}
231
+ if results and results[0].keypoints is not None:
232
+ kp = results[0].keypoints
233
+ if kp.xy is not None and len(kp.xy) > 0:
234
+ xy = kp.xy[0].cpu().numpy()
235
+ conf = kp.conf[0].cpu().numpy()
236
+ for j in range(min(len(xy), 17)):
237
+ kps[j] = {"x": float(xy[j, 0]), "y": float(xy[j, 1]), "conf": float(conf[j])}
238
+ out.append(kps)
239
+ except Exception:
240
+ out.append({})
241
+ return out
242
+
243
+
244
+ # ── MediaPipe backend ────────────────────────────────────────────────────────
245
+
246
+ def _get_mediapipe_sessions(hf_id: str):
247
+ """Return (detector_session, landmark_session) cached by hf_id."""
248
+ cache_key = f"mp:{hf_id}"
249
+ if cache_key not in _model_cache:
250
+ from huggingface_hub import snapshot_download
251
+ import onnxruntime as ort
252
+ from pathlib import Path
253
+
254
+ snap = Path(snapshot_download(hf_id))
255
+ onnx_files = sorted(snap.glob("**/*.onnx"), key=lambda p: p.stat().st_size)
256
+ if len(onnx_files) < 2:
257
+ raise RuntimeError(f"Expected 2 ONNX files in {snap}, found {len(onnx_files)}")
258
+ # Smaller file = pose detector; larger = pose landmark detector
259
+ det_sess = ort.InferenceSession(str(onnx_files[0]))
260
+ lmk_sess = ort.InferenceSession(str(onnx_files[-1]))
261
+ _model_cache[cache_key] = (det_sess, lmk_sess)
262
+ return _model_cache[cache_key]
263
+
264
+
265
+ def _preprocess_mediapipe(frame: np.ndarray, size: int = 256) -> np.ndarray:
266
+ """Resize to size×size, normalize to [0,1], add batch dim → (1,3,H,W)."""
267
+ import cv2
268
+ img = cv2.resize(frame, (size, size)).astype(np.float32) / 255.0
269
+ return img.transpose(2, 0, 1)[None] # (1, 3, 256, 256)
270
+
271
+
272
+ def _run_mediapipe(frames: list, hf_id: str) -> list[dict]:
273
+ try:
274
+ det_sess, lmk_sess = _get_mediapipe_sessions(hf_id)
275
+ except Exception as e:
276
+ logger.warning("mediapipe load failed: %s", e)
277
+ return [{} for _ in frames]
278
+
279
+ import cv2
280
+ h_orig, w_orig = frames[0].shape[:2] if frames else (480, 640)
281
+ out = []
282
+
283
+ for frame in frames:
284
+ try:
285
+ h, w = frame.shape[:2]
286
+ inp = _preprocess_mediapipe(frame)
287
+
288
+ # Run landmark detector directly on full frame (single-person FMS use-case)
289
+ lmk_input_name = lmk_sess.get_inputs()[0].name
290
+ lmk_out = lmk_sess.run(None, {lmk_input_name: inp})
291
+
292
+ # lmk_out[0] shape: (1, 33, 3) — [x, y, visibility] normalized 0..1
293
+ landmarks = lmk_out[0][0] # (33, 3)
294
+
295
+ kps: dict[int, dict] = {}
296
+ for coco_idx, bp_idx in zip(_BP_DST, _BP_SRC):
297
+ if bp_idx < len(landmarks):
298
+ lm = landmarks[bp_idx]
299
+ kps[coco_idx] = {
300
+ "x": float(lm[0] * w),
301
+ "y": float(lm[1] * h),
302
+ "conf": float(lm[2]), # visibility score
303
+ }
304
+ out.append(kps)
305
+ except Exception:
306
+ out.append({})
307
+
308
+ return out
309
+
310
+
311
+ # ── Sapiens2 backend ─────────────────────────────────────────────────────────
312
+
313
+ # COCO-17 keypoint names in order (used to map Sapiens2 named output → COCO index)
314
+ _COCO_NAMES = [
315
+ "nose", "left_eye", "right_eye", "left_ear", "right_ear",
316
+ "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
317
+ "left_wrist", "right_wrist", "left_hip", "right_hip",
318
+ "left_knee", "right_knee", "left_ankle", "right_ankle",
319
+ ]
320
+
321
+
322
+ def _get_sapiens2(hf_id: str) -> object:
323
+ if hf_id not in _model_cache:
324
+ from transformers import pipeline as hf_pipeline
325
+ _model_cache[hf_id] = hf_pipeline("pose-estimation", model=hf_id)
326
+ return _model_cache[hf_id]
327
+
328
+
329
+ def _run_sapiens2(frames: list, hf_id: str) -> list[dict]:
330
+ try:
331
+ pipe = _get_sapiens2(hf_id)
332
+ except Exception as e:
333
+ logger.warning("sapiens2 load failed: %s", e)
334
+ return [{} for _ in frames]
335
+
336
+ from PIL import Image
337
+ out = []
338
+
339
+ for frame in frames:
340
+ try:
341
+ pil_img = Image.fromarray(frame)
342
+ result = pipe(pil_img)
343
+
344
+ # result is a list of person dicts; take the first (highest confidence)
345
+ if not result:
346
+ out.append({})
347
+ continue
348
+
349
+ person = result[0]
350
+ keypoints = person.get("keypoints", [])
351
+ scores = person.get("keypoint_scores", [])
352
+
353
+ # Build name→(x,y,score) lookup from pipeline output
354
+ kp_lookup: dict[str, tuple] = {}
355
+ for i, kp in enumerate(keypoints):
356
+ name = kp.get("label", "") if isinstance(kp, dict) else ""
357
+ x = kp.get("x", 0.0) if isinstance(kp, dict) else float(kp[0])
358
+ y = kp.get("y", 0.0) if isinstance(kp, dict) else float(kp[1])
359
+ score = scores[i] if i < len(scores) else 0.0
360
+ if name:
361
+ kp_lookup[name] = (x, y, float(score))
362
+
363
+ kps: dict[int, dict] = {}
364
+ for coco_idx, name in enumerate(_COCO_NAMES):
365
+ if name in kp_lookup:
366
+ x, y, s = kp_lookup[name]
367
+ kps[coco_idx] = {"x": x, "y": y, "conf": s}
368
+
369
+ out.append(kps)
370
+ except Exception:
371
+ out.append({})
372
+
373
+ return out
374
+
375
+
376
+ # ── Agent ────────────────────────────────────────────────────────────────────
377
+
378
+ class Pose2DAgent:
379
+ """Extracts COCO-17 keypoints per frame; dispatches to YOLO, MediaPipe, or Sapiens2."""
380
+
381
+ def run(self, ingest: IngestResult, model_key: str | None = None) -> Pose2DResult:
382
+ if not ingest.frames:
383
+ return Pose2DResult(keypoints=[], fps=ingest.fps, confidence=0.0, notes="no frames in ingest")
384
+
385
+ key = model_key or config.DEFAULT_POSE_MODEL
386
+ spec = config.POSE_MODELS.get(key)
387
+ if spec is None:
388
+ logger.warning("Unknown model_key %r — falling back to %s", key, config.DEFAULT_POSE_MODEL)
389
+ spec = config.POSE_MODELS[config.DEFAULT_POSE_MODEL]
390
+
391
+ backend = spec["backend"]
392
+ try:
393
+ if backend == "yolo":
394
+ kps_per_frame = _run_yolo(ingest.frames, spec["path"])
395
+ elif backend == "mediapipe":
396
+ kps_per_frame = _run_mediapipe(ingest.frames, spec["hf_id"])
397
+ elif backend == "sapiens2":
398
+ kps_per_frame = _run_sapiens2(ingest.frames, spec["hf_id"])
399
+ else:
400
+ return Pose2DResult(
401
+ keypoints=[{} for _ in ingest.frames],
402
+ fps=ingest.fps, confidence=0.0,
403
+ notes=f"unknown backend: {backend}",
404
+ )
405
+ except Exception as e:
406
+ return Pose2DResult(
407
+ keypoints=[{} for _ in ingest.frames],
408
+ fps=ingest.fps, confidence=0.0,
409
+ notes=str(e),
410
+ )
411
+
412
+ n_detected = sum(1 for f in kps_per_frame if f)
413
+ total_conf = sum(
414
+ sum(kp["conf"] for kp in f.values()) / len(f)
415
+ for f in kps_per_frame if f
416
+ )
417
+ overall_conf = (total_conf / n_detected) if n_detected > 0 else 0.0
418
+ notes = "" if n_detected > 0 else "no person detected in any frame"
419
+
420
+ return Pose2DResult(
421
+ keypoints=kps_per_frame,
422
+ fps=ingest.fps,
423
+ confidence=overall_conf,
424
+ notes=notes,
425
+ )
426
+ ```
427
+
428
+ - [ ] **Step 4: Run the new signature test**
429
+
430
+ ```bash
431
+ pytest tests/test_pose2d.py::TestPose2DAgent::test_run_accepts_model_key -v
432
+ ```
433
+
434
+ Expected: PASS
435
+
436
+ - [ ] **Step 5: Run full existing pose2d test suite**
437
+
438
+ ```bash
439
+ pytest tests/test_pose2d.py -v
440
+ ```
441
+
442
+ Expected: all existing tests pass (they will skip if YOLO model unavailable in env — that's OK).
443
+
444
+ - [ ] **Step 6: Commit and push**
445
+
446
+ ```bash
447
+ git add formscout/agents/pose2d.py tests/test_pose2d.py
448
+ git commit -m "feat: Pose2DAgent — three backends (yolo/mediapipe/sapiens2), model_key dispatch"
449
+ git push
450
+ ```
451
+
452
+ ---
453
+
454
+ ## Task 3: Add `onnxruntime` to requirements
455
+
456
+ **Files:**
457
+ - Modify: `requirements.txt`
458
+
459
+ - [ ] **Step 1: Add onnxruntime**
460
+
461
+ Open `requirements.txt` and add after the existing `transformers` line:
462
+
463
+ ```
464
+ onnxruntime>=1.18
465
+ ```
466
+
467
+ - [ ] **Step 2: Verify it installs**
468
+
469
+ ```bash
470
+ pip install onnxruntime --quiet && python3 -c "import onnxruntime; print(onnxruntime.__version__)"
471
+ ```
472
+
473
+ Expected: version string printed, no errors.
474
+
475
+ - [ ] **Step 3: Commit and push**
476
+
477
+ ```bash
478
+ git add requirements.txt
479
+ git commit -m "chore: add onnxruntime for MediaPipe ONNX backend"
480
+ git push
481
+ ```
482
+
483
+ ---
484
+
485
+ ## Task 4: Update `Director.run()` — `pose_model_path` → `model_key`
486
+
487
+ **Files:**
488
+ - Modify: `formscout/pipeline.py`
489
+
490
+ - [ ] **Step 1: Update the signature and the `pose2d` call**
491
+
492
+ In `formscout/pipeline.py`, change `Director.run()`:
493
+
494
+ ```python
495
+ def run(self, video_path: str, test_name: str = "deep_squat", side: str = "na", model_key: str | None = None) -> PipelineState:
496
+ """
497
+ Run the full pipeline on a single video.
498
+ test_name/side serve as manual override when provided (skips classifier).
499
+ model_key selects the pose backend (see config.POSE_MODELS).
500
+ """
501
+ state = PipelineState(video_path=video_path)
502
+
503
+ # ─── Ingest ───
504
+ state.ingest = self._ingest.run(video_path)
505
+ if state.ingest.confidence < config.MIN_CONFIDENCE:
506
+ state.errors.append("ingest: low confidence — video may be corrupt")
507
+ return state
508
+
509
+ # ─── Pose 2D ───
510
+ state.pose2d = self._pose2d.run(state.ingest, model_key=model_key)
511
+ # ... rest of method unchanged
512
+ ```
513
+
514
+ (Only the signature line and the `self._pose2d.run(...)` call change — everything else stays the same.)
515
+
516
+ - [ ] **Step 2: Verify import is clean**
517
+
518
+ ```bash
519
+ python3 -c "from formscout.pipeline import Director; d = Director(); print('ok')"
520
+ ```
521
+
522
+ Expected: `ok` (models load lazily so no crash here).
523
+
524
+ - [ ] **Step 3: Commit and push**
525
+
526
+ ```bash
527
+ git add formscout/pipeline.py
528
+ git commit -m "feat: Director.run() accepts model_key, threads to Pose2DAgent"
529
+ git push
530
+ ```
531
+
532
+ ---
533
+
534
+ ## Task 5: Wire the UI — pose model dropdown in `app.py`
535
+
536
+ **Files:**
537
+ - Modify: `app.py`
538
+
539
+ - [ ] **Step 1: Update `process_video` to use `model_key` and the unified registry**
540
+
541
+ Replace the existing `process_video` function signature and the old `YOLO_POSE_MODELS.get()` lookup:
542
+
543
+ ```python
544
+ def process_video(video_path: str, test_name: str, side: str, model_key: str):
545
+ """Process an uploaded video through the FormScout pipeline."""
546
+ if not video_path:
547
+ return (
548
+ _render_empty_state(),
549
+ "Upload a video to begin analysis.",
550
+ "",
551
+ "",
552
+ )
553
+
554
+ director = Director()
555
+ state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
556
+ ```
557
+
558
+ (Remove the `pose_model_path = config.YOLO_POSE_MODELS.get(...)` line entirely.)
559
+
560
+ - [ ] **Step 2: Add the `pose_model_dropdown` in `build_app()`**
561
+
562
+ Inside `build_app()`, after the `side_dropdown` block (around line 265) and before `submit_btn`, add:
563
+
564
+ ```python
565
+ pose_model_dropdown = gr.Dropdown(
566
+ choices=list(config.POSE_MODELS.keys()),
567
+ value=config.DEFAULT_POSE_MODEL,
568
+ label="Pose Model",
569
+ )
570
+ ```
571
+
572
+ - [ ] **Step 3: Update `_map_inputs` to pass the model key**
573
+
574
+ Replace the existing `_map_inputs` closure:
575
+
576
+ ```python
577
+ def _map_inputs(video, test_display_name, side_display, pose_model_key):
578
+ """Map UI display values to internal values."""
579
+ test_map = {name: val for name, val in FMS_TESTS}
580
+ test_name = test_map.get(test_display_name, "deep_squat")
581
+ side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
582
+ return process_video(video, test_name, side, pose_model_key)
583
+ ```
584
+
585
+ - [ ] **Step 4: Update `submit_btn.click` to include `pose_model_dropdown`**
586
+
587
+ Replace the existing `.click(...)` call:
588
+
589
+ ```python
590
+ submit_btn.click(
591
+ fn=_map_inputs,
592
+ inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown],
593
+ outputs=[score_html, pipeline_md, score_details, alerts_md],
594
+ )
595
+ ```
596
+
597
+ - [ ] **Step 5: Smoke-test the app starts**
598
+
599
+ ```bash
600
+ python3 -c "from app import build_app; app = build_app(); print('app built ok')"
601
+ ```
602
+
603
+ Expected: `app built ok` — no import or config errors.
604
+
605
+ - [ ] **Step 6: Commit and push**
606
+
607
+ ```bash
608
+ git add app.py
609
+ git commit -m "feat: pose model dropdown in UI, wired through process_video → Director"
610
+ git push
611
+ ```
612
+
613
+ ---
614
+
615
+ ## Task 6: Add mocked backend tests
616
+
617
+ **Files:**
618
+ - Modify: `tests/test_pose2d.py`
619
+
620
+ - [ ] **Step 1: Add mocked YOLO test**
621
+
622
+ Append to `tests/test_pose2d.py`:
623
+
624
+ ```python
625
+ import unittest.mock as mock
626
+ import numpy as np
627
+ from formscout.types import IngestResult, Pose2DResult
628
+
629
+
630
+ def _blank_ingest_3():
631
+ frames = [np.zeros((480, 640, 3), dtype=np.uint8) for _ in range(3)]
632
+ return IngestResult(frames=frames, fps=30.0, duration=0.1, n_people=1, width=640, height=480)
633
+
634
+
635
+ class TestPose2DBackendsMocked:
636
+ """Backend dispatch tests — no real model downloads."""
637
+
638
+ def test_yolo_backend_dispatches(self):
639
+ from formscout.agents.pose2d import Pose2DAgent, _run_yolo
640
+ fake_kps = [{0: {"x": 10.0, "y": 20.0, "conf": 0.9}} for _ in range(3)]
641
+ with mock.patch("formscout.agents.pose2d._run_yolo", return_value=fake_kps) as m:
642
+ agent = Pose2DAgent()
643
+ result = agent.run(_blank_ingest_3(), model_key="YOLO26n — nano (0.7M, fastest)")
644
+ m.assert_called_once()
645
+ assert isinstance(result, Pose2DResult)
646
+ assert len(result.keypoints) == 3
647
+ assert result.confidence > 0.0
648
+
649
+ def test_mediapipe_backend_dispatches(self):
650
+ from formscout.agents.pose2d import Pose2DAgent
651
+ fake_kps = [{i: {"x": float(i), "y": float(i), "conf": 0.8} for i in range(17)} for _ in range(3)]
652
+ with mock.patch("formscout.agents.pose2d._run_mediapipe", return_value=fake_kps) as m:
653
+ agent = Pose2DAgent()
654
+ result = agent.run(_blank_ingest_3(), model_key="MediaPipe-Pose ⬇ ~16 MB, CPU-friendly")
655
+ m.assert_called_once()
656
+ assert isinstance(result, Pose2DResult)
657
+ assert len(result.keypoints) == 3
658
+ assert all(len(f) == 17 for f in result.keypoints)
659
+
660
+ def test_sapiens2_backend_dispatches(self):
661
+ from formscout.agents.pose2d import Pose2DAgent
662
+ fake_kps = [{i: {"x": float(i), "y": float(i), "conf": 0.85} for i in range(17)} for _ in range(3)]
663
+ with mock.patch("formscout.agents.pose2d._run_sapiens2", return_value=fake_kps) as m:
664
+ agent = Pose2DAgent()
665
+ result = agent.run(_blank_ingest_3(), model_key="Sapiens2-0.4B ⬇ ~1.6 GB")
666
+ m.assert_called_once()
667
+ assert isinstance(result, Pose2DResult)
668
+ assert len(result.keypoints) == 3
669
+
670
+ def test_unknown_model_key_falls_back(self):
671
+ from formscout.agents.pose2d import Pose2DAgent
672
+ fake_kps = [{0: {"x": 1.0, "y": 2.0, "conf": 0.7}} for _ in range(3)]
673
+ with mock.patch("formscout.agents.pose2d._run_yolo", return_value=fake_kps):
674
+ agent = Pose2DAgent()
675
+ result = agent.run(_blank_ingest_3(), model_key="nonexistent-model-xyz")
676
+ assert isinstance(result, Pose2DResult) # graceful fallback, no crash
677
+
678
+ def test_confidence_zero_on_empty_keypoints(self):
679
+ from formscout.agents.pose2d import Pose2DAgent
680
+ with mock.patch("formscout.agents.pose2d._run_yolo", return_value=[{}, {}, {}]):
681
+ agent = Pose2DAgent()
682
+ result = agent.run(_blank_ingest_3(), model_key="YOLO26n — nano (0.7M, fastest)")
683
+ assert result.confidence == 0.0
684
+ assert "no person" in result.notes.lower()
685
+ ```
686
+
687
+ - [ ] **Step 2: Run the new tests**
688
+
689
+ ```bash
690
+ pytest tests/test_pose2d.py::TestPose2DBackendsMocked -v
691
+ ```
692
+
693
+ Expected: all 5 tests PASS.
694
+
695
+ - [ ] **Step 3: Run the full test suite to check for regressions**
696
+
697
+ ```bash
698
+ pytest tests/ -v --tb=short 2>&1 | tail -30
699
+ ```
700
+
701
+ Expected: same pass/fail ratio as before (45/46 known passing). The one known failure (`test_unimplemented_test_returns_low_confidence`) is pre-existing — ignore it.
702
+
703
+ - [ ] **Step 4: Commit and push**
704
+
705
+ ```bash
706
+ git add tests/test_pose2d.py
707
+ git commit -m "test: mocked backend dispatch tests for YOLO, MediaPipe, Sapiens2"
708
+ git push
709
+ ```
710
+
711
+ ---
712
+
713
+ ## Self-review
714
+
715
+ **Spec coverage:**
716
+ - ✅ Unified `POSE_MODELS` registry (Task 1)
717
+ - ✅ `DEFAULT_POSE_MODEL = YOLO26n` (Task 1)
718
+ - ✅ Backward-compat `YOLO_POSE_MODEL` / `YOLO_POSE_MODEL_HQ` aliases (Task 1)
719
+ - ✅ `_run_yolo` sub-runner (Task 2)
720
+ - ✅ `_run_mediapipe` with ONNX Runtime + BlazePose→COCO-17 mapping (Task 2)
721
+ - ✅ `_run_sapiens2` with transformers pipeline + named-keypoint→COCO-17 mapping (Task 2)
722
+ - ✅ `Pose2DAgent.run(model_key)` dispatch + fallback on unknown key (Task 2)
723
+ - ✅ `onnxruntime` added to requirements (Task 3)
724
+ - ✅ `Director.run(model_key)` threads key to agent (Task 4)
725
+ - ✅ `pose_model_dropdown` in UI (Task 5)
726
+ - ✅ `_map_inputs` + `submit_btn.click` wired (Task 5)
727
+ - ✅ Error handling: unknown key → warning + fallback; download failure → confidence=0 (Task 2)
728
+ - ✅ Mocked tests for all three backends (Task 6)
729
+
730
+ **Placeholder scan:** None found.
731
+
732
+ **Type consistency:** `model_key: str | None` used consistently across `Pose2DAgent.run`, `Director.run`, `process_video`. `config.POSE_MODELS` and `config.DEFAULT_POSE_MODEL` referenced consistently.
733
+
734
+ **Note on Sapiens2 keypoint format:** The `_run_sapiens2` implementation uses **named keypoint lookup** (by label string) rather than assuming fixed indices 0–16 = COCO. This is the safe approach — the transformers pipeline returns labeled keypoints and the code maps by name. If the pipeline returns unnamed keypoints (index-only), the `kp_lookup` will be empty and the frame will gracefully return `{}`.
docs/superpowers/plans/2026-06-09-pose-visualizer.md CHANGED
@@ -1,914 +1,914 @@
1
- # Pose Overlay Visualizer Implementation Plan
2
-
3
- > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
4
-
5
- **Goal:** Add a pose overlay video output to FormScout with skeleton, motion trails, and velocity arrows, plus a per-joint velocity summary table.
6
-
7
- **Architecture:** A new `formscout/agents/visualizer.py` runs after `director.run()` in `process_video()`; it uses Kalman-filtered per-joint velocity and OpenCV rendering. `app.py` gains a `gr.CheckboxGroup` for layer selection, a new `gr.Video` output tab, and a `gr.Markdown` velocity summary.
8
-
9
- **Tech Stack:** `opencv-python`, `numpy`, `colorsys` (stdlib), `gradio`.
10
-
11
- ---
12
-
13
- ## File map
14
-
15
- | File | Change |
16
- |---|---|
17
- | `formscout/agents/visualizer.py` | Create — Kalman filter, velocity, PoseVisualizer, summary |
18
- | `tests/test_visualizer.py` | Create — all visualizer tests |
19
- | `app.py` | Modify — overlay_layers checkbox, new tab, wiring |
20
-
21
- ---
22
-
23
- ## Task 1: `SimpleKalmanFilter` + `compute_joint_velocity`
24
-
25
- **Files:**
26
- - Create: `formscout/agents/visualizer.py`
27
- - Create: `tests/test_visualizer.py`
28
-
29
- - [ ] **Step 1: Write failing tests**
30
-
31
- Create `tests/test_visualizer.py`:
32
-
33
- ```python
34
- """Tests for PoseVisualizer — no GPU, no model downloads."""
35
- import numpy as np
36
- import pytest
37
- from formscout.types import IngestResult, Pose2DResult
38
-
39
-
40
- def _make_ingest(n=5, h=480, w=640, fps=30.0):
41
- frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
42
- return IngestResult(frames=frames, fps=fps, duration=n/fps, n_people=1, width=w, height=h)
43
-
44
-
45
- def _make_pose(n=5, w=640, h=480):
46
- """Synthetic Pose2DResult: 17 joints at fixed pixel positions, conf=0.9."""
47
- kps_per_frame = []
48
- for i in range(n):
49
- frame_kps = {}
50
- for j in range(17):
51
- frame_kps[j] = {
52
- "x": float(50 + j * 30 + i * 2), # slight movement each frame
53
- "y": float(100 + j * 20),
54
- "conf": 0.9,
55
- }
56
- kps_per_frame.append(frame_kps)
57
- return Pose2DResult(keypoints=kps_per_frame, fps=30.0, confidence=0.9, notes="")
58
-
59
-
60
- class TestComputeJointVelocity:
61
- def test_returns_17_joints(self):
62
- from formscout.agents.visualizer import compute_joint_velocity
63
- pose = _make_pose(n=5)
64
- result = compute_joint_velocity(pose.keypoints, fps=30.0)
65
- assert len(result) == 17
66
-
67
- def test_each_list_has_n_frames(self):
68
- from formscout.agents.visualizer import compute_joint_velocity
69
- pose = _make_pose(n=5)
70
- result = compute_joint_velocity(pose.keypoints, fps=30.0)
71
- for joint_idx, speeds in result.items():
72
- assert len(speeds) == 5, f"joint {joint_idx} has {len(speeds)} speeds, expected 5"
73
-
74
- def test_speeds_are_non_negative(self):
75
- from formscout.agents.visualizer import compute_joint_velocity
76
- pose = _make_pose(n=5)
77
- result = compute_joint_velocity(pose.keypoints, fps=30.0)
78
- for speeds in result.values():
79
- assert all(s >= 0.0 for s in speeds)
80
-
81
- def test_missing_keypoints_give_zero_speed(self):
82
- from formscout.agents.visualizer import compute_joint_velocity
83
- # All frames empty
84
- empty_kps = [{} for _ in range(5)]
85
- result = compute_joint_velocity(empty_kps, fps=30.0)
86
- for speeds in result.values():
87
- assert all(s == 0.0 for s in speeds)
88
- ```
89
-
90
- - [ ] **Step 2: Run to confirm failure**
91
-
92
- ```bash
93
- pytest tests/test_visualizer.py::TestComputeJointVelocity -v
94
- ```
95
-
96
- Expected: `ERROR` — `ModuleNotFoundError: No module named 'formscout.agents.visualizer'`
97
-
98
- - [ ] **Step 3: Create `formscout/agents/visualizer.py` with Kalman + velocity**
99
-
100
- ```python
101
- """
102
- PoseVisualizer — annotated overlay video with skeleton, trails, velocity arrows.
103
-
104
- Input: IngestResult + Pose2DResult
105
- Output: .mp4 path (or None on failure/empty layers)
106
- Failure: returns None, never raises.
107
- """
108
- from __future__ import annotations
109
-
110
- import colorsys
111
- import logging
112
- import math
113
- import tempfile
114
- from collections import deque
115
-
116
- import cv2
117
- import numpy as np
118
-
119
- logger = logging.getLogger(__name__)
120
-
121
- # ── COCO constants ────────────────────────────────────────────────────────────
122
-
123
- COCO_KEYPOINTS = [
124
- "nose", "left_eye", "right_eye", "left_ear", "right_ear",
125
- "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
126
- "left_wrist", "right_wrist", "left_hip", "right_hip",
127
- "left_knee", "right_knee", "left_ankle", "right_ankle",
128
- ]
129
-
130
- COCO_SKELETON = [
131
- (0, 1), (0, 2), (1, 3), (2, 4), # face
132
- (5, 6), (5, 7), (7, 9), (6, 8), (8, 10), # arms
133
- (5, 11), (6, 12), (11, 12), # torso
134
- (11, 13), (13, 15), (12, 14), (14, 16), # legs
135
- ]
136
-
137
- TRAIL_LENGTH = 10
138
- MAX_ARROW_PX = 40
139
- CONF_THRESHOLD = 0.3
140
-
141
-
142
- # ── Kalman filter ─────────────────────────────────────────────────────────────
143
-
144
- class SimpleKalmanFilter:
145
- """4-state Kalman filter (x, y, vx, vy) for joint tracking."""
146
-
147
- def __init__(self, process_noise: float = 0.01, measurement_noise: float = 0.1):
148
- self.is_initialized = False
149
- self.state = np.zeros(4)
150
- self.cov = np.eye(4) * 0.1
151
- self.Q = np.eye(4) * process_noise
152
- self.R = np.eye(2) * measurement_noise
153
- self.H = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], dtype=float)
154
-
155
- def predict(self, dt: float = 1.0):
156
- F = np.array([[1, 0, dt, 0], [0, 1, 0, dt], [0, 0, 1, 0], [0, 0, 0, 1]], dtype=float)
157
- self.state = F @ self.state
158
- self.cov = F @ self.cov @ F.T + self.Q
159
-
160
- def update(self, x: float, y: float):
161
- z = np.array([x, y])
162
- if not self.is_initialized:
163
- self.state[:2] = z
164
- self.is_initialized = True
165
- return
166
- S = self.H @ self.cov @ self.H.T + self.R
167
- K = self.cov @ self.H.T @ np.linalg.inv(S)
168
- self.state = self.state + K @ (z - self.H @ self.state)
169
- self.cov = (np.eye(4) - K @ self.H) @ self.cov
170
-
171
- def velocity_magnitude(self) -> float:
172
- vx, vy = self.state[2], self.state[3]
173
- return math.sqrt(vx * vx + vy * vy)
174
-
175
- def velocity_vector(self) -> tuple[float, float]:
176
- return float(self.state[2]), float(self.state[3])
177
-
178
-
179
- # ── Velocity computation ──────────────────────────────────────────────────────
180
-
181
- def compute_joint_velocity(
182
- keypoints_per_frame: list[dict],
183
- fps: float,
184
- ) -> dict[int, list[float]]:
185
- """
186
- Compute Kalman-filtered per-joint speed (px/s) for each frame.
187
-
188
- Returns dict[joint_idx, [speed_frame0, speed_frame1, ...]] for all 17 COCO joints.
189
- Missing/low-confidence keypoints yield speed=0.0 for that frame.
190
- """
191
- dt = 1.0 / fps if fps > 0 else 1.0
192
- filters: dict[int, SimpleKalmanFilter] = {j: SimpleKalmanFilter() for j in range(17)}
193
- result: dict[int, list[float]] = {j: [] for j in range(17)}
194
-
195
- for frame_kps in keypoints_per_frame:
196
- for j in range(17):
197
- kf = filters[j]
198
- kp = frame_kps.get(j)
199
- kf.predict(dt)
200
- if kp and kp.get("conf", 0.0) >= CONF_THRESHOLD:
201
- kf.update(kp["x"], kp["y"])
202
- speed = kf.velocity_magnitude()
203
- else:
204
- speed = 0.0
205
- result[j].append(speed)
206
-
207
- return result
208
- ```
209
-
210
- - [ ] **Step 4: Run tests**
211
-
212
- ```bash
213
- pytest tests/test_visualizer.py::TestComputeJointVelocity -v
214
- ```
215
-
216
- Expected: 4 PASS
217
-
218
- - [ ] **Step 5: Commit**
219
-
220
- ```bash
221
- git add formscout/agents/visualizer.py tests/test_visualizer.py
222
- git commit -m "feat: SimpleKalmanFilter + compute_joint_velocity (4 tests pass)"
223
- ```
224
-
225
- ---
226
-
227
- ## Task 2: `PoseVisualizer._draw_skeleton`
228
-
229
- **Files:**
230
- - Modify: `formscout/agents/visualizer.py`
231
- - Modify: `tests/test_visualizer.py`
232
-
233
- - [ ] **Step 1: Write failing test**
234
-
235
- Append to `tests/test_visualizer.py`:
236
-
237
- ```python
238
- class TestDrawSkeleton:
239
- def test_skeleton_draws_without_error(self):
240
- from formscout.agents.visualizer import PoseVisualizer
241
- vis = PoseVisualizer()
242
- frame = np.zeros((480, 640, 3), dtype=np.uint8)
243
- kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
244
- for j in range(17)}
245
- result = vis._draw_skeleton(frame.copy(), kps)
246
- assert result.shape == frame.shape
247
- # Frame must be modified (not all zeros after drawing)
248
- assert not np.array_equal(result, frame)
249
-
250
- def test_low_confidence_keypoints_not_drawn(self):
251
- from formscout.agents.visualizer import PoseVisualizer
252
- vis = PoseVisualizer()
253
- frame = np.zeros((480, 640, 3), dtype=np.uint8)
254
- # All keypoints below threshold
255
- kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.1} for j in range(17)}
256
- result = vis._draw_skeleton(frame.copy(), kps)
257
- # Nothing drawn — frame stays all zeros
258
- assert np.array_equal(result, frame)
259
- ```
260
-
261
- - [ ] **Step 2: Run to confirm failure**
262
-
263
- ```bash
264
- pytest tests/test_visualizer.py::TestDrawSkeleton -v
265
- ```
266
-
267
- Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute '_draw_skeleton'`
268
-
269
- - [ ] **Step 3: Add `PoseVisualizer` class with `_draw_skeleton` to `visualizer.py`**
270
-
271
- Append after `compute_joint_velocity`:
272
-
273
- ```python
274
- # ── Helpers ───────────────────────────────────────────────────────────────────
275
-
276
- def _conf_to_bgr(conf: float) -> tuple[int, int, int]:
277
- """Map confidence 0→1 to BGR color red→green via HSV."""
278
- hue = conf * 120.0 / 360.0
279
- r, g, b = colorsys.hsv_to_rgb(hue, 1.0, 1.0)
280
- return (int(b * 255), int(g * 255), int(r * 255))
281
-
282
-
283
- # ── PoseVisualizer ────────────────────────────────────────────────────────────
284
-
285
- class PoseVisualizer:
286
- """Renders skeleton, trails, and velocity arrows onto video frames."""
287
-
288
- def __init__(self):
289
- self.last_velocities: dict[int, list[float]] = {}
290
-
291
- # ── Skeleton ──────────────────────────────────────────────────────────────
292
-
293
- def _draw_skeleton(self, frame: np.ndarray, kps: dict) -> np.ndarray:
294
- """Draw COCO-17 bones (white) and joints (confidence-colored) onto frame."""
295
- visible = {j: kp for j, kp in kps.items() if kp.get("conf", 0.0) >= CONF_THRESHOLD}
296
-
297
- # Bones
298
- for j1, j2 in COCO_SKELETON:
299
- if j1 in visible and j2 in visible:
300
- p1 = (int(visible[j1]["x"]), int(visible[j1]["y"]))
301
- p2 = (int(visible[j2]["x"]), int(visible[j2]["y"]))
302
- cv2.line(frame, p1, p2, (255, 255, 255), 2)
303
-
304
- # Joints
305
- for j, kp in visible.items():
306
- pt = (int(kp["x"]), int(kp["y"]))
307
- color = _conf_to_bgr(kp["conf"])
308
- cv2.circle(frame, pt, 4, color, -1)
309
- cv2.circle(frame, pt, 5, (255, 255, 255), 1)
310
-
311
- return frame
312
- ```
313
-
314
- - [ ] **Step 4: Run tests**
315
-
316
- ```bash
317
- pytest tests/test_visualizer.py::TestDrawSkeleton -v
318
- ```
319
-
320
- Expected: 2 PASS
321
-
322
- - [ ] **Step 5: Commit**
323
-
324
- ```bash
325
- git add formscout/agents/visualizer.py tests/test_visualizer.py
326
- git commit -m "feat: PoseVisualizer._draw_skeleton with confidence-colored joints"
327
- ```
328
-
329
- ---
330
-
331
- ## Task 3: `PoseVisualizer._draw_trails`
332
-
333
- **Files:**
334
- - Modify: `formscout/agents/visualizer.py`
335
- - Modify: `tests/test_visualizer.py`
336
-
337
- - [ ] **Step 1: Write failing test**
338
-
339
- Append to `tests/test_visualizer.py`:
340
-
341
- ```python
342
- class TestDrawTrails:
343
- def test_trails_draw_without_error(self):
344
- from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
345
- from collections import deque
346
- vis = PoseVisualizer()
347
- frame = np.zeros((480, 640, 3), dtype=np.uint8)
348
- # Build a trail history for joint 0 with 5 positions
349
- trail_history = {
350
- 0: deque([(100 + i * 5, 200 + i * 3) for i in range(5)], maxlen=TRAIL_LENGTH)
351
- }
352
- result = vis._draw_trails(frame.copy(), trail_history)
353
- assert result.shape == frame.shape
354
- # Trail should modify at least some pixels
355
- assert not np.array_equal(result, frame)
356
-
357
- def test_short_trail_no_crash(self):
358
- from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
359
- from collections import deque
360
- vis = PoseVisualizer()
361
- frame = np.zeros((480, 640, 3), dtype=np.uint8)
362
- # Only one point — no line possible
363
- trail_history = {0: deque([(100, 200)], maxlen=TRAIL_LENGTH)}
364
- result = vis._draw_trails(frame.copy(), trail_history)
365
- # No crash, frame unchanged (single point = no segment)
366
- assert np.array_equal(result, frame)
367
- ```
368
-
369
- - [ ] **Step 2: Run to confirm failure**
370
-
371
- ```bash
372
- pytest tests/test_visualizer.py::TestDrawTrails -v
373
- ```
374
-
375
- Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute '_draw_trails'`
376
-
377
- - [ ] **Step 3: Add `_draw_trails` to `PoseVisualizer`**
378
-
379
- Inside the `PoseVisualizer` class, after `_draw_skeleton`:
380
-
381
- ```python
382
- # ── Trails ───────────────────────────────────────────────────────────────
383
-
384
- def _draw_trails(self, frame: np.ndarray, trail_history: dict) -> np.ndarray:
385
- """Draw fading motion trails for each joint."""
386
- for joint_idx, trail in trail_history.items():
387
- pts = list(trail)
388
- if len(pts) < 2:
389
- continue
390
- for i in range(1, len(pts)):
391
- alpha = i / len(pts)
392
- brightness = int(255 * alpha)
393
- color = (brightness, brightness, brightness)
394
- thickness = max(1, int(3 * alpha))
395
- p1 = (int(pts[i - 1][0]), int(pts[i - 1][1]))
396
- p2 = (int(pts[i][0]), int(pts[i][1]))
397
- cv2.line(frame, p1, p2, color, thickness)
398
- return frame
399
- ```
400
-
401
- - [ ] **Step 4: Run tests**
402
-
403
- ```bash
404
- pytest tests/test_visualizer.py::TestDrawTrails -v
405
- ```
406
-
407
- Expected: 2 PASS
408
-
409
- - [ ] **Step 5: Commit**
410
-
411
- ```bash
412
- git add formscout/agents/visualizer.py tests/test_visualizer.py
413
- git commit -m "feat: PoseVisualizer._draw_trails with fading alpha"
414
- ```
415
-
416
- ---
417
-
418
- ## Task 4: `PoseVisualizer._draw_velocity_arrows`
419
-
420
- **Files:**
421
- - Modify: `formscout/agents/visualizer.py`
422
- - Modify: `tests/test_visualizer.py`
423
-
424
- - [ ] **Step 1: Write failing test**
425
-
426
- Append to `tests/test_visualizer.py`:
427
-
428
- ```python
429
- class TestDrawVelocityArrows:
430
- def test_arrows_draw_without_error(self):
431
- from formscout.agents.visualizer import PoseVisualizer
432
- vis = PoseVisualizer()
433
- frame = np.zeros((480, 640, 3), dtype=np.uint8)
434
- kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
435
- for j in range(17)}
436
- prev_kps = {j: {"x": float(48 + j * 30), "y": float(98 + j * 20), "conf": 0.9}
437
- for j in range(17)}
438
- # velocities: joint 5 moving fast
439
- velocities = {j: [0.0] * 5 for j in range(17)}
440
- velocities[5] = [0.0, 10.0, 50.0, 80.0, 120.0]
441
- result = vis._draw_velocity_arrows(frame.copy(), kps, prev_kps, velocities, frame_idx=4)
442
- assert result.shape == frame.shape
443
-
444
- def test_no_prev_kps_no_crash(self):
445
- from formscout.agents.visualizer import PoseVisualizer
446
- vis = PoseVisualizer()
447
- frame = np.zeros((480, 640, 3), dtype=np.uint8)
448
- kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.9} for j in range(17)}
449
- velocities = {j: [50.0] * 5 for j in range(17)}
450
- # prev_kps is None — should skip without crash
451
- result = vis._draw_velocity_arrows(frame.copy(), kps, None, velocities, frame_idx=0)
452
- assert result.shape == frame.shape
453
- ```
454
-
455
- - [ ] **Step 2: Run to confirm failure**
456
-
457
- ```bash
458
- pytest tests/test_visualizer.py::TestDrawVelocityArrows -v
459
- ```
460
-
461
- Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute '_draw_velocity_arrows'`
462
-
463
- - [ ] **Step 3: Add `_draw_velocity_arrows` to `PoseVisualizer`**
464
-
465
- Inside the `PoseVisualizer` class, after `_draw_trails`:
466
-
467
- ```python
468
- # ── Velocity arrows ───────────────────────────────────────────────────────
469
-
470
- def _draw_velocity_arrows(
471
- self,
472
- frame: np.ndarray,
473
- kps: dict,
474
- prev_kps: dict | None,
475
- velocities: dict[int, list[float]],
476
- frame_idx: int,
477
- ) -> np.ndarray:
478
- """Draw per-joint velocity arrows scaled by speed."""
479
- if prev_kps is None:
480
- return frame
481
-
482
- all_speeds = [velocities[j][frame_idx] for j in range(17) if frame_idx < len(velocities.get(j, []))]
483
- peak = max(all_speeds) if all_speeds else 1.0
484
- if peak == 0.0:
485
- return frame
486
-
487
- for j in range(17):
488
- kp = kps.get(j)
489
- pk = prev_kps.get(j)
490
- if not kp or not pk:
491
- continue
492
- if kp.get("conf", 0.0) < CONF_THRESHOLD:
493
- continue
494
- speeds = velocities.get(j, [])
495
- if frame_idx >= len(speeds):
496
- continue
497
- speed = speeds[frame_idx]
498
- if speed == 0.0:
499
- continue
500
-
501
- dx = kp["x"] - pk["x"]
502
- dy = kp["y"] - pk["y"]
503
- mag = math.sqrt(dx * dx + dy * dy)
504
- if mag < 1e-6:
505
- continue
506
-
507
- # Normalize direction, scale to arrow length
508
- length = min(speed / peak * MAX_ARROW_PX, MAX_ARROW_PX)
509
- nx, ny = dx / mag, dy / mag
510
- start = (int(kp["x"]), int(kp["y"]))
511
- end = (int(kp["x"] + nx * length), int(kp["y"] + ny * length))
512
-
513
- ratio = speed / peak
514
- if ratio < 0.33:
515
- color = (0, 200, 0) # green
516
- elif ratio < 0.66:
517
- color = (0, 140, 255) # orange
518
- else:
519
- color = (0, 0, 255) # red
520
-
521
- cv2.arrowedLine(frame, start, end, color, 2, tipLength=0.35)
522
-
523
- return frame
524
- ```
525
-
526
- - [ ] **Step 4: Run tests**
527
-
528
- ```bash
529
- pytest tests/test_visualizer.py::TestDrawVelocityArrows -v
530
- ```
531
-
532
- Expected: 2 PASS
533
-
534
- - [ ] **Step 5: Commit**
535
-
536
- ```bash
537
- git add formscout/agents/visualizer.py tests/test_visualizer.py
538
- git commit -m "feat: PoseVisualizer._draw_velocity_arrows speed-colored"
539
- ```
540
-
541
- ---
542
-
543
- ## Task 5: `render_video` + `build_velocity_summary`
544
-
545
- **Files:**
546
- - Modify: `formscout/agents/visualizer.py`
547
- - Modify: `tests/test_visualizer.py`
548
-
549
- - [ ] **Step 1: Write failing tests**
550
-
551
- Append to `tests/test_visualizer.py`:
552
-
553
- ```python
554
- class TestRenderVideo:
555
- def test_creates_mp4_file(self, tmp_path):
556
- from formscout.agents.visualizer import PoseVisualizer
557
- vis = PoseVisualizer()
558
- ingest = _make_ingest(n=5)
559
- pose = _make_pose(n=5)
560
- out = str(tmp_path / "out.mp4")
561
- result = vis.render_video(ingest, pose, {"skeleton"}, out)
562
- assert result is not None
563
- import os
564
- assert os.path.exists(result)
565
- assert os.path.getsize(result) > 0
566
-
567
- def test_empty_layers_returns_none(self, tmp_path):
568
- from formscout.agents.visualizer import PoseVisualizer
569
- vis = PoseVisualizer()
570
- out = str(tmp_path / "out.mp4")
571
- result = vis.render_video(_make_ingest(), _make_pose(), set(), out)
572
- assert result is None
573
-
574
- def test_no_detections_returns_none(self, tmp_path):
575
- from formscout.agents.visualizer import PoseVisualizer
576
- vis = PoseVisualizer()
577
- ingest = _make_ingest(n=5)
578
- empty_pose = Pose2DResult(
579
- keypoints=[{} for _ in range(5)], fps=30.0, confidence=0.0, notes=""
580
- )
581
- out = str(tmp_path / "out.mp4")
582
- result = vis.render_video(ingest, empty_pose, {"skeleton"}, out)
583
- assert result is None
584
-
585
- def test_last_velocities_set_after_render(self, tmp_path):
586
- from formscout.agents.visualizer import PoseVisualizer
587
- vis = PoseVisualizer()
588
- out = str(tmp_path / "out.mp4")
589
- vis.render_video(_make_ingest(n=5), _make_pose(n=5), {"skeleton"}, out)
590
- assert len(vis.last_velocities) == 17
591
-
592
-
593
- class TestBuildVelocitySummary:
594
- def test_returns_markdown_table(self):
595
- from formscout.agents.visualizer import build_velocity_summary, compute_joint_velocity
596
- pose = _make_pose(n=10)
597
- vels = compute_joint_velocity(pose.keypoints, fps=30.0)
598
- result = build_velocity_summary(pose.keypoints, vels)
599
- assert "|" in result
600
- # At least one COCO joint name appears
601
- assert any(name in result for name in ["knee", "shoulder", "hip", "ankle"])
602
-
603
- def test_empty_keypoints_returns_empty_string(self):
604
- from formscout.agents.visualizer import build_velocity_summary
605
- empty_kps = [{} for _ in range(5)]
606
- vels = {j: [0.0] * 5 for j in range(17)}
607
- result = build_velocity_summary(empty_kps, vels)
608
- assert result == ""
609
- ```
610
-
611
- - [ ] **Step 2: Run to confirm failure**
612
-
613
- ```bash
614
- pytest tests/test_visualizer.py::TestRenderVideo tests/test_visualizer.py::TestBuildVelocitySummary -v
615
- ```
616
-
617
- Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute 'render_video'`
618
-
619
- - [ ] **Step 3: Add `render_video` to `PoseVisualizer`**
620
-
621
- Inside the `PoseVisualizer` class, after `_draw_velocity_arrows`:
622
-
623
- ```python
624
- # ── Public ────────────────────────────────────────────────────────────────
625
-
626
- def render_video(
627
- self,
628
- ingest,
629
- pose2d,
630
- layers: set[str],
631
- output_path: str,
632
- ) -> str | None:
633
- """
634
- Render annotated video. Returns output_path on success, None otherwise.
635
- layers: subset of {"skeleton", "trails", "velocity_arrows"}
636
- """
637
- if not layers:
638
- return None
639
-
640
- # Require at least one detected frame
641
- if not any(pose2d.keypoints):
642
- return None
643
-
644
- try:
645
- velocities = compute_joint_velocity(pose2d.keypoints, ingest.fps)
646
- self.last_velocities = velocities
647
-
648
- frames = ingest.frames
649
- h, w = frames[0].shape[:2]
650
- fps = ingest.fps or 30.0
651
-
652
- fourcc = cv2.VideoWriter_fourcc(*"mp4v")
653
- writer = cv2.VideoWriter(output_path, fourcc, fps, (w, h))
654
- if not writer.isOpened():
655
- logger.warning("VideoWriter failed to open: %s", output_path)
656
- return None
657
-
658
- trail_history: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
659
- prev_kps: dict | None = None
660
-
661
- for frame_idx, (frame, kps) in enumerate(zip(frames, pose2d.keypoints)):
662
- out_frame = frame.copy()
663
-
664
- if "trails" in layers:
665
- # Update trail history before drawing
666
- for j, kp in kps.items():
667
- if kp.get("conf", 0.0) >= CONF_THRESHOLD:
668
- trail_history[j].append((kp["x"], kp["y"]))
669
- out_frame = self._draw_trails(out_frame, trail_history)
670
-
671
- if "skeleton" in layers:
672
- out_frame = self._draw_skeleton(out_frame, kps)
673
-
674
- if "velocity_arrows" in layers:
675
- out_frame = self._draw_velocity_arrows(
676
- out_frame, kps, prev_kps, velocities, frame_idx
677
- )
678
-
679
- writer.write(out_frame)
680
- prev_kps = kps
681
-
682
- writer.release()
683
- return output_path
684
-
685
- except Exception as e:
686
- logger.warning("render_video failed: %s", e)
687
- return None
688
- ```
689
-
690
- - [ ] **Step 4: Add `build_velocity_summary` after the class**
691
-
692
- After the `PoseVisualizer` class definition, add:
693
-
694
- ```python
695
- # ── Velocity summary ──────────────────────────────────────────────────────────
696
-
697
- def build_velocity_summary(
698
- keypoints_per_frame: list[dict],
699
- velocities: dict[int, list[float]],
700
- ) -> str:
701
- """Return markdown table of per-joint avg/peak velocity. Empty string if no valid joints."""
702
- n_frames = len(keypoints_per_frame)
703
- if n_frames == 0:
704
- return ""
705
-
706
- rows = []
707
- for j in range(17):
708
- # Count frames where this joint is detected
709
- detected = sum(
710
- 1 for kps in keypoints_per_frame
711
- if kps.get(j, {}).get("conf", 0.0) >= CONF_THRESHOLD
712
- )
713
- if detected < n_frames * 0.5:
714
- continue # skip joints present in <50% of frames
715
-
716
- speeds = velocities.get(j, [])
717
- if not speeds:
718
- continue
719
-
720
- avg_speed = sum(speeds) / len(speeds)
721
- peak_speed = max(speeds)
722
- rows.append((COCO_KEYPOINTS[j], avg_speed, peak_speed))
723
-
724
- if not rows:
725
- return ""
726
-
727
- rows.sort(key=lambda r: r[2], reverse=True) # sort by peak descending
728
- lines = [
729
- "| Joint | Avg (px/s) | Peak (px/s) |",
730
- "|---|---|---|",
731
- ]
732
- for name, avg, peak in rows:
733
- lines.append(f"| {name} | {avg:.1f} | {peak:.1f} |")
734
- return "\n".join(lines)
735
- ```
736
-
737
- - [ ] **Step 5: Run all visualizer tests**
738
-
739
- ```bash
740
- pytest tests/test_visualizer.py -v
741
- ```
742
-
743
- Expected: all tests PASS (4 + 2 + 2 + 2 + 4 + 2 = 16 total)
744
-
745
- - [ ] **Step 6: Commit**
746
-
747
- ```bash
748
- git add formscout/agents/visualizer.py tests/test_visualizer.py
749
- git commit -m "feat: PoseVisualizer.render_video + build_velocity_summary (16 tests pass)"
750
- ```
751
-
752
- ---
753
-
754
- ## Task 6: Wire `app.py`
755
-
756
- **Files:**
757
- - Modify: `app.py`
758
-
759
- - [ ] **Step 1: Add `import tempfile` if not present and import visualizer in `process_video`**
760
-
761
- Check the top of `app.py` for `import tempfile`. If missing, add it alongside the other stdlib imports. (Look at the existing import block and add `import tempfile` there.)
762
-
763
- - [ ] **Step 2: Update `process_video()` signature and body**
764
-
765
- Replace the existing `process_video` function (lines 46–83) with:
766
-
767
- ```python
768
- def process_video(video_path: str, test_name: str, side: str, model_key: str, layers: list[str]):
769
- """Process an uploaded video through the FormScout pipeline."""
770
- if not video_path:
771
- return (
772
- _render_empty_state(),
773
- "Upload a video to begin analysis.",
774
- "",
775
- "",
776
- None,
777
- "",
778
- )
779
-
780
- director = Director()
781
- state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
782
-
783
- # ─── Score card ───
784
- score_html = _render_empty_state()
785
- score_details = ""
786
-
787
- if state.features:
788
- result = score_test(state.features)
789
- judge = state.judge
790
- if judge and judge.score is not None:
791
- score_html = _render_score_card(judge.score, judge.confidence, judge.needs_human)
792
- score_details = _render_score_details_judge(judge, result, state.features)
793
- elif judge and judge.needs_human:
794
- score_html = _render_score_card(0, 0, True)
795
- score_details = f"### Needs Clinician Review\n{judge.rationale}"
796
- else:
797
- score_html = _render_score_card(result.score, result.confidence, result.needs_human)
798
- score_details = _render_score_details(result, state.features)
799
-
800
- # ─── Pipeline info ───
801
- pipeline_md = _render_pipeline_status(state)
802
-
803
- # ─── Warnings/errors ───
804
- alerts = _render_alerts(state)
805
-
806
- # ─── Overlay video ───
807
- overlay_path = None
808
- vel_summary = ""
809
- layer_set = {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
810
- if layer_set and state.ingest and state.pose2d:
811
- try:
812
- from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
813
- vis = PoseVisualizer()
814
- with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
815
- out_path = f.name
816
- overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
817
- if overlay_path:
818
- vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
819
- except Exception as e:
820
- alerts = (alerts or "") + f"\n⚠️ Visualizer error: {e}"
821
-
822
- return score_html, pipeline_md, score_details, alerts, overlay_path, vel_summary
823
- ```
824
-
825
- - [ ] **Step 3: Add `overlay_layers` CheckboxGroup in `build_app()`**
826
-
827
- After the `pose_model_dropdown` block (around line 270), and before `submit_btn`:
828
-
829
- ```python
830
- overlay_layers = gr.CheckboxGroup(
831
- choices=["Skeleton", "Trails", "Velocity arrows"],
832
- value=["Skeleton", "Trails"],
833
- label="Overlay Layers",
834
- )
835
- ```
836
-
837
- - [ ] **Step 4: Add overlay tab in the results panel**
838
-
839
- Inside the `with gr.Tabs():` block (after the `⚠️ Alerts` tab):
840
-
841
- ```python
842
- with gr.TabItem("🎬 Overlay Video"):
843
- overlay_video = gr.Video(label="Annotated Movement")
844
- velocity_md = gr.Markdown("")
845
- ```
846
-
847
- - [ ] **Step 5: Update `_map_inputs` and `submit_btn.click`**
848
-
849
- Replace the `_map_inputs` closure and `submit_btn.click` call:
850
-
851
- ```python
852
- def _map_inputs(video, test_display_name, side_display, pose_model_key, overlay_layers):
853
- """Map UI display values to internal values."""
854
- test_map = {name: val for name, val in FMS_TESTS}
855
- test_name = test_map.get(test_display_name, "deep_squat")
856
- side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
857
- return process_video(video, test_name, side, pose_model_key, overlay_layers)
858
-
859
- submit_btn.click(
860
- fn=_map_inputs,
861
- inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown, overlay_layers],
862
- outputs=[score_html, pipeline_md, score_details, alerts_md, overlay_video, velocity_md],
863
- )
864
- ```
865
-
866
- - [ ] **Step 6: Smoke-test the app builds**
867
-
868
- ```bash
869
- python3 -c "from app import build_app; build_app(); print('ok')"
870
- ```
871
-
872
- Expected: `ok` (Gradio UserWarning about theme is fine, not an error)
873
-
874
- - [ ] **Step 7: Run full test suite to check for regressions**
875
-
876
- ```bash
877
- pytest tests/ -v --tb=short 2>&1 | tail -15
878
- ```
879
-
880
- Expected: all previous tests still pass (62 passing, 1 pre-existing fail in biomechanics), plus 16 new visualizer tests = 78 passing.
881
-
882
- - [ ] **Step 8: Commit**
883
-
884
- ```bash
885
- git add app.py
886
- git commit -m "feat: overlay video tab + velocity summary wired in Gradio UI"
887
- ```
888
-
889
- ---
890
-
891
- ## Self-review
892
-
893
- **Spec coverage:**
894
- - ✅ `SimpleKalmanFilter` 4-state (Task 1)
895
- - ✅ `compute_joint_velocity` Kalman-filtered px/s (Task 1)
896
- - ✅ `_draw_skeleton` COCO bones, confidence-colored joints (Task 2)
897
- - ✅ `_draw_trails` fading deque-based trails (Task 3)
898
- - ✅ `_draw_velocity_arrows` speed-colored, direction from consecutive frames (Task 4)
899
- - ✅ `render_video` layer dispatch, trail history, VideoWriter (Task 5)
900
- - ✅ `build_velocity_summary` markdown table, >50% detection filter (Task 5)
901
- - ✅ `overlay_layers` CheckboxGroup in UI (Task 6)
902
- - ✅ New `🎬 Overlay Video` tab with `gr.Video` + `gr.Markdown` (Task 6)
903
- - ✅ `process_video` wired with layers param (Task 6)
904
- - ✅ `vis.last_velocities` stored on instance after `render_video` (Task 5)
905
- - ✅ Error handling: empty layers → None, empty detections → None, exception → alerts (Task 5 + 6)
906
- - ✅ All 5 spec test cases covered across Tasks 1–5
907
-
908
- **Placeholder scan:** None found. All code blocks are complete.
909
-
910
- **Type consistency:**
911
- - `compute_joint_velocity` returns `dict[int, list[float]]` — used identically in `render_video`, `_draw_velocity_arrows`, and `build_velocity_summary`. ✓
912
- - `layers: set[str]` in `render_video`; converted from `list[str]` in `process_video` via set comprehension. ✓
913
- - `vis.last_velocities` set in `render_video`, read in `process_video`. ✓
914
- - `_draw_velocity_arrows(frame, kps, prev_kps, velocities, frame_idx)` — signature matches call in `render_video`. ✓
 
1
+ # Pose Overlay Visualizer Implementation Plan
2
+
3
+ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
4
+
5
+ **Goal:** Add a pose overlay video output to FormScout with skeleton, motion trails, and velocity arrows, plus a per-joint velocity summary table.
6
+
7
+ **Architecture:** A new `formscout/agents/visualizer.py` runs after `director.run()` in `process_video()`; it uses Kalman-filtered per-joint velocity and OpenCV rendering. `app.py` gains a `gr.CheckboxGroup` for layer selection, a new `gr.Video` output tab, and a `gr.Markdown` velocity summary.
8
+
9
+ **Tech Stack:** `opencv-python`, `numpy`, `colorsys` (stdlib), `gradio`.
10
+
11
+ ---
12
+
13
+ ## File map
14
+
15
+ | File | Change |
16
+ |---|---|
17
+ | `formscout/agents/visualizer.py` | Create — Kalman filter, velocity, PoseVisualizer, summary |
18
+ | `tests/test_visualizer.py` | Create — all visualizer tests |
19
+ | `app.py` | Modify — overlay_layers checkbox, new tab, wiring |
20
+
21
+ ---
22
+
23
+ ## Task 1: `SimpleKalmanFilter` + `compute_joint_velocity`
24
+
25
+ **Files:**
26
+ - Create: `formscout/agents/visualizer.py`
27
+ - Create: `tests/test_visualizer.py`
28
+
29
+ - [ ] **Step 1: Write failing tests**
30
+
31
+ Create `tests/test_visualizer.py`:
32
+
33
+ ```python
34
+ """Tests for PoseVisualizer — no GPU, no model downloads."""
35
+ import numpy as np
36
+ import pytest
37
+ from formscout.types import IngestResult, Pose2DResult
38
+
39
+
40
+ def _make_ingest(n=5, h=480, w=640, fps=30.0):
41
+ frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
42
+ return IngestResult(frames=frames, fps=fps, duration=n/fps, n_people=1, width=w, height=h)
43
+
44
+
45
+ def _make_pose(n=5, w=640, h=480):
46
+ """Synthetic Pose2DResult: 17 joints at fixed pixel positions, conf=0.9."""
47
+ kps_per_frame = []
48
+ for i in range(n):
49
+ frame_kps = {}
50
+ for j in range(17):
51
+ frame_kps[j] = {
52
+ "x": float(50 + j * 30 + i * 2), # slight movement each frame
53
+ "y": float(100 + j * 20),
54
+ "conf": 0.9,
55
+ }
56
+ kps_per_frame.append(frame_kps)
57
+ return Pose2DResult(keypoints=kps_per_frame, fps=30.0, confidence=0.9, notes="")
58
+
59
+
60
+ class TestComputeJointVelocity:
61
+ def test_returns_17_joints(self):
62
+ from formscout.agents.visualizer import compute_joint_velocity
63
+ pose = _make_pose(n=5)
64
+ result = compute_joint_velocity(pose.keypoints, fps=30.0)
65
+ assert len(result) == 17
66
+
67
+ def test_each_list_has_n_frames(self):
68
+ from formscout.agents.visualizer import compute_joint_velocity
69
+ pose = _make_pose(n=5)
70
+ result = compute_joint_velocity(pose.keypoints, fps=30.0)
71
+ for joint_idx, speeds in result.items():
72
+ assert len(speeds) == 5, f"joint {joint_idx} has {len(speeds)} speeds, expected 5"
73
+
74
+ def test_speeds_are_non_negative(self):
75
+ from formscout.agents.visualizer import compute_joint_velocity
76
+ pose = _make_pose(n=5)
77
+ result = compute_joint_velocity(pose.keypoints, fps=30.0)
78
+ for speeds in result.values():
79
+ assert all(s >= 0.0 for s in speeds)
80
+
81
+ def test_missing_keypoints_give_zero_speed(self):
82
+ from formscout.agents.visualizer import compute_joint_velocity
83
+ # All frames empty
84
+ empty_kps = [{} for _ in range(5)]
85
+ result = compute_joint_velocity(empty_kps, fps=30.0)
86
+ for speeds in result.values():
87
+ assert all(s == 0.0 for s in speeds)
88
+ ```
89
+
90
+ - [ ] **Step 2: Run to confirm failure**
91
+
92
+ ```bash
93
+ pytest tests/test_visualizer.py::TestComputeJointVelocity -v
94
+ ```
95
+
96
+ Expected: `ERROR` — `ModuleNotFoundError: No module named 'formscout.agents.visualizer'`
97
+
98
+ - [ ] **Step 3: Create `formscout/agents/visualizer.py` with Kalman + velocity**
99
+
100
+ ```python
101
+ """
102
+ PoseVisualizer — annotated overlay video with skeleton, trails, velocity arrows.
103
+
104
+ Input: IngestResult + Pose2DResult
105
+ Output: .mp4 path (or None on failure/empty layers)
106
+ Failure: returns None, never raises.
107
+ """
108
+ from __future__ import annotations
109
+
110
+ import colorsys
111
+ import logging
112
+ import math
113
+ import tempfile
114
+ from collections import deque
115
+
116
+ import cv2
117
+ import numpy as np
118
+
119
+ logger = logging.getLogger(__name__)
120
+
121
+ # ── COCO constants ────────────────────────────────────────────────────────────
122
+
123
+ COCO_KEYPOINTS = [
124
+ "nose", "left_eye", "right_eye", "left_ear", "right_ear",
125
+ "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
126
+ "left_wrist", "right_wrist", "left_hip", "right_hip",
127
+ "left_knee", "right_knee", "left_ankle", "right_ankle",
128
+ ]
129
+
130
+ COCO_SKELETON = [
131
+ (0, 1), (0, 2), (1, 3), (2, 4), # face
132
+ (5, 6), (5, 7), (7, 9), (6, 8), (8, 10), # arms
133
+ (5, 11), (6, 12), (11, 12), # torso
134
+ (11, 13), (13, 15), (12, 14), (14, 16), # legs
135
+ ]
136
+
137
+ TRAIL_LENGTH = 10
138
+ MAX_ARROW_PX = 40
139
+ CONF_THRESHOLD = 0.3
140
+
141
+
142
+ # ── Kalman filter ─────────────────────────────────────────────────────────────
143
+
144
+ class SimpleKalmanFilter:
145
+ """4-state Kalman filter (x, y, vx, vy) for joint tracking."""
146
+
147
+ def __init__(self, process_noise: float = 0.01, measurement_noise: float = 0.1):
148
+ self.is_initialized = False
149
+ self.state = np.zeros(4)
150
+ self.cov = np.eye(4) * 0.1
151
+ self.Q = np.eye(4) * process_noise
152
+ self.R = np.eye(2) * measurement_noise
153
+ self.H = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], dtype=float)
154
+
155
+ def predict(self, dt: float = 1.0):
156
+ F = np.array([[1, 0, dt, 0], [0, 1, 0, dt], [0, 0, 1, 0], [0, 0, 0, 1]], dtype=float)
157
+ self.state = F @ self.state
158
+ self.cov = F @ self.cov @ F.T + self.Q
159
+
160
+ def update(self, x: float, y: float):
161
+ z = np.array([x, y])
162
+ if not self.is_initialized:
163
+ self.state[:2] = z
164
+ self.is_initialized = True
165
+ return
166
+ S = self.H @ self.cov @ self.H.T + self.R
167
+ K = self.cov @ self.H.T @ np.linalg.inv(S)
168
+ self.state = self.state + K @ (z - self.H @ self.state)
169
+ self.cov = (np.eye(4) - K @ self.H) @ self.cov
170
+
171
+ def velocity_magnitude(self) -> float:
172
+ vx, vy = self.state[2], self.state[3]
173
+ return math.sqrt(vx * vx + vy * vy)
174
+
175
+ def velocity_vector(self) -> tuple[float, float]:
176
+ return float(self.state[2]), float(self.state[3])
177
+
178
+
179
+ # ── Velocity computation ──────────────────────────────────────────────────────
180
+
181
+ def compute_joint_velocity(
182
+ keypoints_per_frame: list[dict],
183
+ fps: float,
184
+ ) -> dict[int, list[float]]:
185
+ """
186
+ Compute Kalman-filtered per-joint speed (px/s) for each frame.
187
+
188
+ Returns dict[joint_idx, [speed_frame0, speed_frame1, ...]] for all 17 COCO joints.
189
+ Missing/low-confidence keypoints yield speed=0.0 for that frame.
190
+ """
191
+ dt = 1.0 / fps if fps > 0 else 1.0
192
+ filters: dict[int, SimpleKalmanFilter] = {j: SimpleKalmanFilter() for j in range(17)}
193
+ result: dict[int, list[float]] = {j: [] for j in range(17)}
194
+
195
+ for frame_kps in keypoints_per_frame:
196
+ for j in range(17):
197
+ kf = filters[j]
198
+ kp = frame_kps.get(j)
199
+ kf.predict(dt)
200
+ if kp and kp.get("conf", 0.0) >= CONF_THRESHOLD:
201
+ kf.update(kp["x"], kp["y"])
202
+ speed = kf.velocity_magnitude()
203
+ else:
204
+ speed = 0.0
205
+ result[j].append(speed)
206
+
207
+ return result
208
+ ```
209
+
210
+ - [ ] **Step 4: Run tests**
211
+
212
+ ```bash
213
+ pytest tests/test_visualizer.py::TestComputeJointVelocity -v
214
+ ```
215
+
216
+ Expected: 4 PASS
217
+
218
+ - [ ] **Step 5: Commit**
219
+
220
+ ```bash
221
+ git add formscout/agents/visualizer.py tests/test_visualizer.py
222
+ git commit -m "feat: SimpleKalmanFilter + compute_joint_velocity (4 tests pass)"
223
+ ```
224
+
225
+ ---
226
+
227
+ ## Task 2: `PoseVisualizer._draw_skeleton`
228
+
229
+ **Files:**
230
+ - Modify: `formscout/agents/visualizer.py`
231
+ - Modify: `tests/test_visualizer.py`
232
+
233
+ - [ ] **Step 1: Write failing test**
234
+
235
+ Append to `tests/test_visualizer.py`:
236
+
237
+ ```python
238
+ class TestDrawSkeleton:
239
+ def test_skeleton_draws_without_error(self):
240
+ from formscout.agents.visualizer import PoseVisualizer
241
+ vis = PoseVisualizer()
242
+ frame = np.zeros((480, 640, 3), dtype=np.uint8)
243
+ kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
244
+ for j in range(17)}
245
+ result = vis._draw_skeleton(frame.copy(), kps)
246
+ assert result.shape == frame.shape
247
+ # Frame must be modified (not all zeros after drawing)
248
+ assert not np.array_equal(result, frame)
249
+
250
+ def test_low_confidence_keypoints_not_drawn(self):
251
+ from formscout.agents.visualizer import PoseVisualizer
252
+ vis = PoseVisualizer()
253
+ frame = np.zeros((480, 640, 3), dtype=np.uint8)
254
+ # All keypoints below threshold
255
+ kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.1} for j in range(17)}
256
+ result = vis._draw_skeleton(frame.copy(), kps)
257
+ # Nothing drawn — frame stays all zeros
258
+ assert np.array_equal(result, frame)
259
+ ```
260
+
261
+ - [ ] **Step 2: Run to confirm failure**
262
+
263
+ ```bash
264
+ pytest tests/test_visualizer.py::TestDrawSkeleton -v
265
+ ```
266
+
267
+ Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute '_draw_skeleton'`
268
+
269
+ - [ ] **Step 3: Add `PoseVisualizer` class with `_draw_skeleton` to `visualizer.py`**
270
+
271
+ Append after `compute_joint_velocity`:
272
+
273
+ ```python
274
+ # ── Helpers ───────────────────────────────────────────────────────────────────
275
+
276
+ def _conf_to_bgr(conf: float) -> tuple[int, int, int]:
277
+ """Map confidence 0→1 to BGR color red→green via HSV."""
278
+ hue = conf * 120.0 / 360.0
279
+ r, g, b = colorsys.hsv_to_rgb(hue, 1.0, 1.0)
280
+ return (int(b * 255), int(g * 255), int(r * 255))
281
+
282
+
283
+ # ── PoseVisualizer ────────────────────────────────────────────────────────────
284
+
285
+ class PoseVisualizer:
286
+ """Renders skeleton, trails, and velocity arrows onto video frames."""
287
+
288
+ def __init__(self):
289
+ self.last_velocities: dict[int, list[float]] = {}
290
+
291
+ # ── Skeleton ──────────────────────────────────────────────────────────────
292
+
293
+ def _draw_skeleton(self, frame: np.ndarray, kps: dict) -> np.ndarray:
294
+ """Draw COCO-17 bones (white) and joints (confidence-colored) onto frame."""
295
+ visible = {j: kp for j, kp in kps.items() if kp.get("conf", 0.0) >= CONF_THRESHOLD}
296
+
297
+ # Bones
298
+ for j1, j2 in COCO_SKELETON:
299
+ if j1 in visible and j2 in visible:
300
+ p1 = (int(visible[j1]["x"]), int(visible[j1]["y"]))
301
+ p2 = (int(visible[j2]["x"]), int(visible[j2]["y"]))
302
+ cv2.line(frame, p1, p2, (255, 255, 255), 2)
303
+
304
+ # Joints
305
+ for j, kp in visible.items():
306
+ pt = (int(kp["x"]), int(kp["y"]))
307
+ color = _conf_to_bgr(kp["conf"])
308
+ cv2.circle(frame, pt, 4, color, -1)
309
+ cv2.circle(frame, pt, 5, (255, 255, 255), 1)
310
+
311
+ return frame
312
+ ```
313
+
314
+ - [ ] **Step 4: Run tests**
315
+
316
+ ```bash
317
+ pytest tests/test_visualizer.py::TestDrawSkeleton -v
318
+ ```
319
+
320
+ Expected: 2 PASS
321
+
322
+ - [ ] **Step 5: Commit**
323
+
324
+ ```bash
325
+ git add formscout/agents/visualizer.py tests/test_visualizer.py
326
+ git commit -m "feat: PoseVisualizer._draw_skeleton with confidence-colored joints"
327
+ ```
328
+
329
+ ---
330
+
331
+ ## Task 3: `PoseVisualizer._draw_trails`
332
+
333
+ **Files:**
334
+ - Modify: `formscout/agents/visualizer.py`
335
+ - Modify: `tests/test_visualizer.py`
336
+
337
+ - [ ] **Step 1: Write failing test**
338
+
339
+ Append to `tests/test_visualizer.py`:
340
+
341
+ ```python
342
+ class TestDrawTrails:
343
+ def test_trails_draw_without_error(self):
344
+ from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
345
+ from collections import deque
346
+ vis = PoseVisualizer()
347
+ frame = np.zeros((480, 640, 3), dtype=np.uint8)
348
+ # Build a trail history for joint 0 with 5 positions
349
+ trail_history = {
350
+ 0: deque([(100 + i * 5, 200 + i * 3) for i in range(5)], maxlen=TRAIL_LENGTH)
351
+ }
352
+ result = vis._draw_trails(frame.copy(), trail_history)
353
+ assert result.shape == frame.shape
354
+ # Trail should modify at least some pixels
355
+ assert not np.array_equal(result, frame)
356
+
357
+ def test_short_trail_no_crash(self):
358
+ from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
359
+ from collections import deque
360
+ vis = PoseVisualizer()
361
+ frame = np.zeros((480, 640, 3), dtype=np.uint8)
362
+ # Only one point — no line possible
363
+ trail_history = {0: deque([(100, 200)], maxlen=TRAIL_LENGTH)}
364
+ result = vis._draw_trails(frame.copy(), trail_history)
365
+ # No crash, frame unchanged (single point = no segment)
366
+ assert np.array_equal(result, frame)
367
+ ```
368
+
369
+ - [ ] **Step 2: Run to confirm failure**
370
+
371
+ ```bash
372
+ pytest tests/test_visualizer.py::TestDrawTrails -v
373
+ ```
374
+
375
+ Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute '_draw_trails'`
376
+
377
+ - [ ] **Step 3: Add `_draw_trails` to `PoseVisualizer`**
378
+
379
+ Inside the `PoseVisualizer` class, after `_draw_skeleton`:
380
+
381
+ ```python
382
+ # ── Trails ───────────────────────────────────────────────────────────────
383
+
384
+ def _draw_trails(self, frame: np.ndarray, trail_history: dict) -> np.ndarray:
385
+ """Draw fading motion trails for each joint."""
386
+ for joint_idx, trail in trail_history.items():
387
+ pts = list(trail)
388
+ if len(pts) < 2:
389
+ continue
390
+ for i in range(1, len(pts)):
391
+ alpha = i / len(pts)
392
+ brightness = int(255 * alpha)
393
+ color = (brightness, brightness, brightness)
394
+ thickness = max(1, int(3 * alpha))
395
+ p1 = (int(pts[i - 1][0]), int(pts[i - 1][1]))
396
+ p2 = (int(pts[i][0]), int(pts[i][1]))
397
+ cv2.line(frame, p1, p2, color, thickness)
398
+ return frame
399
+ ```
400
+
401
+ - [ ] **Step 4: Run tests**
402
+
403
+ ```bash
404
+ pytest tests/test_visualizer.py::TestDrawTrails -v
405
+ ```
406
+
407
+ Expected: 2 PASS
408
+
409
+ - [ ] **Step 5: Commit**
410
+
411
+ ```bash
412
+ git add formscout/agents/visualizer.py tests/test_visualizer.py
413
+ git commit -m "feat: PoseVisualizer._draw_trails with fading alpha"
414
+ ```
415
+
416
+ ---
417
+
418
+ ## Task 4: `PoseVisualizer._draw_velocity_arrows`
419
+
420
+ **Files:**
421
+ - Modify: `formscout/agents/visualizer.py`
422
+ - Modify: `tests/test_visualizer.py`
423
+
424
+ - [ ] **Step 1: Write failing test**
425
+
426
+ Append to `tests/test_visualizer.py`:
427
+
428
+ ```python
429
+ class TestDrawVelocityArrows:
430
+ def test_arrows_draw_without_error(self):
431
+ from formscout.agents.visualizer import PoseVisualizer
432
+ vis = PoseVisualizer()
433
+ frame = np.zeros((480, 640, 3), dtype=np.uint8)
434
+ kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
435
+ for j in range(17)}
436
+ prev_kps = {j: {"x": float(48 + j * 30), "y": float(98 + j * 20), "conf": 0.9}
437
+ for j in range(17)}
438
+ # velocities: joint 5 moving fast
439
+ velocities = {j: [0.0] * 5 for j in range(17)}
440
+ velocities[5] = [0.0, 10.0, 50.0, 80.0, 120.0]
441
+ result = vis._draw_velocity_arrows(frame.copy(), kps, prev_kps, velocities, frame_idx=4)
442
+ assert result.shape == frame.shape
443
+
444
+ def test_no_prev_kps_no_crash(self):
445
+ from formscout.agents.visualizer import PoseVisualizer
446
+ vis = PoseVisualizer()
447
+ frame = np.zeros((480, 640, 3), dtype=np.uint8)
448
+ kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.9} for j in range(17)}
449
+ velocities = {j: [50.0] * 5 for j in range(17)}
450
+ # prev_kps is None — should skip without crash
451
+ result = vis._draw_velocity_arrows(frame.copy(), kps, None, velocities, frame_idx=0)
452
+ assert result.shape == frame.shape
453
+ ```
454
+
455
+ - [ ] **Step 2: Run to confirm failure**
456
+
457
+ ```bash
458
+ pytest tests/test_visualizer.py::TestDrawVelocityArrows -v
459
+ ```
460
+
461
+ Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute '_draw_velocity_arrows'`
462
+
463
+ - [ ] **Step 3: Add `_draw_velocity_arrows` to `PoseVisualizer`**
464
+
465
+ Inside the `PoseVisualizer` class, after `_draw_trails`:
466
+
467
+ ```python
468
+ # ── Velocity arrows ───────────────────────────────────────────────────────
469
+
470
+ def _draw_velocity_arrows(
471
+ self,
472
+ frame: np.ndarray,
473
+ kps: dict,
474
+ prev_kps: dict | None,
475
+ velocities: dict[int, list[float]],
476
+ frame_idx: int,
477
+ ) -> np.ndarray:
478
+ """Draw per-joint velocity arrows scaled by speed."""
479
+ if prev_kps is None:
480
+ return frame
481
+
482
+ all_speeds = [velocities[j][frame_idx] for j in range(17) if frame_idx < len(velocities.get(j, []))]
483
+ peak = max(all_speeds) if all_speeds else 1.0
484
+ if peak == 0.0:
485
+ return frame
486
+
487
+ for j in range(17):
488
+ kp = kps.get(j)
489
+ pk = prev_kps.get(j)
490
+ if not kp or not pk:
491
+ continue
492
+ if kp.get("conf", 0.0) < CONF_THRESHOLD:
493
+ continue
494
+ speeds = velocities.get(j, [])
495
+ if frame_idx >= len(speeds):
496
+ continue
497
+ speed = speeds[frame_idx]
498
+ if speed == 0.0:
499
+ continue
500
+
501
+ dx = kp["x"] - pk["x"]
502
+ dy = kp["y"] - pk["y"]
503
+ mag = math.sqrt(dx * dx + dy * dy)
504
+ if mag < 1e-6:
505
+ continue
506
+
507
+ # Normalize direction, scale to arrow length
508
+ length = min(speed / peak * MAX_ARROW_PX, MAX_ARROW_PX)
509
+ nx, ny = dx / mag, dy / mag
510
+ start = (int(kp["x"]), int(kp["y"]))
511
+ end = (int(kp["x"] + nx * length), int(kp["y"] + ny * length))
512
+
513
+ ratio = speed / peak
514
+ if ratio < 0.33:
515
+ color = (0, 200, 0) # green
516
+ elif ratio < 0.66:
517
+ color = (0, 140, 255) # orange
518
+ else:
519
+ color = (0, 0, 255) # red
520
+
521
+ cv2.arrowedLine(frame, start, end, color, 2, tipLength=0.35)
522
+
523
+ return frame
524
+ ```
525
+
526
+ - [ ] **Step 4: Run tests**
527
+
528
+ ```bash
529
+ pytest tests/test_visualizer.py::TestDrawVelocityArrows -v
530
+ ```
531
+
532
+ Expected: 2 PASS
533
+
534
+ - [ ] **Step 5: Commit**
535
+
536
+ ```bash
537
+ git add formscout/agents/visualizer.py tests/test_visualizer.py
538
+ git commit -m "feat: PoseVisualizer._draw_velocity_arrows speed-colored"
539
+ ```
540
+
541
+ ---
542
+
543
+ ## Task 5: `render_video` + `build_velocity_summary`
544
+
545
+ **Files:**
546
+ - Modify: `formscout/agents/visualizer.py`
547
+ - Modify: `tests/test_visualizer.py`
548
+
549
+ - [ ] **Step 1: Write failing tests**
550
+
551
+ Append to `tests/test_visualizer.py`:
552
+
553
+ ```python
554
+ class TestRenderVideo:
555
+ def test_creates_mp4_file(self, tmp_path):
556
+ from formscout.agents.visualizer import PoseVisualizer
557
+ vis = PoseVisualizer()
558
+ ingest = _make_ingest(n=5)
559
+ pose = _make_pose(n=5)
560
+ out = str(tmp_path / "out.mp4")
561
+ result = vis.render_video(ingest, pose, {"skeleton"}, out)
562
+ assert result is not None
563
+ import os
564
+ assert os.path.exists(result)
565
+ assert os.path.getsize(result) > 0
566
+
567
+ def test_empty_layers_returns_none(self, tmp_path):
568
+ from formscout.agents.visualizer import PoseVisualizer
569
+ vis = PoseVisualizer()
570
+ out = str(tmp_path / "out.mp4")
571
+ result = vis.render_video(_make_ingest(), _make_pose(), set(), out)
572
+ assert result is None
573
+
574
+ def test_no_detections_returns_none(self, tmp_path):
575
+ from formscout.agents.visualizer import PoseVisualizer
576
+ vis = PoseVisualizer()
577
+ ingest = _make_ingest(n=5)
578
+ empty_pose = Pose2DResult(
579
+ keypoints=[{} for _ in range(5)], fps=30.0, confidence=0.0, notes=""
580
+ )
581
+ out = str(tmp_path / "out.mp4")
582
+ result = vis.render_video(ingest, empty_pose, {"skeleton"}, out)
583
+ assert result is None
584
+
585
+ def test_last_velocities_set_after_render(self, tmp_path):
586
+ from formscout.agents.visualizer import PoseVisualizer
587
+ vis = PoseVisualizer()
588
+ out = str(tmp_path / "out.mp4")
589
+ vis.render_video(_make_ingest(n=5), _make_pose(n=5), {"skeleton"}, out)
590
+ assert len(vis.last_velocities) == 17
591
+
592
+
593
+ class TestBuildVelocitySummary:
594
+ def test_returns_markdown_table(self):
595
+ from formscout.agents.visualizer import build_velocity_summary, compute_joint_velocity
596
+ pose = _make_pose(n=10)
597
+ vels = compute_joint_velocity(pose.keypoints, fps=30.0)
598
+ result = build_velocity_summary(pose.keypoints, vels)
599
+ assert "|" in result
600
+ # At least one COCO joint name appears
601
+ assert any(name in result for name in ["knee", "shoulder", "hip", "ankle"])
602
+
603
+ def test_empty_keypoints_returns_empty_string(self):
604
+ from formscout.agents.visualizer import build_velocity_summary
605
+ empty_kps = [{} for _ in range(5)]
606
+ vels = {j: [0.0] * 5 for j in range(17)}
607
+ result = build_velocity_summary(empty_kps, vels)
608
+ assert result == ""
609
+ ```
610
+
611
+ - [ ] **Step 2: Run to confirm failure**
612
+
613
+ ```bash
614
+ pytest tests/test_visualizer.py::TestRenderVideo tests/test_visualizer.py::TestBuildVelocitySummary -v
615
+ ```
616
+
617
+ Expected: FAIL — `AttributeError: 'PoseVisualizer' object has no attribute 'render_video'`
618
+
619
+ - [ ] **Step 3: Add `render_video` to `PoseVisualizer`**
620
+
621
+ Inside the `PoseVisualizer` class, after `_draw_velocity_arrows`:
622
+
623
+ ```python
624
+ # ── Public ────────────────────────────────────────────────────────────────
625
+
626
+ def render_video(
627
+ self,
628
+ ingest,
629
+ pose2d,
630
+ layers: set[str],
631
+ output_path: str,
632
+ ) -> str | None:
633
+ """
634
+ Render annotated video. Returns output_path on success, None otherwise.
635
+ layers: subset of {"skeleton", "trails", "velocity_arrows"}
636
+ """
637
+ if not layers:
638
+ return None
639
+
640
+ # Require at least one detected frame
641
+ if not any(pose2d.keypoints):
642
+ return None
643
+
644
+ try:
645
+ velocities = compute_joint_velocity(pose2d.keypoints, ingest.fps)
646
+ self.last_velocities = velocities
647
+
648
+ frames = ingest.frames
649
+ h, w = frames[0].shape[:2]
650
+ fps = ingest.fps or 30.0
651
+
652
+ fourcc = cv2.VideoWriter_fourcc(*"mp4v")
653
+ writer = cv2.VideoWriter(output_path, fourcc, fps, (w, h))
654
+ if not writer.isOpened():
655
+ logger.warning("VideoWriter failed to open: %s", output_path)
656
+ return None
657
+
658
+ trail_history: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
659
+ prev_kps: dict | None = None
660
+
661
+ for frame_idx, (frame, kps) in enumerate(zip(frames, pose2d.keypoints)):
662
+ out_frame = frame.copy()
663
+
664
+ if "trails" in layers:
665
+ # Update trail history before drawing
666
+ for j, kp in kps.items():
667
+ if kp.get("conf", 0.0) >= CONF_THRESHOLD:
668
+ trail_history[j].append((kp["x"], kp["y"]))
669
+ out_frame = self._draw_trails(out_frame, trail_history)
670
+
671
+ if "skeleton" in layers:
672
+ out_frame = self._draw_skeleton(out_frame, kps)
673
+
674
+ if "velocity_arrows" in layers:
675
+ out_frame = self._draw_velocity_arrows(
676
+ out_frame, kps, prev_kps, velocities, frame_idx
677
+ )
678
+
679
+ writer.write(out_frame)
680
+ prev_kps = kps
681
+
682
+ writer.release()
683
+ return output_path
684
+
685
+ except Exception as e:
686
+ logger.warning("render_video failed: %s", e)
687
+ return None
688
+ ```
689
+
690
+ - [ ] **Step 4: Add `build_velocity_summary` after the class**
691
+
692
+ After the `PoseVisualizer` class definition, add:
693
+
694
+ ```python
695
+ # ── Velocity summary ──────────────────────────────────────────────────────────
696
+
697
+ def build_velocity_summary(
698
+ keypoints_per_frame: list[dict],
699
+ velocities: dict[int, list[float]],
700
+ ) -> str:
701
+ """Return markdown table of per-joint avg/peak velocity. Empty string if no valid joints."""
702
+ n_frames = len(keypoints_per_frame)
703
+ if n_frames == 0:
704
+ return ""
705
+
706
+ rows = []
707
+ for j in range(17):
708
+ # Count frames where this joint is detected
709
+ detected = sum(
710
+ 1 for kps in keypoints_per_frame
711
+ if kps.get(j, {}).get("conf", 0.0) >= CONF_THRESHOLD
712
+ )
713
+ if detected < n_frames * 0.5:
714
+ continue # skip joints present in <50% of frames
715
+
716
+ speeds = velocities.get(j, [])
717
+ if not speeds:
718
+ continue
719
+
720
+ avg_speed = sum(speeds) / len(speeds)
721
+ peak_speed = max(speeds)
722
+ rows.append((COCO_KEYPOINTS[j], avg_speed, peak_speed))
723
+
724
+ if not rows:
725
+ return ""
726
+
727
+ rows.sort(key=lambda r: r[2], reverse=True) # sort by peak descending
728
+ lines = [
729
+ "| Joint | Avg (px/s) | Peak (px/s) |",
730
+ "|---|---|---|",
731
+ ]
732
+ for name, avg, peak in rows:
733
+ lines.append(f"| {name} | {avg:.1f} | {peak:.1f} |")
734
+ return "\n".join(lines)
735
+ ```
736
+
737
+ - [ ] **Step 5: Run all visualizer tests**
738
+
739
+ ```bash
740
+ pytest tests/test_visualizer.py -v
741
+ ```
742
+
743
+ Expected: all tests PASS (4 + 2 + 2 + 2 + 4 + 2 = 16 total)
744
+
745
+ - [ ] **Step 6: Commit**
746
+
747
+ ```bash
748
+ git add formscout/agents/visualizer.py tests/test_visualizer.py
749
+ git commit -m "feat: PoseVisualizer.render_video + build_velocity_summary (16 tests pass)"
750
+ ```
751
+
752
+ ---
753
+
754
+ ## Task 6: Wire `app.py`
755
+
756
+ **Files:**
757
+ - Modify: `app.py`
758
+
759
+ - [ ] **Step 1: Add `import tempfile` if not present and import visualizer in `process_video`**
760
+
761
+ Check the top of `app.py` for `import tempfile`. If missing, add it alongside the other stdlib imports. (Look at the existing import block and add `import tempfile` there.)
762
+
763
+ - [ ] **Step 2: Update `process_video()` signature and body**
764
+
765
+ Replace the existing `process_video` function (lines 46–83) with:
766
+
767
+ ```python
768
+ def process_video(video_path: str, test_name: str, side: str, model_key: str, layers: list[str]):
769
+ """Process an uploaded video through the FormScout pipeline."""
770
+ if not video_path:
771
+ return (
772
+ _render_empty_state(),
773
+ "Upload a video to begin analysis.",
774
+ "",
775
+ "",
776
+ None,
777
+ "",
778
+ )
779
+
780
+ director = Director()
781
+ state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
782
+
783
+ # ─── Score card ───
784
+ score_html = _render_empty_state()
785
+ score_details = ""
786
+
787
+ if state.features:
788
+ result = score_test(state.features)
789
+ judge = state.judge
790
+ if judge and judge.score is not None:
791
+ score_html = _render_score_card(judge.score, judge.confidence, judge.needs_human)
792
+ score_details = _render_score_details_judge(judge, result, state.features)
793
+ elif judge and judge.needs_human:
794
+ score_html = _render_score_card(0, 0, True)
795
+ score_details = f"### Needs Clinician Review\n{judge.rationale}"
796
+ else:
797
+ score_html = _render_score_card(result.score, result.confidence, result.needs_human)
798
+ score_details = _render_score_details(result, state.features)
799
+
800
+ # ─── Pipeline info ───
801
+ pipeline_md = _render_pipeline_status(state)
802
+
803
+ # ─── Warnings/errors ───
804
+ alerts = _render_alerts(state)
805
+
806
+ # ─── Overlay video ───
807
+ overlay_path = None
808
+ vel_summary = ""
809
+ layer_set = {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
810
+ if layer_set and state.ingest and state.pose2d:
811
+ try:
812
+ from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
813
+ vis = PoseVisualizer()
814
+ with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
815
+ out_path = f.name
816
+ overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
817
+ if overlay_path:
818
+ vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
819
+ except Exception as e:
820
+ alerts = (alerts or "") + f"\n⚠️ Visualizer error: {e}"
821
+
822
+ return score_html, pipeline_md, score_details, alerts, overlay_path, vel_summary
823
+ ```
824
+
825
+ - [ ] **Step 3: Add `overlay_layers` CheckboxGroup in `build_app()`**
826
+
827
+ After the `pose_model_dropdown` block (around line 270), and before `submit_btn`:
828
+
829
+ ```python
830
+ overlay_layers = gr.CheckboxGroup(
831
+ choices=["Skeleton", "Trails", "Velocity arrows"],
832
+ value=["Skeleton", "Trails"],
833
+ label="Overlay Layers",
834
+ )
835
+ ```
836
+
837
+ - [ ] **Step 4: Add overlay tab in the results panel**
838
+
839
+ Inside the `with gr.Tabs():` block (after the `⚠️ Alerts` tab):
840
+
841
+ ```python
842
+ with gr.TabItem("🎬 Overlay Video"):
843
+ overlay_video = gr.Video(label="Annotated Movement")
844
+ velocity_md = gr.Markdown("")
845
+ ```
846
+
847
+ - [ ] **Step 5: Update `_map_inputs` and `submit_btn.click`**
848
+
849
+ Replace the `_map_inputs` closure and `submit_btn.click` call:
850
+
851
+ ```python
852
+ def _map_inputs(video, test_display_name, side_display, pose_model_key, overlay_layers):
853
+ """Map UI display values to internal values."""
854
+ test_map = {name: val for name, val in FMS_TESTS}
855
+ test_name = test_map.get(test_display_name, "deep_squat")
856
+ side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
857
+ return process_video(video, test_name, side, pose_model_key, overlay_layers)
858
+
859
+ submit_btn.click(
860
+ fn=_map_inputs,
861
+ inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown, overlay_layers],
862
+ outputs=[score_html, pipeline_md, score_details, alerts_md, overlay_video, velocity_md],
863
+ )
864
+ ```
865
+
866
+ - [ ] **Step 6: Smoke-test the app builds**
867
+
868
+ ```bash
869
+ python3 -c "from app import build_app; build_app(); print('ok')"
870
+ ```
871
+
872
+ Expected: `ok` (Gradio UserWarning about theme is fine, not an error)
873
+
874
+ - [ ] **Step 7: Run full test suite to check for regressions**
875
+
876
+ ```bash
877
+ pytest tests/ -v --tb=short 2>&1 | tail -15
878
+ ```
879
+
880
+ Expected: all previous tests still pass (62 passing, 1 pre-existing fail in biomechanics), plus 16 new visualizer tests = 78 passing.
881
+
882
+ - [ ] **Step 8: Commit**
883
+
884
+ ```bash
885
+ git add app.py
886
+ git commit -m "feat: overlay video tab + velocity summary wired in Gradio UI"
887
+ ```
888
+
889
+ ---
890
+
891
+ ## Self-review
892
+
893
+ **Spec coverage:**
894
+ - ✅ `SimpleKalmanFilter` 4-state (Task 1)
895
+ - ✅ `compute_joint_velocity` Kalman-filtered px/s (Task 1)
896
+ - ✅ `_draw_skeleton` COCO bones, confidence-colored joints (Task 2)
897
+ - ✅ `_draw_trails` fading deque-based trails (Task 3)
898
+ - ✅ `_draw_velocity_arrows` speed-colored, direction from consecutive frames (Task 4)
899
+ - ✅ `render_video` layer dispatch, trail history, VideoWriter (Task 5)
900
+ - ✅ `build_velocity_summary` markdown table, >50% detection filter (Task 5)
901
+ - ✅ `overlay_layers` CheckboxGroup in UI (Task 6)
902
+ - ✅ New `🎬 Overlay Video` tab with `gr.Video` + `gr.Markdown` (Task 6)
903
+ - ✅ `process_video` wired with layers param (Task 6)
904
+ - ✅ `vis.last_velocities` stored on instance after `render_video` (Task 5)
905
+ - ✅ Error handling: empty layers → None, empty detections → None, exception → alerts (Task 5 + 6)
906
+ - ✅ All 5 spec test cases covered across Tasks 1–5
907
+
908
+ **Placeholder scan:** None found. All code blocks are complete.
909
+
910
+ **Type consistency:**
911
+ - `compute_joint_velocity` returns `dict[int, list[float]]` — used identically in `render_video`, `_draw_velocity_arrows`, and `build_velocity_summary`. ✓
912
+ - `layers: set[str]` in `render_video`; converted from `list[str]` in `process_video` via set comprehension. ✓
913
+ - `vis.last_velocities` set in `render_video`, read in `process_video`. ✓
914
+ - `_draw_velocity_arrows(frame, kps, prev_kps, velocities, frame_idx)` — signature matches call in `render_video`. ✓
docs/superpowers/plans/2026-06-13-full-fms-session-pdf.md CHANGED
@@ -1,1209 +1,1209 @@
1
- # Full FMS Session + PDF Report — Implementation Plan
2
-
3
- > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
4
-
5
- **Goal:** Turn FormScout's one-clip scorer into a screening session that accumulates analyzed clips into a composite 0–21 report and exports a branded PDF with annotated worst-moment key-frame stills.
6
-
7
- **Architecture:** A new `formscout/session.py` accumulates typed `SessionEntry` objects (one per analyzed clip), persisting each to a temp session dir. `PoseVisualizer.render_frame()` captures the governing frame (already computed by `BiomechanicsAgent` and stored in `features.timing`) as an annotated PNG. On "Finish", the existing `ReportAgent` computes composite + asymmetries, and a new `PdfReportAgent` renders a ReportLab PDF. The UI (`app.py`) gains `gr.State` session accumulation with "Analyse new clip" / "Finish & generate PDF" buttons.
8
-
9
- **Tech Stack:** Python 3.13, ReportLab (new dep), OpenCV (existing), Gradio 5, pytest. No model downloads in tests.
10
-
11
- ---
12
-
13
- ## File Structure
14
-
15
- - `requirements.txt` — add `reportlab`.
16
- - `formscout/types.py` — add `SessionEntry` frozen dataclass.
17
- - `formscout/agents/biomechanics.py` — add `max_sag_frame` to `trunk_stability_pushup` timing (rotary already has `peak_extension_frame`).
18
- - `formscout/agents/visualizer.py` — add `PoseVisualizer.render_frame()`.
19
- - `formscout/session.py` — **new**: session accumulator (new/add/finish + persistence + key-frame helpers).
20
- - `formscout/agents/pdf_report.py` — **new**: `PdfReportAgent` (ReportLab).
21
- - `app.py` — wire `gr.State`, two buttons, "Session so far" table, finish handler.
22
- - `tests/test_session.py`, `tests/test_keyframe.py`, `tests/test_pdf_report.py` — **new**.
23
-
24
- ---
25
-
26
- ## Task 1: Add ReportLab dependency
27
-
28
- **Files:**
29
- - Modify: `requirements.txt`
30
-
31
- - [ ] **Step 1: Add the dependency**
32
-
33
- Add this line to `requirements.txt` (after `pillow>=10.3`):
34
-
35
- ```
36
- reportlab>=4.0
37
- ```
38
-
39
- - [ ] **Step 2: Install it**
40
-
41
- Run: `pip install 'reportlab>=4.0'`
42
- Expected: `Successfully installed reportlab-4.x.x`
43
-
44
- - [ ] **Step 3: Verify import**
45
-
46
- Run: `python3 -c "import reportlab; print(reportlab.Version)"`
47
- Expected: prints a version like `4.x.x`
48
-
49
- - [ ] **Step 4: Commit**
50
-
51
- ```bash
52
- git add requirements.txt
53
- git commit -m "build: add reportlab for PDF report generation"
54
- ```
55
-
56
- ---
57
-
58
- ## Task 2: Add `SessionEntry` dataclass
59
-
60
- **Files:**
61
- - Modify: `formscout/types.py` (after `ReportResult`, before `PipelineState`)
62
- - Test: `tests/test_session.py`
63
-
64
- - [ ] **Step 1: Write the failing test**
65
-
66
- Create `tests/test_session.py` with:
67
-
68
- ```python
69
- """Tests for the FMS session accumulator — no GPU, no model downloads."""
70
- import numpy as np
71
-
72
- from formscout.types import (
73
- IngestResult, Pose2DResult, BiomechFeatures, ScoreResult, JudgeResult,
74
- MovementResult, SessionEntry,
75
- )
76
-
77
-
78
- def test_session_entry_holds_typed_objects():
79
- movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
80
- features = BiomechFeatures(
81
- test_name="deep_squat", view="2d", side="na",
82
- angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": True},
83
- symmetry_delta=None, timing={"deepest_frame": 2}, confidence=0.9,
84
- )
85
- rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
86
- judge = JudgeResult(score=2, rationale="ok", compensation_tags=["heels elevated"],
87
- corrective_hint="ankle mobility", confidence=0.85)
88
- entry = SessionEntry(
89
- test_name="deep_squat", side="na", score=2, needs_human=False,
90
- rationale="ok", compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
91
- measurements={"left_knee_flexion_deg": 95.0}, confidence=0.85, view="2d",
92
- keyframe_path=None, movement=movement, features=features,
93
- rubric_score=rubric, judge=judge,
94
- )
95
- assert entry.score == 2
96
- assert entry.movement.test_name == "deep_squat"
97
- assert entry.rubric_score.score == 2
98
- assert entry.judge.compensation_tags == ["heels elevated"]
99
- ```
100
-
101
- - [ ] **Step 2: Run test to verify it fails**
102
-
103
- Run: `pytest tests/test_session.py::test_session_entry_holds_typed_objects -v`
104
- Expected: FAIL with `ImportError: cannot import name 'SessionEntry'`
105
-
106
- - [ ] **Step 3: Add the dataclass**
107
-
108
- In `formscout/types.py`, insert after the `ReportResult` class (line ~142) and before `PipelineState`:
109
-
110
- ```python
111
- @dataclass(frozen=True)
112
- class SessionEntry:
113
- """One accumulated analysis in a screening session.
114
-
115
- Display fields (test_name…keyframe_path) feed the PDF/JSON/MD artifacts;
116
- the trailing typed objects (movement…judge) feed ReportAgent.run().
117
- """
118
- test_name: str
119
- side: str
120
- score: int | None
121
- needs_human: bool
122
- rationale: str
123
- compensation_tags: list
124
- corrective_hint: str
125
- measurements: dict
126
- confidence: float
127
- view: str
128
- keyframe_path: str | None
129
- movement: MovementResult
130
- features: BiomechFeatures
131
- rubric_score: ScoreResult
132
- judge: JudgeResult | None
133
- ```
134
-
135
- - [ ] **Step 4: Run test to verify it passes**
136
-
137
- Run: `pytest tests/test_session.py::test_session_entry_holds_typed_objects -v`
138
- Expected: PASS
139
-
140
- - [ ] **Step 5: Commit**
141
-
142
- ```bash
143
- git add formscout/types.py tests/test_session.py
144
- git commit -m "feat: add SessionEntry typed contract for screening sessions"
145
- ```
146
-
147
- ---
148
-
149
- ## Task 3: Add governing-frame index to push-up biomechanics
150
-
151
- **Files:**
152
- - Modify: `formscout/agents/biomechanics.py:468-529` (`_trunk_stability_pushup`)
153
- - Test: `tests/test_biomechanics.py` (append a test)
154
-
155
- The other six tests already store a governing frame index in `features.timing`
156
- (`deepest_frame`, `peak_step_frame`, `deepest_lunge_frame`, `measure_frame`,
157
- `peak_raise_frame`, `peak_extension_frame`). Only `trunk_stability_pushup` is missing one.
158
-
159
- - [ ] **Step 1: Write the failing test**
160
-
161
- Append to `tests/test_biomechanics.py`:
162
-
163
- ```python
164
- def test_pushup_timing_has_max_sag_frame():
165
- from formscout.agents.biomechanics import BiomechanicsAgent
166
- from formscout.types import Pose2DResult, Body3DResult, MovementResult
167
-
168
- # 4 frames; frame 2 has the largest hip sag (hip far below shoulder/ankle midline)
169
- def kps(hip_y):
170
- base = {
171
- 5: {"x": 200, "y": 200, "conf": 0.9}, # L shoulder
172
- 6: {"x": 220, "y": 200, "conf": 0.9}, # R shoulder
173
- 11: {"x": 300, "y": hip_y, "conf": 0.9}, # L hip
174
- 12: {"x": 320, "y": hip_y, "conf": 0.9}, # R hip
175
- 15: {"x": 400, "y": 200, "conf": 0.9}, # L ankle
176
- 16: {"x": 420, "y": 200, "conf": 0.9}, # R ankle
177
- }
178
- return base
179
-
180
- frames = [kps(200), kps(210), kps(260), kps(205)]
181
- pose = Pose2DResult(keypoints=frames, fps=30.0, confidence=0.9)
182
- body3d = Body3DResult(used=False, joints_3d=[])
183
- movement = MovementResult(test_name="trunk_stability_pushup", side="na", confidence=1.0)
184
-
185
- feats = BiomechanicsAgent().run(pose, body3d, movement)
186
- assert "max_sag_frame" in feats.timing
187
- assert feats.timing["max_sag_frame"] == 2
188
- ```
189
-
190
- - [ ] **Step 2: Run test to verify it fails**
191
-
192
- Run: `pytest tests/test_biomechanics.py::test_pushup_timing_has_max_sag_frame -v`
193
- Expected: FAIL with `assert 'max_sag_frame' in {...}` (KeyError-style assertion failure)
194
-
195
- - [ ] **Step 3: Track the max-sag frame index**
196
-
197
- In `formscout/agents/biomechanics.py`, replace the body of `_trunk_stability_pushup` from the
198
- `trunk_angles_over_time = []` loop through the `if trunk_angles_over_time:` block. Replace:
199
-
200
- ```python
201
- # Analyze multiple frames to detect sag/lag
202
- trunk_angles_over_time = []
203
- for i, kps in enumerate(pose2d.keypoints):
204
- ```
205
-
206
- …down to and including the `alignments["no_sag"] = max_sag < 30` line, with:
207
-
208
- ```python
209
- # Analyze multiple frames to detect sag/lag
210
- trunk_sags: list[tuple[int, float]] = [] # (frame_idx, sag_px)
211
- for i, kps in enumerate(pose2d.keypoints):
212
- l_sh = _get_joint(kps, L_SHOULDER)
213
- r_sh = _get_joint(kps, R_SHOULDER)
214
- l_hip = _get_joint(kps, L_HIP)
215
- r_hip = _get_joint(kps, R_HIP)
216
- l_ankle = _get_joint(kps, L_ANKLE)
217
- r_ankle = _get_joint(kps, R_ANKLE)
218
-
219
- if l_sh and r_sh and l_hip and r_hip and l_ankle and r_ankle:
220
- sh_y = (l_sh[1] + r_sh[1]) / 2
221
- hip_y = (l_hip[1] + r_hip[1]) / 2
222
- ankle_y = (l_ankle[1] + r_ankle[1]) / 2
223
- expected_hip_y = (sh_y + ankle_y) / 2
224
- sag_px = hip_y - expected_hip_y
225
- trunk_sags.append((i, sag_px))
226
-
227
- max_sag_frame = 0
228
- if trunk_sags:
229
- sags = [s for _, s in trunk_sags]
230
- max_sag_frame = max(trunk_sags, key=lambda t: t[1])[0]
231
- mean = sum(sags) / len(sags)
232
- variance = (sum((x - mean) ** 2 for x in sags) / len(sags)) ** 0.5
233
- max_sag = max(sags)
234
- angles["max_sag_px"] = max_sag
235
- angles["trunk_variance_px"] = variance
236
- alignments["body_rigid"] = max_sag < 30 and variance < 15
237
- alignments["no_sag"] = max_sag < 30
238
- else:
239
- notes_parts.append("insufficient landmarks for trunk analysis")
240
- ```
241
-
242
- Then update the `return BiomechFeatures(...)` `timing=` argument at the end of the method from:
243
-
244
- ```python
245
- timing={"n_frames_analyzed": len(trunk_angles_over_time)},
246
- ```
247
-
248
- to:
249
-
250
- ```python
251
- timing={"n_frames_analyzed": len(trunk_sags), "max_sag_frame": max_sag_frame},
252
- ```
253
-
254
- - [ ] **Step 4: Run test to verify it passes**
255
-
256
- Run: `pytest tests/test_biomechanics.py::test_pushup_timing_has_max_sag_frame -v`
257
- Expected: PASS
258
-
259
- - [ ] **Step 5: Run the full biomechanics suite (no regressions)**
260
-
261
- Run: `pytest tests/test_biomechanics.py -v`
262
- Expected: all previously-passing tests still pass (the pre-existing `test_unimplemented_test_returns_low_confidence` known-failure may remain failing — that is unrelated and documented in CLAUDE.md).
263
-
264
- - [ ] **Step 6: Commit**
265
-
266
- ```bash
267
- git add formscout/agents/biomechanics.py tests/test_biomechanics.py
268
- git commit -m "feat: track max-sag frame index in push-up biomechanics for key-frame capture"
269
- ```
270
-
271
- ---
272
-
273
- ## Task 4: Add `PoseVisualizer.render_frame()`
274
-
275
- **Files:**
276
- - Modify: `formscout/agents/visualizer.py` (add method to `PoseVisualizer`, after `render_video`)
277
- - Test: `tests/test_keyframe.py`
278
-
279
- - [ ] **Step 1: Write the failing test**
280
-
281
- Create `tests/test_keyframe.py`:
282
-
283
- ```python
284
- """Tests for PoseVisualizer.render_frame — single annotated still."""
285
- import os
286
- import numpy as np
287
-
288
- from formscout.types import IngestResult, Pose2DResult
289
-
290
-
291
- def _ingest(n=5, h=480, w=640):
292
- frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
293
- return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
294
-
295
-
296
- def _pose(n=5):
297
- kps = []
298
- for i in range(n):
299
- kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
300
- for j in range(17)})
301
- return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
302
-
303
-
304
- def test_render_frame_writes_png(tmp_path):
305
- from formscout.agents.visualizer import PoseVisualizer
306
- out = str(tmp_path / "key.png")
307
- path = PoseVisualizer().render_frame(_ingest(), _pose(), frame_idx=2,
308
- layers={"skeleton"}, caption="Deep Squat — heels elevated",
309
- out_png=out)
310
- assert path == out
311
- assert os.path.exists(out)
312
- assert os.path.getsize(out) > 0
313
-
314
-
315
- def test_render_frame_bad_index_returns_none(tmp_path):
316
- from formscout.agents.visualizer import PoseVisualizer
317
- out = str(tmp_path / "key.png")
318
- path = PoseVisualizer().render_frame(_ingest(n=3), _pose(n=3), frame_idx=99,
319
- layers={"skeleton"}, caption="", out_png=out)
320
- assert path is None
321
- ```
322
-
323
- - [ ] **Step 2: Run test to verify it fails**
324
-
325
- Run: `pytest tests/test_keyframe.py -v`
326
- Expected: FAIL with `AttributeError: 'PoseVisualizer' object has no attribute 'render_frame'`
327
-
328
- - [ ] **Step 3: Add the method**
329
-
330
- In `formscout/agents/visualizer.py`, inside the `PoseVisualizer` class, add this method
331
- immediately after `render_video` (before the closing of the class / the module-level
332
- `build_velocity_summary`):
333
-
334
- ```python
335
- def render_frame(
336
- self,
337
- ingest,
338
- pose2d,
339
- frame_idx: int,
340
- layers: set[str],
341
- caption: str = "",
342
- out_png: str | None = None,
343
- ) -> str | None:
344
- """Render a single annotated still (skeleton + optional trails + caption).
345
-
346
- frame_idx is typically the governing frame from BiomechFeatures.timing.
347
- Returns the PNG path on success, None on any failure. Never raises.
348
- """
349
- try:
350
- if not (0 <= frame_idx < len(ingest.frames)) or frame_idx >= len(pose2d.keypoints):
351
- return None
352
-
353
- frame = ingest.frames[frame_idx].copy()
354
- kps = pose2d.keypoints[frame_idx]
355
-
356
- if "trails" in layers:
357
- trail: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
358
- start = max(0, frame_idx - TRAIL_LENGTH)
359
- for fi in range(start, frame_idx + 1):
360
- for j, kp in pose2d.keypoints[fi].items():
361
- if kp.get("conf", 0.0) >= CONF_THRESHOLD:
362
- trail[j].append((kp["x"], kp["y"]))
363
- frame = self._draw_trails(frame, trail)
364
-
365
- if "skeleton" in layers:
366
- frame = self._draw_skeleton(frame, kps)
367
-
368
- if caption:
369
- cv2.rectangle(frame, (0, 0), (frame.shape[1], 28), (0, 0, 0), -1)
370
- cv2.putText(frame, caption[:80], (8, 20), cv2.FONT_HERSHEY_SIMPLEX,
371
- 0.55, (255, 255, 255), 1, cv2.LINE_AA)
372
-
373
- if out_png is None:
374
- out_png = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
375
-
376
- ok = cv2.imwrite(out_png, frame)
377
- return out_png if ok else None
378
- except Exception as e:
379
- logger.warning("render_frame failed: %s", e)
380
- return None
381
- ```
382
-
383
- (`deque`, `cv2`, `tempfile`, `logger`, `TRAIL_LENGTH`, `CONF_THRESHOLD` are all already imported at the top of this file.)
384
-
385
- - [ ] **Step 4: Run test to verify it passes**
386
-
387
- Run: `pytest tests/test_keyframe.py -v`
388
- Expected: both tests PASS
389
-
390
- - [ ] **Step 5: Commit**
391
-
392
- ```bash
393
- git add formscout/agents/visualizer.py tests/test_keyframe.py
394
- git commit -m "feat: add PoseVisualizer.render_frame for annotated key-frame stills"
395
- ```
396
-
397
- ---
398
-
399
- ## Task 5: Create the session accumulator
400
-
401
- **Files:**
402
- - Create: `formscout/session.py`
403
- - Test: `tests/test_session.py` (append tests)
404
-
405
- - [ ] **Step 1: Write the failing tests**
406
-
407
- Append to `tests/test_session.py`:
408
-
409
- ```python
410
- def _ingest(n=5, h=480, w=640):
411
- frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
412
- return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
413
-
414
-
415
- def _pose(n=5):
416
- kps = []
417
- for i in range(n):
418
- kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
419
- for j in range(17)})
420
- return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
421
-
422
-
423
- def _features(test_name="deep_squat", side="na", frame_key="deepest_frame"):
424
- return BiomechFeatures(
425
- test_name=test_name, view="2d", side=side,
426
- angles={"left_knee_flexion_deg": 95.0},
427
- alignments={"knees_tracking_over_feet": False},
428
- symmetry_delta=None, timing={frame_key: 2}, confidence=0.9,
429
- )
430
-
431
-
432
- def _judge(score=2, needs_human=False):
433
- return JudgeResult(
434
- score=None if needs_human else score, rationale="r",
435
- compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
436
- confidence=0.85, needs_human=needs_human,
437
- )
438
-
439
-
440
- def test_add_analysis_appends_entry_and_writes_files():
441
- import os
442
- from formscout import session as S
443
- sess = S.new_session()
444
- entry = S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
445
- features=_features(), judge=_judge(), test_name="deep_squat", side="na")
446
- assert len(sess.entries) == 1
447
- assert entry.score == 2
448
- assert os.path.exists(os.path.join(sess.session_dir, "session.json"))
449
- assert os.path.exists(os.path.join(sess.session_dir, "analysis.md"))
450
- # key-frame still written (deepest_frame=2 is valid)
451
- assert entry.keyframe_path and os.path.exists(entry.keyframe_path)
452
-
453
-
454
- def test_finish_composite_null_when_needs_human():
455
- from formscout import session as S
456
- sess = S.new_session()
457
- S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(), features=_features(),
458
- judge=_judge(score=3), test_name="deep_squat", side="na")
459
- S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
460
- features=_features("trunk_stability_pushup", frame_key="max_sag_frame"),
461
- judge=_judge(needs_human=True), test_name="trunk_stability_pushup", side="na")
462
- report, pdf_path = S.finish_session(sess)
463
- assert report is not None
464
- assert report.composite is None # one test needs_human
465
-
466
-
467
- def test_finish_empty_session_returns_none():
468
- from formscout import session as S
469
- sess = S.new_session()
470
- report, pdf_path = S.finish_session(sess)
471
- assert report is None and pdf_path is None
472
- ```
473
-
474
- - [ ] **Step 2: Run tests to verify they fail**
475
-
476
- Run: `pytest tests/test_session.py -v`
477
- Expected: the three new tests FAIL with `ModuleNotFoundError: No module named 'formscout.session'`
478
-
479
- - [ ] **Step 3: Create the module**
480
-
481
- Create `formscout/session.py`:
482
-
483
- ```python
484
- """
485
- Screening-session accumulator.
486
-
487
- Accumulates one SessionEntry per analyzed clip, persists each to a temp session
488
- dir (session.json + analysis.md + key-frame PNGs), and on finish builds a
489
- ReportResult (via ReportAgent) + a PDF (via PdfReportAgent).
490
-
491
- Pure orchestration — no Gradio imports. Disk writes tolerate failure with a
492
- logged warning and never block scoring.
493
- """
494
- from __future__ import annotations
495
-
496
- import json
497
- import logging
498
- import os
499
- import tempfile
500
- import uuid
501
- from dataclasses import dataclass, replace
502
-
503
- from formscout.rubric import score_test
504
- from formscout.types import MovementResult, ReportResult, SessionEntry
505
-
506
- logger = logging.getLogger(__name__)
507
-
508
- # Maps each test to the BiomechFeatures.timing key holding its governing frame.
509
- TIMING_KEY = {
510
- "deep_squat": "deepest_frame",
511
- "hurdle_step": "peak_step_frame",
512
- "inline_lunge": "deepest_lunge_frame",
513
- "shoulder_mobility": "measure_frame",
514
- "active_slr": "peak_raise_frame",
515
- "trunk_stability_pushup": "max_sag_frame",
516
- "rotary_stability": "peak_extension_frame",
517
- }
518
-
519
-
520
- @dataclass
521
- class Session:
522
- """Mutable session: an id, its temp dir, and accumulated entries."""
523
- session_id: str
524
- session_dir: str
525
- entries: list # list[SessionEntry]
526
-
527
-
528
- def new_session() -> Session:
529
- sid = uuid.uuid4().hex[:12]
530
- base = os.path.join(tempfile.gettempdir(), "formscout_sessions", sid)
531
- try:
532
- os.makedirs(os.path.join(base, "keyframes"), exist_ok=True)
533
- except Exception as e:
534
- logger.warning("session dir create failed: %s", e)
535
- return Session(session_id=sid, session_dir=base, entries=[])
536
-
537
-
538
- def governing_frame_index(features) -> int | None:
539
- """Return the governing frame index for this test, or None."""
540
- key = TIMING_KEY.get(features.test_name)
541
- if key is None:
542
- return None
543
- idx = features.timing.get(key)
544
- return int(idx) if isinstance(idx, (int, float)) else None
545
-
546
-
547
- def worst_compensation_caption(judge, features) -> str:
548
- """Short caption naming the worst compensation for the key-frame still."""
549
- if judge and getattr(judge, "compensation_tags", None):
550
- return ", ".join(judge.compensation_tags)
551
- failed = [k.replace("_", " ") for k, v in features.alignments.items() if v is False]
552
- return ("compensation: " + ", ".join(failed)) if failed else "key position"
553
-
554
-
555
- def add_analysis(session, *, ingest, pose2d, features, judge, test_name, side,
556
- draw_trails: bool = False) -> SessionEntry:
557
- """Build a SessionEntry from a completed analysis, render its key-frame,
558
- persist the session, append, and return the entry."""
559
- movement = MovementResult(test_name=test_name, side=side, confidence=1.0)
560
- rubric = score_test(features)
561
-
562
- needs_human = bool((judge and judge.needs_human) or rubric.needs_human)
563
- if needs_human:
564
- score = None
565
- elif judge and judge.score is not None:
566
- score = judge.score
567
- else:
568
- score = rubric.score
569
-
570
- keyframe_path = None
571
- idx = governing_frame_index(features)
572
- if idx is not None and 0 <= idx < len(pose2d.keypoints):
573
- from formscout.agents.visualizer import PoseVisualizer
574
- caption = (f"{test_name.replace('_', ' ').title()} "
575
- f"({side}) — {worst_compensation_caption(judge, features)}")
576
- layers = {"skeleton", "trails"} if draw_trails else {"skeleton"}
577
- out_png = os.path.join(session.session_dir, "keyframes", f"{test_name}_{side}.png")
578
- try:
579
- keyframe_path = PoseVisualizer().render_frame(ingest, pose2d, idx, layers, caption, out_png)
580
- except Exception as e:
581
- logger.warning("keyframe render failed: %s", e)
582
-
583
- measurements = {}
584
- measurements.update(features.angles)
585
- measurements.update(features.alignments)
586
-
587
- entry = SessionEntry(
588
- test_name=test_name, side=side, score=score, needs_human=needs_human,
589
- rationale=(judge.rationale if judge else rubric.rationale),
590
- compensation_tags=list(judge.compensation_tags) if judge else [],
591
- corrective_hint=(judge.corrective_hint if judge else ""),
592
- measurements=measurements,
593
- confidence=(judge.confidence if judge else rubric.confidence),
594
- view=features.view,
595
- keyframe_path=keyframe_path,
596
- movement=movement, features=features, rubric_score=rubric, judge=judge,
597
- )
598
- session.entries.append(entry)
599
- _persist(session)
600
- return entry
601
-
602
-
603
- def finish_session(session) -> tuple[ReportResult | None, str | None]:
604
- """Build the composite report + PDF. Returns (report, pdf_path).
605
- Returns (None, None) for an empty session."""
606
- if not session.entries:
607
- return None, None
608
-
609
- from formscout.agents.report import ReportAgent
610
- report_inputs = [{
611
- "movement": e.movement, "features": e.features,
612
- "rubric_score": e.rubric_score, "judge": e.judge, "side": e.side,
613
- } for e in session.entries]
614
- report = ReportAgent().run(report_inputs)
615
-
616
- pdf_path = None
617
- try:
618
- from formscout.agents.pdf_report import PdfReportAgent
619
- pdf_path = PdfReportAgent().run(report, session.entries, session.session_dir)
620
- except Exception as e:
621
- logger.warning("pdf generation failed: %s", e)
622
-
623
- report = replace(report, pdf_path=pdf_path)
624
- return report, pdf_path
625
-
626
-
627
- # ── Persistence ───────────────────────────────────────────────────────────────
628
-
629
- def _jsonable(d: dict) -> dict:
630
- out = {}
631
- for k, v in d.items():
632
- if isinstance(v, float):
633
- out[k] = round(v, 2)
634
- elif isinstance(v, (int, str, bool)) or v is None:
635
- out[k] = v
636
- else:
637
- out[k] = str(v)
638
- return out
639
-
640
-
641
- def _entry_display(e: SessionEntry) -> dict:
642
- return {
643
- "test_name": e.test_name, "side": e.side, "score": e.score,
644
- "needs_human": e.needs_human, "rationale": e.rationale,
645
- "compensation_tags": list(e.compensation_tags), "corrective_hint": e.corrective_hint,
646
- "measurements": _jsonable(e.measurements), "confidence": round(e.confidence, 2),
647
- "view": e.view, "keyframe_path": e.keyframe_path,
648
- }
649
-
650
-
651
- def _render_markdown(session: Session) -> str:
652
- lines = ["# FormScout — Session Log", ""]
653
- for e in session.entries:
654
- title = e.test_name.replace("_", " ").title()
655
- if e.side in ("left", "right"):
656
- title += f" ({e.side})"
657
- score = "Clinician review required" if e.needs_human else f"{e.score}/3"
658
- lines.append(f"## {title} ��� {score}")
659
- lines.append(e.rationale or "")
660
- if e.compensation_tags:
661
- lines.append(f"- Compensations: {', '.join(e.compensation_tags)}")
662
- if e.corrective_hint:
663
- lines.append(f"- Corrective: {e.corrective_hint}")
664
- if e.keyframe_path:
665
- lines.append(f"- Key frame: `{e.keyframe_path}`")
666
- lines.append("")
667
- return "\n".join(lines)
668
-
669
-
670
- def _persist(session: Session) -> None:
671
- try:
672
- with open(os.path.join(session.session_dir, "session.json"), "w") as f:
673
- json.dump([_entry_display(e) for e in session.entries], f, indent=2)
674
- with open(os.path.join(session.session_dir, "analysis.md"), "w") as f:
675
- f.write(_render_markdown(session))
676
- except Exception as e:
677
- logger.warning("session persist failed: %s", e)
678
- ```
679
-
680
- - [ ] **Step 4: Run tests to verify they pass**
681
-
682
- Run: `pytest tests/test_session.py -v`
683
- Expected: all session tests PASS (Task 6 provides `PdfReportAgent`; `finish_session` tolerates its
684
- absence via the try/except, so these pass now — `pdf_path` may be `None` until Task 6).
685
-
686
- - [ ] **Step 5: Commit**
687
-
688
- ```bash
689
- git add formscout/session.py tests/test_session.py
690
- git commit -m "feat: add screening-session accumulator with key-frame capture and persistence"
691
- ```
692
-
693
- ---
694
-
695
- ## Task 6: Create `PdfReportAgent`
696
-
697
- **Files:**
698
- - Create: `formscout/agents/pdf_report.py`
699
- - Test: `tests/test_pdf_report.py`
700
-
701
- - [ ] **Step 1: Write the failing test**
702
-
703
- Create `tests/test_pdf_report.py`:
704
-
705
- ```python
706
- """Tests for PdfReportAgent — no GPU, no model downloads."""
707
- import os
708
-
709
- from formscout.types import (
710
- ReportResult, SessionEntry, MovementResult, BiomechFeatures, ScoreResult, JudgeResult,
711
- )
712
-
713
-
714
- def _entry(test_name="deep_squat", score=2, needs_human=False):
715
- movement = MovementResult(test_name=test_name, side="na", confidence=1.0)
716
- features = BiomechFeatures(
717
- test_name=test_name, view="2d", side="na",
718
- angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": False},
719
- symmetry_delta=None, timing={"deepest_frame": 1}, confidence=0.9,
720
- )
721
- rubric = ScoreResult(score=2, rationale="rubric ok", confidence=0.8)
722
- judge = JudgeResult(score=None if needs_human else score, rationale="judge rationale",
723
- compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
724
- confidence=0.85, needs_human=needs_human)
725
- return SessionEntry(
726
- test_name=test_name, side="na", score=None if needs_human else score,
727
- needs_human=needs_human, rationale="judge rationale",
728
- compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
729
- measurements={"left_knee_flexion_deg": 95.0, "knees_tracking_over_feet": False},
730
- confidence=0.85, view="2d", keyframe_path=None,
731
- movement=movement, features=features, rubric_score=rubric, judge=judge,
732
- )
733
-
734
-
735
- def _report(composite=2):
736
- return ReportResult(
737
- per_test=[], composite=composite, asymmetries=[],
738
- overlay_video_path=None, pdf_path=None,
739
- low_confidence_flags=[], disagreement_flags=[],
740
- )
741
-
742
-
743
- def test_pdf_is_created(tmp_path):
744
- from formscout.agents.pdf_report import PdfReportAgent
745
- path = PdfReportAgent().run(_report(2), [_entry()], str(tmp_path))
746
- assert path is not None
747
- assert os.path.exists(path)
748
- assert os.path.getsize(path) > 1000 # a real PDF, not an empty file
749
- with open(path, "rb") as f:
750
- assert f.read(5) == b"%PDF-"
751
-
752
-
753
- def test_pdf_handles_incomplete_composite(tmp_path):
754
- from formscout.agents.pdf_report import PdfReportAgent
755
- path = PdfReportAgent().run(_report(None), [_entry(needs_human=True)], str(tmp_path))
756
- assert path is not None and os.path.exists(path)
757
- ```
758
-
759
- - [ ] **Step 2: Run test to verify it fails**
760
-
761
- Run: `pytest tests/test_pdf_report.py -v`
762
- Expected: FAIL with `ModuleNotFoundError: No module named 'formscout.agents.pdf_report'`
763
-
764
- - [ ] **Step 3: Create the agent**
765
-
766
- Create `formscout/agents/pdf_report.py`:
767
-
768
- ```python
769
- """
770
- PdfReportAgent — renders a ReportResult + session entries to a branded PDF.
771
-
772
- Input: ReportResult, list[SessionEntry], session_dir (str)
773
- Output: path to the written PDF (str), or None on failure.
774
- Failure: returns None, never raises.
775
- Params: 0 (pure rendering — no model).
776
- License: n/a.
777
- Gated: no.
778
- """
779
- from __future__ import annotations
780
-
781
- import logging
782
- import os
783
-
784
- from formscout.types import ReportResult
785
-
786
- logger = logging.getLogger(__name__)
787
-
788
- DISCLAIMER = "Screening aid — not a diagnosis. Pain or clearing tests require a clinician."
789
-
790
-
791
- class PdfReportAgent:
792
- """Assembles the screening-session PDF via ReportLab."""
793
-
794
- def run(self, report: ReportResult, entries: list, session_dir: str) -> str | None:
795
- try:
796
- from reportlab.lib import colors
797
- from reportlab.lib.pagesizes import LETTER
798
- from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
799
- from reportlab.lib.units import inch
800
- from reportlab.platypus import (
801
- Image, Paragraph, SimpleDocTemplate, Spacer, Table, TableStyle,
802
- )
803
- except Exception as e:
804
- logger.warning("reportlab unavailable: %s", e)
805
- return None
806
-
807
- out_path = os.path.join(session_dir, "formscout_report.pdf")
808
- try:
809
- styles = getSampleStyleSheet()
810
- banner = ParagraphStyle(
811
- "banner", parent=styles["Normal"], fontSize=9, textColor=colors.white,
812
- backColor=colors.HexColor("#b45309"), alignment=1, borderPadding=6, spaceAfter=12,
813
- )
814
- story = []
815
- story.append(Paragraph(f"<b>&#9888; {DISCLAIMER}</b>", banner))
816
- story.append(Paragraph("FormScout — FMS Screening Report", styles["Title"]))
817
-
818
- if report.composite is not None:
819
- comp = f"Composite: <b>{report.composite} / 21</b>"
820
- else:
821
- comp = f"Composite: <b>Incomplete</b> — {len(entries)}/7 tests scored"
822
- story.append(Paragraph(comp, styles["Heading2"]))
823
- story.append(Spacer(1, 0.2 * inch))
824
-
825
- for e in entries:
826
- title = e.test_name.replace("_", " ").title()
827
- if e.side in ("left", "right"):
828
- title += f" ({e.side})"
829
- score_txt = "Clinician review required" if e.needs_human else f"Score: {e.score}/3"
830
- story.append(Paragraph(f"<b>{title}</b> — {score_txt}", styles["Heading3"]))
831
- if e.rationale:
832
- story.append(Paragraph(e.rationale, styles["Normal"]))
833
- if e.compensation_tags:
834
- story.append(Paragraph("Compensations: " + ", ".join(e.compensation_tags),
835
- styles["Normal"]))
836
- if e.corrective_hint:
837
- story.append(Paragraph("Corrective: " + e.corrective_hint, styles["Normal"]))
838
-
839
- items = list(e.measurements.items())[:6]
840
- if items:
841
- rows = [[k.replace("_", " "),
842
- (f"{v:.1f}" if isinstance(v, float) else str(v))] for k, v in items]
843
- tbl = Table(rows, colWidths=[3 * inch, 1.5 * inch])
844
- tbl.setStyle(TableStyle([
845
- ("FONTSIZE", (0, 0), (-1, -1), 8),
846
- ("TEXTCOLOR", (0, 0), (-1, -1), colors.HexColor("#334155")),
847
- ]))
848
- story.append(tbl)
849
-
850
- if e.keyframe_path and os.path.exists(e.keyframe_path):
851
- try:
852
- story.append(Image(e.keyframe_path, width=3.0 * inch, height=2.25 * inch))
853
- except Exception:
854
- story.append(Paragraph("<i>(key-frame image unavailable)</i>", styles["Normal"]))
855
- else:
856
- story.append(Paragraph("<i>(key-frame image unavailable)</i>", styles["Normal"]))
857
-
858
- story.append(Spacer(1, 0.2 * inch))
859
-
860
- if report.asymmetries:
861
- story.append(Paragraph("Asymmetries", styles["Heading2"]))
862
- for a in report.asymmetries:
863
- story.append(Paragraph(
864
- f"{a['test'].replace('_', ' ').title()}: "
865
- f"L={a['left_score']} R={a['right_score']} (&#916; {a['delta']})",
866
- styles["Normal"]))
867
-
868
- flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
869
- if flags:
870
- story.append(Paragraph("Flags", styles["Heading2"]))
871
- for fl in flags:
872
- story.append(Paragraph(fl, styles["Normal"]))
873
-
874
- story.append(Spacer(1, 0.3 * inch))
875
- story.append(Paragraph(f"<b>&#9888; {DISCLAIMER}</b>", banner))
876
-
877
- doc = SimpleDocTemplate(out_path, pagesize=LETTER,
878
- topMargin=0.6 * inch, bottomMargin=0.6 * inch)
879
- doc.build(story)
880
- return out_path
881
- except Exception as e:
882
- logger.warning("pdf build failed: %s", e)
883
- return None
884
- ```
885
-
886
- - [ ] **Step 4: Run test to verify it passes**
887
-
888
- Run: `pytest tests/test_pdf_report.py -v`
889
- Expected: both tests PASS
890
-
891
- - [ ] **Step 5: Re-run the session suite (pdf_path now populated)**
892
-
893
- Run: `pytest tests/test_session.py -v`
894
- Expected: all PASS (now `finish_session` returns a real `pdf_path`).
895
-
896
- - [ ] **Step 6: Commit**
897
-
898
- ```bash
899
- git add formscout/agents/pdf_report.py tests/test_pdf_report.py
900
- git commit -m "feat: add PdfReportAgent — branded ReportLab session PDF"
901
- ```
902
-
903
- ---
904
-
905
- ## Task 7: Wire the session UI in `app.py`
906
-
907
- **Files:**
908
- - Modify: `app.py` (`process_video`, `build_app`, event wiring)
909
-
910
- This task is verified by running the app (Gradio event wiring is not unit-tested; the
911
- orchestration it calls is already covered by `tests/test_session.py`).
912
-
913
- - [ ] **Step 1: Import the session module**
914
-
915
- In `app.py`, add to the imports block (after `from formscout.startup import ensure_checkpoints`):
916
-
917
- ```python
918
- from formscout import session as session_mod
919
- ```
920
-
921
- - [ ] **Step 2: Refactor `process_video` to accumulate into a session**
922
-
923
- Replace the entire `process_video` function (lines ~51-105) with a version that takes and
924
- returns the session, appends an entry on success, and builds the "Session so far" table.
925
- Replace from `def process_video(` through its final `return ...` with:
926
-
927
- ```python
928
- def process_video(video_path: str, test_name: str, side: str, model_key: str,
929
- layers: list[str], session_state):
930
- """Analyse one clip and accumulate it into the screening session."""
931
- if not video_path:
932
- return (
933
- session_state, _render_empty_state(), "Upload a video to begin analysis.",
934
- "", "", None, "", _render_session_table(session_state),
935
- gr.update(visible=False), gr.update(visible=False),
936
- )
937
-
938
- if session_state is None:
939
- session_state = session_mod.new_session()
940
-
941
- director = Director()
942
- state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
943
-
944
- score_html = _render_empty_state()
945
- score_details = ""
946
-
947
- if state.features:
948
- result = score_test(state.features)
949
- judge = state.judge
950
- if judge and judge.score is not None:
951
- score_html = _render_score_card(judge.score, judge.confidence, judge.needs_human)
952
- score_details = _render_score_details_judge(judge, result, state.features)
953
- elif judge and judge.needs_human:
954
- score_html = _render_score_card(0, 0, True)
955
- score_details = f"### Needs Clinician Review\n{judge.rationale}"
956
- else:
957
- score_html = _render_score_card(result.score, result.confidence, result.needs_human)
958
- score_details = _render_score_details(result, state.features)
959
-
960
- # Accumulate into the session (only when we have a real analysis)
961
- if state.ingest and state.pose2d and state.judge:
962
- draw_trails = "trails" in {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
963
- try:
964
- session_mod.add_analysis(
965
- session_state, ingest=state.ingest, pose2d=state.pose2d,
966
- features=state.features, judge=state.judge,
967
- test_name=test_name, side=side, draw_trails=draw_trails,
968
- )
969
- except Exception as e:
970
- state.warnings.append(f"session accumulation failed: {e}")
971
-
972
- pipeline_md = _render_pipeline_status(state)
973
- alerts = _render_alerts(state)
974
-
975
- overlay_path = None
976
- vel_summary = ""
977
- layer_set = {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
978
- if layer_set and state.ingest and state.pose2d:
979
- try:
980
- from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
981
- vis = PoseVisualizer()
982
- with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
983
- out_path = f.name
984
- overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
985
- if overlay_path:
986
- vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
987
- except Exception as e:
988
- alerts = (alerts or "") + f"\n⚠️ Visualizer error: {e}"
989
-
990
- has_entries = bool(session_state and session_state.entries)
991
- return (
992
- session_state, score_html, pipeline_md, score_details, alerts,
993
- overlay_path, vel_summary, _render_session_table(session_state),
994
- gr.update(visible=has_entries), gr.update(visible=has_entries),
995
- )
996
- ```
997
-
998
- - [ ] **Step 3: Add the session-table renderer and finish handler**
999
-
1000
- In `app.py`, add these two functions just before `def build_app()`:
1001
-
1002
- ```python
1003
- def _render_session_table(session_state) -> str:
1004
- """Render the accumulated 'Session so far' table as markdown."""
1005
- if not session_state or not session_state.entries:
1006
- return "*No clips analysed yet.*"
1007
- lines = ["| Test | Side | Score | Status |", "|---|---|---|---|"]
1008
- for e in session_state.entries:
1009
- test = e.test_name.replace("_", " ").title()
1010
- side = e.side if e.side in ("left", "right") else "—"
1011
- if e.needs_human:
1012
- score, status = "—", "⚠️ Clinician review"
1013
- else:
1014
- score, status = f"{e.score}/3", "✓ scored"
1015
- lines.append(f"| {test} | {side} | {score} | {status} |")
1016
- return "\n".join(lines)
1017
-
1018
-
1019
- def _finish_session(session_state):
1020
- """Build the composite report + PDF for the whole session."""
1021
- if not session_state or not session_state.entries:
1022
- return ("⚠️ No clips analysed yet — analyse at least one clip first.",
1023
- None, None)
1024
-
1025
- report, pdf_path = session_mod.finish_session(session_state)
1026
- if report is None:
1027
- return ("⚠️ Nothing to report.", None, None)
1028
-
1029
- if report.composite is not None:
1030
- summary = [f"## Composite: {report.composite} / 21"]
1031
- else:
1032
- n = len(session_state.entries)
1033
- summary = [f"## Composite: Incomplete — {n}/7 tests scored",
1034
- "*(One or more tests need clinician review or were unscored.)*"]
1035
-
1036
- if report.asymmetries:
1037
- summary.append("\n### Asymmetries")
1038
- for a in report.asymmetries:
1039
- test = a["test"].replace("_", " ").title()
1040
- summary.append(f"- **{test}:** L={a['left_score']} R={a['right_score']} (Δ {a['delta']})")
1041
-
1042
- flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
1043
- if flags:
1044
- summary.append("\n### Flags")
1045
- for fl in flags:
1046
- summary.append(f"- {fl}")
1047
-
1048
- md_path = os.path.join(session_state.session_dir, "analysis.md")
1049
- md_out = md_path if os.path.exists(md_path) else None
1050
- return "\n".join(summary), pdf_path, md_out
1051
- ```
1052
-
1053
- Also add `import os` to the top of `app.py` if not already present (it currently imports only
1054
- `tempfile` and `gradio`). Add after `import tempfile`:
1055
-
1056
- ```python
1057
- import os
1058
- ```
1059
-
1060
- - [ ] **Step 4: Add the session state, buttons, and outputs to `build_app`**
1061
-
1062
- In `build_app`, inside the `with gr.Blocks(...) as app:` block, immediately after the line
1063
- `with gr.Blocks(title="FormScout — FMS Screening Aid") as app:` add:
1064
-
1065
- ```python
1066
- session_state = gr.State(None)
1067
- ```
1068
-
1069
- Then, in the left input column, replace the single submit button block:
1070
-
1071
- ```python
1072
- submit_btn = gr.Button(
1073
- "🎯 Score Movement",
1074
- variant="primary",
1075
- size="lg",
1076
- )
1077
- ```
1078
-
1079
- with:
1080
-
1081
- ```python
1082
- submit_btn = gr.Button(
1083
- "🎯 Score Movement",
1084
- variant="primary",
1085
- size="lg",
1086
- )
1087
- with gr.Row():
1088
- new_clip_btn = gr.Button("➕ Analyse new clip", visible=False)
1089
- finish_btn = gr.Button("✅ Finish & generate PDF",
1090
- variant="primary", visible=False)
1091
- ```
1092
-
1093
- In the right results column, add a "Session" tab and a finish-output area. Inside `with gr.Tabs():`
1094
- add a new tab after the "🎬 Overlay Video" tab:
1095
-
1096
- ```python
1097
- with gr.TabItem("🗂️ Session"):
1098
- session_table = gr.Markdown("*No clips analysed yet.*")
1099
- finish_summary = gr.Markdown("")
1100
- pdf_file = gr.File(label="Screening Report (PDF)", visible=True)
1101
- md_file = gr.File(label="Analysis Log (Markdown)", visible=True)
1102
- ```
1103
-
1104
- - [ ] **Step 5: Update event wiring**
1105
-
1106
- Replace the `_map_inputs` function and `submit_btn.click(...)` block at the bottom of `build_app`
1107
- with:
1108
-
1109
- ```python
1110
- def _map_inputs(video, test_display_name, side_display, pose_model_key, overlay_layers, sess):
1111
- """Map UI display values to internal values and accumulate into the session."""
1112
- test_map = {name: val for name, val in FMS_TESTS}
1113
- test_name = test_map.get(test_display_name, "deep_squat")
1114
- side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
1115
- return process_video(video, test_name, side, pose_model_key, overlay_layers, sess)
1116
-
1117
- submit_btn.click(
1118
- fn=_map_inputs,
1119
- inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown,
1120
- overlay_layers, session_state],
1121
- outputs=[session_state, score_html, pipeline_md, score_details, alerts_md,
1122
- overlay_video, velocity_md, session_table, new_clip_btn, finish_btn],
1123
- )
1124
-
1125
- def _new_clip():
1126
- """Clear inputs for the next clip; keep the session intact."""
1127
- return None, _render_empty_state(), ""
1128
-
1129
- new_clip_btn.click(
1130
- fn=_new_clip,
1131
- inputs=[],
1132
- outputs=[video_input, score_html, score_details],
1133
- )
1134
-
1135
- finish_btn.click(
1136
- fn=_finish_session,
1137
- inputs=[session_state],
1138
- outputs=[finish_summary, pdf_file, md_file],
1139
- )
1140
- ```
1141
-
1142
- - [ ] **Step 6: Verify the full test suite still passes**
1143
-
1144
- Run: `pytest tests/ -q`
1145
- Expected: all tests pass except the single pre-existing known failure documented in CLAUDE.md
1146
- (`test_unimplemented_test_returns_low_confidence`). No new failures.
1147
-
1148
- - [ ] **Step 7: Manually verify the app**
1149
-
1150
- Run: `python3 app.py`
1151
- Then in the browser:
1152
- 1. Upload a clip, pick a test, click **Score Movement** → score card appears; the **Session** tab
1153
- shows one row; the two new buttons appear.
1154
- 2. Click **➕ Analyse new clip** → the video input clears, the session row persists.
1155
- 3. Analyse a second test → a second row appears.
1156
- 4. Click **✅ Finish & generate PDF** → the Session tab shows the composite summary and a
1157
- downloadable PDF (open it: disclaimer top + bottom, per-test blocks with key-frame images,
1158
- composite or "Incomplete"). The Markdown log is also downloadable.
1159
-
1160
- Expected: all four steps work; PDF opens and contains the disclaimer, composite, and per-test sections.
1161
-
1162
- - [ ] **Step 8: Commit**
1163
-
1164
- ```bash
1165
- git add app.py
1166
- git commit -m "feat: accumulate FMS clips into a session with composite report + PDF export"
1167
- ```
1168
-
1169
- ---
1170
-
1171
- ## Task 8: Update docs
1172
-
1173
- **Files:**
1174
- - Modify: `CLAUDE.md` (Build phases / status)
1175
- - Modify: `MODEL_BUDGET.md` (no param change — note PDF agent adds 0 params, for completeness)
1176
-
1177
- - [ ] **Step 1: Update the Phase 4 line in CLAUDE.md**
1178
-
1179
- In `CLAUDE.md`, in the "Build phases" section, update the Phase 4 line from:
1180
-
1181
- ```
1182
- 4. **Phase 4 — Polish + ship:** Custom Svelte UI components, PDF export, agent trace to Hub, blog post. (Overlay video already done via `PoseVisualizer`.)
1183
- ```
1184
-
1185
- to:
1186
-
1187
- ```
1188
- 4. **Phase 4 — Polish + ship:** Custom Svelte UI components, agent trace to Hub, blog post. (Overlay video done via `PoseVisualizer`; full 7-test session + PDF export done via `formscout/session.py` + `PdfReportAgent`.)
1189
- ```
1190
-
1191
- - [ ] **Step 2: Note the PDF agent in the architecture section**
1192
-
1193
- In `CLAUDE.md`, under "### Rubric scorers" or near the ReportAgent description, this is optional
1194
- context; no required change. Skip if no natural home.
1195
-
1196
- - [ ] **Step 3: Commit**
1197
-
1198
- ```bash
1199
- git add CLAUDE.md
1200
- git commit -m "docs: mark full FMS session + PDF export complete in build phases"
1201
- ```
1202
-
1203
- ---
1204
-
1205
- ## Self-Review Notes (already applied)
1206
-
1207
- - **Spec coverage:** session accumulation (Task 5), two-button UX (Task 7), on-disk MD/JSON/keyframes (Task 5), key-frame from `features.timing` (Tasks 3–5), ReportLab PDF top/bottom disclaimer + composite + per-test + asymmetry + flags (Task 6), `SessionEntry` type (Task 2), `ReportAgent` reuse (Task 5 `finish_session`), composite-null-on-needs-human (Task 5 test), error tolerance / never-raise (Tasks 4–6). All covered.
1208
- - **Type consistency:** `SessionEntry` field names are identical across Tasks 2, 5, 6, 7. `finish_session` returns `(ReportResult | None, str | None)` and is consumed that way in Task 7. `render_frame(ingest, pose2d, frame_idx, layers, caption, out_png)` signature matches its callers.
1209
- - **No placeholders:** every code step shows complete code; every run step states the exact command + expected outcome.
 
1
+ # Full FMS Session + PDF Report — Implementation Plan
2
+
3
+ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
4
+
5
+ **Goal:** Turn FormScout's one-clip scorer into a screening session that accumulates analyzed clips into a composite 0–21 report and exports a branded PDF with annotated worst-moment key-frame stills.
6
+
7
+ **Architecture:** A new `formscout/session.py` accumulates typed `SessionEntry` objects (one per analyzed clip), persisting each to a temp session dir. `PoseVisualizer.render_frame()` captures the governing frame (already computed by `BiomechanicsAgent` and stored in `features.timing`) as an annotated PNG. On "Finish", the existing `ReportAgent` computes composite + asymmetries, and a new `PdfReportAgent` renders a ReportLab PDF. The UI (`app.py`) gains `gr.State` session accumulation with "Analyse new clip" / "Finish & generate PDF" buttons.
8
+
9
+ **Tech Stack:** Python 3.13, ReportLab (new dep), OpenCV (existing), Gradio 5, pytest. No model downloads in tests.
10
+
11
+ ---
12
+
13
+ ## File Structure
14
+
15
+ - `requirements.txt` — add `reportlab`.
16
+ - `formscout/types.py` — add `SessionEntry` frozen dataclass.
17
+ - `formscout/agents/biomechanics.py` — add `max_sag_frame` to `trunk_stability_pushup` timing (rotary already has `peak_extension_frame`).
18
+ - `formscout/agents/visualizer.py` — add `PoseVisualizer.render_frame()`.
19
+ - `formscout/session.py` — **new**: session accumulator (new/add/finish + persistence + key-frame helpers).
20
+ - `formscout/agents/pdf_report.py` — **new**: `PdfReportAgent` (ReportLab).
21
+ - `app.py` — wire `gr.State`, two buttons, "Session so far" table, finish handler.
22
+ - `tests/test_session.py`, `tests/test_keyframe.py`, `tests/test_pdf_report.py` — **new**.
23
+
24
+ ---
25
+
26
+ ## Task 1: Add ReportLab dependency
27
+
28
+ **Files:**
29
+ - Modify: `requirements.txt`
30
+
31
+ - [ ] **Step 1: Add the dependency**
32
+
33
+ Add this line to `requirements.txt` (after `pillow>=10.3`):
34
+
35
+ ```
36
+ reportlab>=4.0
37
+ ```
38
+
39
+ - [ ] **Step 2: Install it**
40
+
41
+ Run: `pip install 'reportlab>=4.0'`
42
+ Expected: `Successfully installed reportlab-4.x.x`
43
+
44
+ - [ ] **Step 3: Verify import**
45
+
46
+ Run: `python3 -c "import reportlab; print(reportlab.Version)"`
47
+ Expected: prints a version like `4.x.x`
48
+
49
+ - [ ] **Step 4: Commit**
50
+
51
+ ```bash
52
+ git add requirements.txt
53
+ git commit -m "build: add reportlab for PDF report generation"
54
+ ```
55
+
56
+ ---
57
+
58
+ ## Task 2: Add `SessionEntry` dataclass
59
+
60
+ **Files:**
61
+ - Modify: `formscout/types.py` (after `ReportResult`, before `PipelineState`)
62
+ - Test: `tests/test_session.py`
63
+
64
+ - [ ] **Step 1: Write the failing test**
65
+
66
+ Create `tests/test_session.py` with:
67
+
68
+ ```python
69
+ """Tests for the FMS session accumulator — no GPU, no model downloads."""
70
+ import numpy as np
71
+
72
+ from formscout.types import (
73
+ IngestResult, Pose2DResult, BiomechFeatures, ScoreResult, JudgeResult,
74
+ MovementResult, SessionEntry,
75
+ )
76
+
77
+
78
+ def test_session_entry_holds_typed_objects():
79
+ movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
80
+ features = BiomechFeatures(
81
+ test_name="deep_squat", view="2d", side="na",
82
+ angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": True},
83
+ symmetry_delta=None, timing={"deepest_frame": 2}, confidence=0.9,
84
+ )
85
+ rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
86
+ judge = JudgeResult(score=2, rationale="ok", compensation_tags=["heels elevated"],
87
+ corrective_hint="ankle mobility", confidence=0.85)
88
+ entry = SessionEntry(
89
+ test_name="deep_squat", side="na", score=2, needs_human=False,
90
+ rationale="ok", compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
91
+ measurements={"left_knee_flexion_deg": 95.0}, confidence=0.85, view="2d",
92
+ keyframe_path=None, movement=movement, features=features,
93
+ rubric_score=rubric, judge=judge,
94
+ )
95
+ assert entry.score == 2
96
+ assert entry.movement.test_name == "deep_squat"
97
+ assert entry.rubric_score.score == 2
98
+ assert entry.judge.compensation_tags == ["heels elevated"]
99
+ ```
100
+
101
+ - [ ] **Step 2: Run test to verify it fails**
102
+
103
+ Run: `pytest tests/test_session.py::test_session_entry_holds_typed_objects -v`
104
+ Expected: FAIL with `ImportError: cannot import name 'SessionEntry'`
105
+
106
+ - [ ] **Step 3: Add the dataclass**
107
+
108
+ In `formscout/types.py`, insert after the `ReportResult` class (line ~142) and before `PipelineState`:
109
+
110
+ ```python
111
+ @dataclass(frozen=True)
112
+ class SessionEntry:
113
+ """One accumulated analysis in a screening session.
114
+
115
+ Display fields (test_name…keyframe_path) feed the PDF/JSON/MD artifacts;
116
+ the trailing typed objects (movement…judge) feed ReportAgent.run().
117
+ """
118
+ test_name: str
119
+ side: str
120
+ score: int | None
121
+ needs_human: bool
122
+ rationale: str
123
+ compensation_tags: list
124
+ corrective_hint: str
125
+ measurements: dict
126
+ confidence: float
127
+ view: str
128
+ keyframe_path: str | None
129
+ movement: MovementResult
130
+ features: BiomechFeatures
131
+ rubric_score: ScoreResult
132
+ judge: JudgeResult | None
133
+ ```
134
+
135
+ - [ ] **Step 4: Run test to verify it passes**
136
+
137
+ Run: `pytest tests/test_session.py::test_session_entry_holds_typed_objects -v`
138
+ Expected: PASS
139
+
140
+ - [ ] **Step 5: Commit**
141
+
142
+ ```bash
143
+ git add formscout/types.py tests/test_session.py
144
+ git commit -m "feat: add SessionEntry typed contract for screening sessions"
145
+ ```
146
+
147
+ ---
148
+
149
+ ## Task 3: Add governing-frame index to push-up biomechanics
150
+
151
+ **Files:**
152
+ - Modify: `formscout/agents/biomechanics.py:468-529` (`_trunk_stability_pushup`)
153
+ - Test: `tests/test_biomechanics.py` (append a test)
154
+
155
+ The other six tests already store a governing frame index in `features.timing`
156
+ (`deepest_frame`, `peak_step_frame`, `deepest_lunge_frame`, `measure_frame`,
157
+ `peak_raise_frame`, `peak_extension_frame`). Only `trunk_stability_pushup` is missing one.
158
+
159
+ - [ ] **Step 1: Write the failing test**
160
+
161
+ Append to `tests/test_biomechanics.py`:
162
+
163
+ ```python
164
+ def test_pushup_timing_has_max_sag_frame():
165
+ from formscout.agents.biomechanics import BiomechanicsAgent
166
+ from formscout.types import Pose2DResult, Body3DResult, MovementResult
167
+
168
+ # 4 frames; frame 2 has the largest hip sag (hip far below shoulder/ankle midline)
169
+ def kps(hip_y):
170
+ base = {
171
+ 5: {"x": 200, "y": 200, "conf": 0.9}, # L shoulder
172
+ 6: {"x": 220, "y": 200, "conf": 0.9}, # R shoulder
173
+ 11: {"x": 300, "y": hip_y, "conf": 0.9}, # L hip
174
+ 12: {"x": 320, "y": hip_y, "conf": 0.9}, # R hip
175
+ 15: {"x": 400, "y": 200, "conf": 0.9}, # L ankle
176
+ 16: {"x": 420, "y": 200, "conf": 0.9}, # R ankle
177
+ }
178
+ return base
179
+
180
+ frames = [kps(200), kps(210), kps(260), kps(205)]
181
+ pose = Pose2DResult(keypoints=frames, fps=30.0, confidence=0.9)
182
+ body3d = Body3DResult(used=False, joints_3d=[])
183
+ movement = MovementResult(test_name="trunk_stability_pushup", side="na", confidence=1.0)
184
+
185
+ feats = BiomechanicsAgent().run(pose, body3d, movement)
186
+ assert "max_sag_frame" in feats.timing
187
+ assert feats.timing["max_sag_frame"] == 2
188
+ ```
189
+
190
+ - [ ] **Step 2: Run test to verify it fails**
191
+
192
+ Run: `pytest tests/test_biomechanics.py::test_pushup_timing_has_max_sag_frame -v`
193
+ Expected: FAIL with `assert 'max_sag_frame' in {...}` (KeyError-style assertion failure)
194
+
195
+ - [ ] **Step 3: Track the max-sag frame index**
196
+
197
+ In `formscout/agents/biomechanics.py`, replace the body of `_trunk_stability_pushup` from the
198
+ `trunk_angles_over_time = []` loop through the `if trunk_angles_over_time:` block. Replace:
199
+
200
+ ```python
201
+ # Analyze multiple frames to detect sag/lag
202
+ trunk_angles_over_time = []
203
+ for i, kps in enumerate(pose2d.keypoints):
204
+ ```
205
+
206
+ …down to and including the `alignments["no_sag"] = max_sag < 30` line, with:
207
+
208
+ ```python
209
+ # Analyze multiple frames to detect sag/lag
210
+ trunk_sags: list[tuple[int, float]] = [] # (frame_idx, sag_px)
211
+ for i, kps in enumerate(pose2d.keypoints):
212
+ l_sh = _get_joint(kps, L_SHOULDER)
213
+ r_sh = _get_joint(kps, R_SHOULDER)
214
+ l_hip = _get_joint(kps, L_HIP)
215
+ r_hip = _get_joint(kps, R_HIP)
216
+ l_ankle = _get_joint(kps, L_ANKLE)
217
+ r_ankle = _get_joint(kps, R_ANKLE)
218
+
219
+ if l_sh and r_sh and l_hip and r_hip and l_ankle and r_ankle:
220
+ sh_y = (l_sh[1] + r_sh[1]) / 2
221
+ hip_y = (l_hip[1] + r_hip[1]) / 2
222
+ ankle_y = (l_ankle[1] + r_ankle[1]) / 2
223
+ expected_hip_y = (sh_y + ankle_y) / 2
224
+ sag_px = hip_y - expected_hip_y
225
+ trunk_sags.append((i, sag_px))
226
+
227
+ max_sag_frame = 0
228
+ if trunk_sags:
229
+ sags = [s for _, s in trunk_sags]
230
+ max_sag_frame = max(trunk_sags, key=lambda t: t[1])[0]
231
+ mean = sum(sags) / len(sags)
232
+ variance = (sum((x - mean) ** 2 for x in sags) / len(sags)) ** 0.5
233
+ max_sag = max(sags)
234
+ angles["max_sag_px"] = max_sag
235
+ angles["trunk_variance_px"] = variance
236
+ alignments["body_rigid"] = max_sag < 30 and variance < 15
237
+ alignments["no_sag"] = max_sag < 30
238
+ else:
239
+ notes_parts.append("insufficient landmarks for trunk analysis")
240
+ ```
241
+
242
+ Then update the `return BiomechFeatures(...)` `timing=` argument at the end of the method from:
243
+
244
+ ```python
245
+ timing={"n_frames_analyzed": len(trunk_angles_over_time)},
246
+ ```
247
+
248
+ to:
249
+
250
+ ```python
251
+ timing={"n_frames_analyzed": len(trunk_sags), "max_sag_frame": max_sag_frame},
252
+ ```
253
+
254
+ - [ ] **Step 4: Run test to verify it passes**
255
+
256
+ Run: `pytest tests/test_biomechanics.py::test_pushup_timing_has_max_sag_frame -v`
257
+ Expected: PASS
258
+
259
+ - [ ] **Step 5: Run the full biomechanics suite (no regressions)**
260
+
261
+ Run: `pytest tests/test_biomechanics.py -v`
262
+ Expected: all previously-passing tests still pass (the pre-existing `test_unimplemented_test_returns_low_confidence` known-failure may remain failing — that is unrelated and documented in CLAUDE.md).
263
+
264
+ - [ ] **Step 6: Commit**
265
+
266
+ ```bash
267
+ git add formscout/agents/biomechanics.py tests/test_biomechanics.py
268
+ git commit -m "feat: track max-sag frame index in push-up biomechanics for key-frame capture"
269
+ ```
270
+
271
+ ---
272
+
273
+ ## Task 4: Add `PoseVisualizer.render_frame()`
274
+
275
+ **Files:**
276
+ - Modify: `formscout/agents/visualizer.py` (add method to `PoseVisualizer`, after `render_video`)
277
+ - Test: `tests/test_keyframe.py`
278
+
279
+ - [ ] **Step 1: Write the failing test**
280
+
281
+ Create `tests/test_keyframe.py`:
282
+
283
+ ```python
284
+ """Tests for PoseVisualizer.render_frame — single annotated still."""
285
+ import os
286
+ import numpy as np
287
+
288
+ from formscout.types import IngestResult, Pose2DResult
289
+
290
+
291
+ def _ingest(n=5, h=480, w=640):
292
+ frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
293
+ return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
294
+
295
+
296
+ def _pose(n=5):
297
+ kps = []
298
+ for i in range(n):
299
+ kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
300
+ for j in range(17)})
301
+ return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
302
+
303
+
304
+ def test_render_frame_writes_png(tmp_path):
305
+ from formscout.agents.visualizer import PoseVisualizer
306
+ out = str(tmp_path / "key.png")
307
+ path = PoseVisualizer().render_frame(_ingest(), _pose(), frame_idx=2,
308
+ layers={"skeleton"}, caption="Deep Squat — heels elevated",
309
+ out_png=out)
310
+ assert path == out
311
+ assert os.path.exists(out)
312
+ assert os.path.getsize(out) > 0
313
+
314
+
315
+ def test_render_frame_bad_index_returns_none(tmp_path):
316
+ from formscout.agents.visualizer import PoseVisualizer
317
+ out = str(tmp_path / "key.png")
318
+ path = PoseVisualizer().render_frame(_ingest(n=3), _pose(n=3), frame_idx=99,
319
+ layers={"skeleton"}, caption="", out_png=out)
320
+ assert path is None
321
+ ```
322
+
323
+ - [ ] **Step 2: Run test to verify it fails**
324
+
325
+ Run: `pytest tests/test_keyframe.py -v`
326
+ Expected: FAIL with `AttributeError: 'PoseVisualizer' object has no attribute 'render_frame'`
327
+
328
+ - [ ] **Step 3: Add the method**
329
+
330
+ In `formscout/agents/visualizer.py`, inside the `PoseVisualizer` class, add this method
331
+ immediately after `render_video` (before the closing of the class / the module-level
332
+ `build_velocity_summary`):
333
+
334
+ ```python
335
+ def render_frame(
336
+ self,
337
+ ingest,
338
+ pose2d,
339
+ frame_idx: int,
340
+ layers: set[str],
341
+ caption: str = "",
342
+ out_png: str | None = None,
343
+ ) -> str | None:
344
+ """Render a single annotated still (skeleton + optional trails + caption).
345
+
346
+ frame_idx is typically the governing frame from BiomechFeatures.timing.
347
+ Returns the PNG path on success, None on any failure. Never raises.
348
+ """
349
+ try:
350
+ if not (0 <= frame_idx < len(ingest.frames)) or frame_idx >= len(pose2d.keypoints):
351
+ return None
352
+
353
+ frame = ingest.frames[frame_idx].copy()
354
+ kps = pose2d.keypoints[frame_idx]
355
+
356
+ if "trails" in layers:
357
+ trail: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
358
+ start = max(0, frame_idx - TRAIL_LENGTH)
359
+ for fi in range(start, frame_idx + 1):
360
+ for j, kp in pose2d.keypoints[fi].items():
361
+ if kp.get("conf", 0.0) >= CONF_THRESHOLD:
362
+ trail[j].append((kp["x"], kp["y"]))
363
+ frame = self._draw_trails(frame, trail)
364
+
365
+ if "skeleton" in layers:
366
+ frame = self._draw_skeleton(frame, kps)
367
+
368
+ if caption:
369
+ cv2.rectangle(frame, (0, 0), (frame.shape[1], 28), (0, 0, 0), -1)
370
+ cv2.putText(frame, caption[:80], (8, 20), cv2.FONT_HERSHEY_SIMPLEX,
371
+ 0.55, (255, 255, 255), 1, cv2.LINE_AA)
372
+
373
+ if out_png is None:
374
+ out_png = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
375
+
376
+ ok = cv2.imwrite(out_png, frame)
377
+ return out_png if ok else None
378
+ except Exception as e:
379
+ logger.warning("render_frame failed: %s", e)
380
+ return None
381
+ ```
382
+
383
+ (`deque`, `cv2`, `tempfile`, `logger`, `TRAIL_LENGTH`, `CONF_THRESHOLD` are all already imported at the top of this file.)
384
+
385
+ - [ ] **Step 4: Run test to verify it passes**
386
+
387
+ Run: `pytest tests/test_keyframe.py -v`
388
+ Expected: both tests PASS
389
+
390
+ - [ ] **Step 5: Commit**
391
+
392
+ ```bash
393
+ git add formscout/agents/visualizer.py tests/test_keyframe.py
394
+ git commit -m "feat: add PoseVisualizer.render_frame for annotated key-frame stills"
395
+ ```
396
+
397
+ ---
398
+
399
+ ## Task 5: Create the session accumulator
400
+
401
+ **Files:**
402
+ - Create: `formscout/session.py`
403
+ - Test: `tests/test_session.py` (append tests)
404
+
405
+ - [ ] **Step 1: Write the failing tests**
406
+
407
+ Append to `tests/test_session.py`:
408
+
409
+ ```python
410
+ def _ingest(n=5, h=480, w=640):
411
+ frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
412
+ return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
413
+
414
+
415
+ def _pose(n=5):
416
+ kps = []
417
+ for i in range(n):
418
+ kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
419
+ for j in range(17)})
420
+ return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
421
+
422
+
423
+ def _features(test_name="deep_squat", side="na", frame_key="deepest_frame"):
424
+ return BiomechFeatures(
425
+ test_name=test_name, view="2d", side=side,
426
+ angles={"left_knee_flexion_deg": 95.0},
427
+ alignments={"knees_tracking_over_feet": False},
428
+ symmetry_delta=None, timing={frame_key: 2}, confidence=0.9,
429
+ )
430
+
431
+
432
+ def _judge(score=2, needs_human=False):
433
+ return JudgeResult(
434
+ score=None if needs_human else score, rationale="r",
435
+ compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
436
+ confidence=0.85, needs_human=needs_human,
437
+ )
438
+
439
+
440
+ def test_add_analysis_appends_entry_and_writes_files():
441
+ import os
442
+ from formscout import session as S
443
+ sess = S.new_session()
444
+ entry = S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
445
+ features=_features(), judge=_judge(), test_name="deep_squat", side="na")
446
+ assert len(sess.entries) == 1
447
+ assert entry.score == 2
448
+ assert os.path.exists(os.path.join(sess.session_dir, "session.json"))
449
+ assert os.path.exists(os.path.join(sess.session_dir, "analysis.md"))
450
+ # key-frame still written (deepest_frame=2 is valid)
451
+ assert entry.keyframe_path and os.path.exists(entry.keyframe_path)
452
+
453
+
454
+ def test_finish_composite_null_when_needs_human():
455
+ from formscout import session as S
456
+ sess = S.new_session()
457
+ S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(), features=_features(),
458
+ judge=_judge(score=3), test_name="deep_squat", side="na")
459
+ S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
460
+ features=_features("trunk_stability_pushup", frame_key="max_sag_frame"),
461
+ judge=_judge(needs_human=True), test_name="trunk_stability_pushup", side="na")
462
+ report, pdf_path = S.finish_session(sess)
463
+ assert report is not None
464
+ assert report.composite is None # one test needs_human
465
+
466
+
467
+ def test_finish_empty_session_returns_none():
468
+ from formscout import session as S
469
+ sess = S.new_session()
470
+ report, pdf_path = S.finish_session(sess)
471
+ assert report is None and pdf_path is None
472
+ ```
473
+
474
+ - [ ] **Step 2: Run tests to verify they fail**
475
+
476
+ Run: `pytest tests/test_session.py -v`
477
+ Expected: the three new tests FAIL with `ModuleNotFoundError: No module named 'formscout.session'`
478
+
479
+ - [ ] **Step 3: Create the module**
480
+
481
+ Create `formscout/session.py`:
482
+
483
+ ```python
484
+ """
485
+ Screening-session accumulator.
486
+
487
+ Accumulates one SessionEntry per analyzed clip, persists each to a temp session
488
+ dir (session.json + analysis.md + key-frame PNGs), and on finish builds a
489
+ ReportResult (via ReportAgent) + a PDF (via PdfReportAgent).
490
+
491
+ Pure orchestration — no Gradio imports. Disk writes tolerate failure with a
492
+ logged warning and never block scoring.
493
+ """
494
+ from __future__ import annotations
495
+
496
+ import json
497
+ import logging
498
+ import os
499
+ import tempfile
500
+ import uuid
501
+ from dataclasses import dataclass, replace
502
+
503
+ from formscout.rubric import score_test
504
+ from formscout.types import MovementResult, ReportResult, SessionEntry
505
+
506
+ logger = logging.getLogger(__name__)
507
+
508
+ # Maps each test to the BiomechFeatures.timing key holding its governing frame.
509
+ TIMING_KEY = {
510
+ "deep_squat": "deepest_frame",
511
+ "hurdle_step": "peak_step_frame",
512
+ "inline_lunge": "deepest_lunge_frame",
513
+ "shoulder_mobility": "measure_frame",
514
+ "active_slr": "peak_raise_frame",
515
+ "trunk_stability_pushup": "max_sag_frame",
516
+ "rotary_stability": "peak_extension_frame",
517
+ }
518
+
519
+
520
+ @dataclass
521
+ class Session:
522
+ """Mutable session: an id, its temp dir, and accumulated entries."""
523
+ session_id: str
524
+ session_dir: str
525
+ entries: list # list[SessionEntry]
526
+
527
+
528
+ def new_session() -> Session:
529
+ sid = uuid.uuid4().hex[:12]
530
+ base = os.path.join(tempfile.gettempdir(), "formscout_sessions", sid)
531
+ try:
532
+ os.makedirs(os.path.join(base, "keyframes"), exist_ok=True)
533
+ except Exception as e:
534
+ logger.warning("session dir create failed: %s", e)
535
+ return Session(session_id=sid, session_dir=base, entries=[])
536
+
537
+
538
+ def governing_frame_index(features) -> int | None:
539
+ """Return the governing frame index for this test, or None."""
540
+ key = TIMING_KEY.get(features.test_name)
541
+ if key is None:
542
+ return None
543
+ idx = features.timing.get(key)
544
+ return int(idx) if isinstance(idx, (int, float)) else None
545
+
546
+
547
+ def worst_compensation_caption(judge, features) -> str:
548
+ """Short caption naming the worst compensation for the key-frame still."""
549
+ if judge and getattr(judge, "compensation_tags", None):
550
+ return ", ".join(judge.compensation_tags)
551
+ failed = [k.replace("_", " ") for k, v in features.alignments.items() if v is False]
552
+ return ("compensation: " + ", ".join(failed)) if failed else "key position"
553
+
554
+
555
+ def add_analysis(session, *, ingest, pose2d, features, judge, test_name, side,
556
+ draw_trails: bool = False) -> SessionEntry:
557
+ """Build a SessionEntry from a completed analysis, render its key-frame,
558
+ persist the session, append, and return the entry."""
559
+ movement = MovementResult(test_name=test_name, side=side, confidence=1.0)
560
+ rubric = score_test(features)
561
+
562
+ needs_human = bool((judge and judge.needs_human) or rubric.needs_human)
563
+ if needs_human:
564
+ score = None
565
+ elif judge and judge.score is not None:
566
+ score = judge.score
567
+ else:
568
+ score = rubric.score
569
+
570
+ keyframe_path = None
571
+ idx = governing_frame_index(features)
572
+ if idx is not None and 0 <= idx < len(pose2d.keypoints):
573
+ from formscout.agents.visualizer import PoseVisualizer
574
+ caption = (f"{test_name.replace('_', ' ').title()} "
575
+ f"({side}) — {worst_compensation_caption(judge, features)}")
576
+ layers = {"skeleton", "trails"} if draw_trails else {"skeleton"}
577
+ out_png = os.path.join(session.session_dir, "keyframes", f"{test_name}_{side}.png")
578
+ try:
579
+ keyframe_path = PoseVisualizer().render_frame(ingest, pose2d, idx, layers, caption, out_png)
580
+ except Exception as e:
581
+ logger.warning("keyframe render failed: %s", e)
582
+
583
+ measurements = {}
584
+ measurements.update(features.angles)
585
+ measurements.update(features.alignments)
586
+
587
+ entry = SessionEntry(
588
+ test_name=test_name, side=side, score=score, needs_human=needs_human,
589
+ rationale=(judge.rationale if judge else rubric.rationale),
590
+ compensation_tags=list(judge.compensation_tags) if judge else [],
591
+ corrective_hint=(judge.corrective_hint if judge else ""),
592
+ measurements=measurements,
593
+ confidence=(judge.confidence if judge else rubric.confidence),
594
+ view=features.view,
595
+ keyframe_path=keyframe_path,
596
+ movement=movement, features=features, rubric_score=rubric, judge=judge,
597
+ )
598
+ session.entries.append(entry)
599
+ _persist(session)
600
+ return entry
601
+
602
+
603
+ def finish_session(session) -> tuple[ReportResult | None, str | None]:
604
+ """Build the composite report + PDF. Returns (report, pdf_path).
605
+ Returns (None, None) for an empty session."""
606
+ if not session.entries:
607
+ return None, None
608
+
609
+ from formscout.agents.report import ReportAgent
610
+ report_inputs = [{
611
+ "movement": e.movement, "features": e.features,
612
+ "rubric_score": e.rubric_score, "judge": e.judge, "side": e.side,
613
+ } for e in session.entries]
614
+ report = ReportAgent().run(report_inputs)
615
+
616
+ pdf_path = None
617
+ try:
618
+ from formscout.agents.pdf_report import PdfReportAgent
619
+ pdf_path = PdfReportAgent().run(report, session.entries, session.session_dir)
620
+ except Exception as e:
621
+ logger.warning("pdf generation failed: %s", e)
622
+
623
+ report = replace(report, pdf_path=pdf_path)
624
+ return report, pdf_path
625
+
626
+
627
+ # ── Persistence ───────────────────────────────────────────────────────────────
628
+
629
+ def _jsonable(d: dict) -> dict:
630
+ out = {}
631
+ for k, v in d.items():
632
+ if isinstance(v, float):
633
+ out[k] = round(v, 2)
634
+ elif isinstance(v, (int, str, bool)) or v is None:
635
+ out[k] = v
636
+ else:
637
+ out[k] = str(v)
638
+ return out
639
+
640
+
641
+ def _entry_display(e: SessionEntry) -> dict:
642
+ return {
643
+ "test_name": e.test_name, "side": e.side, "score": e.score,
644
+ "needs_human": e.needs_human, "rationale": e.rationale,
645
+ "compensation_tags": list(e.compensation_tags), "corrective_hint": e.corrective_hint,
646
+ "measurements": _jsonable(e.measurements), "confidence": round(e.confidence, 2),
647
+ "view": e.view, "keyframe_path": e.keyframe_path,
648
+ }
649
+
650
+
651
+ def _render_markdown(session: Session) -> str:
652
+ lines = ["# FormScout — Session Log", ""]
653
+ for e in session.entries:
654
+ title = e.test_name.replace("_", " ").title()
655
+ if e.side in ("left", "right"):
656
+ title += f" ({e.side})"
657
+ score = "Clinician review required" if e.needs_human else f"{e.score}/3"
658
+ lines.append(f"## {title} {score}")
659
+ lines.append(e.rationale or "")
660
+ if e.compensation_tags:
661
+ lines.append(f"- Compensations: {', '.join(e.compensation_tags)}")
662
+ if e.corrective_hint:
663
+ lines.append(f"- Corrective: {e.corrective_hint}")
664
+ if e.keyframe_path:
665
+ lines.append(f"- Key frame: `{e.keyframe_path}`")
666
+ lines.append("")
667
+ return "\n".join(lines)
668
+
669
+
670
+ def _persist(session: Session) -> None:
671
+ try:
672
+ with open(os.path.join(session.session_dir, "session.json"), "w") as f:
673
+ json.dump([_entry_display(e) for e in session.entries], f, indent=2)
674
+ with open(os.path.join(session.session_dir, "analysis.md"), "w") as f:
675
+ f.write(_render_markdown(session))
676
+ except Exception as e:
677
+ logger.warning("session persist failed: %s", e)
678
+ ```
679
+
680
+ - [ ] **Step 4: Run tests to verify they pass**
681
+
682
+ Run: `pytest tests/test_session.py -v`
683
+ Expected: all session tests PASS (Task 6 provides `PdfReportAgent`; `finish_session` tolerates its
684
+ absence via the try/except, so these pass now — `pdf_path` may be `None` until Task 6).
685
+
686
+ - [ ] **Step 5: Commit**
687
+
688
+ ```bash
689
+ git add formscout/session.py tests/test_session.py
690
+ git commit -m "feat: add screening-session accumulator with key-frame capture and persistence"
691
+ ```
692
+
693
+ ---
694
+
695
+ ## Task 6: Create `PdfReportAgent`
696
+
697
+ **Files:**
698
+ - Create: `formscout/agents/pdf_report.py`
699
+ - Test: `tests/test_pdf_report.py`
700
+
701
+ - [ ] **Step 1: Write the failing test**
702
+
703
+ Create `tests/test_pdf_report.py`:
704
+
705
+ ```python
706
+ """Tests for PdfReportAgent — no GPU, no model downloads."""
707
+ import os
708
+
709
+ from formscout.types import (
710
+ ReportResult, SessionEntry, MovementResult, BiomechFeatures, ScoreResult, JudgeResult,
711
+ )
712
+
713
+
714
+ def _entry(test_name="deep_squat", score=2, needs_human=False):
715
+ movement = MovementResult(test_name=test_name, side="na", confidence=1.0)
716
+ features = BiomechFeatures(
717
+ test_name=test_name, view="2d", side="na",
718
+ angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": False},
719
+ symmetry_delta=None, timing={"deepest_frame": 1}, confidence=0.9,
720
+ )
721
+ rubric = ScoreResult(score=2, rationale="rubric ok", confidence=0.8)
722
+ judge = JudgeResult(score=None if needs_human else score, rationale="judge rationale",
723
+ compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
724
+ confidence=0.85, needs_human=needs_human)
725
+ return SessionEntry(
726
+ test_name=test_name, side="na", score=None if needs_human else score,
727
+ needs_human=needs_human, rationale="judge rationale",
728
+ compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
729
+ measurements={"left_knee_flexion_deg": 95.0, "knees_tracking_over_feet": False},
730
+ confidence=0.85, view="2d", keyframe_path=None,
731
+ movement=movement, features=features, rubric_score=rubric, judge=judge,
732
+ )
733
+
734
+
735
+ def _report(composite=2):
736
+ return ReportResult(
737
+ per_test=[], composite=composite, asymmetries=[],
738
+ overlay_video_path=None, pdf_path=None,
739
+ low_confidence_flags=[], disagreement_flags=[],
740
+ )
741
+
742
+
743
+ def test_pdf_is_created(tmp_path):
744
+ from formscout.agents.pdf_report import PdfReportAgent
745
+ path = PdfReportAgent().run(_report(2), [_entry()], str(tmp_path))
746
+ assert path is not None
747
+ assert os.path.exists(path)
748
+ assert os.path.getsize(path) > 1000 # a real PDF, not an empty file
749
+ with open(path, "rb") as f:
750
+ assert f.read(5) == b"%PDF-"
751
+
752
+
753
+ def test_pdf_handles_incomplete_composite(tmp_path):
754
+ from formscout.agents.pdf_report import PdfReportAgent
755
+ path = PdfReportAgent().run(_report(None), [_entry(needs_human=True)], str(tmp_path))
756
+ assert path is not None and os.path.exists(path)
757
+ ```
758
+
759
+ - [ ] **Step 2: Run test to verify it fails**
760
+
761
+ Run: `pytest tests/test_pdf_report.py -v`
762
+ Expected: FAIL with `ModuleNotFoundError: No module named 'formscout.agents.pdf_report'`
763
+
764
+ - [ ] **Step 3: Create the agent**
765
+
766
+ Create `formscout/agents/pdf_report.py`:
767
+
768
+ ```python
769
+ """
770
+ PdfReportAgent — renders a ReportResult + session entries to a branded PDF.
771
+
772
+ Input: ReportResult, list[SessionEntry], session_dir (str)
773
+ Output: path to the written PDF (str), or None on failure.
774
+ Failure: returns None, never raises.
775
+ Params: 0 (pure rendering — no model).
776
+ License: n/a.
777
+ Gated: no.
778
+ """
779
+ from __future__ import annotations
780
+
781
+ import logging
782
+ import os
783
+
784
+ from formscout.types import ReportResult
785
+
786
+ logger = logging.getLogger(__name__)
787
+
788
+ DISCLAIMER = "Screening aid — not a diagnosis. Pain or clearing tests require a clinician."
789
+
790
+
791
+ class PdfReportAgent:
792
+ """Assembles the screening-session PDF via ReportLab."""
793
+
794
+ def run(self, report: ReportResult, entries: list, session_dir: str) -> str | None:
795
+ try:
796
+ from reportlab.lib import colors
797
+ from reportlab.lib.pagesizes import LETTER
798
+ from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
799
+ from reportlab.lib.units import inch
800
+ from reportlab.platypus import (
801
+ Image, Paragraph, SimpleDocTemplate, Spacer, Table, TableStyle,
802
+ )
803
+ except Exception as e:
804
+ logger.warning("reportlab unavailable: %s", e)
805
+ return None
806
+
807
+ out_path = os.path.join(session_dir, "formscout_report.pdf")
808
+ try:
809
+ styles = getSampleStyleSheet()
810
+ banner = ParagraphStyle(
811
+ "banner", parent=styles["Normal"], fontSize=9, textColor=colors.white,
812
+ backColor=colors.HexColor("#b45309"), alignment=1, borderPadding=6, spaceAfter=12,
813
+ )
814
+ story = []
815
+ story.append(Paragraph(f"<b>&#9888; {DISCLAIMER}</b>", banner))
816
+ story.append(Paragraph("FormScout — FMS Screening Report", styles["Title"]))
817
+
818
+ if report.composite is not None:
819
+ comp = f"Composite: <b>{report.composite} / 21</b>"
820
+ else:
821
+ comp = f"Composite: <b>Incomplete</b> — {len(entries)}/7 tests scored"
822
+ story.append(Paragraph(comp, styles["Heading2"]))
823
+ story.append(Spacer(1, 0.2 * inch))
824
+
825
+ for e in entries:
826
+ title = e.test_name.replace("_", " ").title()
827
+ if e.side in ("left", "right"):
828
+ title += f" ({e.side})"
829
+ score_txt = "Clinician review required" if e.needs_human else f"Score: {e.score}/3"
830
+ story.append(Paragraph(f"<b>{title}</b> — {score_txt}", styles["Heading3"]))
831
+ if e.rationale:
832
+ story.append(Paragraph(e.rationale, styles["Normal"]))
833
+ if e.compensation_tags:
834
+ story.append(Paragraph("Compensations: " + ", ".join(e.compensation_tags),
835
+ styles["Normal"]))
836
+ if e.corrective_hint:
837
+ story.append(Paragraph("Corrective: " + e.corrective_hint, styles["Normal"]))
838
+
839
+ items = list(e.measurements.items())[:6]
840
+ if items:
841
+ rows = [[k.replace("_", " "),
842
+ (f"{v:.1f}" if isinstance(v, float) else str(v))] for k, v in items]
843
+ tbl = Table(rows, colWidths=[3 * inch, 1.5 * inch])
844
+ tbl.setStyle(TableStyle([
845
+ ("FONTSIZE", (0, 0), (-1, -1), 8),
846
+ ("TEXTCOLOR", (0, 0), (-1, -1), colors.HexColor("#334155")),
847
+ ]))
848
+ story.append(tbl)
849
+
850
+ if e.keyframe_path and os.path.exists(e.keyframe_path):
851
+ try:
852
+ story.append(Image(e.keyframe_path, width=3.0 * inch, height=2.25 * inch))
853
+ except Exception:
854
+ story.append(Paragraph("<i>(key-frame image unavailable)</i>", styles["Normal"]))
855
+ else:
856
+ story.append(Paragraph("<i>(key-frame image unavailable)</i>", styles["Normal"]))
857
+
858
+ story.append(Spacer(1, 0.2 * inch))
859
+
860
+ if report.asymmetries:
861
+ story.append(Paragraph("Asymmetries", styles["Heading2"]))
862
+ for a in report.asymmetries:
863
+ story.append(Paragraph(
864
+ f"{a['test'].replace('_', ' ').title()}: "
865
+ f"L={a['left_score']} R={a['right_score']} (&#916; {a['delta']})",
866
+ styles["Normal"]))
867
+
868
+ flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
869
+ if flags:
870
+ story.append(Paragraph("Flags", styles["Heading2"]))
871
+ for fl in flags:
872
+ story.append(Paragraph(fl, styles["Normal"]))
873
+
874
+ story.append(Spacer(1, 0.3 * inch))
875
+ story.append(Paragraph(f"<b>&#9888; {DISCLAIMER}</b>", banner))
876
+
877
+ doc = SimpleDocTemplate(out_path, pagesize=LETTER,
878
+ topMargin=0.6 * inch, bottomMargin=0.6 * inch)
879
+ doc.build(story)
880
+ return out_path
881
+ except Exception as e:
882
+ logger.warning("pdf build failed: %s", e)
883
+ return None
884
+ ```
885
+
886
+ - [ ] **Step 4: Run test to verify it passes**
887
+
888
+ Run: `pytest tests/test_pdf_report.py -v`
889
+ Expected: both tests PASS
890
+
891
+ - [ ] **Step 5: Re-run the session suite (pdf_path now populated)**
892
+
893
+ Run: `pytest tests/test_session.py -v`
894
+ Expected: all PASS (now `finish_session` returns a real `pdf_path`).
895
+
896
+ - [ ] **Step 6: Commit**
897
+
898
+ ```bash
899
+ git add formscout/agents/pdf_report.py tests/test_pdf_report.py
900
+ git commit -m "feat: add PdfReportAgent — branded ReportLab session PDF"
901
+ ```
902
+
903
+ ---
904
+
905
+ ## Task 7: Wire the session UI in `app.py`
906
+
907
+ **Files:**
908
+ - Modify: `app.py` (`process_video`, `build_app`, event wiring)
909
+
910
+ This task is verified by running the app (Gradio event wiring is not unit-tested; the
911
+ orchestration it calls is already covered by `tests/test_session.py`).
912
+
913
+ - [ ] **Step 1: Import the session module**
914
+
915
+ In `app.py`, add to the imports block (after `from formscout.startup import ensure_checkpoints`):
916
+
917
+ ```python
918
+ from formscout import session as session_mod
919
+ ```
920
+
921
+ - [ ] **Step 2: Refactor `process_video` to accumulate into a session**
922
+
923
+ Replace the entire `process_video` function (lines ~51-105) with a version that takes and
924
+ returns the session, appends an entry on success, and builds the "Session so far" table.
925
+ Replace from `def process_video(` through its final `return ...` with:
926
+
927
+ ```python
928
+ def process_video(video_path: str, test_name: str, side: str, model_key: str,
929
+ layers: list[str], session_state):
930
+ """Analyse one clip and accumulate it into the screening session."""
931
+ if not video_path:
932
+ return (
933
+ session_state, _render_empty_state(), "Upload a video to begin analysis.",
934
+ "", "", None, "", _render_session_table(session_state),
935
+ gr.update(visible=False), gr.update(visible=False),
936
+ )
937
+
938
+ if session_state is None:
939
+ session_state = session_mod.new_session()
940
+
941
+ director = Director()
942
+ state = director.run(video_path, test_name=test_name, side=side, model_key=model_key)
943
+
944
+ score_html = _render_empty_state()
945
+ score_details = ""
946
+
947
+ if state.features:
948
+ result = score_test(state.features)
949
+ judge = state.judge
950
+ if judge and judge.score is not None:
951
+ score_html = _render_score_card(judge.score, judge.confidence, judge.needs_human)
952
+ score_details = _render_score_details_judge(judge, result, state.features)
953
+ elif judge and judge.needs_human:
954
+ score_html = _render_score_card(0, 0, True)
955
+ score_details = f"### Needs Clinician Review\n{judge.rationale}"
956
+ else:
957
+ score_html = _render_score_card(result.score, result.confidence, result.needs_human)
958
+ score_details = _render_score_details(result, state.features)
959
+
960
+ # Accumulate into the session (only when we have a real analysis)
961
+ if state.ingest and state.pose2d and state.judge:
962
+ draw_trails = "trails" in {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
963
+ try:
964
+ session_mod.add_analysis(
965
+ session_state, ingest=state.ingest, pose2d=state.pose2d,
966
+ features=state.features, judge=state.judge,
967
+ test_name=test_name, side=side, draw_trails=draw_trails,
968
+ )
969
+ except Exception as e:
970
+ state.warnings.append(f"session accumulation failed: {e}")
971
+
972
+ pipeline_md = _render_pipeline_status(state)
973
+ alerts = _render_alerts(state)
974
+
975
+ overlay_path = None
976
+ vel_summary = ""
977
+ layer_set = {lbl.lower().replace(" ", "_") for lbl in (layers or [])}
978
+ if layer_set and state.ingest and state.pose2d:
979
+ try:
980
+ from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
981
+ vis = PoseVisualizer()
982
+ with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
983
+ out_path = f.name
984
+ overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
985
+ if overlay_path:
986
+ vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
987
+ except Exception as e:
988
+ alerts = (alerts or "") + f"\n⚠️ Visualizer error: {e}"
989
+
990
+ has_entries = bool(session_state and session_state.entries)
991
+ return (
992
+ session_state, score_html, pipeline_md, score_details, alerts,
993
+ overlay_path, vel_summary, _render_session_table(session_state),
994
+ gr.update(visible=has_entries), gr.update(visible=has_entries),
995
+ )
996
+ ```
997
+
998
+ - [ ] **Step 3: Add the session-table renderer and finish handler**
999
+
1000
+ In `app.py`, add these two functions just before `def build_app()`:
1001
+
1002
+ ```python
1003
+ def _render_session_table(session_state) -> str:
1004
+ """Render the accumulated 'Session so far' table as markdown."""
1005
+ if not session_state or not session_state.entries:
1006
+ return "*No clips analysed yet.*"
1007
+ lines = ["| Test | Side | Score | Status |", "|---|---|---|---|"]
1008
+ for e in session_state.entries:
1009
+ test = e.test_name.replace("_", " ").title()
1010
+ side = e.side if e.side in ("left", "right") else "—"
1011
+ if e.needs_human:
1012
+ score, status = "—", "⚠️ Clinician review"
1013
+ else:
1014
+ score, status = f"{e.score}/3", "✓ scored"
1015
+ lines.append(f"| {test} | {side} | {score} | {status} |")
1016
+ return "\n".join(lines)
1017
+
1018
+
1019
+ def _finish_session(session_state):
1020
+ """Build the composite report + PDF for the whole session."""
1021
+ if not session_state or not session_state.entries:
1022
+ return ("⚠️ No clips analysed yet — analyse at least one clip first.",
1023
+ None, None)
1024
+
1025
+ report, pdf_path = session_mod.finish_session(session_state)
1026
+ if report is None:
1027
+ return ("⚠️ Nothing to report.", None, None)
1028
+
1029
+ if report.composite is not None:
1030
+ summary = [f"## Composite: {report.composite} / 21"]
1031
+ else:
1032
+ n = len(session_state.entries)
1033
+ summary = [f"## Composite: Incomplete — {n}/7 tests scored",
1034
+ "*(One or more tests need clinician review or were unscored.)*"]
1035
+
1036
+ if report.asymmetries:
1037
+ summary.append("\n### Asymmetries")
1038
+ for a in report.asymmetries:
1039
+ test = a["test"].replace("_", " ").title()
1040
+ summary.append(f"- **{test}:** L={a['left_score']} R={a['right_score']} (Δ {a['delta']})")
1041
+
1042
+ flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
1043
+ if flags:
1044
+ summary.append("\n### Flags")
1045
+ for fl in flags:
1046
+ summary.append(f"- {fl}")
1047
+
1048
+ md_path = os.path.join(session_state.session_dir, "analysis.md")
1049
+ md_out = md_path if os.path.exists(md_path) else None
1050
+ return "\n".join(summary), pdf_path, md_out
1051
+ ```
1052
+
1053
+ Also add `import os` to the top of `app.py` if not already present (it currently imports only
1054
+ `tempfile` and `gradio`). Add after `import tempfile`:
1055
+
1056
+ ```python
1057
+ import os
1058
+ ```
1059
+
1060
+ - [ ] **Step 4: Add the session state, buttons, and outputs to `build_app`**
1061
+
1062
+ In `build_app`, inside the `with gr.Blocks(...) as app:` block, immediately after the line
1063
+ `with gr.Blocks(title="FormScout — FMS Screening Aid") as app:` add:
1064
+
1065
+ ```python
1066
+ session_state = gr.State(None)
1067
+ ```
1068
+
1069
+ Then, in the left input column, replace the single submit button block:
1070
+
1071
+ ```python
1072
+ submit_btn = gr.Button(
1073
+ "🎯 Score Movement",
1074
+ variant="primary",
1075
+ size="lg",
1076
+ )
1077
+ ```
1078
+
1079
+ with:
1080
+
1081
+ ```python
1082
+ submit_btn = gr.Button(
1083
+ "🎯 Score Movement",
1084
+ variant="primary",
1085
+ size="lg",
1086
+ )
1087
+ with gr.Row():
1088
+ new_clip_btn = gr.Button("➕ Analyse new clip", visible=False)
1089
+ finish_btn = gr.Button("✅ Finish & generate PDF",
1090
+ variant="primary", visible=False)
1091
+ ```
1092
+
1093
+ In the right results column, add a "Session" tab and a finish-output area. Inside `with gr.Tabs():`
1094
+ add a new tab after the "🎬 Overlay Video" tab:
1095
+
1096
+ ```python
1097
+ with gr.TabItem("🗂️ Session"):
1098
+ session_table = gr.Markdown("*No clips analysed yet.*")
1099
+ finish_summary = gr.Markdown("")
1100
+ pdf_file = gr.File(label="Screening Report (PDF)", visible=True)
1101
+ md_file = gr.File(label="Analysis Log (Markdown)", visible=True)
1102
+ ```
1103
+
1104
+ - [ ] **Step 5: Update event wiring**
1105
+
1106
+ Replace the `_map_inputs` function and `submit_btn.click(...)` block at the bottom of `build_app`
1107
+ with:
1108
+
1109
+ ```python
1110
+ def _map_inputs(video, test_display_name, side_display, pose_model_key, overlay_layers, sess):
1111
+ """Map UI display values to internal values and accumulate into the session."""
1112
+ test_map = {name: val for name, val in FMS_TESTS}
1113
+ test_name = test_map.get(test_display_name, "deep_squat")
1114
+ side = {"N/A": "na", "Left": "left", "Right": "right"}.get(side_display, "na")
1115
+ return process_video(video, test_name, side, pose_model_key, overlay_layers, sess)
1116
+
1117
+ submit_btn.click(
1118
+ fn=_map_inputs,
1119
+ inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown,
1120
+ overlay_layers, session_state],
1121
+ outputs=[session_state, score_html, pipeline_md, score_details, alerts_md,
1122
+ overlay_video, velocity_md, session_table, new_clip_btn, finish_btn],
1123
+ )
1124
+
1125
+ def _new_clip():
1126
+ """Clear inputs for the next clip; keep the session intact."""
1127
+ return None, _render_empty_state(), ""
1128
+
1129
+ new_clip_btn.click(
1130
+ fn=_new_clip,
1131
+ inputs=[],
1132
+ outputs=[video_input, score_html, score_details],
1133
+ )
1134
+
1135
+ finish_btn.click(
1136
+ fn=_finish_session,
1137
+ inputs=[session_state],
1138
+ outputs=[finish_summary, pdf_file, md_file],
1139
+ )
1140
+ ```
1141
+
1142
+ - [ ] **Step 6: Verify the full test suite still passes**
1143
+
1144
+ Run: `pytest tests/ -q`
1145
+ Expected: all tests pass except the single pre-existing known failure documented in CLAUDE.md
1146
+ (`test_unimplemented_test_returns_low_confidence`). No new failures.
1147
+
1148
+ - [ ] **Step 7: Manually verify the app**
1149
+
1150
+ Run: `python3 app.py`
1151
+ Then in the browser:
1152
+ 1. Upload a clip, pick a test, click **Score Movement** → score card appears; the **Session** tab
1153
+ shows one row; the two new buttons appear.
1154
+ 2. Click **➕ Analyse new clip** → the video input clears, the session row persists.
1155
+ 3. Analyse a second test → a second row appears.
1156
+ 4. Click **✅ Finish & generate PDF** → the Session tab shows the composite summary and a
1157
+ downloadable PDF (open it: disclaimer top + bottom, per-test blocks with key-frame images,
1158
+ composite or "Incomplete"). The Markdown log is also downloadable.
1159
+
1160
+ Expected: all four steps work; PDF opens and contains the disclaimer, composite, and per-test sections.
1161
+
1162
+ - [ ] **Step 8: Commit**
1163
+
1164
+ ```bash
1165
+ git add app.py
1166
+ git commit -m "feat: accumulate FMS clips into a session with composite report + PDF export"
1167
+ ```
1168
+
1169
+ ---
1170
+
1171
+ ## Task 8: Update docs
1172
+
1173
+ **Files:**
1174
+ - Modify: `CLAUDE.md` (Build phases / status)
1175
+ - Modify: `MODEL_BUDGET.md` (no param change — note PDF agent adds 0 params, for completeness)
1176
+
1177
+ - [ ] **Step 1: Update the Phase 4 line in CLAUDE.md**
1178
+
1179
+ In `CLAUDE.md`, in the "Build phases" section, update the Phase 4 line from:
1180
+
1181
+ ```
1182
+ 4. **Phase 4 — Polish + ship:** Custom Svelte UI components, PDF export, agent trace to Hub, blog post. (Overlay video already done via `PoseVisualizer`.)
1183
+ ```
1184
+
1185
+ to:
1186
+
1187
+ ```
1188
+ 4. **Phase 4 — Polish + ship:** Custom Svelte UI components, agent trace to Hub, blog post. (Overlay video done via `PoseVisualizer`; full 7-test session + PDF export done via `formscout/session.py` + `PdfReportAgent`.)
1189
+ ```
1190
+
1191
+ - [ ] **Step 2: Note the PDF agent in the architecture section**
1192
+
1193
+ In `CLAUDE.md`, under "### Rubric scorers" or near the ReportAgent description, this is optional
1194
+ context; no required change. Skip if no natural home.
1195
+
1196
+ - [ ] **Step 3: Commit**
1197
+
1198
+ ```bash
1199
+ git add CLAUDE.md
1200
+ git commit -m "docs: mark full FMS session + PDF export complete in build phases"
1201
+ ```
1202
+
1203
+ ---
1204
+
1205
+ ## Self-Review Notes (already applied)
1206
+
1207
+ - **Spec coverage:** session accumulation (Task 5), two-button UX (Task 7), on-disk MD/JSON/keyframes (Task 5), key-frame from `features.timing` (Tasks 3–5), ReportLab PDF top/bottom disclaimer + composite + per-test + asymmetry + flags (Task 6), `SessionEntry` type (Task 2), `ReportAgent` reuse (Task 5 `finish_session`), composite-null-on-needs-human (Task 5 test), error tolerance / never-raise (Tasks 4–6). All covered.
1208
+ - **Type consistency:** `SessionEntry` field names are identical across Tasks 2, 5, 6, 7. `finish_session` returns `(ReportResult | None, str | None)` and is consumed that way in Task 7. `render_frame(ingest, pose2d, frame_idx, layers, caption, out_png)` signature matches its callers.
1209
+ - **No placeholders:** every code step shows complete code; every run step states the exact command + expected outcome.
docs/superpowers/specs/2026-06-09-pose-model-selector-design.md CHANGED
@@ -1,171 +1,171 @@
1
- # Pose Model Selector — Design Spec
2
-
3
- **Date:** 2026-06-09
4
- **Status:** Approved
5
-
6
- ## Goal
7
-
8
- Expose all available pose estimation models as a selectable dropdown in the Gradio UI, replacing the hard-coded YOLO26l default. Supported families: MediaPipe (Qualcomm HF/ONNX), YOLO26 n→x (local), Sapiens2 0.4B→5B (HF/transformers).
9
-
10
- ---
11
-
12
- ## Architecture
13
-
14
- ### Unified model registry (`config.py`)
15
-
16
- Replace `YOLO_POSE_MODELS` with a single `POSE_MODELS` dict. Each entry:
17
-
18
- ```python
19
- {
20
- "backend": "yolo" | "mediapipe" | "sapiens2",
21
- "path": str, # yolo only — absolute path to local .pt
22
- "hf_id": str, # mediapipe + sapiens2 — HuggingFace repo id
23
- "params_m": float, # millions of parameters
24
- }
25
- ```
26
-
27
- Ordered as displayed in the UI:
28
-
29
- | Label | backend | source |
30
- |---|---|---|
31
- | `MediaPipe-Pose ⬇ ~16 MB, CPU-friendly` | mediapipe | `qualcomm/MediaPipe-Pose-Estimation` |
32
- | `YOLO26n — nano (0.7M, fastest)` ★ default | yolo | local checkpoint |
33
- | `YOLO26s — small (3.5M)` | yolo | local checkpoint |
34
- | `YOLO26m — medium (9M)` | yolo | local checkpoint |
35
- | `YOLO26l — large (25.9M)` | yolo | local checkpoint |
36
- | `YOLO26x — extra-large (57.6M)` | yolo | local checkpoint |
37
- | `Sapiens2-0.4B ⬇ ~1.6 GB` | sapiens2 | `facebook/sapiens2-pose-0.4b` |
38
- | `Sapiens2-0.8B ⬇ ~3.2 GB` | sapiens2 | `facebook/sapiens2-pose-0.8b` |
39
- | `Sapiens2-1B ⬇ ~4 GB` | sapiens2 | `facebook/sapiens2-pose-1b` |
40
- | `Sapiens2-5B ⬇ ~20 GB, large GPU` | sapiens2 | `facebook/sapiens2-pose-5b` |
41
-
42
- ```python
43
- DEFAULT_POSE_MODEL = "YOLO26n — nano (0.7M, fastest)"
44
- ```
45
-
46
- Keep `YOLO_POSE_MODEL` and `YOLO_POSE_MODEL_HQ` as string aliases for backward compat with any direct references outside the agent.
47
-
48
- ---
49
-
50
- ### Pose2DAgent (`formscout/agents/pose2d.py`)
51
-
52
- Three private sub-runners, all returning `list[dict[int, dict]]` (COCO 17 keypoints per frame, same format as today):
53
-
54
- #### `_run_yolo(frames, path) -> list[dict]`
55
- Existing logic, lifted into a named function. Model cached in `_model_cache[path]`.
56
-
57
- #### `_run_mediapipe(frames, hf_id) -> list[dict]`
58
- - Download repo snapshot via `huggingface_hub.snapshot_download(hf_id)`
59
- - Locate the pose landmark `.onnx` file in the snapshot
60
- - Load with `onnxruntime.InferenceSession`
61
- - Preprocess each frame: resize to 256×256, normalize
62
- - Run inference → 33 BlazePose landmarks
63
- - Map BlazePose 33 → COCO 17 via fixed index table:
64
- ```
65
- COCO 0=nose → BlazePose 0
66
- COCO 1=left_eye → BlazePose 2
67
- COCO 2=right_eye → BlazePose 5
68
- COCO 3=left_ear → BlazePose 7
69
- COCO 4=right_ear → BlazePose 8
70
- COCO 5=left_shld → BlazePose 11
71
- COCO 6=right_shld → BlazePose 12
72
- COCO 7=left_elbow → BlazePose 13
73
- COCO 8=right_elbow → BlazePose 14
74
- COCO 9=left_wrist → BlazePose 15
75
- COCO 10=right_wrist → BlazePose 16
76
- COCO 11=left_hip → BlazePose 23
77
- COCO 12=right_hip → BlazePose 24
78
- COCO 13=left_knee → BlazePose 25
79
- COCO 14=right_knee → BlazePose 26
80
- COCO 15=left_ankle → BlazePose 27
81
- COCO 16=right_ankle → BlazePose 28
82
- ```
83
- - Session cached in `_model_cache[hf_id]`
84
-
85
- #### `_run_sapiens2(frames, hf_id) -> list[dict]`
86
- - Load via `transformers.pipeline("pose-estimation", model=hf_id)`
87
- - Sapiens2 outputs 308 whole-body keypoints; map first 17 (indices 0–16) to COCO 17 — Sapiens2 preserves COCO ordering for the body subset
88
- - Pipeline cached in `_model_cache[hf_id]`
89
-
90
- #### `Pose2DAgent.run(ingest, model_key)`
91
- - `model_key: str` replaces `model_path: str` (old param)
92
- - Looks up `config.POSE_MODELS[model_key]` (falls back to `DEFAULT_POSE_MODEL` if key missing)
93
- - Dispatches to the appropriate sub-runner
94
- - Returns `Pose2DResult` — identical contract as today
95
-
96
- ---
97
-
98
- ### UI (`app.py`)
99
-
100
- Add `gr.Dropdown` for pose model in the input column, below the test/side row:
101
-
102
- ```python
103
- pose_model_dropdown = gr.Dropdown(
104
- choices=list(config.POSE_MODELS.keys()),
105
- value=config.DEFAULT_POSE_MODEL,
106
- label="Pose Model",
107
- )
108
- ```
109
-
110
- Update `_map_inputs` to accept and forward `pose_model_key`:
111
-
112
- ```python
113
- def _map_inputs(video, test_display_name, side_display, pose_model_key):
114
- ...
115
- return process_video(video, test_name, side, pose_model_key)
116
- ```
117
-
118
- Update `submit_btn.click` inputs to include `pose_model_dropdown`.
119
-
120
- `process_video(video_path, test_name, side, pose_model_key)` passes `pose_model_key` through to `director.run()`, which passes it to `Pose2DAgent.run()`. Remove the old `YOLO_POSE_MODELS.get()` lookup from `process_video`.
121
-
122
- ---
123
-
124
- ## Data flow
125
-
126
- ```
127
- UI dropdown (pose_model_key: str)
128
- → process_video()
129
- → Director.run(pose_model_key=...)
130
- → Pose2DAgent.run(ingest, model_key=pose_model_key)
131
- → config.POSE_MODELS[model_key] → {backend, path|hf_id}
132
- → _run_yolo / _run_mediapipe / _run_sapiens2
133
- → list[dict[int, {x, y, conf}]] (COCO 17, same contract)
134
- ��� Pose2DResult
135
- ```
136
-
137
- ---
138
-
139
- ## Error handling
140
-
141
- - Unknown `model_key`: log warning, fall back to `DEFAULT_POSE_MODEL`
142
- - ONNX file not found in MediaPipe snapshot: `Pose2DResult(confidence=0.0, notes="mediapipe onnx not found")`
143
- - Sapiens2 / MediaPipe download failure: `Pose2DResult(confidence=0.0, notes=str(e))`
144
- - All failures are non-fatal; pipeline continues with 0-confidence result and surfaces alert in UI
145
-
146
- ---
147
-
148
- ## Dependencies to add (`requirements.txt`)
149
-
150
- - `onnxruntime` — MediaPipe ONNX inference
151
- - `huggingface_hub` — snapshot download for MediaPipe (already likely present via transformers)
152
-
153
- Sapiens2 uses `transformers`, already a dependency.
154
-
155
- ---
156
-
157
- ## Testing
158
-
159
- Each new backend gets a pytest in `tests/test_pose2d.py` that:
160
- - Mocks the model load (no actual HF download in CI)
161
- - Passes a 3-frame synthetic IngestResult
162
- - Asserts `Pose2DResult.keypoints` has 3 entries, each a dict with at most 17 int keys
163
- - Asserts `confidence` is a float in [0, 1]
164
-
165
- ---
166
-
167
- ## Out of scope
168
-
169
- - Sapiens2 / MediaPipe accuracy benchmarking
170
- - Automatic backend selection based on hardware
171
- - Downloading Sapiens2/MediaPipe checkpoints to local `checkpoints/` directory
 
1
+ # Pose Model Selector — Design Spec
2
+
3
+ **Date:** 2026-06-09
4
+ **Status:** Approved
5
+
6
+ ## Goal
7
+
8
+ Expose all available pose estimation models as a selectable dropdown in the Gradio UI, replacing the hard-coded YOLO26l default. Supported families: MediaPipe (Qualcomm HF/ONNX), YOLO26 n→x (local), Sapiens2 0.4B→5B (HF/transformers).
9
+
10
+ ---
11
+
12
+ ## Architecture
13
+
14
+ ### Unified model registry (`config.py`)
15
+
16
+ Replace `YOLO_POSE_MODELS` with a single `POSE_MODELS` dict. Each entry:
17
+
18
+ ```python
19
+ {
20
+ "backend": "yolo" | "mediapipe" | "sapiens2",
21
+ "path": str, # yolo only — absolute path to local .pt
22
+ "hf_id": str, # mediapipe + sapiens2 — HuggingFace repo id
23
+ "params_m": float, # millions of parameters
24
+ }
25
+ ```
26
+
27
+ Ordered as displayed in the UI:
28
+
29
+ | Label | backend | source |
30
+ |---|---|---|
31
+ | `MediaPipe-Pose ⬇ ~16 MB, CPU-friendly` | mediapipe | `qualcomm/MediaPipe-Pose-Estimation` |
32
+ | `YOLO26n — nano (0.7M, fastest)` ★ default | yolo | local checkpoint |
33
+ | `YOLO26s — small (3.5M)` | yolo | local checkpoint |
34
+ | `YOLO26m — medium (9M)` | yolo | local checkpoint |
35
+ | `YOLO26l — large (25.9M)` | yolo | local checkpoint |
36
+ | `YOLO26x — extra-large (57.6M)` | yolo | local checkpoint |
37
+ | `Sapiens2-0.4B ⬇ ~1.6 GB` | sapiens2 | `facebook/sapiens2-pose-0.4b` |
38
+ | `Sapiens2-0.8B ⬇ ~3.2 GB` | sapiens2 | `facebook/sapiens2-pose-0.8b` |
39
+ | `Sapiens2-1B ⬇ ~4 GB` | sapiens2 | `facebook/sapiens2-pose-1b` |
40
+ | `Sapiens2-5B ⬇ ~20 GB, large GPU` | sapiens2 | `facebook/sapiens2-pose-5b` |
41
+
42
+ ```python
43
+ DEFAULT_POSE_MODEL = "YOLO26n — nano (0.7M, fastest)"
44
+ ```
45
+
46
+ Keep `YOLO_POSE_MODEL` and `YOLO_POSE_MODEL_HQ` as string aliases for backward compat with any direct references outside the agent.
47
+
48
+ ---
49
+
50
+ ### Pose2DAgent (`formscout/agents/pose2d.py`)
51
+
52
+ Three private sub-runners, all returning `list[dict[int, dict]]` (COCO 17 keypoints per frame, same format as today):
53
+
54
+ #### `_run_yolo(frames, path) -> list[dict]`
55
+ Existing logic, lifted into a named function. Model cached in `_model_cache[path]`.
56
+
57
+ #### `_run_mediapipe(frames, hf_id) -> list[dict]`
58
+ - Download repo snapshot via `huggingface_hub.snapshot_download(hf_id)`
59
+ - Locate the pose landmark `.onnx` file in the snapshot
60
+ - Load with `onnxruntime.InferenceSession`
61
+ - Preprocess each frame: resize to 256×256, normalize
62
+ - Run inference → 33 BlazePose landmarks
63
+ - Map BlazePose 33 → COCO 17 via fixed index table:
64
+ ```
65
+ COCO 0=nose → BlazePose 0
66
+ COCO 1=left_eye → BlazePose 2
67
+ COCO 2=right_eye → BlazePose 5
68
+ COCO 3=left_ear → BlazePose 7
69
+ COCO 4=right_ear → BlazePose 8
70
+ COCO 5=left_shld → BlazePose 11
71
+ COCO 6=right_shld → BlazePose 12
72
+ COCO 7=left_elbow → BlazePose 13
73
+ COCO 8=right_elbow → BlazePose 14
74
+ COCO 9=left_wrist → BlazePose 15
75
+ COCO 10=right_wrist → BlazePose 16
76
+ COCO 11=left_hip → BlazePose 23
77
+ COCO 12=right_hip → BlazePose 24
78
+ COCO 13=left_knee → BlazePose 25
79
+ COCO 14=right_knee → BlazePose 26
80
+ COCO 15=left_ankle → BlazePose 27
81
+ COCO 16=right_ankle → BlazePose 28
82
+ ```
83
+ - Session cached in `_model_cache[hf_id]`
84
+
85
+ #### `_run_sapiens2(frames, hf_id) -> list[dict]`
86
+ - Load via `transformers.pipeline("pose-estimation", model=hf_id)`
87
+ - Sapiens2 outputs 308 whole-body keypoints; map first 17 (indices 0–16) to COCO 17 — Sapiens2 preserves COCO ordering for the body subset
88
+ - Pipeline cached in `_model_cache[hf_id]`
89
+
90
+ #### `Pose2DAgent.run(ingest, model_key)`
91
+ - `model_key: str` replaces `model_path: str` (old param)
92
+ - Looks up `config.POSE_MODELS[model_key]` (falls back to `DEFAULT_POSE_MODEL` if key missing)
93
+ - Dispatches to the appropriate sub-runner
94
+ - Returns `Pose2DResult` — identical contract as today
95
+
96
+ ---
97
+
98
+ ### UI (`app.py`)
99
+
100
+ Add `gr.Dropdown` for pose model in the input column, below the test/side row:
101
+
102
+ ```python
103
+ pose_model_dropdown = gr.Dropdown(
104
+ choices=list(config.POSE_MODELS.keys()),
105
+ value=config.DEFAULT_POSE_MODEL,
106
+ label="Pose Model",
107
+ )
108
+ ```
109
+
110
+ Update `_map_inputs` to accept and forward `pose_model_key`:
111
+
112
+ ```python
113
+ def _map_inputs(video, test_display_name, side_display, pose_model_key):
114
+ ...
115
+ return process_video(video, test_name, side, pose_model_key)
116
+ ```
117
+
118
+ Update `submit_btn.click` inputs to include `pose_model_dropdown`.
119
+
120
+ `process_video(video_path, test_name, side, pose_model_key)` passes `pose_model_key` through to `director.run()`, which passes it to `Pose2DAgent.run()`. Remove the old `YOLO_POSE_MODELS.get()` lookup from `process_video`.
121
+
122
+ ---
123
+
124
+ ## Data flow
125
+
126
+ ```
127
+ UI dropdown (pose_model_key: str)
128
+ → process_video()
129
+ → Director.run(pose_model_key=...)
130
+ → Pose2DAgent.run(ingest, model_key=pose_model_key)
131
+ → config.POSE_MODELS[model_key] → {backend, path|hf_id}
132
+ → _run_yolo / _run_mediapipe / _run_sapiens2
133
+ → list[dict[int, {x, y, conf}]] (COCO 17, same contract)
134
+ Pose2DResult
135
+ ```
136
+
137
+ ---
138
+
139
+ ## Error handling
140
+
141
+ - Unknown `model_key`: log warning, fall back to `DEFAULT_POSE_MODEL`
142
+ - ONNX file not found in MediaPipe snapshot: `Pose2DResult(confidence=0.0, notes="mediapipe onnx not found")`
143
+ - Sapiens2 / MediaPipe download failure: `Pose2DResult(confidence=0.0, notes=str(e))`
144
+ - All failures are non-fatal; pipeline continues with 0-confidence result and surfaces alert in UI
145
+
146
+ ---
147
+
148
+ ## Dependencies to add (`requirements.txt`)
149
+
150
+ - `onnxruntime` — MediaPipe ONNX inference
151
+ - `huggingface_hub` — snapshot download for MediaPipe (already likely present via transformers)
152
+
153
+ Sapiens2 uses `transformers`, already a dependency.
154
+
155
+ ---
156
+
157
+ ## Testing
158
+
159
+ Each new backend gets a pytest in `tests/test_pose2d.py` that:
160
+ - Mocks the model load (no actual HF download in CI)
161
+ - Passes a 3-frame synthetic IngestResult
162
+ - Asserts `Pose2DResult.keypoints` has 3 entries, each a dict with at most 17 int keys
163
+ - Asserts `confidence` is a float in [0, 1]
164
+
165
+ ---
166
+
167
+ ## Out of scope
168
+
169
+ - Sapiens2 / MediaPipe accuracy benchmarking
170
+ - Automatic backend selection based on hardware
171
+ - Downloading Sapiens2/MediaPipe checkpoints to local `checkpoints/` directory
docs/superpowers/specs/2026-06-09-pose-visualizer-design.md CHANGED
@@ -1,197 +1,197 @@
1
- # Pose Overlay Visualizer — Design Spec
2
-
3
- **Date:** 2026-06-09
4
- **Status:** Approved
5
-
6
- ## Goal
7
-
8
- Add an annotated overlay video output to the FormScout UI showing skeleton, motion trails, and velocity arrows on top of the original footage, alongside a per-joint velocity summary table. Overlay layers are user-selectable via checkboxes. Adapted from the Laban Movement Analysis project.
9
-
10
- ---
11
-
12
- ## Architecture
13
-
14
- Three files change or are created. No changes to `pipeline.py`, `types.py`, or any existing agent.
15
-
16
- ```
17
- formscout/agents/visualizer.py ← new
18
- tests/test_visualizer.py ← new
19
- app.py ← overlay_layers checkbox, new tab, wiring
20
- ```
21
-
22
- The visualizer runs **after** `director.run()` returns in `process_video()` — it is a pure post-processing step, never on the critical scoring path.
23
-
24
- ---
25
-
26
- ## Module: `formscout/agents/visualizer.py`
27
-
28
- ### `compute_joint_velocity(keypoints_per_frame, fps) → dict[int, list[float]]`
29
-
30
- - Input: `list[dict[int, {x, y, conf}]]` (COCO-17 pixel coords per frame), `fps: float`
31
- - Output: `dict[int, list[float]]` — per-joint per-frame speed in **px/s**
32
- - Method: for each joint index, run a `SimpleKalmanFilter` (1D per axis, constant-velocity model, same structure as Laban's engine) over the (x, y) series. Speed = `sqrt(vx² + vy²)` from the filter's velocity state.
33
- - Missing keypoints (conf < 0.3 or absent) → speed = 0.0 for that frame, filter state held.
34
-
35
- ### `SimpleKalmanFilter`
36
-
37
- Minimal 4-state Kalman (x, y, vx, vy), identical in structure to the Laban `SimpleKalmanFilter`:
38
- - Transition: constant-velocity model
39
- - Measurement: position only (x, y)
40
- - One instance per joint per video run
41
-
42
- ### `PoseVisualizer`
43
-
44
- #### Constants
45
- ```python
46
- COCO_SKELETON = [
47
- (0,1),(0,2),(1,3),(2,4), # face
48
- (5,6),(5,7),(7,9),(6,8),(8,10), # arms
49
- (5,11),(6,12),(11,12), # torso
50
- (11,13),(13,15),(12,14),(14,16), # legs
51
- ]
52
- TRAIL_LENGTH = 10 # frames of trail history
53
- MAX_ARROW_PX = 40 # arrow scaled so peak velocity → 40px length
54
- CONF_THRESHOLD = 0.3 # min confidence to draw a keypoint
55
- ```
56
-
57
- #### Private methods
58
-
59
- **`_draw_skeleton(frame, kps)`**
60
- - Draw each COCO bone as a line if both endpoints have conf > CONF_THRESHOLD
61
- - Joint dots: color green→red by confidence using HSV (same as Laban `_confidence_to_color`)
62
- - Bone color: white
63
-
64
- **`_draw_trails(frame, trail_history, frame_idx)`**
65
- - `trail_history: dict[int, deque(maxlen=TRAIL_LENGTH)]` keyed by joint index
66
- - Each deque holds `(x, y)` pixel positions from previous frames
67
- - Draw fading line segments: alpha = segment_position / TRAIL_LENGTH, color white
68
-
69
- **`_draw_velocity_arrows(frame, kps, velocities, frame_idx)`**
70
- - `velocities: dict[int, list[float]]` — speeds per joint per frame
71
- - Direction vector from consecutive keypoint positions (x[t] - x[t-1], y[t] - y[t-1])
72
- - Arrow length = `speed / peak_speed * MAX_ARROW_PX` (clamped)
73
- - Drawn only for joints with conf > CONF_THRESHOLD and speed > 0
74
- - Color: green=slow, orange=medium, red=fast (same thresholds as Laban intensity)
75
-
76
- #### Public method
77
-
78
- **`render_video(ingest, pose2d, layers: set[str], output_path: str) → str | None`**
79
- - `layers`: subset of `{"skeleton", "trails", "velocity_arrows"}`
80
- - If `layers` is empty → return `None` immediately
81
- - Pre-computes `compute_joint_velocity(pose2d.keypoints, ingest.fps)`
82
- - Iterates frames, updates `trail_history`, calls selected `_draw_*` methods
83
- - Writes output via `cv2.VideoWriter` (codec: `mp4v`, same fps as ingest)
84
- - Returns output path on success; `None` on any exception (logs warning)
85
-
86
- #### Velocity summary
87
-
88
- **`build_velocity_summary(keypoints_per_frame, velocities) → str`**
89
- - For each joint with conf > 0.3 in >50% of frames:
90
- - Compute avg and peak speed (px/s)
91
- - Return markdown table sorted by peak speed descending:
92
- ```
93
- | Joint | Avg (px/s) | Peak (px/s) |
94
- |---------------|-----------|-------------|
95
- | left_knee | 42.3 | 118.7 |
96
- ```
97
- - Returns empty string if no valid joints
98
-
99
- ---
100
-
101
- ## UI changes: `app.py`
102
-
103
- ### Input column — overlay layer checkboxes
104
-
105
- Below `pose_model_dropdown`, add:
106
-
107
- ```python
108
- overlay_layers = gr.CheckboxGroup(
109
- choices=["Skeleton", "Trails", "Velocity arrows"],
110
- value=["Skeleton", "Trails"],
111
- label="Overlay Layers",
112
- )
113
- ```
114
-
115
- ### Results panel — new tab
116
-
117
- Inside the existing `gr.Tabs()` block, add a fourth tab:
118
-
119
- ```python
120
- with gr.TabItem("🎬 Overlay Video"):
121
- overlay_video = gr.Video(label="Annotated Movement")
122
- velocity_md = gr.Markdown("")
123
- ```
124
-
125
- ### `process_video()` signature
126
-
127
- ```python
128
- def process_video(video_path, test_name, side, model_key, layers: list[str]):
129
- ```
130
-
131
- After `director.run()`:
132
- ```python
133
- from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
134
- layer_set = {l.lower().replace(" ", "_") for l in layers}
135
- # map UI labels to internal names:
136
- # "Skeleton" → "skeleton", "Trails" → "trails", "Velocity arrows" → "velocity_arrows"
137
- overlay_path = None
138
- vel_summary = ""
139
- if layer_set and state.ingest and state.pose2d:
140
- try:
141
- vis = PoseVisualizer()
142
- with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
143
- out_path = f.name
144
- overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
145
- if overlay_path:
146
- vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
147
- except Exception as e:
148
- alerts += f"\n⚠️ Visualizer error: {e}"
149
- return score_html, pipeline_md, score_details, alerts, overlay_path, vel_summary
150
- ```
151
-
152
- `vis.last_velocities` is stored on the instance after `render_video()` to avoid recomputing.
153
-
154
- ### Event wiring
155
-
156
- ```python
157
- submit_btn.click(
158
- fn=_map_inputs,
159
- inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown, overlay_layers],
160
- outputs=[score_html, pipeline_md, score_details, alerts_md, overlay_video, velocity_md],
161
- )
162
- ```
163
-
164
- `_map_inputs` gains `overlay_layers` as fifth parameter.
165
-
166
- ---
167
-
168
- ## Error handling
169
-
170
- | Failure | Behaviour |
171
- |---|---|
172
- | All frames have no detections | `render_video()` returns `None`, tab empty, no crash |
173
- | `cv2.VideoWriter` fails | logs warning, returns `None` |
174
- | Any exception in visualizer | caught in `process_video()`, appended to alerts, `overlay_path = None` |
175
- | `layers` is empty | returns `None` immediately, no processing |
176
-
177
- The score is always returned regardless of visualizer outcome.
178
-
179
- ---
180
-
181
- ## Testing: `tests/test_visualizer.py`
182
-
183
- - Synthetic `IngestResult`: 5 blank 480×640 BGR frames, fps=30
184
- - Synthetic `Pose2DResult`: 17 keypoints per frame at fixed positions with conf=0.9
185
- - `test_render_video_creates_file`: assert output `.mp4` exists and size > 0
186
- - `test_compute_joint_velocity_shape`: assert 17-key dict, each list length == 5
187
- - `test_empty_layers_returns_none`: assert `render_video(..., layers=set())` returns `None`
188
- - `test_no_detections_returns_none`: all-empty keypoints → `None`
189
- - `test_velocity_summary_markdown`: assert output contains `|` (table) and at least one joint name
190
-
191
- ---
192
-
193
- ## Out of scope
194
-
195
- - Frame-by-frame metrics synced to video playback (Phase 4 / custom Svelte)
196
- - Multi-person tracking
197
- - Saving overlay video to Hugging Face Hub (tracing feature, Phase 4)
 
1
+ # Pose Overlay Visualizer — Design Spec
2
+
3
+ **Date:** 2026-06-09
4
+ **Status:** Approved
5
+
6
+ ## Goal
7
+
8
+ Add an annotated overlay video output to the FormScout UI showing skeleton, motion trails, and velocity arrows on top of the original footage, alongside a per-joint velocity summary table. Overlay layers are user-selectable via checkboxes. Adapted from the Laban Movement Analysis project.
9
+
10
+ ---
11
+
12
+ ## Architecture
13
+
14
+ Three files change or are created. No changes to `pipeline.py`, `types.py`, or any existing agent.
15
+
16
+ ```
17
+ formscout/agents/visualizer.py ← new
18
+ tests/test_visualizer.py ← new
19
+ app.py ← overlay_layers checkbox, new tab, wiring
20
+ ```
21
+
22
+ The visualizer runs **after** `director.run()` returns in `process_video()` — it is a pure post-processing step, never on the critical scoring path.
23
+
24
+ ---
25
+
26
+ ## Module: `formscout/agents/visualizer.py`
27
+
28
+ ### `compute_joint_velocity(keypoints_per_frame, fps) → dict[int, list[float]]`
29
+
30
+ - Input: `list[dict[int, {x, y, conf}]]` (COCO-17 pixel coords per frame), `fps: float`
31
+ - Output: `dict[int, list[float]]` — per-joint per-frame speed in **px/s**
32
+ - Method: for each joint index, run a `SimpleKalmanFilter` (1D per axis, constant-velocity model, same structure as Laban's engine) over the (x, y) series. Speed = `sqrt(vx² + vy²)` from the filter's velocity state.
33
+ - Missing keypoints (conf < 0.3 or absent) → speed = 0.0 for that frame, filter state held.
34
+
35
+ ### `SimpleKalmanFilter`
36
+
37
+ Minimal 4-state Kalman (x, y, vx, vy), identical in structure to the Laban `SimpleKalmanFilter`:
38
+ - Transition: constant-velocity model
39
+ - Measurement: position only (x, y)
40
+ - One instance per joint per video run
41
+
42
+ ### `PoseVisualizer`
43
+
44
+ #### Constants
45
+ ```python
46
+ COCO_SKELETON = [
47
+ (0,1),(0,2),(1,3),(2,4), # face
48
+ (5,6),(5,7),(7,9),(6,8),(8,10), # arms
49
+ (5,11),(6,12),(11,12), # torso
50
+ (11,13),(13,15),(12,14),(14,16), # legs
51
+ ]
52
+ TRAIL_LENGTH = 10 # frames of trail history
53
+ MAX_ARROW_PX = 40 # arrow scaled so peak velocity → 40px length
54
+ CONF_THRESHOLD = 0.3 # min confidence to draw a keypoint
55
+ ```
56
+
57
+ #### Private methods
58
+
59
+ **`_draw_skeleton(frame, kps)`**
60
+ - Draw each COCO bone as a line if both endpoints have conf > CONF_THRESHOLD
61
+ - Joint dots: color green→red by confidence using HSV (same as Laban `_confidence_to_color`)
62
+ - Bone color: white
63
+
64
+ **`_draw_trails(frame, trail_history, frame_idx)`**
65
+ - `trail_history: dict[int, deque(maxlen=TRAIL_LENGTH)]` keyed by joint index
66
+ - Each deque holds `(x, y)` pixel positions from previous frames
67
+ - Draw fading line segments: alpha = segment_position / TRAIL_LENGTH, color white
68
+
69
+ **`_draw_velocity_arrows(frame, kps, velocities, frame_idx)`**
70
+ - `velocities: dict[int, list[float]]` — speeds per joint per frame
71
+ - Direction vector from consecutive keypoint positions (x[t] - x[t-1], y[t] - y[t-1])
72
+ - Arrow length = `speed / peak_speed * MAX_ARROW_PX` (clamped)
73
+ - Drawn only for joints with conf > CONF_THRESHOLD and speed > 0
74
+ - Color: green=slow, orange=medium, red=fast (same thresholds as Laban intensity)
75
+
76
+ #### Public method
77
+
78
+ **`render_video(ingest, pose2d, layers: set[str], output_path: str) → str | None`**
79
+ - `layers`: subset of `{"skeleton", "trails", "velocity_arrows"}`
80
+ - If `layers` is empty → return `None` immediately
81
+ - Pre-computes `compute_joint_velocity(pose2d.keypoints, ingest.fps)`
82
+ - Iterates frames, updates `trail_history`, calls selected `_draw_*` methods
83
+ - Writes output via `cv2.VideoWriter` (codec: `mp4v`, same fps as ingest)
84
+ - Returns output path on success; `None` on any exception (logs warning)
85
+
86
+ #### Velocity summary
87
+
88
+ **`build_velocity_summary(keypoints_per_frame, velocities) → str`**
89
+ - For each joint with conf > 0.3 in >50% of frames:
90
+ - Compute avg and peak speed (px/s)
91
+ - Return markdown table sorted by peak speed descending:
92
+ ```
93
+ | Joint | Avg (px/s) | Peak (px/s) |
94
+ |---------------|-----------|-------------|
95
+ | left_knee | 42.3 | 118.7 |
96
+ ```
97
+ - Returns empty string if no valid joints
98
+
99
+ ---
100
+
101
+ ## UI changes: `app.py`
102
+
103
+ ### Input column — overlay layer checkboxes
104
+
105
+ Below `pose_model_dropdown`, add:
106
+
107
+ ```python
108
+ overlay_layers = gr.CheckboxGroup(
109
+ choices=["Skeleton", "Trails", "Velocity arrows"],
110
+ value=["Skeleton", "Trails"],
111
+ label="Overlay Layers",
112
+ )
113
+ ```
114
+
115
+ ### Results panel — new tab
116
+
117
+ Inside the existing `gr.Tabs()` block, add a fourth tab:
118
+
119
+ ```python
120
+ with gr.TabItem("🎬 Overlay Video"):
121
+ overlay_video = gr.Video(label="Annotated Movement")
122
+ velocity_md = gr.Markdown("")
123
+ ```
124
+
125
+ ### `process_video()` signature
126
+
127
+ ```python
128
+ def process_video(video_path, test_name, side, model_key, layers: list[str]):
129
+ ```
130
+
131
+ After `director.run()`:
132
+ ```python
133
+ from formscout.agents.visualizer import PoseVisualizer, build_velocity_summary
134
+ layer_set = {l.lower().replace(" ", "_") for l in layers}
135
+ # map UI labels to internal names:
136
+ # "Skeleton" → "skeleton", "Trails" → "trails", "Velocity arrows" → "velocity_arrows"
137
+ overlay_path = None
138
+ vel_summary = ""
139
+ if layer_set and state.ingest and state.pose2d:
140
+ try:
141
+ vis = PoseVisualizer()
142
+ with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as f:
143
+ out_path = f.name
144
+ overlay_path = vis.render_video(state.ingest, state.pose2d, layer_set, out_path)
145
+ if overlay_path:
146
+ vel_summary = build_velocity_summary(state.pose2d.keypoints, vis.last_velocities)
147
+ except Exception as e:
148
+ alerts += f"\n⚠️ Visualizer error: {e}"
149
+ return score_html, pipeline_md, score_details, alerts, overlay_path, vel_summary
150
+ ```
151
+
152
+ `vis.last_velocities` is stored on the instance after `render_video()` to avoid recomputing.
153
+
154
+ ### Event wiring
155
+
156
+ ```python
157
+ submit_btn.click(
158
+ fn=_map_inputs,
159
+ inputs=[video_input, test_dropdown, side_dropdown, pose_model_dropdown, overlay_layers],
160
+ outputs=[score_html, pipeline_md, score_details, alerts_md, overlay_video, velocity_md],
161
+ )
162
+ ```
163
+
164
+ `_map_inputs` gains `overlay_layers` as fifth parameter.
165
+
166
+ ---
167
+
168
+ ## Error handling
169
+
170
+ | Failure | Behaviour |
171
+ |---|---|
172
+ | All frames have no detections | `render_video()` returns `None`, tab empty, no crash |
173
+ | `cv2.VideoWriter` fails | logs warning, returns `None` |
174
+ | Any exception in visualizer | caught in `process_video()`, appended to alerts, `overlay_path = None` |
175
+ | `layers` is empty | returns `None` immediately, no processing |
176
+
177
+ The score is always returned regardless of visualizer outcome.
178
+
179
+ ---
180
+
181
+ ## Testing: `tests/test_visualizer.py`
182
+
183
+ - Synthetic `IngestResult`: 5 blank 480×640 BGR frames, fps=30
184
+ - Synthetic `Pose2DResult`: 17 keypoints per frame at fixed positions with conf=0.9
185
+ - `test_render_video_creates_file`: assert output `.mp4` exists and size > 0
186
+ - `test_compute_joint_velocity_shape`: assert 17-key dict, each list length == 5
187
+ - `test_empty_layers_returns_none`: assert `render_video(..., layers=set())` returns `None`
188
+ - `test_no_detections_returns_none`: all-empty keypoints → `None`
189
+ - `test_velocity_summary_markdown`: assert output contains `|` (table) and at least one joint name
190
+
191
+ ---
192
+
193
+ ## Out of scope
194
+
195
+ - Frame-by-frame metrics synced to video playback (Phase 4 / custom Svelte)
196
+ - Multi-person tracking
197
+ - Saving overlay video to Hugging Face Hub (tracing feature, Phase 4)
docs/superpowers/specs/2026-06-13-full-fms-session-pdf-design.md CHANGED
@@ -1,154 +1,154 @@
1
- # Full FMS Session + PDF Report — Design
2
-
3
- **Date:** 2026-06-13
4
- **Status:** Approved (brainstorming) — pending implementation plan
5
- **Owner:** FormScout
6
- **Related:** `formscout/agents/report.py`, `formscout/agents/visualizer.py`, `formscout/agents/biomechanics.py`, `app.py`, `formscout/types.py`
7
-
8
- ## Problem
9
-
10
- FormScout today scores **one** FMS test per upload. A real Functional Movement Screen is **all 7 tests** producing a single **composite 0–21** with asymmetry flags. `ReportAgent` and `ReportResult.composite` already support a multi-test report, but the UI never accumulates more than one test, and `ReportResult.pdf_path` is a hardcoded `None` stub.
11
-
12
- This feature turns the one-clip scorer into a **screening session**: each analyzed clip accumulates; a "Finish" action produces the composite report plus a downloadable, brand-consistent **PDF**. Each clip's worst-moment frame is captured as an annotated still embedded in both an on-disk log and the PDF.
13
-
14
- ## Goals
15
-
16
- - Accumulate multiple analyzed clips into one session, then emit a composite 0–21 report.
17
- - Generate a clinician/client-facing **PDF** handout (ReportLab) with scores, rationale, asymmetries, key-frame images, and the safety disclaimer.
18
- - Capture and annotate the **worst-moment frame** per test (the governing/peak frame already computed by `BiomechanicsAgent`).
19
- - Persist each analysis incrementally to disk (`session.json`, `analysis.md`, key-frame PNGs) until "Finish" is clicked.
20
-
21
- ## Non-goals (YAGNI)
22
-
23
- - No cross-restart session reload — the session lives in `gr.State` + a temp dir for the browser session.
24
- - No PDF styling beyond a clean branded layout (no HTML/CSS engine; ReportLab only).
25
- - No RAG / exemplar-clip citations (separate future spec).
26
- - No changes to the scoring pipeline, rubric functions, or Director flow.
27
-
28
- ## UX
29
-
30
- The current one-clip-at-a-time flow is preserved. Two new buttons appear after an analysis completes:
31
-
32
- - **➕ Analyse new clip** — clears the video/test inputs for the next upload; **keeps** the session.
33
- - **✅ Finish & generate PDF** — runs the report + PDF over everything accumulated so far.
34
-
35
- After each analysis, a **"Session so far"** table updates: `test · side · score · status`. Finish renders an on-screen composite scorecard + asymmetry summary and exposes the PDF (and `analysis.md`) via `gr.File` for download.
36
-
37
- Guard: Finish with zero analyses → warning, no PDF.
38
-
39
- ## Components
40
-
41
- ### 1. Session state + on-disk store
42
-
43
- A per-session temp directory `<tmpdir>/formscout_sessions/<session_id>/`:
44
-
45
- - `session.json` — structured list of entries; **source of truth** for the PDF.
46
- - `analysis.md` — human-readable log, appended after each clip.
47
- - `keyframes/<test_name>_<side>.png` — annotated worst-frame stills.
48
-
49
- Session identity lives in a `gr.State`. Each entry carries:
50
-
51
- - `test_name`, `side`, `score` (judge score, else rubric), `needs_human`
52
- - `rationale`, `compensation_tags`, `corrective_hint`
53
- - key measurements (selected `angles` / `alignments`)
54
- - `confidence`, `view` (`"2d"`/`"3d"`)
55
- - `keyframe_path`
56
- - the `movement` / `features` / `rubric_score` / `judge` objects that `ReportAgent.run()` consumes
57
-
58
- Persistence lasts until Finish; files are kept afterward for download. Cross-restart cleanup is best-effort and out of scope.
59
-
60
- ### 2. Key-frame capture
61
-
62
- New method on `PoseVisualizer`:
63
-
64
- ```python
65
- def render_frame(self, ingest, pose2d, frame_idx: int,
66
- layers: set[str], caption: str, out_png: str) -> str | None
67
- ```
68
-
69
- - `frame_idx` comes from `features.timing`, which already stores the governing frame per test:
70
- `deep_squat → deepest_frame`, `hurdle_step → peak_step_frame`,
71
- `inline_lunge → deepest_lunge_frame`, `shoulder_mobility → measure_frame`,
72
- `active_slr → peak_raise_frame`.
73
- - `trunk_stability_pushup` and `rotary_stability` currently store only counts in `timing`. Add the worst-sag-frame and peak-extension-frame index to their `timing` dicts (one-line change in each `BiomechanicsAgent` method).
74
- - Reuses `_draw_skeleton` (+ optional `_draw_trails`) on the single frame, overlays a caption naming the worst compensation, writes a PNG.
75
- - Returns `None` on any failure — never raises, never blocks the entry.
76
-
77
- The "worst compensation" caption is derived from `judge.compensation_tags` (preferred) or the failed `alignments` (fallback).
78
-
79
- ### 3. PDF generator
80
-
81
- New module `formscout/agents/pdf_report.py`:
82
-
83
- ```python
84
- class PdfReportAgent:
85
- def run(self, report_result: ReportResult,
86
- entries: list[SessionEntry], session_dir: str) -> str | None
87
- ```
88
-
89
- Uses **ReportLab** (pure-Python, no system deps — safe on HF Spaces/ZeroGPU). Layout:
90
-
91
- - Safety disclaimer banner at **top and bottom** (mirrors the UI invariant).
92
- - Title/brand header + date.
93
- - Composite **0–21** badge, or "Incomplete — N/7 tests scored" when `composite is None`.
94
- - Per-test block: score, rationale, key measurements, compensation tags, corrective hint, the annotated key-frame image, asymmetry delta (bilateral).
95
- - Flags section: low-confidence, rubric↔judge disagreement, needs-human.
96
- - Populates `ReportResult.pdf_path`.
97
-
98
- Returns the PDF path, or `None` on failure (UI surfaces the error and keeps the session for retry). Image embedding tolerates a missing/`None` `keyframe_path` with a placeholder line.
99
-
100
- ### 4. ReportAgent reuse
101
-
102
- At Finish, build the entry list and call the existing `ReportAgent.run()` for composite + asymmetries + flags. The bilateral lower-score + asymmetry-delta logic and the null-composite rule already exist and are not rewritten. A small adapter converts `SessionEntry` objects to the dict schema `ReportAgent.run()` expects (or `ReportAgent` gains overload tolerance — implementer's choice, keep it minimal).
103
-
104
- ### 5. Types
105
-
106
- Add a `SessionEntry` frozen dataclass to `formscout/types.py` (consistent with the "every agent I/O is a typed dataclass" standard), including `keyframe_path: str | None`. Populate the existing `ReportResult.pdf_path` (and optionally `overlay_video_path`). No other type changes.
107
-
108
- ### 6. UI (`app.py`)
109
-
110
- - Add a `gr.State` holding the session (id + entries).
111
- - After each analysis: render the scorecard as today, append the entry, write `session.json`/`analysis.md`/keyframe PNG, refresh the "Session so far" table, and reveal the two buttons.
112
- - **Analyse new clip**: reset the video/test/side inputs; keep session state.
113
- - **Finish & generate PDF**: `ReportAgent.run` → `PdfReportAgent.run` → display composite + asymmetry summary + `gr.File` downloads (PDF + `analysis.md`).
114
- - Guard: Finish with zero analyses → warning.
115
-
116
- ## Data flow
117
-
118
- ```
119
- upload → Director.run → score
120
- → build SessionEntry (+ render_frame keyframe png)
121
- → append to gr.State + write session.json / analysis.md / keyframe png
122
- → refresh "Session so far" table
123
-
124
- Finish → ReportAgent.run(entries) → composite / asymmetries / flags
125
- → PdfReportAgent.run(...) → pdf_path
126
- → on-screen composite + gr.File (PDF, analysis.md)
127
- ```
128
-
129
- ## Error handling
130
-
131
- - Key-frame render fails → entry still saved; PDF shows an image placeholder.
132
- - PDF generation fails → surface the error, keep the session intact for retry.
133
- - `needs_human` entry → no numeric score; PDF shows "Clinician review required"; composite null.
134
- - Composite is `None` whenever any test is unscored or needs human review (existing rule — never show a partial 0–21 as complete).
135
- - All disk writes tolerate failure with a logged warning; a write failure degrades the artifact but never blocks scoring.
136
-
137
- ## Testing (must run without model downloads)
138
-
139
- - `tests/test_pdf_report.py` — synthetic `ReportResult` + entries → PDF file created, non-zero size, contains the disclaimer text and composite line.
140
- - `tests/test_session.py` — accumulation; composite math; bilateral lower-score + asymmetry delta; null composite when one entry `needs_human`.
141
- - `tests/test_keyframe.py` — `render_frame` returns a real PNG path (file exists) for a synthetic frame; returns `None` gracefully on bad input.
142
-
143
- ## Invariants preserved
144
-
145
- - Pipeline stays headless — no Gradio imports in agent files (`PdfReportAgent` is a pure agent; key-frame capture stays in `visualizer.py`, the existing UI-layer component).
146
- - Safety disclaimer present top and bottom of the PDF, mirroring the UI.
147
- - Pain / clearing / needs-human is never auto-scored; composite null when any test unscored.
148
- - New code follows the engineering standards: one public entrypoint per agent, typed dataclass I/O, `confidence`/`notes` where applicable, module docstring stating purpose/inputs/outputs/failure/params/license/gated.
149
-
150
- ## Open implementation choices (left to the plan)
151
-
152
- - Exact `SessionEntry` → `ReportAgent` dict adapter shape.
153
- - Which measurements to surface per test in the PDF (a curated subset, not the full `angles` dump).
154
- - PDF assertion strategy in tests (text extraction vs. size/smoke).
 
1
+ # Full FMS Session + PDF Report — Design
2
+
3
+ **Date:** 2026-06-13
4
+ **Status:** Approved (brainstorming) — pending implementation plan
5
+ **Owner:** FormScout
6
+ **Related:** `formscout/agents/report.py`, `formscout/agents/visualizer.py`, `formscout/agents/biomechanics.py`, `app.py`, `formscout/types.py`
7
+
8
+ ## Problem
9
+
10
+ FormScout today scores **one** FMS test per upload. A real Functional Movement Screen is **all 7 tests** producing a single **composite 0–21** with asymmetry flags. `ReportAgent` and `ReportResult.composite` already support a multi-test report, but the UI never accumulates more than one test, and `ReportResult.pdf_path` is a hardcoded `None` stub.
11
+
12
+ This feature turns the one-clip scorer into a **screening session**: each analyzed clip accumulates; a "Finish" action produces the composite report plus a downloadable, brand-consistent **PDF**. Each clip's worst-moment frame is captured as an annotated still embedded in both an on-disk log and the PDF.
13
+
14
+ ## Goals
15
+
16
+ - Accumulate multiple analyzed clips into one session, then emit a composite 0–21 report.
17
+ - Generate a clinician/client-facing **PDF** handout (ReportLab) with scores, rationale, asymmetries, key-frame images, and the safety disclaimer.
18
+ - Capture and annotate the **worst-moment frame** per test (the governing/peak frame already computed by `BiomechanicsAgent`).
19
+ - Persist each analysis incrementally to disk (`session.json`, `analysis.md`, key-frame PNGs) until "Finish" is clicked.
20
+
21
+ ## Non-goals (YAGNI)
22
+
23
+ - No cross-restart session reload — the session lives in `gr.State` + a temp dir for the browser session.
24
+ - No PDF styling beyond a clean branded layout (no HTML/CSS engine; ReportLab only).
25
+ - No RAG / exemplar-clip citations (separate future spec).
26
+ - No changes to the scoring pipeline, rubric functions, or Director flow.
27
+
28
+ ## UX
29
+
30
+ The current one-clip-at-a-time flow is preserved. Two new buttons appear after an analysis completes:
31
+
32
+ - **➕ Analyse new clip** — clears the video/test inputs for the next upload; **keeps** the session.
33
+ - **✅ Finish & generate PDF** — runs the report + PDF over everything accumulated so far.
34
+
35
+ After each analysis, a **"Session so far"** table updates: `test · side · score · status`. Finish renders an on-screen composite scorecard + asymmetry summary and exposes the PDF (and `analysis.md`) via `gr.File` for download.
36
+
37
+ Guard: Finish with zero analyses → warning, no PDF.
38
+
39
+ ## Components
40
+
41
+ ### 1. Session state + on-disk store
42
+
43
+ A per-session temp directory `<tmpdir>/formscout_sessions/<session_id>/`:
44
+
45
+ - `session.json` — structured list of entries; **source of truth** for the PDF.
46
+ - `analysis.md` — human-readable log, appended after each clip.
47
+ - `keyframes/<test_name>_<side>.png` — annotated worst-frame stills.
48
+
49
+ Session identity lives in a `gr.State`. Each entry carries:
50
+
51
+ - `test_name`, `side`, `score` (judge score, else rubric), `needs_human`
52
+ - `rationale`, `compensation_tags`, `corrective_hint`
53
+ - key measurements (selected `angles` / `alignments`)
54
+ - `confidence`, `view` (`"2d"`/`"3d"`)
55
+ - `keyframe_path`
56
+ - the `movement` / `features` / `rubric_score` / `judge` objects that `ReportAgent.run()` consumes
57
+
58
+ Persistence lasts until Finish; files are kept afterward for download. Cross-restart cleanup is best-effort and out of scope.
59
+
60
+ ### 2. Key-frame capture
61
+
62
+ New method on `PoseVisualizer`:
63
+
64
+ ```python
65
+ def render_frame(self, ingest, pose2d, frame_idx: int,
66
+ layers: set[str], caption: str, out_png: str) -> str | None
67
+ ```
68
+
69
+ - `frame_idx` comes from `features.timing`, which already stores the governing frame per test:
70
+ `deep_squat → deepest_frame`, `hurdle_step → peak_step_frame`,
71
+ `inline_lunge → deepest_lunge_frame`, `shoulder_mobility → measure_frame`,
72
+ `active_slr → peak_raise_frame`.
73
+ - `trunk_stability_pushup` and `rotary_stability` currently store only counts in `timing`. Add the worst-sag-frame and peak-extension-frame index to their `timing` dicts (one-line change in each `BiomechanicsAgent` method).
74
+ - Reuses `_draw_skeleton` (+ optional `_draw_trails`) on the single frame, overlays a caption naming the worst compensation, writes a PNG.
75
+ - Returns `None` on any failure — never raises, never blocks the entry.
76
+
77
+ The "worst compensation" caption is derived from `judge.compensation_tags` (preferred) or the failed `alignments` (fallback).
78
+
79
+ ### 3. PDF generator
80
+
81
+ New module `formscout/agents/pdf_report.py`:
82
+
83
+ ```python
84
+ class PdfReportAgent:
85
+ def run(self, report_result: ReportResult,
86
+ entries: list[SessionEntry], session_dir: str) -> str | None
87
+ ```
88
+
89
+ Uses **ReportLab** (pure-Python, no system deps — safe on HF Spaces/ZeroGPU). Layout:
90
+
91
+ - Safety disclaimer banner at **top and bottom** (mirrors the UI invariant).
92
+ - Title/brand header + date.
93
+ - Composite **0–21** badge, or "Incomplete — N/7 tests scored" when `composite is None`.
94
+ - Per-test block: score, rationale, key measurements, compensation tags, corrective hint, the annotated key-frame image, asymmetry delta (bilateral).
95
+ - Flags section: low-confidence, rubric↔judge disagreement, needs-human.
96
+ - Populates `ReportResult.pdf_path`.
97
+
98
+ Returns the PDF path, or `None` on failure (UI surfaces the error and keeps the session for retry). Image embedding tolerates a missing/`None` `keyframe_path` with a placeholder line.
99
+
100
+ ### 4. ReportAgent reuse
101
+
102
+ At Finish, build the entry list and call the existing `ReportAgent.run()` for composite + asymmetries + flags. The bilateral lower-score + asymmetry-delta logic and the null-composite rule already exist and are not rewritten. A small adapter converts `SessionEntry` objects to the dict schema `ReportAgent.run()` expects (or `ReportAgent` gains overload tolerance — implementer's choice, keep it minimal).
103
+
104
+ ### 5. Types
105
+
106
+ Add a `SessionEntry` frozen dataclass to `formscout/types.py` (consistent with the "every agent I/O is a typed dataclass" standard), including `keyframe_path: str | None`. Populate the existing `ReportResult.pdf_path` (and optionally `overlay_video_path`). No other type changes.
107
+
108
+ ### 6. UI (`app.py`)
109
+
110
+ - Add a `gr.State` holding the session (id + entries).
111
+ - After each analysis: render the scorecard as today, append the entry, write `session.json`/`analysis.md`/keyframe PNG, refresh the "Session so far" table, and reveal the two buttons.
112
+ - **Analyse new clip**: reset the video/test/side inputs; keep session state.
113
+ - **Finish & generate PDF**: `ReportAgent.run` → `PdfReportAgent.run` → display composite + asymmetry summary + `gr.File` downloads (PDF + `analysis.md`).
114
+ - Guard: Finish with zero analyses → warning.
115
+
116
+ ## Data flow
117
+
118
+ ```
119
+ upload → Director.run → score
120
+ → build SessionEntry (+ render_frame keyframe png)
121
+ → append to gr.State + write session.json / analysis.md / keyframe png
122
+ → refresh "Session so far" table
123
+
124
+ Finish → ReportAgent.run(entries) → composite / asymmetries / flags
125
+ → PdfReportAgent.run(...) → pdf_path
126
+ → on-screen composite + gr.File (PDF, analysis.md)
127
+ ```
128
+
129
+ ## Error handling
130
+
131
+ - Key-frame render fails → entry still saved; PDF shows an image placeholder.
132
+ - PDF generation fails → surface the error, keep the session intact for retry.
133
+ - `needs_human` entry → no numeric score; PDF shows "Clinician review required"; composite null.
134
+ - Composite is `None` whenever any test is unscored or needs human review (existing rule — never show a partial 0–21 as complete).
135
+ - All disk writes tolerate failure with a logged warning; a write failure degrades the artifact but never blocks scoring.
136
+
137
+ ## Testing (must run without model downloads)
138
+
139
+ - `tests/test_pdf_report.py` — synthetic `ReportResult` + entries → PDF file created, non-zero size, contains the disclaimer text and composite line.
140
+ - `tests/test_session.py` — accumulation; composite math; bilateral lower-score + asymmetry delta; null composite when one entry `needs_human`.
141
+ - `tests/test_keyframe.py` — `render_frame` returns a real PNG path (file exists) for a synthetic frame; returns `None` gracefully on bad input.
142
+
143
+ ## Invariants preserved
144
+
145
+ - Pipeline stays headless — no Gradio imports in agent files (`PdfReportAgent` is a pure agent; key-frame capture stays in `visualizer.py`, the existing UI-layer component).
146
+ - Safety disclaimer present top and bottom of the PDF, mirroring the UI.
147
+ - Pain / clearing / needs-human is never auto-scored; composite null when any test unscored.
148
+ - New code follows the engineering standards: one public entrypoint per agent, typed dataclass I/O, `confidence`/`notes` where applicable, module docstring stating purpose/inputs/outputs/failure/params/license/gated.
149
+
150
+ ## Open implementation choices (left to the plan)
151
+
152
+ - Exact `SessionEntry` → `ReportAgent` dict adapter shape.
153
+ - Which measurements to surface per test in the PDF (a curated subset, not the full `angles` dump).
154
+ - PDF assertion strategy in tests (text extraction vs. size/smoke).
formscout/agents/classifier.py CHANGED
@@ -1,102 +1,102 @@
1
- """
2
- MovementClassifierAgent — identifies which FMS test is in the clip.
3
-
4
- Input: IngestResult (keyframes), Pose2DResult (skeleton context)
5
- Output: MovementResult(test_name, side, confidence)
6
- Failure: returns MovementResult(test_name="unknown") — pipeline stops and asks for manual override.
7
- Model: Qwen3-VL-8B-Instruct via llama.cpp (8B params, Apache-2.0).
8
- Gated: No.
9
- """
10
- from __future__ import annotations
11
-
12
- import logging
13
- from pathlib import Path
14
-
15
- from formscout import config
16
- from formscout.types import IngestResult, Pose2DResult, MovementResult
17
- from formscout.serving.llama_cpp import LlamaCppClient
18
-
19
- logger = logging.getLogger(__name__)
20
-
21
- _PROMPT_PATH = Path(__file__).parent / "prompts" / "c1_classifier.md"
22
-
23
-
24
- class MovementClassifierAgent:
25
- """Classifies which FMS test is being performed via VLM or manual override."""
26
-
27
- def __init__(self):
28
- self._client = LlamaCppClient(port=config.LLAMA_CPP_PORT_VLM)
29
- self._system_prompt = _PROMPT_PATH.read_text(encoding="utf-8")
30
-
31
- def run(
32
- self,
33
- ingest: IngestResult,
34
- pose2d: Pose2DResult | None = None,
35
- manual_override: str | None = None,
36
- ) -> MovementResult:
37
- """
38
- Classify the movement. If manual_override is provided, use it directly.
39
- Otherwise, use VLM inference on keyframes.
40
- """
41
- if manual_override and manual_override != "unknown":
42
- return MovementResult(
43
- test_name=manual_override, side="na",
44
- confidence=1.0, notes="manual override",
45
- )
46
-
47
- if not self._client.available:
48
- return MovementResult(
49
- test_name="unknown", side="na", confidence=0.0,
50
- notes="VLM server unavailable — use manual override",
51
- )
52
-
53
- # Select keyframes for classification (3 evenly spaced)
54
- n = len(ingest.frames)
55
- indices = [0, n // 2, n - 1] if n >= 3 else list(range(n))
56
- images = self._encode_frames(ingest.frames, indices)
57
-
58
- prompt = f"{self._system_prompt}\n\nClassify this movement from the keyframes shown."
59
- result = self._client.complete(prompt, images=images, max_tokens=256, temperature=0.1)
60
-
61
- return self._parse_response(result)
62
-
63
- def _encode_frames(self, frames: list, indices: list[int]) -> list[str]:
64
- """Encode selected frames as base64 JPEG for the VLM."""
65
- import cv2
66
- import base64
67
-
68
- encoded = []
69
- for idx in indices:
70
- if idx < len(frames):
71
- _, buf = cv2.imencode(".jpg", frames[idx], [cv2.IMWRITE_JPEG_QUALITY, 80])
72
- encoded.append(base64.b64encode(buf.tobytes()).decode())
73
- return encoded
74
-
75
- def _parse_response(self, result: dict) -> MovementResult:
76
- """Parse VLM JSON response into MovementResult."""
77
- if "error" in result:
78
- return MovementResult(
79
- test_name="unknown", side="na", confidence=0.0,
80
- notes=f"VLM error: {result['error']}",
81
- )
82
-
83
- test = result.get("test", "unknown")
84
- side = result.get("side", "na")
85
- confidence = float(result.get("confidence", 0.0))
86
- reason = result.get("reason", "")
87
-
88
- valid_tests = {
89
- "deep_squat", "hurdle_step", "inline_lunge",
90
- "shoulder_mobility", "active_slr",
91
- "trunk_stability_pushup", "rotary_stability", "unknown",
92
- }
93
- if test not in valid_tests:
94
- test = "unknown"
95
-
96
- if side not in ("left", "right", "na"):
97
- side = "na"
98
-
99
- return MovementResult(
100
- test_name=test, side=side,
101
- confidence=confidence, notes=reason,
102
- )
 
1
+ """
2
+ MovementClassifierAgent — identifies which FMS test is in the clip.
3
+
4
+ Input: IngestResult (keyframes), Pose2DResult (skeleton context)
5
+ Output: MovementResult(test_name, side, confidence)
6
+ Failure: returns MovementResult(test_name="unknown") — pipeline stops and asks for manual override.
7
+ Model: Qwen3-VL-8B-Instruct via llama.cpp (8B params, Apache-2.0).
8
+ Gated: No.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import logging
13
+ from pathlib import Path
14
+
15
+ from formscout import config
16
+ from formscout.types import IngestResult, Pose2DResult, MovementResult
17
+ from formscout.serving.llama_cpp import LlamaCppClient
18
+
19
+ logger = logging.getLogger(__name__)
20
+
21
+ _PROMPT_PATH = Path(__file__).parent / "prompts" / "c1_classifier.md"
22
+
23
+
24
+ class MovementClassifierAgent:
25
+ """Classifies which FMS test is being performed via VLM or manual override."""
26
+
27
+ def __init__(self):
28
+ self._client = LlamaCppClient(port=config.LLAMA_CPP_PORT_VLM)
29
+ self._system_prompt = _PROMPT_PATH.read_text(encoding="utf-8")
30
+
31
+ def run(
32
+ self,
33
+ ingest: IngestResult,
34
+ pose2d: Pose2DResult | None = None,
35
+ manual_override: str | None = None,
36
+ ) -> MovementResult:
37
+ """
38
+ Classify the movement. If manual_override is provided, use it directly.
39
+ Otherwise, use VLM inference on keyframes.
40
+ """
41
+ if manual_override and manual_override != "unknown":
42
+ return MovementResult(
43
+ test_name=manual_override, side="na",
44
+ confidence=1.0, notes="manual override",
45
+ )
46
+
47
+ if not self._client.available:
48
+ return MovementResult(
49
+ test_name="unknown", side="na", confidence=0.0,
50
+ notes="VLM server unavailable — use manual override",
51
+ )
52
+
53
+ # Select keyframes for classification (3 evenly spaced)
54
+ n = len(ingest.frames)
55
+ indices = [0, n // 2, n - 1] if n >= 3 else list(range(n))
56
+ images = self._encode_frames(ingest.frames, indices)
57
+
58
+ prompt = f"{self._system_prompt}\n\nClassify this movement from the keyframes shown."
59
+ result = self._client.complete(prompt, images=images, max_tokens=256, temperature=0.1)
60
+
61
+ return self._parse_response(result)
62
+
63
+ def _encode_frames(self, frames: list, indices: list[int]) -> list[str]:
64
+ """Encode selected frames as base64 JPEG for the VLM."""
65
+ import cv2
66
+ import base64
67
+
68
+ encoded = []
69
+ for idx in indices:
70
+ if idx < len(frames):
71
+ _, buf = cv2.imencode(".jpg", frames[idx], [cv2.IMWRITE_JPEG_QUALITY, 80])
72
+ encoded.append(base64.b64encode(buf.tobytes()).decode())
73
+ return encoded
74
+
75
+ def _parse_response(self, result: dict) -> MovementResult:
76
+ """Parse VLM JSON response into MovementResult."""
77
+ if "error" in result:
78
+ return MovementResult(
79
+ test_name="unknown", side="na", confidence=0.0,
80
+ notes=f"VLM error: {result['error']}",
81
+ )
82
+
83
+ test = result.get("test", "unknown")
84
+ side = result.get("side", "na")
85
+ confidence = float(result.get("confidence", 0.0))
86
+ reason = result.get("reason", "")
87
+
88
+ valid_tests = {
89
+ "deep_squat", "hurdle_step", "inline_lunge",
90
+ "shoulder_mobility", "active_slr",
91
+ "trunk_stability_pushup", "rotary_stability", "unknown",
92
+ }
93
+ if test not in valid_tests:
94
+ test = "unknown"
95
+
96
+ if side not in ("left", "right", "na"):
97
+ side = "na"
98
+
99
+ return MovementResult(
100
+ test_name=test, side=side,
101
+ confidence=confidence, notes=reason,
102
+ )
formscout/agents/ingest.py CHANGED
@@ -49,32 +49,22 @@ class IngestAgent:
49
  total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
50
  w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
51
  h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
 
52
 
53
  notes_parts: list[str] = []
 
 
 
 
54
 
55
- # Frame count is unreliable for webm/mkv (browser recordings often
56
- # report 0 or a wrong value). Detect this and fall back to reading
57
- # every frame up to the cap, subsampling only when the count is trusted.
58
- count_reliable = total > 0
59
- if count_reliable:
60
- step = max(1, total // config.MAX_FRAMES)
61
- else:
62
- step = 1 # unknown length — keep frames, cap at MAX_FRAMES
63
-
64
- # Read frames; tolerate a truncated/premature end gracefully.
65
  frames: list = []
66
  idx = 0
67
- read_failures = 0
68
  while True:
69
  ret, frame = cap.read()
70
  if not ret:
71
- # A truncated final chunk yields ret=False; stop cleanly.
72
  break
73
- if frame is None:
74
- read_failures += 1
75
- if read_failures > 5:
76
- break
77
- continue
78
  if idx % step == 0:
79
  frames.append(frame)
80
  idx += 1
@@ -82,17 +72,6 @@ class IngestAgent:
82
  break
83
  cap.release()
84
 
85
- # Compute duration from what we actually decoded, not the (possibly
86
- # bogus) header frame count.
87
- decoded_total = idx if idx > 0 else len(frames)
88
- duration = decoded_total / fps if fps > 0 else 0.0
89
- if duration > config.MAX_DURATION_SEC:
90
- notes_parts.append(
91
- f"video is {duration:.1f}s (>{config.MAX_DURATION_SEC}s) — capping frames"
92
- )
93
- if not count_reliable and frames:
94
- notes_parts.append("frame count unreliable (webm/mkv) — read sequentially")
95
-
96
  if not frames:
97
  return IngestResult(
98
  frames=[], fps=fps, duration=duration, n_people=0,
 
49
  total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
50
  w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
51
  h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
52
+ duration = total / fps if fps > 0 else 0.0
53
 
54
  notes_parts: list[str] = []
55
+ if duration > config.MAX_DURATION_SEC:
56
+ notes_parts.append(
57
+ f"video is {duration:.1f}s (>{config.MAX_DURATION_SEC}s) — capping frames"
58
+ )
59
 
60
+ # Sample frames evenly, capped at MAX_FRAMES
61
+ step = max(1, total // config.MAX_FRAMES)
 
 
 
 
 
 
 
 
62
  frames: list = []
63
  idx = 0
 
64
  while True:
65
  ret, frame = cap.read()
66
  if not ret:
 
67
  break
 
 
 
 
 
68
  if idx % step == 0:
69
  frames.append(frame)
70
  idx += 1
 
72
  break
73
  cap.release()
74
 
 
 
 
 
 
 
 
 
 
 
 
75
  if not frames:
76
  return IngestResult(
77
  frames=[], fps=fps, duration=duration, n_people=0,
formscout/agents/judge.py CHANGED
@@ -1,136 +1,125 @@
1
- """
2
- JudgeAgent — VLM-based final scorer with rationale, compensation tags, pain detection.
3
-
4
- Input: BiomechFeatures, ScoreResult (rubric candidate), MovementResult, keyframes
5
- Output: JudgeResult(score, rationale, compensation_tags, corrective_hint, needs_human)
6
- Failure: returns JudgeResult(needs_human=True, score=None) when uncertain.
7
- Model: Qwen3-VL-8B-Instruct via llama.cpp (8B params, Apache-2.0).
8
- Gated: No.
9
-
10
- Safety: NEVER auto-scores pain. If any indication of pain/clearing test,
11
- sets needs_human=True and score=None.
12
- """
13
- from __future__ import annotations
14
-
15
- import json
16
- import logging
17
- from pathlib import Path
18
-
19
- from formscout import config
20
- from formscout.types import (
21
- BiomechFeatures, ScoreResult, MovementResult,
22
- IngestResult, JudgeResult,
23
- )
24
- from formscout.serving.llama_cpp import LlamaCppClient
25
-
26
- logger = logging.getLogger(__name__)
27
-
28
- _PROMPT_PATH = Path(__file__).parent / "prompts" / "c2_judge.md"
29
-
30
-
31
- class JudgeAgent:
32
- """VLM judge that produces the final FMS score with rationale."""
33
-
34
- def __init__(self):
35
- self._client = LlamaCppClient(port=config.LLAMA_CPP_PORT_VLM)
36
- self._system_prompt = _PROMPT_PATH.read_text(encoding="utf-8")
37
-
38
- def run(
39
- self,
40
- features: BiomechFeatures,
41
- rubric_score: ScoreResult,
42
- movement: MovementResult,
43
- ingest: IngestResult | None = None,
44
- ) -> JudgeResult:
45
- """
46
- Produce final score. Falls back to rubric score if VLM unavailable.
47
- """
48
- if not config.ENABLE_JUDGE:
49
- return self._fallback_from_rubric(rubric_score, features)
50
-
51
- if not self._client.available:
52
- logger.warning("JudgeAgent: VLM unavailable, using rubric score as final")
53
- return self._fallback_from_rubric(rubric_score, features)
54
-
55
- # Build context for the judge
56
- context = {
57
- "test": features.test_name,
58
- "side": features.side,
59
- "view": features.view,
60
- "features": {"angles": features.angles, "alignments": features.alignments},
61
- "candidate_score": rubric_score.score,
62
- "candidate_confidence": rubric_score.confidence,
63
- "exemplars": [], # Phase 3: populated by RetrievalAgent
64
- }
65
-
66
- prompt = f"{self._system_prompt}\n\n{json.dumps(context, indent=2)}"
67
-
68
- # Optionally include keyframes
69
- images = None
70
- if ingest and ingest.frames:
71
- images = self._encode_keyframes(ingest.frames)
72
-
73
- result = self._client.complete(prompt, images=images, max_tokens=512, temperature=0.1)
74
- return self._parse_response(result)
75
-
76
- def _encode_keyframes(self, frames: list) -> list[str]:
77
- """Encode 3 keyframes for VLM context (downscaled to max 768px)."""
78
- import cv2
79
- import base64
80
-
81
- n = len(frames)
82
- indices = [0, n // 2, n - 1] if n >= 3 else list(range(n))
83
- encoded = []
84
- for idx in indices:
85
- frame = self._downscale(frames[idx], max_dim=768)
86
- _, buf = cv2.imencode(".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, 70])
87
- encoded.append(base64.b64encode(buf.tobytes()).decode())
88
- return encoded
89
-
90
- @staticmethod
91
- def _downscale(frame, max_dim: int = 768):
92
- """Resize frame so its largest dimension is at most max_dim."""
93
- import cv2
94
-
95
- h, w = frame.shape[:2]
96
- longest = max(h, w)
97
- if longest <= max_dim:
98
- return frame
99
- scale = max_dim / longest
100
- new_size = (int(w * scale), int(h * scale))
101
- return cv2.resize(frame, new_size, interpolation=cv2.INTER_AREA)
102
-
103
- def _parse_response(self, result: dict) -> JudgeResult:
104
- """Parse VLM JSON response into JudgeResult."""
105
- if "error" in result:
106
- return JudgeResult(
107
- score=None, rationale=f"VLM error: {result['error']}",
108
- compensation_tags=[], corrective_hint="",
109
- confidence=0.0, needs_human=True,
110
- )
111
-
112
- needs_human = result.get("needs_human", False)
113
- score = result.get("score") if not needs_human else None
114
- if score is not None:
115
- score = max(0, min(3, int(score)))
116
-
117
- return JudgeResult(
118
- score=score,
119
- rationale=result.get("rationale", ""),
120
- compensation_tags=result.get("compensation_tags", []),
121
- corrective_hint=result.get("corrective_hint", ""),
122
- confidence=float(result.get("confidence", 0.5)),
123
- needs_human=needs_human,
124
- )
125
-
126
- def _fallback_from_rubric(self, rubric: ScoreResult, features: BiomechFeatures) -> JudgeResult:
127
- """When VLM is unavailable, promote the rubric score as the final score."""
128
- return JudgeResult(
129
- score=rubric.score,
130
- rationale=f"[rubric-only] {rubric.rationale}",
131
- compensation_tags=[],
132
- corrective_hint="",
133
- confidence=rubric.confidence * 0.8,
134
- needs_human=rubric.needs_human,
135
- notes="VLM unavailable — rubric score used as final",
136
- )
 
1
+ """
2
+ JudgeAgent — VLM-based final scorer with rationale, compensation tags, pain detection.
3
+
4
+ Input: BiomechFeatures, ScoreResult (rubric candidate), MovementResult, keyframes
5
+ Output: JudgeResult(score, rationale, compensation_tags, corrective_hint, needs_human)
6
+ Failure: returns JudgeResult(needs_human=True, score=None) when uncertain.
7
+ Model: Qwen3-VL-8B-Instruct via llama.cpp (8B params, Apache-2.0).
8
+ Gated: No.
9
+
10
+ Safety: NEVER auto-scores pain. If any indication of pain/clearing test,
11
+ sets needs_human=True and score=None.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import logging
17
+ from pathlib import Path
18
+
19
+ from formscout import config
20
+ from formscout.types import (
21
+ BiomechFeatures, ScoreResult, MovementResult,
22
+ IngestResult, JudgeResult,
23
+ )
24
+ from formscout.serving import get_vlm_client
25
+
26
+ logger = logging.getLogger(__name__)
27
+
28
+ _PROMPT_PATH = Path(__file__).parent / "prompts" / "c2_judge.md"
29
+
30
+
31
+ class JudgeAgent:
32
+ """VLM judge that produces the final FMS score with rationale."""
33
+
34
+ def __init__(self):
35
+ self._client = get_vlm_client()
36
+ self._system_prompt = _PROMPT_PATH.read_text(encoding="utf-8")
37
+
38
+ def run(
39
+ self,
40
+ features: BiomechFeatures,
41
+ rubric_score: ScoreResult,
42
+ movement: MovementResult,
43
+ ingest: IngestResult | None = None,
44
+ ) -> JudgeResult:
45
+ """
46
+ Produce final score. Falls back to rubric score if VLM unavailable.
47
+ """
48
+ if not config.ENABLE_JUDGE:
49
+ return self._fallback_from_rubric(rubric_score, features)
50
+
51
+ if not self._client.available:
52
+ logger.warning("JudgeAgent: VLM unavailable, using rubric score as final")
53
+ return self._fallback_from_rubric(rubric_score, features)
54
+
55
+ # Build context for the judge
56
+ context = {
57
+ "test": features.test_name,
58
+ "side": features.side,
59
+ "view": features.view,
60
+ "features": {"angles": features.angles, "alignments": features.alignments},
61
+ "candidate_score": rubric_score.score,
62
+ "candidate_confidence": rubric_score.confidence,
63
+ "exemplars": [], # Phase 3: populated by RetrievalAgent
64
+ }
65
+
66
+ prompt = f"{self._system_prompt}\n\n{json.dumps(context, indent=2)}"
67
+
68
+ # Optionally include keyframes
69
+ images = None
70
+ if ingest and ingest.frames:
71
+ images = self._encode_keyframes(ingest.frames)
72
+
73
+ result = self._client.complete(prompt, images=images, max_tokens=512, temperature=0.1)
74
+ if result.get("fallback"):
75
+ # transformers backend couldn't load/run — use the deterministic rubric
76
+ return self._fallback_from_rubric(rubric_score, features)
77
+ return self._parse_response(result)
78
+
79
+ def _encode_keyframes(self, frames: list) -> list[str]:
80
+ """Encode 3 keyframes for VLM context."""
81
+ import cv2
82
+ import base64
83
+
84
+ n = len(frames)
85
+ indices = [0, n // 2, n - 1] if n >= 3 else list(range(n))
86
+ encoded = []
87
+ for idx in indices:
88
+ _, buf = cv2.imencode(".jpg", frames[idx], [cv2.IMWRITE_JPEG_QUALITY, 70])
89
+ encoded.append(base64.b64encode(buf.tobytes()).decode())
90
+ return encoded
91
+
92
+ def _parse_response(self, result: dict) -> JudgeResult:
93
+ """Parse VLM JSON response into JudgeResult."""
94
+ if "error" in result:
95
+ return JudgeResult(
96
+ score=None, rationale=f"VLM error: {result['error']}",
97
+ compensation_tags=[], corrective_hint="",
98
+ confidence=0.0, needs_human=True,
99
+ )
100
+
101
+ needs_human = result.get("needs_human", False)
102
+ score = result.get("score") if not needs_human else None
103
+ if score is not None:
104
+ score = max(0, min(3, int(score)))
105
+
106
+ return JudgeResult(
107
+ score=score,
108
+ rationale=result.get("rationale", ""),
109
+ compensation_tags=result.get("compensation_tags", []),
110
+ corrective_hint=result.get("corrective_hint", ""),
111
+ confidence=float(result.get("confidence", 0.5)),
112
+ needs_human=needs_human,
113
+ )
114
+
115
+ def _fallback_from_rubric(self, rubric: ScoreResult, features: BiomechFeatures) -> JudgeResult:
116
+ """When VLM is unavailable, promote the rubric score as the final score."""
117
+ return JudgeResult(
118
+ score=rubric.score,
119
+ rationale=f"[rubric-only] {rubric.rationale}",
120
+ compensation_tags=[],
121
+ corrective_hint="",
122
+ confidence=rubric.confidence * 0.8,
123
+ needs_human=rubric.needs_human,
124
+ notes="VLM unavailable — rubric score used as final",
125
+ )
 
 
 
 
 
 
 
 
 
 
 
formscout/agents/pdf_report.py CHANGED
@@ -1,115 +1,175 @@
1
- """
2
- PdfReportAgent — renders a ReportResult + session entries to a branded PDF.
3
-
4
- Input: ReportResult, list[SessionEntry], session_dir (str)
5
- Output: path to the written PDF (str), or None on failure.
6
- Failure: returns None, never raises.
7
- Params: 0 (pure rendering — no model).
8
- License: n/a.
9
- Gated: no.
10
- """
11
- from __future__ import annotations
12
-
13
- import logging
14
- import os
15
-
16
- from formscout.types import ReportResult
17
-
18
- logger = logging.getLogger(__name__)
19
-
20
- DISCLAIMER = "Screening aid — not a diagnosis. Pain or clearing tests require a clinician."
21
-
22
-
23
- class PdfReportAgent:
24
- """Assembles the screening-session PDF via ReportLab."""
25
-
26
- def run(self, report: ReportResult, entries: list, session_dir: str) -> str | None:
27
- try:
28
- from reportlab.lib import colors
29
- from reportlab.lib.pagesizes import LETTER
30
- from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
31
- from reportlab.lib.units import inch
32
- from reportlab.platypus import (
33
- Image, Paragraph, SimpleDocTemplate, Spacer, Table, TableStyle,
34
- )
35
- except Exception as e:
36
- logger.warning("reportlab unavailable: %s", e)
37
- return None
38
-
39
- out_path = os.path.join(session_dir, "formscout_report.pdf")
40
- try:
41
- styles = getSampleStyleSheet()
42
- banner = ParagraphStyle(
43
- "banner", parent=styles["Normal"], fontSize=9, textColor=colors.white,
44
- backColor=colors.HexColor("#b45309"), alignment=1, borderPadding=6, spaceAfter=12,
45
- )
46
- story = []
47
- story.append(Paragraph(f"<b>&#9888; {DISCLAIMER}</b>", banner))
48
- story.append(Paragraph("FormScout — FMS Screening Report", styles["Title"]))
49
-
50
- if report.composite is not None:
51
- comp = f"Composite: <b>{report.composite} / 21</b>"
52
- else:
53
- comp = f"Composite: <b>Incomplete</b> {len(entries)}/7 tests scored"
54
- story.append(Paragraph(comp, styles["Heading2"]))
55
- story.append(Spacer(1, 0.2 * inch))
56
-
57
- for e in entries:
58
- title = e.test_name.replace("_", " ").title()
59
- if e.side in ("left", "right"):
60
- title += f" ({e.side})"
61
- score_txt = "Clinician review required" if e.needs_human else f"Score: {e.score}/3"
62
- story.append(Paragraph(f"<b>{title}</b> — {score_txt}", styles["Heading3"]))
63
- if e.rationale:
64
- story.append(Paragraph(e.rationale, styles["Normal"]))
65
- if e.compensation_tags:
66
- story.append(Paragraph("Compensations: " + ", ".join(e.compensation_tags),
67
- styles["Normal"]))
68
- if e.corrective_hint:
69
- story.append(Paragraph("Corrective: " + e.corrective_hint, styles["Normal"]))
70
-
71
- items = list(e.measurements.items())[:6]
72
- if items:
73
- rows = [[k.replace("_", " "),
74
- (f"{v:.1f}" if isinstance(v, float) else str(v))] for k, v in items]
75
- tbl = Table(rows, colWidths=[3 * inch, 1.5 * inch])
76
- tbl.setStyle(TableStyle([
77
- ("FONTSIZE", (0, 0), (-1, -1), 8),
78
- ("TEXTCOLOR", (0, 0), (-1, -1), colors.HexColor("#334155")),
79
- ]))
80
- story.append(tbl)
81
-
82
- if e.keyframe_path and os.path.exists(e.keyframe_path):
83
- try:
84
- story.append(Image(e.keyframe_path, width=3.0 * inch, height=2.25 * inch))
85
- except Exception:
86
- story.append(Paragraph("<i>(key-frame image unavailable)</i>", styles["Normal"]))
87
- else:
88
- story.append(Paragraph("<i>(key-frame image unavailable)</i>", styles["Normal"]))
89
-
90
- story.append(Spacer(1, 0.2 * inch))
91
-
92
- if report.asymmetries:
93
- story.append(Paragraph("Asymmetries", styles["Heading2"]))
94
- for a in report.asymmetries:
95
- story.append(Paragraph(
96
- f"{a['test'].replace('_', ' ').title()}: "
97
- f"L={a['left_score']} R={a['right_score']} (&#916; {a['delta']})",
98
- styles["Normal"]))
99
-
100
- flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
101
- if flags:
102
- story.append(Paragraph("Flags", styles["Heading2"]))
103
- for fl in flags:
104
- story.append(Paragraph(fl, styles["Normal"]))
105
-
106
- story.append(Spacer(1, 0.3 * inch))
107
- story.append(Paragraph(f"<b>&#9888; {DISCLAIMER}</b>", banner))
108
-
109
- doc = SimpleDocTemplate(out_path, pagesize=LETTER,
110
- topMargin=0.6 * inch, bottomMargin=0.6 * inch)
111
- doc.build(story)
112
- return out_path
113
- except Exception as e:
114
- logger.warning("pdf build failed: %s", e)
115
- return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ PdfReportAgent — renders a ReportResult + session entries to a branded PDF.
3
+
4
+ Input: ReportResult, list[SessionEntry], session_dir (str)
5
+ Output: path to the written PDF (str), or None on failure.
6
+ Failure: returns None, never raises.
7
+ Params: 0 (pure rendering — no model).
8
+ License: n/a.
9
+ Gated: no.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import logging
14
+ import os
15
+
16
+ from formscout.types import ReportResult
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+ DISCLAIMER = "Screening aid — not a diagnosis. Pain or clearing tests require a clinician."
21
+
22
+
23
+ class PdfReportAgent:
24
+ """Assembles the screening-session PDF via ReportLab."""
25
+
26
+ def run(self, report: ReportResult, entries: list, session_dir: str) -> str | None:
27
+ try:
28
+ from reportlab.lib import colors
29
+ from reportlab.lib.pagesizes import LETTER
30
+ from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
31
+ from reportlab.lib.units import inch
32
+ from reportlab.platypus import (
33
+ Image, PageBreak, Paragraph, SimpleDocTemplate, Spacer, Table, TableStyle,
34
+ )
35
+ except Exception as e:
36
+ logger.warning("reportlab unavailable: %s", e)
37
+ return None
38
+
39
+ out_path = os.path.join(session_dir, "formscout_report.pdf")
40
+ try:
41
+ styles = getSampleStyleSheet()
42
+ banner = ParagraphStyle(
43
+ "banner", parent=styles["Normal"], fontSize=9, textColor=colors.white,
44
+ backColor=colors.HexColor("#cf922a"), alignment=1, borderPadding=6, spaceAfter=12,
45
+ )
46
+ ink = colors.HexColor("#243a34")
47
+
48
+ def _meas_table(pairs, col0=3.0, col1=1.6):
49
+ rows = [[str(k).replace("_", " "),
50
+ (f"{v:.2f}" if isinstance(v, float) else str(v))] for k, v in pairs]
51
+ tbl = Table(rows, colWidths=[col0 * inch, col1 * inch])
52
+ tbl.setStyle(TableStyle([
53
+ ("FONTSIZE", (0, 0), (-1, -1), 8),
54
+ ("TEXTCOLOR", (0, 0), (-1, -1), ink),
55
+ ("ROWBACKGROUNDS", (0, 0), (-1, -1),
56
+ [colors.HexColor("#f7eedd"), colors.white]),
57
+ ]))
58
+ return tbl
59
+
60
+ def _img(path, w=3.0, h=2.25):
61
+ if path and os.path.exists(path):
62
+ try:
63
+ return Image(path, width=w * inch, height=h * inch)
64
+ except Exception:
65
+ return None
66
+ return None
67
+ story = []
68
+ story.append(Paragraph(f"<b>&#9888; {DISCLAIMER}</b>", banner))
69
+ story.append(Paragraph("FormScout FMS Screening Report", styles["Title"]))
70
+
71
+ if report.composite is not None:
72
+ comp = f"Composite: <b>{report.composite} / 21</b>"
73
+ else:
74
+ comp = f"Composite: <b>Incomplete</b> {len(entries)}/7 tests scored"
75
+ story.append(Paragraph(comp, styles["Heading2"]))
76
+ story.append(Spacer(1, 0.2 * inch))
77
+
78
+ for ei, e in enumerate(entries):
79
+ if ei > 0:
80
+ story.append(PageBreak())
81
+ title = e.test_name.replace("_", " ").title()
82
+ if e.side in ("left", "right"):
83
+ title += f" ({e.side})"
84
+ score_txt = "Clinician review required" if e.needs_human else f"Score: {e.score}/3"
85
+ story.append(Paragraph(f"<b>{title}</b> — {score_txt}", styles["Heading3"]))
86
+ story.append(Paragraph(f"<font size=8>view: {e.view} · confidence: "
87
+ f"{e.confidence:.0%}</font>", styles["Normal"]))
88
+ if e.rationale:
89
+ story.append(Paragraph(e.rationale, styles["Normal"]))
90
+ if e.compensation_tags:
91
+ story.append(Paragraph("<b>Compensations:</b> " + ", ".join(e.compensation_tags),
92
+ styles["Normal"]))
93
+ if e.corrective_hint:
94
+ story.append(Paragraph("<b>Corrective:</b> " + e.corrective_hint, styles["Normal"]))
95
+
96
+ # Key frame + flexion chart side by side
97
+ kf, fb = _img(e.keyframe_path), _img((e.chart_paths or {}).get("flexion"), w=3.2, h=2.0)
98
+ if kf or fb:
99
+ cells = [c for c in (kf, fb) if c] or [Paragraph("<i>(images unavailable)</i>",
100
+ styles["Normal"])]
101
+ story.append(Table([cells], hAlign="LEFT"))
102
+
103
+ # Relevant-joint flexion table
104
+ if e.flexion:
105
+ story.append(Paragraph("<b>Relevant joint flexion (key frame)</b>", styles["Normal"]))
106
+ story.append(_meas_table(
107
+ [(n, f"{v['deg']:.1f}° {v['openness']}") for n, v in e.flexion.items()],
108
+ col0=2.6, col1=2.6))
109
+
110
+ # Laban Effort + radar
111
+ if e.laban:
112
+ eff, lab = e.laban.get("effort", {}), e.laban.get("labels", {})
113
+ story.append(Spacer(1, 0.08 * inch))
114
+ story.append(Paragraph("<b>Laban Effort (kinematic estimate)</b>", styles["Normal"]))
115
+ laban_tbl = _meas_table(
116
+ [(k.title(), f"{eff.get(k, 0):.2f} — {lab.get(k, '')}")
117
+ for k in ("space", "weight", "time", "flow")], col0=2.6, col1=2.6)
118
+ radar = _img((e.chart_paths or {}).get("radar"), w=2.6, h=2.6)
119
+ if radar:
120
+ story.append(Table([[laban_tbl, radar]], hAlign="LEFT"))
121
+ else:
122
+ story.append(laban_tbl)
123
+ if e.laban.get("body_emphasis"):
124
+ emph = ", ".join(f"{n}" for n, _ in e.laban["body_emphasis"])
125
+ story.append(Paragraph(f"<font size=8>Body emphasis: {emph} · "
126
+ f"{e.laban.get('notes', '')}</font>", styles["Normal"]))
127
+
128
+ # Angle + velocity charts
129
+ for kind in ("angle", "velocity"):
130
+ chart = _img((e.chart_paths or {}).get(kind), w=5.0, h=2.5)
131
+ if chart:
132
+ story.append(chart)
133
+
134
+ # Full measurement dump
135
+ if e.measurements:
136
+ story.append(Paragraph("<b>All measurements</b>", styles["Normal"]))
137
+ story.append(_meas_table(list(e.measurements.items())))
138
+
139
+ story.append(Spacer(1, 0.15 * inch))
140
+
141
+ if report.asymmetries:
142
+ story.append(PageBreak())
143
+ story.append(Paragraph("Asymmetries", styles["Heading2"]))
144
+ for a in report.asymmetries:
145
+ story.append(Paragraph(
146
+ f"{a['test'].replace('_', ' ').title()}: "
147
+ f"L={a['left_score']} R={a['right_score']} (&#916; {a['delta']})",
148
+ styles["Normal"]))
149
+ try:
150
+ from formscout.analysis.charts import symmetry_bars
151
+ os.makedirs(os.path.join(session_dir, "charts"), exist_ok=True)
152
+ sym_png = symmetry_bars(report.asymmetries,
153
+ os.path.join(session_dir, "charts", "symmetry.png"))
154
+ sym_img = _img(sym_png, w=5.5, h=2.75)
155
+ if sym_img:
156
+ story.append(sym_img)
157
+ except Exception:
158
+ pass
159
+
160
+ flags = list(report.low_confidence_flags) + list(report.disagreement_flags)
161
+ if flags:
162
+ story.append(Paragraph("Flags", styles["Heading2"]))
163
+ for fl in flags:
164
+ story.append(Paragraph(fl, styles["Normal"]))
165
+
166
+ story.append(Spacer(1, 0.3 * inch))
167
+ story.append(Paragraph(f"<b>&#9888; {DISCLAIMER}</b>", banner))
168
+
169
+ doc = SimpleDocTemplate(out_path, pagesize=LETTER,
170
+ topMargin=0.6 * inch, bottomMargin=0.6 * inch)
171
+ doc.build(story)
172
+ return out_path
173
+ except Exception as e:
174
+ logger.warning("pdf build failed: %s", e)
175
+ return None
formscout/agents/pose2d.py CHANGED
@@ -1,232 +1,232 @@
1
- """
2
- Pose2DAgent — 2D per-frame keypoint extraction.
3
-
4
- Backends: yolo (local checkpoints, ultralytics), mediapipe (official Tasks API,
5
- local .task checkpoint), sapiens2 (Meta HF/transformers).
6
- All backends output COCO-17 keypoints: dict[int, {x, y, conf}] per frame.
7
-
8
- Input: IngestResult
9
- Output: Pose2DResult(keypoints per frame, fps, confidence)
10
- Failure: Pose2DResult(confidence=0.0, notes=<reason>) — never raises.
11
- Gated: yolo=no; mediapipe=no (local checkpoint); sapiens2=yes (access accepted).
12
- """
13
- from __future__ import annotations
14
-
15
- import logging
16
- import numpy as np
17
-
18
- from formscout import config
19
- from formscout.types import IngestResult, Pose2DResult
20
-
21
- logger = logging.getLogger(__name__)
22
-
23
- COCO_KEYPOINTS = [
24
- "nose", "left_eye", "right_eye", "left_ear", "right_ear",
25
- "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
26
- "left_wrist", "right_wrist", "left_hip", "right_hip",
27
- "left_knee", "right_knee", "left_ankle", "right_ankle",
28
- ]
29
-
30
- # BlazePose-33 source indices → COCO-17 target indices
31
- # BlazePose: 0=nose, 2=left_eye, 5=right_eye, 7=left_ear, 8=right_ear,
32
- # 11=left_shoulder, 12=right_shoulder, 13=left_elbow, 14=right_elbow,
33
- # 15=left_wrist, 16=right_wrist, 23=left_hip, 24=right_hip,
34
- # 25=left_knee, 26=right_knee, 27=left_ankle, 28=right_ankle
35
- _BP_SRC = [0, 2, 5, 7, 8, 11, 12, 13, 14, 15, 16, 23, 24, 25, 26, 27, 28]
36
- _BP_DST = list(range(17)) # COCO indices 0..16
37
-
38
- _model_cache: dict[str, object] = {}
39
-
40
-
41
- # ── YOLO backend ──────────────────────────────────────────────────────────────
42
-
43
- def _get_yolo(path: str) -> object:
44
- if path not in _model_cache:
45
- from ultralytics import YOLO
46
- _model_cache[path] = YOLO(path)
47
- return _model_cache[path]
48
-
49
-
50
- def _run_yolo(frames: list, path: str) -> list[dict]:
51
- model = _get_yolo(path)
52
- out = []
53
- for frame in frames:
54
- try:
55
- results = model(frame, verbose=False)
56
- kps: dict[int, dict] = {}
57
- if results and results[0].keypoints is not None:
58
- kp = results[0].keypoints
59
- if kp.xy is not None and len(kp.xy) > 0:
60
- xy = kp.xy[0].cpu().numpy()
61
- conf = kp.conf[0].cpu().numpy()
62
- for j in range(min(len(xy), 17)):
63
- kps[j] = {"x": float(xy[j, 0]), "y": float(xy[j, 1]), "conf": float(conf[j])}
64
- out.append(kps)
65
- except Exception:
66
- out.append({})
67
- return out
68
-
69
-
70
- # ── MediaPipe backend (official Tasks API, local .task checkpoint) ────────────
71
-
72
- def _get_mediapipe_landmarker(path: str) -> object:
73
- """Return PoseLandmarker cached by model path."""
74
- cache_key = f"mp:{path}"
75
- if cache_key not in _model_cache:
76
- from mediapipe.tasks import python as mp_tasks
77
- from mediapipe.tasks.python import vision
78
-
79
- options = vision.PoseLandmarkerOptions(
80
- base_options=mp_tasks.BaseOptions(model_asset_path=path),
81
- running_mode=vision.RunningMode.IMAGE,
82
- num_poses=1,
83
- min_pose_detection_confidence=0.4,
84
- min_pose_presence_confidence=0.4,
85
- min_tracking_confidence=0.4,
86
- )
87
- _model_cache[cache_key] = vision.PoseLandmarker.create_from_options(options)
88
- return _model_cache[cache_key]
89
-
90
-
91
- def _run_mediapipe(frames: list, path: str) -> list[dict]:
92
- import cv2
93
- import mediapipe as mp
94
-
95
- try:
96
- landmarker = _get_mediapipe_landmarker(path)
97
- except Exception as e:
98
- logger.warning("mediapipe load failed: %s", e)
99
- return [{} for _ in frames]
100
-
101
- out = []
102
- for frame in frames:
103
- try:
104
- h, w = frame.shape[:2]
105
- rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
106
- mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
107
- detection = landmarker.detect(mp_image)
108
-
109
- kps: dict[int, dict] = {}
110
- if detection.pose_landmarks:
111
- lms = detection.pose_landmarks[0]
112
- for coco_idx, bp_idx in zip(_BP_DST, _BP_SRC):
113
- if bp_idx < len(lms):
114
- lm = lms[bp_idx]
115
- kps[coco_idx] = {
116
- "x": float(lm.x * w),
117
- "y": float(lm.y * h),
118
- "conf": float(lm.visibility),
119
- }
120
- out.append(kps)
121
- except Exception:
122
- out.append({})
123
- return out
124
-
125
-
126
- # ── Sapiens2 backend (Meta HF, transformers) ──────────────────────────────────
127
-
128
- def _get_sapiens2(hf_id: str) -> object:
129
- if hf_id not in _model_cache:
130
- from transformers import pipeline as hf_pipeline
131
- _model_cache[hf_id] = hf_pipeline("pose-estimation", model=hf_id)
132
- return _model_cache[hf_id]
133
-
134
-
135
- def _run_sapiens2(frames: list, hf_id: str) -> list[dict]:
136
- try:
137
- pipe = _get_sapiens2(hf_id)
138
- except Exception as e:
139
- logger.warning("sapiens2 load failed: %s", e)
140
- return [{} for _ in frames]
141
-
142
- from PIL import Image
143
-
144
- out = []
145
- for frame in frames:
146
- try:
147
- pil_img = Image.fromarray(frame)
148
- result = pipe(pil_img)
149
-
150
- if not result:
151
- out.append({})
152
- continue
153
-
154
- # Take highest-confidence person (first result)
155
- person = result[0]
156
- keypoints = person.get("keypoints", [])
157
- scores = person.get("keypoint_scores", [])
158
-
159
- # Build name→(x, y, score) lookup from pipeline output
160
- kp_lookup: dict[str, tuple] = {}
161
- for i, kp in enumerate(keypoints):
162
- if isinstance(kp, dict):
163
- name = kp.get("label", "")
164
- x, y = kp.get("x", 0.0), kp.get("y", 0.0)
165
- else:
166
- name = ""
167
- x, y = float(kp[0]), float(kp[1])
168
- score = float(scores[i]) if i < len(scores) else 0.0
169
- if name:
170
- kp_lookup[name] = (x, y, score)
171
-
172
- kps: dict[int, dict] = {}
173
- for coco_idx, name in enumerate(COCO_KEYPOINTS):
174
- if name in kp_lookup:
175
- x, y, s = kp_lookup[name]
176
- kps[coco_idx] = {"x": x, "y": y, "conf": s}
177
- out.append(kps)
178
- except Exception:
179
- out.append({})
180
- return out
181
-
182
-
183
- # ── Agent ─────────────────────────────────────────────────────────────────────
184
-
185
- class Pose2DAgent:
186
- """Extracts COCO-17 keypoints per frame; dispatches to YOLO, MediaPipe, or Sapiens2."""
187
-
188
- def run(self, ingest: IngestResult, model_key: str | None = None) -> Pose2DResult:
189
- if not ingest.frames:
190
- return Pose2DResult(keypoints=[], fps=ingest.fps, confidence=0.0, notes="no frames in ingest")
191
-
192
- key = model_key or config.DEFAULT_POSE_MODEL
193
- spec = config.POSE_MODELS.get(key)
194
- if spec is None:
195
- logger.warning("Unknown model_key %r — falling back to %s", key, config.DEFAULT_POSE_MODEL)
196
- spec = config.POSE_MODELS[config.DEFAULT_POSE_MODEL]
197
-
198
- backend = spec["backend"]
199
- try:
200
- if backend == "yolo":
201
- kps_per_frame = _run_yolo(ingest.frames, spec["path"])
202
- elif backend == "mediapipe":
203
- kps_per_frame = _run_mediapipe(ingest.frames, spec["path"])
204
- elif backend == "sapiens2":
205
- kps_per_frame = _run_sapiens2(ingest.frames, spec["hf_id"])
206
- else:
207
- return Pose2DResult(
208
- keypoints=[{} for _ in ingest.frames],
209
- fps=ingest.fps, confidence=0.0,
210
- notes=f"unknown backend: {backend}",
211
- )
212
- except Exception as e:
213
- return Pose2DResult(
214
- keypoints=[{} for _ in ingest.frames],
215
- fps=ingest.fps, confidence=0.0,
216
- notes=str(e),
217
- )
218
-
219
- n_detected = sum(1 for f in kps_per_frame if f)
220
- total_conf = sum(
221
- sum(kp["conf"] for kp in f.values()) / len(f)
222
- for f in kps_per_frame if f
223
- )
224
- overall_conf = (total_conf / n_detected) if n_detected > 0 else 0.0
225
- notes = "" if n_detected > 0 else "no person detected in any frame"
226
-
227
- return Pose2DResult(
228
- keypoints=kps_per_frame,
229
- fps=ingest.fps,
230
- confidence=overall_conf,
231
- notes=notes,
232
- )
 
1
+ """
2
+ Pose2DAgent — 2D per-frame keypoint extraction.
3
+
4
+ Backends: yolo (local checkpoints, ultralytics), mediapipe (official Tasks API,
5
+ local .task checkpoint), sapiens2 (Meta HF/transformers).
6
+ All backends output COCO-17 keypoints: dict[int, {x, y, conf}] per frame.
7
+
8
+ Input: IngestResult
9
+ Output: Pose2DResult(keypoints per frame, fps, confidence)
10
+ Failure: Pose2DResult(confidence=0.0, notes=<reason>) — never raises.
11
+ Gated: yolo=no; mediapipe=no (local checkpoint); sapiens2=yes (access accepted).
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import logging
16
+ import numpy as np
17
+
18
+ from formscout import config
19
+ from formscout.types import IngestResult, Pose2DResult
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+ COCO_KEYPOINTS = [
24
+ "nose", "left_eye", "right_eye", "left_ear", "right_ear",
25
+ "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
26
+ "left_wrist", "right_wrist", "left_hip", "right_hip",
27
+ "left_knee", "right_knee", "left_ankle", "right_ankle",
28
+ ]
29
+
30
+ # BlazePose-33 source indices → COCO-17 target indices
31
+ # BlazePose: 0=nose, 2=left_eye, 5=right_eye, 7=left_ear, 8=right_ear,
32
+ # 11=left_shoulder, 12=right_shoulder, 13=left_elbow, 14=right_elbow,
33
+ # 15=left_wrist, 16=right_wrist, 23=left_hip, 24=right_hip,
34
+ # 25=left_knee, 26=right_knee, 27=left_ankle, 28=right_ankle
35
+ _BP_SRC = [0, 2, 5, 7, 8, 11, 12, 13, 14, 15, 16, 23, 24, 25, 26, 27, 28]
36
+ _BP_DST = list(range(17)) # COCO indices 0..16
37
+
38
+ _model_cache: dict[str, object] = {}
39
+
40
+
41
+ # ── YOLO backend ──────────────────────────────────────────────────────────────
42
+
43
+ def _get_yolo(path: str) -> object:
44
+ if path not in _model_cache:
45
+ from ultralytics import YOLO
46
+ _model_cache[path] = YOLO(path)
47
+ return _model_cache[path]
48
+
49
+
50
+ def _run_yolo(frames: list, path: str) -> list[dict]:
51
+ model = _get_yolo(path)
52
+ out = []
53
+ for frame in frames:
54
+ try:
55
+ results = model(frame, verbose=False)
56
+ kps: dict[int, dict] = {}
57
+ if results and results[0].keypoints is not None:
58
+ kp = results[0].keypoints
59
+ if kp.xy is not None and len(kp.xy) > 0:
60
+ xy = kp.xy[0].cpu().numpy()
61
+ conf = kp.conf[0].cpu().numpy()
62
+ for j in range(min(len(xy), 17)):
63
+ kps[j] = {"x": float(xy[j, 0]), "y": float(xy[j, 1]), "conf": float(conf[j])}
64
+ out.append(kps)
65
+ except Exception:
66
+ out.append({})
67
+ return out
68
+
69
+
70
+ # ── MediaPipe backend (official Tasks API, local .task checkpoint) ────────────
71
+
72
+ def _get_mediapipe_landmarker(path: str) -> object:
73
+ """Return PoseLandmarker cached by model path."""
74
+ cache_key = f"mp:{path}"
75
+ if cache_key not in _model_cache:
76
+ from mediapipe.tasks import python as mp_tasks
77
+ from mediapipe.tasks.python import vision
78
+
79
+ options = vision.PoseLandmarkerOptions(
80
+ base_options=mp_tasks.BaseOptions(model_asset_path=path),
81
+ running_mode=vision.RunningMode.IMAGE,
82
+ num_poses=1,
83
+ min_pose_detection_confidence=0.4,
84
+ min_pose_presence_confidence=0.4,
85
+ min_tracking_confidence=0.4,
86
+ )
87
+ _model_cache[cache_key] = vision.PoseLandmarker.create_from_options(options)
88
+ return _model_cache[cache_key]
89
+
90
+
91
+ def _run_mediapipe(frames: list, path: str) -> list[dict]:
92
+ import cv2
93
+ import mediapipe as mp
94
+
95
+ try:
96
+ landmarker = _get_mediapipe_landmarker(path)
97
+ except Exception as e:
98
+ logger.warning("mediapipe load failed: %s", e)
99
+ return [{} for _ in frames]
100
+
101
+ out = []
102
+ for frame in frames:
103
+ try:
104
+ h, w = frame.shape[:2]
105
+ rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
106
+ mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
107
+ detection = landmarker.detect(mp_image)
108
+
109
+ kps: dict[int, dict] = {}
110
+ if detection.pose_landmarks:
111
+ lms = detection.pose_landmarks[0]
112
+ for coco_idx, bp_idx in zip(_BP_DST, _BP_SRC):
113
+ if bp_idx < len(lms):
114
+ lm = lms[bp_idx]
115
+ kps[coco_idx] = {
116
+ "x": float(lm.x * w),
117
+ "y": float(lm.y * h),
118
+ "conf": float(lm.visibility),
119
+ }
120
+ out.append(kps)
121
+ except Exception:
122
+ out.append({})
123
+ return out
124
+
125
+
126
+ # ── Sapiens2 backend (Meta HF, transformers) ──────────────────────────────────
127
+
128
+ def _get_sapiens2(hf_id: str) -> object:
129
+ if hf_id not in _model_cache:
130
+ from transformers import pipeline as hf_pipeline
131
+ _model_cache[hf_id] = hf_pipeline("pose-estimation", model=hf_id)
132
+ return _model_cache[hf_id]
133
+
134
+
135
+ def _run_sapiens2(frames: list, hf_id: str) -> list[dict]:
136
+ try:
137
+ pipe = _get_sapiens2(hf_id)
138
+ except Exception as e:
139
+ logger.warning("sapiens2 load failed: %s", e)
140
+ return [{} for _ in frames]
141
+
142
+ from PIL import Image
143
+
144
+ out = []
145
+ for frame in frames:
146
+ try:
147
+ pil_img = Image.fromarray(frame)
148
+ result = pipe(pil_img)
149
+
150
+ if not result:
151
+ out.append({})
152
+ continue
153
+
154
+ # Take highest-confidence person (first result)
155
+ person = result[0]
156
+ keypoints = person.get("keypoints", [])
157
+ scores = person.get("keypoint_scores", [])
158
+
159
+ # Build name→(x, y, score) lookup from pipeline output
160
+ kp_lookup: dict[str, tuple] = {}
161
+ for i, kp in enumerate(keypoints):
162
+ if isinstance(kp, dict):
163
+ name = kp.get("label", "")
164
+ x, y = kp.get("x", 0.0), kp.get("y", 0.0)
165
+ else:
166
+ name = ""
167
+ x, y = float(kp[0]), float(kp[1])
168
+ score = float(scores[i]) if i < len(scores) else 0.0
169
+ if name:
170
+ kp_lookup[name] = (x, y, score)
171
+
172
+ kps: dict[int, dict] = {}
173
+ for coco_idx, name in enumerate(COCO_KEYPOINTS):
174
+ if name in kp_lookup:
175
+ x, y, s = kp_lookup[name]
176
+ kps[coco_idx] = {"x": x, "y": y, "conf": s}
177
+ out.append(kps)
178
+ except Exception:
179
+ out.append({})
180
+ return out
181
+
182
+
183
+ # ── Agent ─────────────────────────────────────────────────────────────────────
184
+
185
+ class Pose2DAgent:
186
+ """Extracts COCO-17 keypoints per frame; dispatches to YOLO, MediaPipe, or Sapiens2."""
187
+
188
+ def run(self, ingest: IngestResult, model_key: str | None = None) -> Pose2DResult:
189
+ if not ingest.frames:
190
+ return Pose2DResult(keypoints=[], fps=ingest.fps, confidence=0.0, notes="no frames in ingest")
191
+
192
+ key = model_key or config.DEFAULT_POSE_MODEL
193
+ spec = config.POSE_MODELS.get(key)
194
+ if spec is None:
195
+ logger.warning("Unknown model_key %r — falling back to %s", key, config.DEFAULT_POSE_MODEL)
196
+ spec = config.POSE_MODELS[config.DEFAULT_POSE_MODEL]
197
+
198
+ backend = spec["backend"]
199
+ try:
200
+ if backend == "yolo":
201
+ kps_per_frame = _run_yolo(ingest.frames, spec["path"])
202
+ elif backend == "mediapipe":
203
+ kps_per_frame = _run_mediapipe(ingest.frames, spec["path"])
204
+ elif backend == "sapiens2":
205
+ kps_per_frame = _run_sapiens2(ingest.frames, spec["hf_id"])
206
+ else:
207
+ return Pose2DResult(
208
+ keypoints=[{} for _ in ingest.frames],
209
+ fps=ingest.fps, confidence=0.0,
210
+ notes=f"unknown backend: {backend}",
211
+ )
212
+ except Exception as e:
213
+ return Pose2DResult(
214
+ keypoints=[{} for _ in ingest.frames],
215
+ fps=ingest.fps, confidence=0.0,
216
+ notes=str(e),
217
+ )
218
+
219
+ n_detected = sum(1 for f in kps_per_frame if f)
220
+ total_conf = sum(
221
+ sum(kp["conf"] for kp in f.values()) / len(f)
222
+ for f in kps_per_frame if f
223
+ )
224
+ overall_conf = (total_conf / n_detected) if n_detected > 0 else 0.0
225
+ notes = "" if n_detected > 0 else "no person detected in any frame"
226
+
227
+ return Pose2DResult(
228
+ keypoints=kps_per_frame,
229
+ fps=ingest.fps,
230
+ confidence=overall_conf,
231
+ notes=notes,
232
+ )
formscout/agents/report.py CHANGED
@@ -1,139 +1,139 @@
1
- """
2
- ReportAgent — assembles per-test scorecard, composite, asymmetries.
3
-
4
- Input: List of (MovementResult, BiomechFeatures, ScoreResult, JudgeResult) per test
5
- Output: ReportResult(per_test, composite, asymmetries, overlay_video_path, pdf_path)
6
- Failure: returns ReportResult with composite=None if any test unscored.
7
- Params: 0 (pure assembly — no model).
8
- License: n/a.
9
- Gated: no.
10
- """
11
- from __future__ import annotations
12
-
13
- from formscout.types import (
14
- MovementResult, BiomechFeatures, ScoreResult, JudgeResult, ReportResult,
15
- )
16
- from formscout import config
17
-
18
- # Bilateral tests that need L/R scoring
19
- BILATERAL_TESTS = {"hurdle_step", "inline_lunge", "shoulder_mobility", "active_slr"}
20
-
21
-
22
- class ReportAgent:
23
- """Assembles the final screening report from all test results."""
24
-
25
- def run(self, test_results: list[dict]) -> ReportResult:
26
- """
27
- Assemble the report.
28
-
29
- Args:
30
- test_results: list of dicts with keys:
31
- - movement: MovementResult
32
- - features: BiomechFeatures
33
- - rubric_score: ScoreResult
34
- - judge: JudgeResult
35
- - side: str (for bilateral: "left" or "right")
36
- """
37
- per_test = []
38
- asymmetries = []
39
- low_confidence_flags = []
40
- disagreement_flags = []
41
-
42
- # Group bilateral tests by test_name
43
- bilateral_groups: dict[str, list[dict]] = {}
44
- unilateral: list[dict] = []
45
-
46
- for entry in test_results:
47
- test_name = entry["movement"].test_name
48
- if test_name in BILATERAL_TESTS:
49
- bilateral_groups.setdefault(test_name, []).append(entry)
50
- else:
51
- unilateral.append(entry)
52
-
53
- # Process bilateral tests — take the lower score, emit asymmetry
54
- for test_name, entries in bilateral_groups.items():
55
- scores = []
56
- for entry in entries:
57
- judge = entry["judge"]
58
- side = entry.get("side", entry["movement"].side)
59
- score = judge.score if judge.score is not None else None
60
- scores.append({"side": side, "score": score, "entry": entry})
61
-
62
- # Find best entry per side
63
- left = next((s for s in scores if s["side"] == "left"), None)
64
- right = next((s for s in scores if s["side"] == "right"), None)
65
-
66
- left_score = left["score"] if left else None
67
- right_score = right["score"] if right else None
68
-
69
- # Report lower
70
- if left_score is not None and right_score is not None:
71
- final_score = min(left_score, right_score)
72
- delta = abs(left_score - right_score)
73
- asymmetries.append({
74
- "test": test_name,
75
- "left_score": left_score,
76
- "right_score": right_score,
77
- "delta": delta,
78
- })
79
- elif left_score is not None:
80
- final_score = left_score
81
- elif right_score is not None:
82
- final_score = right_score
83
- else:
84
- final_score = None
85
-
86
- # Use the entry with the lower score for details
87
- primary = (left["entry"] if left and (right is None or (left_score or 4) <= (right_score or 4))
88
- else right["entry"] if right else entries[0])
89
-
90
- per_test.append({
91
- "test_name": test_name,
92
- "score": final_score,
93
- "judge": primary["judge"],
94
- "features": primary["features"],
95
- "needs_human": primary["judge"].needs_human,
96
- })
97
-
98
- self._check_flags(primary, low_confidence_flags, disagreement_flags)
99
-
100
- # Process unilateral tests
101
- for entry in unilateral:
102
- judge = entry["judge"]
103
- per_test.append({
104
- "test_name": entry["movement"].test_name,
105
- "score": judge.score,
106
- "judge": judge,
107
- "features": entry["features"],
108
- "needs_human": judge.needs_human,
109
- })
110
- self._check_flags(entry, low_confidence_flags, disagreement_flags)
111
-
112
- # Composite — null if any test unscored
113
- all_scores = [t["score"] for t in per_test]
114
- composite = sum(all_scores) if all(s is not None for s in all_scores) else None
115
-
116
- return ReportResult(
117
- per_test=per_test,
118
- composite=composite,
119
- asymmetries=asymmetries,
120
- overlay_video_path=None, # Phase 4
121
- pdf_path=None, # Phase 4
122
- low_confidence_flags=low_confidence_flags,
123
- disagreement_flags=disagreement_flags,
124
- )
125
-
126
- def _check_flags(self, entry: dict, low_conf: list, disagree: list):
127
- """Check quality gates and populate flag lists."""
128
- judge = entry["judge"]
129
- rubric = entry["rubric_score"]
130
- test_name = entry["movement"].test_name
131
-
132
- if judge.confidence < config.MIN_CONFIDENCE:
133
- low_conf.append(f"{test_name}: judge confidence {judge.confidence:.2f}")
134
-
135
- if (judge.score is not None and rubric.score is not None
136
- and abs(judge.score - rubric.score) >= config.SCORE_DISAGREE_THRESH):
137
- disagree.append(
138
- f"{test_name}: rubric={rubric.score} vs judge={judge.score}"
139
- )
 
1
+ """
2
+ ReportAgent — assembles per-test scorecard, composite, asymmetries.
3
+
4
+ Input: List of (MovementResult, BiomechFeatures, ScoreResult, JudgeResult) per test
5
+ Output: ReportResult(per_test, composite, asymmetries, overlay_video_path, pdf_path)
6
+ Failure: returns ReportResult with composite=None if any test unscored.
7
+ Params: 0 (pure assembly — no model).
8
+ License: n/a.
9
+ Gated: no.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ from formscout.types import (
14
+ MovementResult, BiomechFeatures, ScoreResult, JudgeResult, ReportResult,
15
+ )
16
+ from formscout import config
17
+
18
+ # Bilateral tests that need L/R scoring
19
+ BILATERAL_TESTS = {"hurdle_step", "inline_lunge", "shoulder_mobility", "active_slr"}
20
+
21
+
22
+ class ReportAgent:
23
+ """Assembles the final screening report from all test results."""
24
+
25
+ def run(self, test_results: list[dict]) -> ReportResult:
26
+ """
27
+ Assemble the report.
28
+
29
+ Args:
30
+ test_results: list of dicts with keys:
31
+ - movement: MovementResult
32
+ - features: BiomechFeatures
33
+ - rubric_score: ScoreResult
34
+ - judge: JudgeResult
35
+ - side: str (for bilateral: "left" or "right")
36
+ """
37
+ per_test = []
38
+ asymmetries = []
39
+ low_confidence_flags = []
40
+ disagreement_flags = []
41
+
42
+ # Group bilateral tests by test_name
43
+ bilateral_groups: dict[str, list[dict]] = {}
44
+ unilateral: list[dict] = []
45
+
46
+ for entry in test_results:
47
+ test_name = entry["movement"].test_name
48
+ if test_name in BILATERAL_TESTS:
49
+ bilateral_groups.setdefault(test_name, []).append(entry)
50
+ else:
51
+ unilateral.append(entry)
52
+
53
+ # Process bilateral tests — take the lower score, emit asymmetry
54
+ for test_name, entries in bilateral_groups.items():
55
+ scores = []
56
+ for entry in entries:
57
+ judge = entry["judge"]
58
+ side = entry.get("side", entry["movement"].side)
59
+ score = judge.score if judge.score is not None else None
60
+ scores.append({"side": side, "score": score, "entry": entry})
61
+
62
+ # Find best entry per side
63
+ left = next((s for s in scores if s["side"] == "left"), None)
64
+ right = next((s for s in scores if s["side"] == "right"), None)
65
+
66
+ left_score = left["score"] if left else None
67
+ right_score = right["score"] if right else None
68
+
69
+ # Report lower
70
+ if left_score is not None and right_score is not None:
71
+ final_score = min(left_score, right_score)
72
+ delta = abs(left_score - right_score)
73
+ asymmetries.append({
74
+ "test": test_name,
75
+ "left_score": left_score,
76
+ "right_score": right_score,
77
+ "delta": delta,
78
+ })
79
+ elif left_score is not None:
80
+ final_score = left_score
81
+ elif right_score is not None:
82
+ final_score = right_score
83
+ else:
84
+ final_score = None
85
+
86
+ # Use the entry with the lower score for details
87
+ primary = (left["entry"] if left and (right is None or (left_score or 4) <= (right_score or 4))
88
+ else right["entry"] if right else entries[0])
89
+
90
+ per_test.append({
91
+ "test_name": test_name,
92
+ "score": final_score,
93
+ "judge": primary["judge"],
94
+ "features": primary["features"],
95
+ "needs_human": primary["judge"].needs_human,
96
+ })
97
+
98
+ self._check_flags(primary, low_confidence_flags, disagreement_flags)
99
+
100
+ # Process unilateral tests
101
+ for entry in unilateral:
102
+ judge = entry["judge"]
103
+ per_test.append({
104
+ "test_name": entry["movement"].test_name,
105
+ "score": judge.score,
106
+ "judge": judge,
107
+ "features": entry["features"],
108
+ "needs_human": judge.needs_human,
109
+ })
110
+ self._check_flags(entry, low_confidence_flags, disagreement_flags)
111
+
112
+ # Composite — null if any test unscored
113
+ all_scores = [t["score"] for t in per_test]
114
+ composite = sum(all_scores) if all(s is not None for s in all_scores) else None
115
+
116
+ return ReportResult(
117
+ per_test=per_test,
118
+ composite=composite,
119
+ asymmetries=asymmetries,
120
+ overlay_video_path=None, # Phase 4
121
+ pdf_path=None, # Phase 4
122
+ low_confidence_flags=low_confidence_flags,
123
+ disagreement_flags=disagreement_flags,
124
+ )
125
+
126
+ def _check_flags(self, entry: dict, low_conf: list, disagree: list):
127
+ """Check quality gates and populate flag lists."""
128
+ judge = entry["judge"]
129
+ rubric = entry["rubric_score"]
130
+ test_name = entry["movement"].test_name
131
+
132
+ if judge.confidence < config.MIN_CONFIDENCE:
133
+ low_conf.append(f"{test_name}: judge confidence {judge.confidence:.2f}")
134
+
135
+ if (judge.score is not None and rubric.score is not None
136
+ and abs(judge.score - rubric.score) >= config.SCORE_DISAGREE_THRESH):
137
+ disagree.append(
138
+ f"{test_name}: rubric={rubric.score} vs judge={judge.score}"
139
+ )
formscout/agents/visualizer.py CHANGED
@@ -1,435 +1,418 @@
1
- """
2
- PoseVisualizer — annotated overlay video with skeleton, trails, velocity arrows.
3
-
4
- Input: IngestResult + Pose2DResult
5
- Output: .mp4 path (or None on failure/empty layers)
6
- Failure: returns None, never raises.
7
- """
8
- from __future__ import annotations
9
-
10
- import colorsys
11
- import logging
12
- import math
13
- import tempfile
14
- from collections import deque
15
-
16
- import cv2
17
- import numpy as np
18
-
19
- logger = logging.getLogger(__name__)
20
-
21
- # ── COCO constants ────────────────────────────────────────────────────────────
22
-
23
- COCO_KEYPOINTS = [
24
- "nose", "left_eye", "right_eye", "left_ear", "right_ear",
25
- "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
26
- "left_wrist", "right_wrist", "left_hip", "right_hip",
27
- "left_knee", "right_knee", "left_ankle", "right_ankle",
28
- ]
29
-
30
- COCO_SKELETON = [
31
- (0, 1), (0, 2), (1, 3), (2, 4), # face
32
- (5, 6), (5, 7), (7, 9), (6, 8), (8, 10), # arms
33
- (5, 11), (6, 12), (11, 12), # torso
34
- (11, 13), (13, 15), (12, 14), (14, 16), # legs
35
- ]
36
-
37
- TRAIL_LENGTH = 10
38
- MAX_ARROW_PX = 40
39
- CONF_THRESHOLD = 0.3
40
-
41
-
42
- # ── Kalman filter ─────────────────────────────────────────────────────────────
43
-
44
- class SimpleKalmanFilter:
45
- """4-state Kalman filter (x, y, vx, vy) for joint tracking."""
46
-
47
- def __init__(self, process_noise: float = 0.01, measurement_noise: float = 0.1):
48
- self.is_initialized = False
49
- self.state = np.zeros(4)
50
- self.cov = np.eye(4) * 0.1
51
- self.Q = np.eye(4) * process_noise
52
- self.R = np.eye(2) * measurement_noise
53
- self.H = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], dtype=float)
54
-
55
- def predict(self, dt: float = 1.0):
56
- F = np.array([[1, 0, dt, 0], [0, 1, 0, dt], [0, 0, 1, 0], [0, 0, 0, 1]], dtype=float)
57
- self.state = F @ self.state
58
- self.cov = F @ self.cov @ F.T + self.Q
59
-
60
- def update(self, x: float, y: float):
61
- z = np.array([x, y])
62
- if not self.is_initialized:
63
- self.state[:2] = z
64
- self.is_initialized = True
65
- return
66
- S = self.H @ self.cov @ self.H.T + self.R
67
- K = self.cov @ self.H.T @ np.linalg.inv(S)
68
- self.state = self.state + K @ (z - self.H @ self.state)
69
- self.cov = (np.eye(4) - K @ self.H) @ self.cov
70
-
71
- def velocity_magnitude(self) -> float:
72
- vx, vy = self.state[2], self.state[3]
73
- return math.sqrt(vx * vx + vy * vy)
74
-
75
- def velocity_vector(self) -> tuple[float, float]:
76
- return float(self.state[2]), float(self.state[3])
77
-
78
-
79
- # ── Velocity computation ──────────────────────────────────────────────────────
80
-
81
- def compute_joint_velocity(
82
- keypoints_per_frame: list[dict],
83
- fps: float,
84
- ) -> dict[int, list[float]]:
85
- """
86
- Compute Kalman-filtered per-joint speed (px/s) for each frame.
87
-
88
- Returns dict[joint_idx, [speed_frame0, ...]] for all 17 COCO joints.
89
- Missing/low-confidence keypoints yield speed=0.0 for that frame.
90
- """
91
- dt = 1.0 / fps if fps > 0 else 1.0
92
- filters: dict[int, SimpleKalmanFilter] = {j: SimpleKalmanFilter() for j in range(17)}
93
- result: dict[int, list[float]] = {j: [] for j in range(17)}
94
-
95
- for frame_kps in keypoints_per_frame:
96
- for j in range(17):
97
- kf = filters[j]
98
- kp = frame_kps.get(j)
99
- kf.predict(dt)
100
- if kp and kp.get("conf", 0.0) >= CONF_THRESHOLD:
101
- kf.update(kp["x"], kp["y"])
102
- speed = kf.velocity_magnitude()
103
- else:
104
- speed = 0.0
105
- result[j].append(speed)
106
-
107
- return result
108
-
109
-
110
- # ── Helpers ───────────────────────────────────────────────────────────────────
111
-
112
- def _conf_to_bgr(conf: float) -> tuple[int, int, int]:
113
- """Map confidence 0→1 to BGR color red→green via HSV."""
114
- hue = conf * 120.0 / 360.0
115
- r, g, b = colorsys.hsv_to_rgb(hue, 1.0, 1.0)
116
- return (int(b * 255), int(g * 255), int(r * 255))
117
-
118
-
119
- def _resolve_ffmpeg() -> str | None:
120
- """Return a usable ffmpeg binary path: imageio-ffmpeg's bundle, then system PATH."""
121
- try:
122
- import imageio_ffmpeg
123
- return imageio_ffmpeg.get_ffmpeg_exe()
124
- except Exception:
125
- pass
126
- import shutil
127
- return shutil.which("ffmpeg")
128
-
129
-
130
- # ── PoseVisualizer ────────────────────────────────────────────────────────────
131
-
132
- class PoseVisualizer:
133
- """Renders skeleton, trails, and velocity arrows onto video frames."""
134
-
135
- def __init__(self):
136
- self.last_velocities: dict[int, list[float]] = {}
137
-
138
- # ── Skeleton ──────────────────────────────────────────────────────────────
139
-
140
- def _draw_skeleton(self, frame: np.ndarray, kps: dict) -> np.ndarray:
141
- """Draw COCO-17 bones (white) and joints (confidence-colored) onto frame."""
142
- visible = {j: kp for j, kp in kps.items() if kp.get("conf", 0.0) >= CONF_THRESHOLD}
143
-
144
- # Bones
145
- for j1, j2 in COCO_SKELETON:
146
- if j1 in visible and j2 in visible:
147
- p1 = (int(visible[j1]["x"]), int(visible[j1]["y"]))
148
- p2 = (int(visible[j2]["x"]), int(visible[j2]["y"]))
149
- cv2.line(frame, p1, p2, (255, 255, 255), 2)
150
-
151
- # Joints
152
- for j, kp in visible.items():
153
- pt = (int(kp["x"]), int(kp["y"]))
154
- color = _conf_to_bgr(kp["conf"])
155
- cv2.circle(frame, pt, 4, color, -1)
156
- cv2.circle(frame, pt, 5, (255, 255, 255), 1)
157
-
158
- return frame
159
-
160
- # ── Trails ───────────────────────────────────────────────────────────────
161
-
162
- def _draw_trails(self, frame: np.ndarray, trail_history: dict) -> np.ndarray:
163
- """Draw fading motion trails for each joint."""
164
- for joint_idx, trail in trail_history.items():
165
- pts = list(trail)
166
- if len(pts) < 2:
167
- continue
168
- for i in range(1, len(pts)):
169
- alpha = i / len(pts)
170
- brightness = int(255 * alpha)
171
- color = (brightness, brightness, brightness)
172
- thickness = max(1, int(3 * alpha))
173
- p1 = (int(pts[i - 1][0]), int(pts[i - 1][1]))
174
- p2 = (int(pts[i][0]), int(pts[i][1]))
175
- cv2.line(frame, p1, p2, color, thickness)
176
- return frame
177
-
178
- # ── Velocity arrows ───────────────────────────────────────────────────────
179
-
180
- def _draw_velocity_arrows(
181
- self,
182
- frame: np.ndarray,
183
- kps: dict,
184
- prev_kps: dict | None,
185
- velocities: dict[int, list[float]],
186
- frame_idx: int,
187
- ) -> np.ndarray:
188
- """Draw per-joint velocity arrows scaled by speed."""
189
- if prev_kps is None:
190
- return frame
191
-
192
- all_speeds = [velocities[j][frame_idx] for j in range(17) if frame_idx < len(velocities.get(j, []))]
193
- peak = max(all_speeds) if all_speeds else 1.0
194
- if peak == 0.0:
195
- return frame
196
-
197
- for j in range(17):
198
- kp = kps.get(j)
199
- pk = prev_kps.get(j)
200
- if not kp or not pk:
201
- continue
202
- if kp.get("conf", 0.0) < CONF_THRESHOLD:
203
- continue
204
- speeds = velocities.get(j, [])
205
- if frame_idx >= len(speeds):
206
- continue
207
- speed = speeds[frame_idx]
208
- if speed == 0.0:
209
- continue
210
-
211
- dx = kp["x"] - pk["x"]
212
- dy = kp["y"] - pk["y"]
213
- mag = math.sqrt(dx * dx + dy * dy)
214
- if mag < 1e-6:
215
- continue
216
-
217
- length = min(speed / peak * MAX_ARROW_PX, MAX_ARROW_PX)
218
- nx, ny = dx / mag, dy / mag
219
- start = (int(kp["x"]), int(kp["y"]))
220
- end = (int(kp["x"] + nx * length), int(kp["y"] + ny * length))
221
-
222
- ratio = speed / peak
223
- if ratio < 0.33:
224
- color = (0, 200, 0) # green
225
- elif ratio < 0.66:
226
- color = (0, 140, 255) # orange
227
- else:
228
- color = (0, 0, 255) # red
229
-
230
- cv2.arrowedLine(frame, start, end, color, 2, tipLength=0.35)
231
-
232
- return frame
233
-
234
- # ── Public ────────────────────────────────────────────────────────────────
235
-
236
- def render_video(
237
- self,
238
- ingest,
239
- pose2d,
240
- layers: set[str],
241
- output_path: str,
242
- ) -> str | None:
243
- """
244
- Render annotated video. Returns output_path on success, None otherwise.
245
- layers: subset of {"skeleton", "trails", "velocity_arrows"}
246
- """
247
- if not layers:
248
- return None
249
-
250
- if not any(pose2d.keypoints):
251
- return None
252
-
253
- try:
254
- velocities = compute_joint_velocity(pose2d.keypoints, ingest.fps)
255
- self.last_velocities = velocities
256
-
257
- frames = ingest.frames
258
- orig_h, orig_w = frames[0].shape[:2]
259
- fps = ingest.fps or 30.0
260
-
261
- # Cap at 1280px wide — big frames are slow and don't need to be HQ
262
- max_w = 1280
263
- if orig_w > max_w:
264
- scale = max_w / orig_w
265
- out_w = max_w
266
- out_h = int(orig_h * scale)
267
- else:
268
- scale = 1.0
269
- out_w, out_h = orig_w, orig_h
270
-
271
- # Scale keypoint coordinates to match resized frames
272
- def _scale_kps(kps: dict) -> dict:
273
- if scale == 1.0:
274
- return kps
275
- return {
276
- j: {**kp, "x": kp["x"] * scale, "y": kp["y"] * scale}
277
- for j, kp in kps.items()
278
- }
279
-
280
- scaled_keypoints = [_scale_kps(k) for k in pose2d.keypoints]
281
-
282
- # Write raw mp4v to a temp file, then remux with ffmpeg faststart
283
- import subprocess
284
- import tempfile as _tf
285
- tmp = _tf.NamedTemporaryFile(suffix="_raw.mp4", delete=False)
286
- tmp_path = tmp.name
287
- tmp.close()
288
-
289
- fourcc = cv2.VideoWriter_fourcc(*"mp4v")
290
- writer = cv2.VideoWriter(tmp_path, fourcc, fps, (out_w, out_h))
291
- if not writer.isOpened():
292
- logger.warning("VideoWriter failed to open: %s", tmp_path)
293
- return None
294
-
295
- trail_history: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
296
- prev_kps: dict | None = None
297
-
298
- for frame_idx, (frame, kps) in enumerate(zip(frames, scaled_keypoints)):
299
- if scale != 1.0:
300
- out_frame = cv2.resize(frame, (out_w, out_h), interpolation=cv2.INTER_AREA)
301
- else:
302
- out_frame = frame.copy()
303
-
304
- if "trails" in layers:
305
- for j, kp in kps.items():
306
- if kp.get("conf", 0.0) >= CONF_THRESHOLD:
307
- trail_history[j].append((kp["x"], kp["y"]))
308
- out_frame = self._draw_trails(out_frame, trail_history)
309
-
310
- if "skeleton" in layers:
311
- out_frame = self._draw_skeleton(out_frame, kps)
312
-
313
- if "velocity_arrows" in layers:
314
- out_frame = self._draw_velocity_arrows(
315
- out_frame, kps, prev_kps, velocities, frame_idx
316
- )
317
-
318
- writer.write(out_frame)
319
- prev_kps = kps
320
-
321
- writer.release()
322
-
323
- # Re-encode to H.264 (browsers/Gradio cannot play raw mp4v).
324
- # Prefer imageio-ffmpeg's bundled binary so no system install is needed.
325
- ffmpeg_bin = _resolve_ffmpeg()
326
- if ffmpeg_bin:
327
- try:
328
- subprocess.run(
329
- [ffmpeg_bin, "-y", "-i", tmp_path,
330
- "-c:v", "libx264", "-pix_fmt", "yuv420p",
331
- "-movflags", "+faststart", output_path],
332
- check=True, capture_output=True,
333
- )
334
- import os
335
- os.unlink(tmp_path)
336
- return output_path
337
- except Exception as ffmpeg_err:
338
- logger.warning("ffmpeg H.264 re-encode failed (%s) — using raw mp4v", ffmpeg_err)
339
-
340
- # No ffmpeg available — fall back to raw mp4v (may not play in browser)
341
- import shutil
342
- shutil.move(tmp_path, output_path)
343
- return output_path
344
-
345
- except Exception as e:
346
- logger.warning("render_video failed: %s", e)
347
- return None
348
-
349
- def render_frame(
350
- self,
351
- ingest,
352
- pose2d,
353
- frame_idx: int,
354
- layers: set[str],
355
- caption: str = "",
356
- out_png: str | None = None,
357
- ) -> str | None:
358
- """Render a single annotated still (skeleton + optional trails + caption).
359
-
360
- frame_idx is typically the governing frame from BiomechFeatures.timing.
361
- Returns the PNG path on success, None on any failure. Never raises.
362
- """
363
- try:
364
- if not (0 <= frame_idx < len(ingest.frames)) or frame_idx >= len(pose2d.keypoints):
365
- return None
366
-
367
- frame = ingest.frames[frame_idx].copy()
368
- kps = pose2d.keypoints[frame_idx]
369
-
370
- if "trails" in layers:
371
- trail: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
372
- start = max(0, frame_idx - TRAIL_LENGTH)
373
- for fi in range(start, frame_idx + 1):
374
- for j, kp in pose2d.keypoints[fi].items():
375
- if kp.get("conf", 0.0) >= CONF_THRESHOLD:
376
- trail[j].append((kp["x"], kp["y"]))
377
- frame = self._draw_trails(frame, trail)
378
-
379
- if "skeleton" in layers:
380
- frame = self._draw_skeleton(frame, kps)
381
-
382
- if caption:
383
- cv2.rectangle(frame, (0, 0), (frame.shape[1], 28), (0, 0, 0), -1)
384
- cv2.putText(frame, caption[:80], (8, 20), cv2.FONT_HERSHEY_SIMPLEX,
385
- 0.55, (255, 255, 255), 1, cv2.LINE_AA)
386
-
387
- if out_png is None:
388
- out_png = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
389
-
390
- ok = cv2.imwrite(out_png, frame)
391
- return out_png if ok else None
392
- except Exception as e:
393
- logger.warning("render_frame failed: %s", e)
394
- return None
395
-
396
-
397
- # ── Velocity summary ──────────────────────────────────────────────────────────
398
-
399
- def build_velocity_summary(
400
- keypoints_per_frame: list[dict],
401
- velocities: dict[int, list[float]],
402
- ) -> str:
403
- """Return markdown table of per-joint avg/peak velocity. Empty string if no valid joints."""
404
- n_frames = len(keypoints_per_frame)
405
- if n_frames == 0:
406
- return ""
407
-
408
- rows = []
409
- for j in range(17):
410
- detected = sum(
411
- 1 for kps in keypoints_per_frame
412
- if kps.get(j, {}).get("conf", 0.0) >= CONF_THRESHOLD
413
- )
414
- if detected < n_frames * 0.5:
415
- continue
416
-
417
- speeds = velocities.get(j, [])
418
- if not speeds:
419
- continue
420
-
421
- avg_speed = sum(speeds) / len(speeds)
422
- peak_speed = max(speeds)
423
- rows.append((COCO_KEYPOINTS[j], avg_speed, peak_speed))
424
-
425
- if not rows:
426
- return ""
427
-
428
- rows.sort(key=lambda r: r[2], reverse=True)
429
- lines = [
430
- "| Joint | Avg (px/s) | Peak (px/s) |",
431
- "|---|---|---|",
432
- ]
433
- for name, avg, peak in rows:
434
- lines.append(f"| {name} | {avg:.1f} | {peak:.1f} |")
435
- return "\n".join(lines)
 
1
+ """
2
+ PoseVisualizer — annotated overlay video with skeleton, trails, velocity arrows.
3
+
4
+ Input: IngestResult + Pose2DResult
5
+ Output: .mp4 path (or None on failure/empty layers)
6
+ Failure: returns None, never raises.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import colorsys
11
+ import logging
12
+ import math
13
+ import tempfile
14
+ from collections import deque
15
+
16
+ import cv2
17
+ import numpy as np
18
+
19
+ logger = logging.getLogger(__name__)
20
+
21
+ # ── COCO constants ────────────────────────────────────────────────────────────
22
+
23
+ COCO_KEYPOINTS = [
24
+ "nose", "left_eye", "right_eye", "left_ear", "right_ear",
25
+ "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
26
+ "left_wrist", "right_wrist", "left_hip", "right_hip",
27
+ "left_knee", "right_knee", "left_ankle", "right_ankle",
28
+ ]
29
+
30
+ COCO_SKELETON = [
31
+ (0, 1), (0, 2), (1, 3), (2, 4), # face
32
+ (5, 6), (5, 7), (7, 9), (6, 8), (8, 10), # arms
33
+ (5, 11), (6, 12), (11, 12), # torso
34
+ (11, 13), (13, 15), (12, 14), (14, 16), # legs
35
+ ]
36
+
37
+ TRAIL_LENGTH = 10
38
+ MAX_ARROW_PX = 40
39
+ CONF_THRESHOLD = 0.3
40
+
41
+
42
+ # ── Kalman filter ─────────────────────────────────────────────────────────────
43
+
44
+ class SimpleKalmanFilter:
45
+ """4-state Kalman filter (x, y, vx, vy) for joint tracking."""
46
+
47
+ def __init__(self, process_noise: float = 0.01, measurement_noise: float = 0.1):
48
+ self.is_initialized = False
49
+ self.state = np.zeros(4)
50
+ self.cov = np.eye(4) * 0.1
51
+ self.Q = np.eye(4) * process_noise
52
+ self.R = np.eye(2) * measurement_noise
53
+ self.H = np.array([[1, 0, 0, 0], [0, 1, 0, 0]], dtype=float)
54
+
55
+ def predict(self, dt: float = 1.0):
56
+ F = np.array([[1, 0, dt, 0], [0, 1, 0, dt], [0, 0, 1, 0], [0, 0, 0, 1]], dtype=float)
57
+ self.state = F @ self.state
58
+ self.cov = F @ self.cov @ F.T + self.Q
59
+
60
+ def update(self, x: float, y: float):
61
+ z = np.array([x, y])
62
+ if not self.is_initialized:
63
+ self.state[:2] = z
64
+ self.is_initialized = True
65
+ return
66
+ S = self.H @ self.cov @ self.H.T + self.R
67
+ K = self.cov @ self.H.T @ np.linalg.inv(S)
68
+ self.state = self.state + K @ (z - self.H @ self.state)
69
+ self.cov = (np.eye(4) - K @ self.H) @ self.cov
70
+
71
+ def velocity_magnitude(self) -> float:
72
+ vx, vy = self.state[2], self.state[3]
73
+ return math.sqrt(vx * vx + vy * vy)
74
+
75
+ def velocity_vector(self) -> tuple[float, float]:
76
+ return float(self.state[2]), float(self.state[3])
77
+
78
+
79
+ # ── Velocity computation ──────────────────────────────────────────────────────
80
+
81
+ def compute_joint_velocity(
82
+ keypoints_per_frame: list[dict],
83
+ fps: float,
84
+ ) -> dict[int, list[float]]:
85
+ """
86
+ Compute Kalman-filtered per-joint speed (px/s) for each frame.
87
+
88
+ Returns dict[joint_idx, [speed_frame0, ...]] for all 17 COCO joints.
89
+ Missing/low-confidence keypoints yield speed=0.0 for that frame.
90
+ """
91
+ dt = 1.0 / fps if fps > 0 else 1.0
92
+ filters: dict[int, SimpleKalmanFilter] = {j: SimpleKalmanFilter() for j in range(17)}
93
+ result: dict[int, list[float]] = {j: [] for j in range(17)}
94
+
95
+ for frame_kps in keypoints_per_frame:
96
+ for j in range(17):
97
+ kf = filters[j]
98
+ kp = frame_kps.get(j)
99
+ kf.predict(dt)
100
+ if kp and kp.get("conf", 0.0) >= CONF_THRESHOLD:
101
+ kf.update(kp["x"], kp["y"])
102
+ speed = kf.velocity_magnitude()
103
+ else:
104
+ speed = 0.0
105
+ result[j].append(speed)
106
+
107
+ return result
108
+
109
+
110
+ # ── Helpers ───────────────────────────────────────────────────────────────────
111
+
112
+ def _conf_to_bgr(conf: float) -> tuple[int, int, int]:
113
+ """Map confidence 0→1 to BGR color red→green via HSV."""
114
+ hue = conf * 120.0 / 360.0
115
+ r, g, b = colorsys.hsv_to_rgb(hue, 1.0, 1.0)
116
+ return (int(b * 255), int(g * 255), int(r * 255))
117
+
118
+
119
+ # ── PoseVisualizer ────────────────────────────────────────────────────────────
120
+
121
+ class PoseVisualizer:
122
+ """Renders skeleton, trails, and velocity arrows onto video frames."""
123
+
124
+ def __init__(self):
125
+ self.last_velocities: dict[int, list[float]] = {}
126
+
127
+ # ── Skeleton ──────────────────────────────────────────────────────────────
128
+
129
+ def _draw_skeleton(self, frame: np.ndarray, kps: dict) -> np.ndarray:
130
+ """Draw COCO-17 bones (white) and joints (confidence-colored) onto frame."""
131
+ visible = {j: kp for j, kp in kps.items() if kp.get("conf", 0.0) >= CONF_THRESHOLD}
132
+
133
+ # Bones
134
+ for j1, j2 in COCO_SKELETON:
135
+ if j1 in visible and j2 in visible:
136
+ p1 = (int(visible[j1]["x"]), int(visible[j1]["y"]))
137
+ p2 = (int(visible[j2]["x"]), int(visible[j2]["y"]))
138
+ cv2.line(frame, p1, p2, (255, 255, 255), 2)
139
+
140
+ # Joints
141
+ for j, kp in visible.items():
142
+ pt = (int(kp["x"]), int(kp["y"]))
143
+ color = _conf_to_bgr(kp["conf"])
144
+ cv2.circle(frame, pt, 4, color, -1)
145
+ cv2.circle(frame, pt, 5, (255, 255, 255), 1)
146
+
147
+ return frame
148
+
149
+ # ── Trails ───────────────────────────────────────────────────────────────
150
+
151
+ def _draw_trails(self, frame: np.ndarray, trail_history: dict) -> np.ndarray:
152
+ """Draw fading motion trails for each joint."""
153
+ for joint_idx, trail in trail_history.items():
154
+ pts = list(trail)
155
+ if len(pts) < 2:
156
+ continue
157
+ for i in range(1, len(pts)):
158
+ alpha = i / len(pts)
159
+ brightness = int(255 * alpha)
160
+ color = (brightness, brightness, brightness)
161
+ thickness = max(1, int(3 * alpha))
162
+ p1 = (int(pts[i - 1][0]), int(pts[i - 1][1]))
163
+ p2 = (int(pts[i][0]), int(pts[i][1]))
164
+ cv2.line(frame, p1, p2, color, thickness)
165
+ return frame
166
+
167
+ # ── Velocity arrows ───────────────────────────────────────────────────────
168
+
169
+ def _draw_velocity_arrows(
170
+ self,
171
+ frame: np.ndarray,
172
+ kps: dict,
173
+ prev_kps: dict | None,
174
+ velocities: dict[int, list[float]],
175
+ frame_idx: int,
176
+ ) -> np.ndarray:
177
+ """Draw per-joint velocity arrows scaled by speed."""
178
+ if prev_kps is None:
179
+ return frame
180
+
181
+ all_speeds = [velocities[j][frame_idx] for j in range(17) if frame_idx < len(velocities.get(j, []))]
182
+ peak = max(all_speeds) if all_speeds else 1.0
183
+ if peak == 0.0:
184
+ return frame
185
+
186
+ for j in range(17):
187
+ kp = kps.get(j)
188
+ pk = prev_kps.get(j)
189
+ if not kp or not pk:
190
+ continue
191
+ if kp.get("conf", 0.0) < CONF_THRESHOLD:
192
+ continue
193
+ speeds = velocities.get(j, [])
194
+ if frame_idx >= len(speeds):
195
+ continue
196
+ speed = speeds[frame_idx]
197
+ if speed == 0.0:
198
+ continue
199
+
200
+ dx = kp["x"] - pk["x"]
201
+ dy = kp["y"] - pk["y"]
202
+ mag = math.sqrt(dx * dx + dy * dy)
203
+ if mag < 1e-6:
204
+ continue
205
+
206
+ length = min(speed / peak * MAX_ARROW_PX, MAX_ARROW_PX)
207
+ nx, ny = dx / mag, dy / mag
208
+ start = (int(kp["x"]), int(kp["y"]))
209
+ end = (int(kp["x"] + nx * length), int(kp["y"] + ny * length))
210
+
211
+ ratio = speed / peak
212
+ if ratio < 0.33:
213
+ color = (0, 200, 0) # green
214
+ elif ratio < 0.66:
215
+ color = (0, 140, 255) # orange
216
+ else:
217
+ color = (0, 0, 255) # red
218
+
219
+ cv2.arrowedLine(frame, start, end, color, 2, tipLength=0.35)
220
+
221
+ return frame
222
+
223
+ # ── Public ────────────────────────────────────────────────────────────────
224
+
225
+ def render_video(
226
+ self,
227
+ ingest,
228
+ pose2d,
229
+ layers: set[str],
230
+ output_path: str,
231
+ ) -> str | None:
232
+ """
233
+ Render annotated video. Returns output_path on success, None otherwise.
234
+ layers: subset of {"skeleton", "trails", "velocity_arrows"}
235
+ """
236
+ if not layers:
237
+ return None
238
+
239
+ if not any(pose2d.keypoints):
240
+ return None
241
+
242
+ try:
243
+ velocities = compute_joint_velocity(pose2d.keypoints, ingest.fps)
244
+ self.last_velocities = velocities
245
+
246
+ frames = ingest.frames
247
+ orig_h, orig_w = frames[0].shape[:2]
248
+ fps = ingest.fps or 30.0
249
+
250
+ # Cap at 1280px wide — big frames are slow and don't need to be HQ
251
+ max_w = 1280
252
+ if orig_w > max_w:
253
+ scale = max_w / orig_w
254
+ out_w = max_w
255
+ out_h = int(orig_h * scale)
256
+ else:
257
+ scale = 1.0
258
+ out_w, out_h = orig_w, orig_h
259
+
260
+ # Scale keypoint coordinates to match resized frames
261
+ def _scale_kps(kps: dict) -> dict:
262
+ if scale == 1.0:
263
+ return kps
264
+ return {
265
+ j: {**kp, "x": kp["x"] * scale, "y": kp["y"] * scale}
266
+ for j, kp in kps.items()
267
+ }
268
+
269
+ scaled_keypoints = [_scale_kps(k) for k in pose2d.keypoints]
270
+
271
+ # Write raw mp4v to a temp file, then remux with ffmpeg faststart
272
+ import subprocess
273
+ import tempfile as _tf
274
+ tmp = _tf.NamedTemporaryFile(suffix="_raw.mp4", delete=False)
275
+ tmp_path = tmp.name
276
+ tmp.close()
277
+
278
+ fourcc = cv2.VideoWriter_fourcc(*"mp4v")
279
+ writer = cv2.VideoWriter(tmp_path, fourcc, fps, (out_w, out_h))
280
+ if not writer.isOpened():
281
+ logger.warning("VideoWriter failed to open: %s", tmp_path)
282
+ return None
283
+
284
+ trail_history: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
285
+ prev_kps: dict | None = None
286
+
287
+ for frame_idx, (frame, kps) in enumerate(zip(frames, scaled_keypoints)):
288
+ if scale != 1.0:
289
+ out_frame = cv2.resize(frame, (out_w, out_h), interpolation=cv2.INTER_AREA)
290
+ else:
291
+ out_frame = frame.copy()
292
+
293
+ if "trails" in layers:
294
+ for j, kp in kps.items():
295
+ if kp.get("conf", 0.0) >= CONF_THRESHOLD:
296
+ trail_history[j].append((kp["x"], kp["y"]))
297
+ out_frame = self._draw_trails(out_frame, trail_history)
298
+
299
+ if "skeleton" in layers:
300
+ out_frame = self._draw_skeleton(out_frame, kps)
301
+
302
+ if "velocity_arrows" in layers:
303
+ out_frame = self._draw_velocity_arrows(
304
+ out_frame, kps, prev_kps, velocities, frame_idx
305
+ )
306
+
307
+ writer.write(out_frame)
308
+ prev_kps = kps
309
+
310
+ writer.release()
311
+
312
+ # Remux with faststart so browsers can seek without downloading the whole file
313
+ try:
314
+ subprocess.run(
315
+ ["ffmpeg", "-y", "-i", tmp_path, "-c", "copy",
316
+ "-movflags", "+faststart", output_path],
317
+ check=True, capture_output=True,
318
+ )
319
+ import os
320
+ os.unlink(tmp_path)
321
+ except Exception as ffmpeg_err:
322
+ logger.warning("ffmpeg remux failed (%s) — using raw mp4v", ffmpeg_err)
323
+ import shutil
324
+ shutil.move(tmp_path, output_path)
325
+
326
+ return output_path
327
+
328
+ except Exception as e:
329
+ logger.warning("render_video failed: %s", e)
330
+ return None
331
+
332
+ def render_frame(
333
+ self,
334
+ ingest,
335
+ pose2d,
336
+ frame_idx: int,
337
+ layers: set[str],
338
+ caption: str = "",
339
+ out_png: str | None = None,
340
+ ) -> str | None:
341
+ """Render a single annotated still (skeleton + optional trails + caption).
342
+
343
+ frame_idx is typically the governing frame from BiomechFeatures.timing.
344
+ Returns the PNG path on success, None on any failure. Never raises.
345
+ """
346
+ try:
347
+ if not (0 <= frame_idx < len(ingest.frames)) or frame_idx >= len(pose2d.keypoints):
348
+ return None
349
+
350
+ frame = ingest.frames[frame_idx].copy()
351
+ kps = pose2d.keypoints[frame_idx]
352
+
353
+ if "trails" in layers:
354
+ trail: dict[int, deque] = {j: deque(maxlen=TRAIL_LENGTH) for j in range(17)}
355
+ start = max(0, frame_idx - TRAIL_LENGTH)
356
+ for fi in range(start, frame_idx + 1):
357
+ for j, kp in pose2d.keypoints[fi].items():
358
+ if kp.get("conf", 0.0) >= CONF_THRESHOLD:
359
+ trail[j].append((kp["x"], kp["y"]))
360
+ frame = self._draw_trails(frame, trail)
361
+
362
+ if "skeleton" in layers:
363
+ frame = self._draw_skeleton(frame, kps)
364
+
365
+ if caption:
366
+ cv2.rectangle(frame, (0, 0), (frame.shape[1], 28), (0, 0, 0), -1)
367
+ cv2.putText(frame, caption[:80], (8, 20), cv2.FONT_HERSHEY_SIMPLEX,
368
+ 0.55, (255, 255, 255), 1, cv2.LINE_AA)
369
+
370
+ if out_png is None:
371
+ out_png = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
372
+
373
+ ok = cv2.imwrite(out_png, frame)
374
+ return out_png if ok else None
375
+ except Exception as e:
376
+ logger.warning("render_frame failed: %s", e)
377
+ return None
378
+
379
+
380
+ # ── Velocity summary ──────────────────────────────────────────────────────────
381
+
382
+ def build_velocity_summary(
383
+ keypoints_per_frame: list[dict],
384
+ velocities: dict[int, list[float]],
385
+ ) -> str:
386
+ """Return markdown table of per-joint avg/peak velocity. Empty string if no valid joints."""
387
+ n_frames = len(keypoints_per_frame)
388
+ if n_frames == 0:
389
+ return ""
390
+
391
+ rows = []
392
+ for j in range(17):
393
+ detected = sum(
394
+ 1 for kps in keypoints_per_frame
395
+ if kps.get(j, {}).get("conf", 0.0) >= CONF_THRESHOLD
396
+ )
397
+ if detected < n_frames * 0.5:
398
+ continue
399
+
400
+ speeds = velocities.get(j, [])
401
+ if not speeds:
402
+ continue
403
+
404
+ avg_speed = sum(speeds) / len(speeds)
405
+ peak_speed = max(speeds)
406
+ rows.append((COCO_KEYPOINTS[j], avg_speed, peak_speed))
407
+
408
+ if not rows:
409
+ return ""
410
+
411
+ rows.sort(key=lambda r: r[2], reverse=True)
412
+ lines = [
413
+ "| Joint | Avg (px/s) | Peak (px/s) |",
414
+ "|---|---|---|",
415
+ ]
416
+ for name, avg, peak in rows:
417
+ lines.append(f"| {name} | {avg:.1f} | {peak:.1f} |")
418
+ return "\n".join(lines)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
formscout/analysis/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """FormScout movement-analysis engine: relevant joints, time series, Laban, charts."""
formscout/analysis/charts.py ADDED
@@ -0,0 +1,171 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Matplotlib chart generators for the screening report.
3
+
4
+ Every function returns a PNG path on success or None on failure (never raises),
5
+ so a chart problem degrades the report but never blocks scoring. Charts use the
6
+ Silas palette and an Agg backend so they render headless on the Space.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+
12
+ import matplotlib
13
+
14
+ matplotlib.use("Agg")
15
+ import matplotlib.pyplot as plt # noqa: E402
16
+ import numpy as np # noqa: E402
17
+
18
+ from formscout.analysis.relevant_joints import COCO_NAMES # noqa: E402
19
+
20
+ logger = logging.getLogger(__name__)
21
+
22
+ TEAL = "#2b8a8a"
23
+ GOLD = "#e0a43b"
24
+ SAGE = "#9cbcad"
25
+ INK = "#243a34"
26
+ RED = "#d9534f"
27
+ _PALETTE = [TEAL, GOLD, SAGE, "#7a5ca0", "#c2683c", "#3c8dbc"]
28
+
29
+
30
+ def _save(fig, out_png: str) -> str | None:
31
+ try:
32
+ fig.savefig(out_png, dpi=110, bbox_inches="tight", facecolor="white")
33
+ return out_png
34
+ except Exception as e:
35
+ logger.warning("chart save failed: %s", e)
36
+ return None
37
+ finally:
38
+ plt.close(fig)
39
+
40
+
41
+ def angle_over_time(series: dict, primary: str | None, governing_idx: int | None,
42
+ out_png: str, title: str = "Joint angle over time") -> str | None:
43
+ """Angle-vs-frame for the relevant angles; primary emphasised, key-frame marked."""
44
+ try:
45
+ if not series:
46
+ return None
47
+ fig, ax = plt.subplots(figsize=(6.4, 3.2))
48
+ for i, (name, vals) in enumerate(series.items()):
49
+ arr = np.array(vals, dtype=float)
50
+ is_primary = name == primary
51
+ ax.plot(np.arange(len(arr)), arr,
52
+ color=(TEAL if is_primary else _PALETTE[i % len(_PALETTE)]),
53
+ lw=2.4 if is_primary else 1.3,
54
+ alpha=1.0 if is_primary else 0.6,
55
+ label=name.replace("_", " ") + (" ★" if is_primary else ""))
56
+ if governing_idx is not None:
57
+ ax.axvline(governing_idx, color=GOLD, ls="--", lw=1.5, label="key frame")
58
+ ax.set_xlabel("frame")
59
+ ax.set_ylabel("degrees")
60
+ ax.set_title(title, color=INK)
61
+ ax.legend(fontsize=7, loc="best")
62
+ ax.grid(True, alpha=0.2)
63
+ return _save(fig, out_png)
64
+ except Exception as e:
65
+ logger.warning("angle_over_time failed: %s", e)
66
+ return None
67
+
68
+
69
+ def velocity_profile(keypoints: list, fps: float, joints: list[int],
70
+ out_png: str, title: str = "Joint speed over time") -> str | None:
71
+ """Per-frame speed (px/s) of the relevant joints across the clip."""
72
+ try:
73
+ from formscout.agents.visualizer import compute_joint_velocity
74
+ vel = compute_joint_velocity(keypoints, fps or 30.0)
75
+ plot_joints = [j for j in joints if j in vel] or list(vel.keys())[:4]
76
+ if not plot_joints:
77
+ return None
78
+ fig, ax = plt.subplots(figsize=(6.4, 3.2))
79
+ for i, j in enumerate(plot_joints):
80
+ ax.plot(vel[j], color=_PALETTE[i % len(_PALETTE)], lw=1.6,
81
+ label=COCO_NAMES.get(j, str(j)).replace("_", " "))
82
+ ax.set_xlabel("frame")
83
+ ax.set_ylabel("speed (px/s)")
84
+ ax.set_title(title, color=INK)
85
+ ax.legend(fontsize=7, loc="best")
86
+ ax.grid(True, alpha=0.2)
87
+ return _save(fig, out_png)
88
+ except Exception as e:
89
+ logger.warning("velocity_profile failed: %s", e)
90
+ return None
91
+
92
+
93
+ def laban_radar(effort: dict, out_png: str, title: str = "Laban Effort") -> str | None:
94
+ """4-axis radar of the Effort factors (Space, Weight, Time, Flow)."""
95
+ try:
96
+ axes_order = ["space", "weight", "time", "flow"]
97
+ labels = ["Space\n(direct)", "Weight\n(strong)", "Time\n(sudden)", "Flow\n(free)"]
98
+ vals = [float(effort.get(k, 0.0)) for k in axes_order]
99
+ angles = np.linspace(0, 2 * np.pi, len(axes_order), endpoint=False).tolist()
100
+ vals_loop = vals + vals[:1]
101
+ angles_loop = angles + angles[:1]
102
+
103
+ fig, ax = plt.subplots(figsize=(4.2, 4.2), subplot_kw={"polar": True})
104
+ ax.plot(angles_loop, vals_loop, color=TEAL, lw=2)
105
+ ax.fill(angles_loop, vals_loop, color=TEAL, alpha=0.25)
106
+ ax.set_xticks(angles)
107
+ ax.set_xticklabels(labels, fontsize=8, color=INK)
108
+ ax.set_ylim(0, 1)
109
+ ax.set_yticks([0.25, 0.5, 0.75, 1.0])
110
+ ax.set_yticklabels(["", "0.5", "", "1.0"], fontsize=7)
111
+ ax.set_title(title, color=INK, pad=18)
112
+ return _save(fig, out_png)
113
+ except Exception as e:
114
+ logger.warning("laban_radar failed: %s", e)
115
+ return None
116
+
117
+
118
+ def flexion_bars(flexion: dict, out_png: str,
119
+ title: str = "Relevant joint flexion") -> str | None:
120
+ """Horizontal bars of relevant joint angles (deg) at the key frame."""
121
+ try:
122
+ if not flexion:
123
+ return None
124
+ names = [n.replace("_", " ") for n in flexion]
125
+ degs = [flexion[n]["deg"] for n in flexion]
126
+ colors = [TEAL if d >= 160 else GOLD if d >= 110 else RED for d in degs]
127
+ fig, ax = plt.subplots(figsize=(6.0, max(1.6, 0.5 * len(names) + 0.8)))
128
+ y = np.arange(len(names))
129
+ ax.barh(y, degs, color=colors)
130
+ ax.set_yticks(y)
131
+ ax.set_yticklabels(names, fontsize=8)
132
+ ax.set_xlim(0, 200)
133
+ ax.axvline(160, color=SAGE, ls=":", lw=1)
134
+ for yi, d in zip(y, degs):
135
+ ax.text(d + 3, yi, f"{d:.0f}°", va="center", fontsize=8, color=INK)
136
+ ax.set_xlabel("interior angle (°) · higher = more open")
137
+ ax.set_title(title, color=INK)
138
+ ax.invert_yaxis()
139
+ return _save(fig, out_png)
140
+ except Exception as e:
141
+ logger.warning("flexion_bars failed: %s", e)
142
+ return None
143
+
144
+
145
+ def symmetry_bars(asymmetries: list, out_png: str,
146
+ title: str = "Left / right symmetry") -> str | None:
147
+ """Grouped L vs R score bars for bilateral tests."""
148
+ try:
149
+ rows = [a for a in asymmetries
150
+ if a.get("left_score") is not None and a.get("right_score") is not None]
151
+ if not rows:
152
+ return None
153
+ names = [a["test"].replace("_", " ") for a in rows]
154
+ left = [a["left_score"] for a in rows]
155
+ right = [a["right_score"] for a in rows]
156
+ x = np.arange(len(names))
157
+ w = 0.36
158
+ fig, ax = plt.subplots(figsize=(6.4, 3.2))
159
+ ax.bar(x - w / 2, left, w, color=TEAL, label="left")
160
+ ax.bar(x + w / 2, right, w, color=GOLD, label="right")
161
+ ax.set_xticks(x)
162
+ ax.set_xticklabels(names, fontsize=8, rotation=15, ha="right")
163
+ ax.set_ylim(0, 3.4)
164
+ ax.set_ylabel("score (0–3)")
165
+ ax.set_title(title, color=INK)
166
+ ax.legend(fontsize=8)
167
+ ax.grid(True, axis="y", alpha=0.2)
168
+ return _save(fig, out_png)
169
+ except Exception as e:
170
+ logger.warning("symmetry_bars failed: %s", e)
171
+ return None
formscout/analysis/laban.py ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Laban Movement Analysis — Effort factors from pose kinematics.
3
+
4
+ Computes the four Effort factors over the joints relevant to a screening test:
5
+
6
+ - Space (indirect 0 … direct 1) — path directness of the leading joint
7
+ - Weight (light 0 … strong 1) — motion-energy of the relevant joints
8
+ - Time (sustained 0 … sudden 1) — impulsivity (peak vs mean speed)
9
+ - Flow (bound 0 … free 1) — smoothness (inverse normalised jerk)
10
+
11
+ These are reproducible kinematic heuristics, not clinical LMA notation — the
12
+ report labels them as such. Distances are normalised by torso length so the
13
+ factors are scale-invariant across cameras. Pure function — no model, no I/O.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import math
18
+
19
+ from formscout.agents.biomechanics import _get_joint
20
+ from formscout.analysis.relevant_joints import (
21
+ COCO_NAMES, L_HIP, L_SHOULDER, R_HIP, R_SHOULDER, relevant_joints,
22
+ )
23
+
24
+ # Heuristic calibration references (movement is normalised to torso-lengths/sec).
25
+ _WEIGHT_REF = 1.0 # energy (bl/s)^2 giving ~0.63 weight
26
+ _TIME_LO, _TIME_HI = 1.5, 4.0 # peak/mean speed ratio mapped to [0, 1]
27
+ _FLOW_JERK_REF = 6.0 # normalised jerk (bl/s^3) giving ~0.37 flow
28
+
29
+ _LABELS = {
30
+ "space": ("indirect", "direct"),
31
+ "weight": ("light", "strong"),
32
+ "time": ("sustained", "sudden"),
33
+ "flow": ("bound", "free"),
34
+ }
35
+
36
+
37
+ def _torso_scale(frames) -> float:
38
+ """Median shoulder-hip distance across frames; 1.0 if unmeasurable."""
39
+ lengths = []
40
+ for kps in frames:
41
+ for sh, hip in ((L_SHOULDER, L_HIP), (R_SHOULDER, R_HIP)):
42
+ a, b = _get_joint(kps, sh), _get_joint(kps, hip)
43
+ if a and b:
44
+ lengths.append(math.hypot(a[0] - b[0], a[1] - b[1]))
45
+ if not lengths:
46
+ return 1.0
47
+ lengths.sort()
48
+ med = lengths[len(lengths) // 2]
49
+ return med if med > 1e-6 else 1.0
50
+
51
+
52
+ def _joint_kinematics(frames, joint_id: int, dt: float, scale: float) -> dict | None:
53
+ """Speed/accel/jerk/directness for one joint trajectory (torso-length units)."""
54
+ pts = [_get_joint(kps, joint_id) for kps in frames]
55
+ valid = [(i, p) for i, p in enumerate(pts) if p is not None]
56
+ if len(valid) < 3:
57
+ return None
58
+
59
+ speeds, path_len = [], 0.0
60
+ for (i0, p0), (i1, p1) in zip(valid, valid[1:]):
61
+ d = math.hypot(p1[0] - p0[0], p1[1] - p0[1]) / scale
62
+ path_len += d
63
+ gap = max(1, i1 - i0)
64
+ speeds.append(d / (gap * dt))
65
+ if not speeds:
66
+ return None
67
+
68
+ net = math.hypot(valid[-1][1][0] - valid[0][1][0],
69
+ valid[-1][1][1] - valid[0][1][1]) / scale
70
+ directness = net / path_len if path_len > 1e-6 else 0.0
71
+
72
+ accels = [abs(speeds[i + 1] - speeds[i]) / dt for i in range(len(speeds) - 1)]
73
+ jerks = [abs(accels[i + 1] - accels[i]) / dt for i in range(len(accels) - 1)]
74
+
75
+ mean_speed = sum(speeds) / len(speeds)
76
+ peak_speed = max(speeds)
77
+ return {
78
+ "mean_speed": mean_speed,
79
+ "peak_speed": peak_speed,
80
+ "energy": sum(s * s for s in speeds) / len(speeds),
81
+ "ratio": peak_speed / (mean_speed + 1e-6),
82
+ "mean_jerk": (sum(jerks) / len(jerks)) if jerks else 0.0,
83
+ "directness": min(1.0, max(0.0, directness)),
84
+ }
85
+
86
+
87
+ def _clip01(x: float) -> float:
88
+ return min(1.0, max(0.0, x))
89
+
90
+
91
+ def compute_laban(pose2d, test_name: str, fps: float) -> dict:
92
+ """Return the four Effort factors, their labels, and body emphasis."""
93
+ frames = pose2d.keypoints
94
+ dt = 1.0 / fps if fps and fps > 0 else 1.0 / 30.0
95
+ scale = _torso_scale(frames)
96
+ joints = relevant_joints(test_name) or list(range(17))
97
+
98
+ kin = {j: k for j in joints if (k := _joint_kinematics(frames, j, dt, scale))}
99
+ if not kin:
100
+ return {
101
+ "effort": {"space": 0.0, "weight": 0.0, "time": 0.0, "flow": 0.0},
102
+ "labels": {k: v[0] for k, v in _LABELS.items()},
103
+ "body_emphasis": [],
104
+ "notes": "insufficient motion to estimate Effort",
105
+ }
106
+
107
+ leader = max(kin, key=lambda j: kin[j]["mean_speed"])
108
+ lead = kin[leader]
109
+
110
+ weight = _clip01(1.0 - math.exp(-(sum(k["energy"] for k in kin.values()) / len(kin)) / _WEIGHT_REF))
111
+ time = _clip01((lead["ratio"] - _TIME_LO) / (_TIME_HI - _TIME_LO))
112
+ flow = _clip01(math.exp(-lead["mean_jerk"] / _FLOW_JERK_REF))
113
+ space = _clip01(lead["directness"])
114
+
115
+ effort = {"space": space, "weight": weight, "time": time, "flow": flow}
116
+ labels = {k: _LABELS[k][1] if v >= 0.5 else _LABELS[k][0] for k, v in effort.items()}
117
+
118
+ emphasis = sorted(kin.items(), key=lambda kv: kv[1]["mean_speed"], reverse=True)[:3]
119
+ body_emphasis = [(COCO_NAMES.get(j, str(j)), round(k["mean_speed"], 3)) for j, k in emphasis]
120
+
121
+ return {
122
+ "effort": {k: round(v, 3) for k, v in effort.items()},
123
+ "labels": labels,
124
+ "body_emphasis": body_emphasis,
125
+ "leading_joint": COCO_NAMES.get(leader, str(leader)),
126
+ "notes": "kinematic Effort estimate (heuristic, not clinical LMA notation)",
127
+ }
formscout/analysis/relevant_joints.py ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Per-test relevant-joint and relevant-angle definitions.
3
+
4
+ Drives the "always describe just the joint relevant to the screening action"
5
+ requirement: each FMS test names the joints and the joint-angles that matter for
6
+ its rubric, plus the single primary angle used for the headline angle-over-time
7
+ graph. Angles are COCO (a, b, c) triplets — the angle is measured at joint b.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ # COCO-17 joint indices
12
+ NOSE = 0
13
+ L_SHOULDER, R_SHOULDER = 5, 6
14
+ L_ELBOW, R_ELBOW = 7, 8
15
+ L_WRIST, R_WRIST = 9, 10
16
+ L_HIP, R_HIP = 11, 12
17
+ L_KNEE, R_KNEE = 13, 14
18
+ L_ANKLE, R_ANKLE = 15, 16
19
+
20
+ COCO_NAMES = {
21
+ 0: "nose", 1: "left_eye", 2: "right_eye", 3: "left_ear", 4: "right_ear",
22
+ 5: "left_shoulder", 6: "right_shoulder", 7: "left_elbow", 8: "right_elbow",
23
+ 9: "left_wrist", 10: "right_wrist", 11: "left_hip", 12: "right_hip",
24
+ 13: "left_knee", 14: "right_knee", 15: "left_ankle", 16: "right_ankle",
25
+ }
26
+
27
+ # Each test: the joints that matter, the named angles (a, b, c) measured at b,
28
+ # and the primary angle for the headline graph.
29
+ RELEVANT: dict[str, dict] = {
30
+ "deep_squat": {
31
+ "joints": [L_SHOULDER, R_SHOULDER, L_HIP, R_HIP, L_KNEE, R_KNEE, L_ANKLE, R_ANKLE],
32
+ "angles": {
33
+ "left_knee_flexion": (L_HIP, L_KNEE, L_ANKLE),
34
+ "right_knee_flexion": (R_HIP, R_KNEE, R_ANKLE),
35
+ "left_hip_flexion": (L_SHOULDER, L_HIP, L_KNEE),
36
+ "right_hip_flexion": (R_SHOULDER, R_HIP, R_KNEE),
37
+ },
38
+ "primary_angle": "left_knee_flexion",
39
+ },
40
+ "hurdle_step": {
41
+ "joints": [L_HIP, R_HIP, L_KNEE, R_KNEE, L_ANKLE, R_ANKLE],
42
+ "angles": {
43
+ "left_knee_flexion": (L_HIP, L_KNEE, L_ANKLE),
44
+ "right_knee_flexion": (R_HIP, R_KNEE, R_ANKLE),
45
+ "left_hip_flexion": (L_SHOULDER, L_HIP, L_KNEE),
46
+ "right_hip_flexion": (R_SHOULDER, R_HIP, R_KNEE),
47
+ },
48
+ "primary_angle": "left_hip_flexion",
49
+ },
50
+ "inline_lunge": {
51
+ "joints": [L_HIP, R_HIP, L_KNEE, R_KNEE, L_ANKLE, R_ANKLE],
52
+ "angles": {
53
+ "left_knee_flexion": (L_HIP, L_KNEE, L_ANKLE),
54
+ "right_knee_flexion": (R_HIP, R_KNEE, R_ANKLE),
55
+ },
56
+ "primary_angle": "left_knee_flexion",
57
+ },
58
+ "shoulder_mobility": {
59
+ "joints": [L_SHOULDER, R_SHOULDER, L_ELBOW, R_ELBOW, L_WRIST, R_WRIST],
60
+ "angles": {
61
+ "left_shoulder_angle": (L_ELBOW, L_SHOULDER, L_HIP),
62
+ "right_shoulder_angle": (R_ELBOW, R_SHOULDER, R_HIP),
63
+ "left_elbow_flexion": (L_SHOULDER, L_ELBOW, L_WRIST),
64
+ "right_elbow_flexion": (R_SHOULDER, R_ELBOW, R_WRIST),
65
+ },
66
+ "primary_angle": "left_shoulder_angle",
67
+ },
68
+ "active_slr": {
69
+ "joints": [L_HIP, R_HIP, L_KNEE, R_KNEE, L_ANKLE, R_ANKLE],
70
+ "angles": {
71
+ "left_hip_flexion": (L_SHOULDER, L_HIP, L_KNEE),
72
+ "right_hip_flexion": (R_SHOULDER, R_HIP, R_KNEE),
73
+ "left_knee_flexion": (L_HIP, L_KNEE, L_ANKLE),
74
+ "right_knee_flexion": (R_HIP, R_KNEE, R_ANKLE),
75
+ },
76
+ "primary_angle": "left_hip_flexion",
77
+ },
78
+ "trunk_stability_pushup": {
79
+ "joints": [L_SHOULDER, R_SHOULDER, L_ELBOW, R_ELBOW, L_HIP, R_HIP, L_ANKLE, R_ANKLE],
80
+ "angles": {
81
+ "left_elbow_flexion": (L_SHOULDER, L_ELBOW, L_WRIST),
82
+ "right_elbow_flexion": (R_SHOULDER, R_ELBOW, R_WRIST),
83
+ "left_hip_line": (L_SHOULDER, L_HIP, L_ANKLE),
84
+ "right_hip_line": (R_SHOULDER, R_HIP, R_ANKLE),
85
+ },
86
+ "primary_angle": "left_hip_line",
87
+ },
88
+ "rotary_stability": {
89
+ "joints": [L_SHOULDER, R_SHOULDER, L_ELBOW, R_ELBOW, L_HIP, R_HIP, L_KNEE, R_KNEE],
90
+ "angles": {
91
+ "left_hip_line": (L_SHOULDER, L_HIP, L_KNEE),
92
+ "right_hip_line": (R_SHOULDER, R_HIP, R_KNEE),
93
+ },
94
+ "primary_angle": "left_hip_line",
95
+ },
96
+ }
97
+
98
+
99
+ def relevant_joints(test_name: str) -> list[int]:
100
+ """COCO joint indices relevant to this test (empty for unknown tests)."""
101
+ return list(RELEVANT.get(test_name, {}).get("joints", []))
102
+
103
+
104
+ def relevant_angles(test_name: str) -> dict[str, tuple]:
105
+ """Named (a, b, c) angle triplets relevant to this test."""
106
+ return dict(RELEVANT.get(test_name, {}).get("angles", {}))
107
+
108
+
109
+ def primary_angle(test_name: str) -> str | None:
110
+ """Name of the headline angle for the angle-over-time graph, or None."""
111
+ return RELEVANT.get(test_name, {}).get("primary_angle")
112
+
113
+
114
+ def openness_label(angle_deg: float) -> str:
115
+ """Describe how open/closed a joint is from its interior angle in degrees."""
116
+ if angle_deg >= 160:
117
+ return "open / extended"
118
+ if angle_deg >= 110:
119
+ return "mid-range"
120
+ if angle_deg >= 60:
121
+ return "flexed"
122
+ return "deeply flexed / closed"
formscout/analysis/timeseries.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Per-frame time series for the joints relevant to a screening test.
3
+
4
+ Builds angle-over-time series (used by the angle graph) and a per-joint flexion
5
+ summary at a chosen frame (degrees + open/closed label). Reuses the biomechanics
6
+ geometry helpers so angle definitions never diverge.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import math
11
+
12
+ from formscout.agents.biomechanics import _angle_between_points, _get_joint
13
+ from formscout.analysis.relevant_joints import openness_label, relevant_angles
14
+
15
+
16
+ def angle_series(pose2d, test_name: str) -> dict[str, list[float]]:
17
+ """Return {angle_name: [deg per frame]} for this test's relevant angles.
18
+
19
+ Frames where the angle cannot be measured hold NaN.
20
+ """
21
+ angles = relevant_angles(test_name)
22
+ series: dict[str, list[float]] = {name: [] for name in angles}
23
+ for kps in pose2d.keypoints:
24
+ for name, (a, b, c) in angles.items():
25
+ pa, pb, pc = _get_joint(kps, a), _get_joint(kps, b), _get_joint(kps, c)
26
+ if pa and pb and pc:
27
+ series[name].append(_angle_between_points(pa, pb, pc))
28
+ else:
29
+ series[name].append(float("nan"))
30
+ return series
31
+
32
+
33
+ def relevant_flexion_at(pose2d, test_name: str, frame_idx: int) -> dict[str, dict]:
34
+ """At one frame, the relevant joint angles with degree value + openness label.
35
+
36
+ Returns {angle_name: {"deg": float, "openness": str}}; angles that cannot be
37
+ measured at that frame are omitted.
38
+ """
39
+ out: dict[str, dict] = {}
40
+ if not (0 <= frame_idx < len(pose2d.keypoints)):
41
+ return out
42
+ kps = pose2d.keypoints[frame_idx]
43
+ for name, (a, b, c) in relevant_angles(test_name).items():
44
+ pa, pb, pc = _get_joint(kps, a), _get_joint(kps, b), _get_joint(kps, c)
45
+ if pa and pb and pc:
46
+ deg = _angle_between_points(pa, pb, pc)
47
+ if not math.isnan(deg):
48
+ out[name] = {"deg": deg, "openness": openness_label(deg)}
49
+ return out
formscout/config.py CHANGED
@@ -143,6 +143,18 @@ DEEP_SQUAT_KNEE_TRACKING_MARGIN_PX = 20
143
  LLAMA_CPP_HOST = "127.0.0.1"
144
  LLAMA_CPP_PORT_VLM = 8080
145
  LLAMA_CPP_PORT_EMBED = 8081
146
- # Model id sent in the OpenAI-compatible request. LM Studio uses this for
147
- # JIT auto-loading; native llama-server ignores it. Override with env var.
148
- LLAMA_CPP_MODEL = os.environ.get("FORMSCOUT_VLM_MODEL", "qwen/qwen3-vl-8b")
 
 
 
 
 
 
 
 
 
 
 
 
 
143
  LLAMA_CPP_HOST = "127.0.0.1"
144
  LLAMA_CPP_PORT_VLM = 8080
145
  LLAMA_CPP_PORT_EMBED = 8081
146
+
147
+ # ─── Judge backend selection ────────────────────────────────────────────────
148
+ # "llama_cpp" local llama-server (default for local dev; works perfectly)
149
+ # "transformers"— in-process Qwen3-VL via transformers, GPU on HF Spaces (ZeroGPU)
150
+ # "auto" — transformers on a Space (SPACE_ID set), llama_cpp locally
151
+ JUDGE_BACKEND = os.environ.get("FORMSCOUT_JUDGE_BACKEND", "auto")
152
+ JUDGE_HF_MODEL = os.environ.get("FORMSCOUT_JUDGE_HF_MODEL", "Qwen/Qwen3-VL-8B-Instruct")
153
+ ON_HF_SPACE = bool(os.environ.get("SPACE_ID"))
154
+
155
+
156
+ def resolve_judge_backend() -> str:
157
+ """Resolve the effective judge backend from JUDGE_BACKEND + environment."""
158
+ if JUDGE_BACKEND in ("llama_cpp", "transformers"):
159
+ return JUDGE_BACKEND
160
+ return "transformers" if ON_HF_SPACE else "llama_cpp"
formscout/pipeline.py CHANGED
@@ -1,111 +1,111 @@
1
- """
2
- Director — deterministic state machine orchestrating the FormScout pipeline.
3
-
4
- NOT an LLM. Runs each agent in sequence, applies quality gates, and assembles
5
- the final PipelineState. Exposes run(video_path, config) -> PipelineState.
6
- """
7
- from __future__ import annotations
8
-
9
- from pathlib import Path
10
-
11
- from formscout import config
12
- from formscout.types import (
13
- PipelineState, Body3DResult, MovementResult,
14
- )
15
- from formscout.agents.ingest import IngestAgent
16
- from formscout.agents.pose2d import Pose2DAgent
17
- from formscout.agents.body3d import Body3DAgent
18
- from formscout.agents.biomechanics import BiomechanicsAgent
19
- from formscout.agents.classifier import MovementClassifierAgent
20
- from formscout.agents.judge import JudgeAgent
21
- from formscout.agents.report import ReportAgent
22
- from formscout.rubric import score_test
23
-
24
-
25
- class Director:
26
- """
27
- Orchestrates the FormScout agent pipeline as a deterministic state machine.
28
- Quality gates are applied after each agent — never silently passes bad data.
29
- """
30
-
31
- def __init__(self):
32
- self._ingest = IngestAgent()
33
- self._pose2d = Pose2DAgent()
34
- self._body3d = Body3DAgent()
35
- self._biomechanics = BiomechanicsAgent()
36
- self._classifier = MovementClassifierAgent()
37
- self._judge = JudgeAgent()
38
- self._report = ReportAgent()
39
-
40
- def run(self, video_path: str, test_name: str = "deep_squat", side: str = "na", model_key: str | None = None) -> PipelineState:
41
- """
42
- Run the full pipeline on a single video.
43
- test_name/side serve as manual override when provided (skips classifier).
44
- model_key selects the pose backend (see config.POSE_MODELS).
45
- """
46
- state = PipelineState(video_path=video_path)
47
-
48
- # ─── Ingest ───
49
- state.ingest = self._ingest.run(video_path)
50
- if state.ingest.confidence < config.MIN_CONFIDENCE:
51
- state.errors.append("ingest: low confidence — video may be corrupt")
52
- return state
53
-
54
- # ─── Pose 2D ───
55
- state.pose2d = self._pose2d.run(state.ingest, model_key=model_key)
56
- if state.pose2d.confidence < config.MIN_CONFIDENCE:
57
- state.warnings.append("pose2d: low confidence — no clear person detected")
58
-
59
- # ─── Body 3D (optional) ───
60
- masks = state.segment.masks if state.segment else []
61
- frames = state.ingest.frames if state.ingest else []
62
- state.body3d = self._body3d.run(state.pose2d, masks, frames=frames)
63
-
64
- # ─── Movement classification ───
65
- if test_name and test_name != "unknown":
66
- # Manual override
67
- state.movement = MovementResult(
68
- test_name=test_name, side=side,
69
- confidence=1.0, notes="manually specified",
70
- )
71
- else:
72
- state.movement = self._classifier.run(state.ingest, state.pose2d)
73
-
74
- # Gate: unknown test → stop
75
- if state.movement.test_name == "unknown":
76
- state.errors.append("movement classifier returned 'unknown' — manual override required")
77
- return state
78
-
79
- # ─── Biomechanics ───
80
- state.features = self._biomechanics.run(
81
- state.pose2d,
82
- state.body3d or Body3DResult(used=False, joints_3d=[]),
83
- state.movement,
84
- )
85
- if state.features.confidence < config.MIN_CONFIDENCE:
86
- state.warnings.append(
87
- f"biomechanics: low confidence ({state.features.confidence:.2f}) — physio review recommended"
88
- )
89
-
90
- # ─── Rubric Score ───
91
- rubric_result = score_test(state.features)
92
- state.stgcn_score = rubric_result # Reusing field for rubric until ST-GCN is built
93
-
94
- # ─── Judge ───
95
- state.judge = self._judge.run(
96
- state.features, rubric_result, state.movement, state.ingest,
97
- )
98
-
99
- # ─── Quality gates ───
100
- # Gate: score disagreement
101
- if (state.judge.score is not None and rubric_result.score is not None
102
- and abs(state.judge.score - rubric_result.score) >= config.SCORE_DISAGREE_THRESH):
103
- state.warnings.append(
104
- f"score disagreement: rubric={rubric_result.score} vs judge={state.judge.score} — review recommended"
105
- )
106
-
107
- # Gate: needs_human
108
- if state.judge.needs_human:
109
- state.warnings.append("judge flagged needs_human — no auto-score emitted")
110
-
111
- return state
 
1
+ """
2
+ Director — deterministic state machine orchestrating the FormScout pipeline.
3
+
4
+ NOT an LLM. Runs each agent in sequence, applies quality gates, and assembles
5
+ the final PipelineState. Exposes run(video_path, config) -> PipelineState.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ from pathlib import Path
10
+
11
+ from formscout import config
12
+ from formscout.types import (
13
+ PipelineState, Body3DResult, MovementResult,
14
+ )
15
+ from formscout.agents.ingest import IngestAgent
16
+ from formscout.agents.pose2d import Pose2DAgent
17
+ from formscout.agents.body3d import Body3DAgent
18
+ from formscout.agents.biomechanics import BiomechanicsAgent
19
+ from formscout.agents.classifier import MovementClassifierAgent
20
+ from formscout.agents.judge import JudgeAgent
21
+ from formscout.agents.report import ReportAgent
22
+ from formscout.rubric import score_test
23
+
24
+
25
+ class Director:
26
+ """
27
+ Orchestrates the FormScout agent pipeline as a deterministic state machine.
28
+ Quality gates are applied after each agent — never silently passes bad data.
29
+ """
30
+
31
+ def __init__(self):
32
+ self._ingest = IngestAgent()
33
+ self._pose2d = Pose2DAgent()
34
+ self._body3d = Body3DAgent()
35
+ self._biomechanics = BiomechanicsAgent()
36
+ self._classifier = MovementClassifierAgent()
37
+ self._judge = JudgeAgent()
38
+ self._report = ReportAgent()
39
+
40
+ def run(self, video_path: str, test_name: str = "deep_squat", side: str = "na", model_key: str | None = None) -> PipelineState:
41
+ """
42
+ Run the full pipeline on a single video.
43
+ test_name/side serve as manual override when provided (skips classifier).
44
+ model_key selects the pose backend (see config.POSE_MODELS).
45
+ """
46
+ state = PipelineState(video_path=video_path)
47
+
48
+ # ─── Ingest ───
49
+ state.ingest = self._ingest.run(video_path)
50
+ if state.ingest.confidence < config.MIN_CONFIDENCE:
51
+ state.errors.append("ingest: low confidence — video may be corrupt")
52
+ return state
53
+
54
+ # ─── Pose 2D ───
55
+ state.pose2d = self._pose2d.run(state.ingest, model_key=model_key)
56
+ if state.pose2d.confidence < config.MIN_CONFIDENCE:
57
+ state.warnings.append("pose2d: low confidence — no clear person detected")
58
+
59
+ # ─── Body 3D (optional) ───
60
+ masks = state.segment.masks if state.segment else []
61
+ frames = state.ingest.frames if state.ingest else []
62
+ state.body3d = self._body3d.run(state.pose2d, masks, frames=frames)
63
+
64
+ # ─── Movement classification ───
65
+ if test_name and test_name != "unknown":
66
+ # Manual override
67
+ state.movement = MovementResult(
68
+ test_name=test_name, side=side,
69
+ confidence=1.0, notes="manually specified",
70
+ )
71
+ else:
72
+ state.movement = self._classifier.run(state.ingest, state.pose2d)
73
+
74
+ # Gate: unknown test → stop
75
+ if state.movement.test_name == "unknown":
76
+ state.errors.append("movement classifier returned 'unknown' — manual override required")
77
+ return state
78
+
79
+ # ─── Biomechanics ───
80
+ state.features = self._biomechanics.run(
81
+ state.pose2d,
82
+ state.body3d or Body3DResult(used=False, joints_3d=[]),
83
+ state.movement,
84
+ )
85
+ if state.features.confidence < config.MIN_CONFIDENCE:
86
+ state.warnings.append(
87
+ f"biomechanics: low confidence ({state.features.confidence:.2f}) — physio review recommended"
88
+ )
89
+
90
+ # ─── Rubric Score ───
91
+ rubric_result = score_test(state.features)
92
+ state.stgcn_score = rubric_result # Reusing field for rubric until ST-GCN is built
93
+
94
+ # ─── Judge ───
95
+ state.judge = self._judge.run(
96
+ state.features, rubric_result, state.movement, state.ingest,
97
+ )
98
+
99
+ # ─── Quality gates ───
100
+ # Gate: score disagreement
101
+ if (state.judge.score is not None and rubric_result.score is not None
102
+ and abs(state.judge.score - rubric_result.score) >= config.SCORE_DISAGREE_THRESH):
103
+ state.warnings.append(
104
+ f"score disagreement: rubric={rubric_result.score} vs judge={state.judge.score} — review recommended"
105
+ )
106
+
107
+ # Gate: needs_human
108
+ if state.judge.needs_human:
109
+ state.warnings.append("judge flagged needs_human — no auto-score emitted")
110
+
111
+ return state
formscout/rubric/__init__.py CHANGED
@@ -1,32 +1,32 @@
1
- """
2
- FormScout rubric scorers — one pure-function scorer per FMS test.
3
- """
4
- from formscout.rubric.deep_squat import score_deep_squat
5
- from formscout.rubric.hurdle_step import score_hurdle_step
6
- from formscout.rubric.inline_lunge import score_inline_lunge
7
- from formscout.rubric.shoulder_mobility import score_shoulder_mobility
8
- from formscout.rubric.active_slr import score_active_slr
9
- from formscout.rubric.trunk_stability_pushup import score_trunk_stability_pushup
10
- from formscout.rubric.rotary_stability import score_rotary_stability
11
- from formscout.types import BiomechFeatures, ScoreResult
12
-
13
- SCORERS = {
14
- "deep_squat": score_deep_squat,
15
- "hurdle_step": score_hurdle_step,
16
- "inline_lunge": score_inline_lunge,
17
- "shoulder_mobility": score_shoulder_mobility,
18
- "active_slr": score_active_slr,
19
- "trunk_stability_pushup": score_trunk_stability_pushup,
20
- "rotary_stability": score_rotary_stability,
21
- }
22
-
23
-
24
- def score_test(features: BiomechFeatures) -> ScoreResult:
25
- """Dispatch to the appropriate rubric scorer by test name."""
26
- fn = SCORERS.get(features.test_name)
27
- if fn is None:
28
- return ScoreResult(
29
- score=1, rationale=f"No rubric for test '{features.test_name}'",
30
- confidence=0.0, notes="unknown test",
31
- )
32
- return fn(features)
 
1
+ """
2
+ FormScout rubric scorers — one pure-function scorer per FMS test.
3
+ """
4
+ from formscout.rubric.deep_squat import score_deep_squat
5
+ from formscout.rubric.hurdle_step import score_hurdle_step
6
+ from formscout.rubric.inline_lunge import score_inline_lunge
7
+ from formscout.rubric.shoulder_mobility import score_shoulder_mobility
8
+ from formscout.rubric.active_slr import score_active_slr
9
+ from formscout.rubric.trunk_stability_pushup import score_trunk_stability_pushup
10
+ from formscout.rubric.rotary_stability import score_rotary_stability
11
+ from formscout.types import BiomechFeatures, ScoreResult
12
+
13
+ SCORERS = {
14
+ "deep_squat": score_deep_squat,
15
+ "hurdle_step": score_hurdle_step,
16
+ "inline_lunge": score_inline_lunge,
17
+ "shoulder_mobility": score_shoulder_mobility,
18
+ "active_slr": score_active_slr,
19
+ "trunk_stability_pushup": score_trunk_stability_pushup,
20
+ "rotary_stability": score_rotary_stability,
21
+ }
22
+
23
+
24
+ def score_test(features: BiomechFeatures) -> ScoreResult:
25
+ """Dispatch to the appropriate rubric scorer by test name."""
26
+ fn = SCORERS.get(features.test_name)
27
+ if fn is None:
28
+ return ScoreResult(
29
+ score=1, rationale=f"No rubric for test '{features.test_name}'",
30
+ confidence=0.0, notes="unknown test",
31
+ )
32
+ return fn(features)
formscout/rubric/active_slr.py CHANGED
@@ -1,51 +1,51 @@
1
- """
2
- Active Straight-Leg Raise rubric scorer — pure function, no model calls.
3
-
4
- FMS ASLR Criteria (bilateral):
5
- - Score 3: raised leg malleolus past contralateral knee (>70°), down leg flat.
6
- - Score 2: malleolus between mid-thigh and knee (45-70°).
7
- - Score 1: malleolus below mid-thigh (<45°).
8
- - Score 0: PAIN — never auto-scored.
9
- """
10
- from __future__ import annotations
11
-
12
- from formscout.types import BiomechFeatures, ScoreResult
13
-
14
-
15
- def score_active_slr(features: BiomechFeatures) -> ScoreResult:
16
- """Pure rubric scorer for active straight-leg raise."""
17
- angles = features.angles
18
- alignments = features.alignments
19
-
20
- has_angle = "raised_leg_angle_deg" in angles
21
- if not has_angle:
22
- return ScoreResult(
23
- score=1, rationale="Insufficient data: leg raise angle not measurable",
24
- confidence=0.3, notes="missing key measurements",
25
- )
26
-
27
- angle = angles["raised_leg_angle_deg"]
28
- past_knee = alignments.get("past_contralateral_knee", False)
29
- past_mid = alignments.get("past_mid_thigh", False)
30
- down_flat = alignments.get("down_leg_flat", True)
31
-
32
- rationale_parts = []
33
-
34
- if past_knee and down_flat:
35
- score = 3
36
- rationale_parts.append(f"Raised leg at {angle:.0f}° (past contralateral knee)")
37
- elif past_mid:
38
- score = 2
39
- rationale_parts.append(f"Raised leg at {angle:.0f}° (between mid-thigh and knee)")
40
- if not down_flat:
41
- rationale_parts.append("down leg lifted off surface")
42
- else:
43
- score = 1
44
- rationale_parts.append(f"Raised leg only {angle:.0f}° (below mid-thigh)")
45
-
46
- confidence = features.confidence * 0.9
47
-
48
- return ScoreResult(
49
- score=score, rationale="; ".join(rationale_parts),
50
- confidence=confidence, notes="",
51
- )
 
1
+ """
2
+ Active Straight-Leg Raise rubric scorer — pure function, no model calls.
3
+
4
+ FMS ASLR Criteria (bilateral):
5
+ - Score 3: raised leg malleolus past contralateral knee (>70°), down leg flat.
6
+ - Score 2: malleolus between mid-thigh and knee (45-70°).
7
+ - Score 1: malleolus below mid-thigh (<45°).
8
+ - Score 0: PAIN — never auto-scored.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ from formscout.types import BiomechFeatures, ScoreResult
13
+
14
+
15
+ def score_active_slr(features: BiomechFeatures) -> ScoreResult:
16
+ """Pure rubric scorer for active straight-leg raise."""
17
+ angles = features.angles
18
+ alignments = features.alignments
19
+
20
+ has_angle = "raised_leg_angle_deg" in angles
21
+ if not has_angle:
22
+ return ScoreResult(
23
+ score=1, rationale="Insufficient data: leg raise angle not measurable",
24
+ confidence=0.3, notes="missing key measurements",
25
+ )
26
+
27
+ angle = angles["raised_leg_angle_deg"]
28
+ past_knee = alignments.get("past_contralateral_knee", False)
29
+ past_mid = alignments.get("past_mid_thigh", False)
30
+ down_flat = alignments.get("down_leg_flat", True)
31
+
32
+ rationale_parts = []
33
+
34
+ if past_knee and down_flat:
35
+ score = 3
36
+ rationale_parts.append(f"Raised leg at {angle:.0f}° (past contralateral knee)")
37
+ elif past_mid:
38
+ score = 2
39
+ rationale_parts.append(f"Raised leg at {angle:.0f}° (between mid-thigh and knee)")
40
+ if not down_flat:
41
+ rationale_parts.append("down leg lifted off surface")
42
+ else:
43
+ score = 1
44
+ rationale_parts.append(f"Raised leg only {angle:.0f}° (below mid-thigh)")
45
+
46
+ confidence = features.confidence * 0.9
47
+
48
+ return ScoreResult(
49
+ score=score, rationale="; ".join(rationale_parts),
50
+ confidence=confidence, notes="",
51
+ )
formscout/rubric/hurdle_step.py CHANGED
@@ -1,60 +1,60 @@
1
- """
2
- Hurdle Step rubric scorer — pure function, no model calls.
3
-
4
- FMS Hurdle Step Criteria (bilateral — score each side, report lower):
5
- - Score 3: hips/knees/ankles aligned, minimal trunk movement, dowel/posture stable,
6
- no contact with hurdle.
7
- - Score 2: movement completed with compensation (trunk lean, loss of alignment).
8
- - Score 1: contact with hurdle, loss of balance, or inability to maintain alignment.
9
- - Score 0: PAIN — never auto-scored.
10
- """
11
- from __future__ import annotations
12
-
13
- from formscout.types import BiomechFeatures, ScoreResult
14
-
15
-
16
- def score_hurdle_step(features: BiomechFeatures) -> ScoreResult:
17
- """Pure rubric scorer for hurdle step."""
18
- angles = features.angles
19
- alignments = features.alignments
20
-
21
- has_hip_flex = "step_hip_flexion_deg" in angles
22
- if not has_hip_flex:
23
- return ScoreResult(
24
- score=1, rationale="Insufficient data: hip flexion not measurable",
25
- confidence=0.3, notes="missing key measurements",
26
- )
27
-
28
- trunk_stable = alignments.get("trunk_stable", False)
29
- stance_extended = alignments.get("stance_knee_extended", False)
30
- hip_flex = angles.get("step_hip_flexion_deg", 0)
31
-
32
- rationale_parts = []
33
-
34
- # Score 3: good hip flexion, trunk stable, stance solid
35
- if hip_flex > 90 and trunk_stable and stance_extended:
36
- score = 3
37
- rationale_parts.append("Hip flexion adequate, trunk stable, stance knee extended")
38
- elif hip_flex > 70 or (trunk_stable and stance_extended):
39
- score = 2
40
- if not trunk_stable:
41
- rationale_parts.append("trunk lean detected")
42
- if not stance_extended:
43
- rationale_parts.append("stance knee flexion")
44
- if hip_flex <= 90:
45
- rationale_parts.append(f"hip flexion {hip_flex:.0f}° (borderline)")
46
- rationale_parts.insert(0, "Movement completed with compensation")
47
- else:
48
- score = 1
49
- rationale_parts.append("Unable to maintain alignment")
50
- if not trunk_stable:
51
- rationale_parts.append("significant trunk lean")
52
- if not stance_extended:
53
- rationale_parts.append("stance knee collapse")
54
-
55
- confidence = features.confidence * 0.85
56
-
57
- return ScoreResult(
58
- score=score, rationale="; ".join(rationale_parts),
59
- confidence=confidence, notes="",
60
- )
 
1
+ """
2
+ Hurdle Step rubric scorer — pure function, no model calls.
3
+
4
+ FMS Hurdle Step Criteria (bilateral — score each side, report lower):
5
+ - Score 3: hips/knees/ankles aligned, minimal trunk movement, dowel/posture stable,
6
+ no contact with hurdle.
7
+ - Score 2: movement completed with compensation (trunk lean, loss of alignment).
8
+ - Score 1: contact with hurdle, loss of balance, or inability to maintain alignment.
9
+ - Score 0: PAIN — never auto-scored.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ from formscout.types import BiomechFeatures, ScoreResult
14
+
15
+
16
+ def score_hurdle_step(features: BiomechFeatures) -> ScoreResult:
17
+ """Pure rubric scorer for hurdle step."""
18
+ angles = features.angles
19
+ alignments = features.alignments
20
+
21
+ has_hip_flex = "step_hip_flexion_deg" in angles
22
+ if not has_hip_flex:
23
+ return ScoreResult(
24
+ score=1, rationale="Insufficient data: hip flexion not measurable",
25
+ confidence=0.3, notes="missing key measurements",
26
+ )
27
+
28
+ trunk_stable = alignments.get("trunk_stable", False)
29
+ stance_extended = alignments.get("stance_knee_extended", False)
30
+ hip_flex = angles.get("step_hip_flexion_deg", 0)
31
+
32
+ rationale_parts = []
33
+
34
+ # Score 3: good hip flexion, trunk stable, stance solid
35
+ if hip_flex > 90 and trunk_stable and stance_extended:
36
+ score = 3
37
+ rationale_parts.append("Hip flexion adequate, trunk stable, stance knee extended")
38
+ elif hip_flex > 70 or (trunk_stable and stance_extended):
39
+ score = 2
40
+ if not trunk_stable:
41
+ rationale_parts.append("trunk lean detected")
42
+ if not stance_extended:
43
+ rationale_parts.append("stance knee flexion")
44
+ if hip_flex <= 90:
45
+ rationale_parts.append(f"hip flexion {hip_flex:.0f}° (borderline)")
46
+ rationale_parts.insert(0, "Movement completed with compensation")
47
+ else:
48
+ score = 1
49
+ rationale_parts.append("Unable to maintain alignment")
50
+ if not trunk_stable:
51
+ rationale_parts.append("significant trunk lean")
52
+ if not stance_extended:
53
+ rationale_parts.append("stance knee collapse")
54
+
55
+ confidence = features.confidence * 0.85
56
+
57
+ return ScoreResult(
58
+ score=score, rationale="; ".join(rationale_parts),
59
+ confidence=confidence, notes="",
60
+ )
formscout/rubric/inline_lunge.py CHANGED
@@ -1,58 +1,58 @@
1
- """
2
- In-Line Lunge rubric scorer — pure function, no model calls.
3
-
4
- FMS In-Line Lunge Criteria (bilateral):
5
- - Score 3: dowel contacts maintained, no torso movement, knee touches behind heel.
6
- - Score 2: movement completed with compensation (trunk lean, loss of balance).
7
- - Score 1: loss of balance, inability to maintain foot contact or posture.
8
- - Score 0: PAIN — never auto-scored.
9
- """
10
- from __future__ import annotations
11
-
12
- from formscout.types import BiomechFeatures, ScoreResult
13
-
14
-
15
- def score_inline_lunge(features: BiomechFeatures) -> ScoreResult:
16
- """Pure rubric scorer for in-line lunge."""
17
- angles = features.angles
18
- alignments = features.alignments
19
-
20
- has_knee = "front_knee_flexion_deg" in angles
21
- if not has_knee:
22
- return ScoreResult(
23
- score=1, rationale="Insufficient data: knee flexion not measurable",
24
- confidence=0.3, notes="missing key measurements",
25
- )
26
-
27
- knee_flex = angles.get("front_knee_flexion_deg", 180)
28
- trunk_upright = alignments.get("trunk_upright", False)
29
- knee_over_ankle = alignments.get("knee_over_ankle", False)
30
-
31
- rationale_parts = []
32
-
33
- # Good lunge: knee flexion < 90° (deep), trunk upright, knee aligned
34
- deep_enough = knee_flex < 100
35
- if deep_enough and trunk_upright and knee_over_ankle:
36
- score = 3
37
- rationale_parts.append("Deep lunge with trunk upright and knee aligned")
38
- elif deep_enough or (trunk_upright and knee_over_ankle):
39
- score = 2
40
- if not trunk_upright:
41
- rationale_parts.append(f"trunk lean {angles.get('trunk_lean_from_vertical_deg', '?')}°")
42
- if not knee_over_ankle:
43
- rationale_parts.append("knee drifts past ankle")
44
- if not deep_enough:
45
- rationale_parts.append(f"knee flexion {knee_flex:.0f}° (insufficient depth)")
46
- rationale_parts.insert(0, "Completed with compensation")
47
- else:
48
- score = 1
49
- rationale_parts.append("Unable to complete lunge pattern")
50
- if not deep_enough:
51
- rationale_parts.append(f"knee flexion only {knee_flex:.0f}°")
52
-
53
- confidence = features.confidence * 0.85
54
-
55
- return ScoreResult(
56
- score=score, rationale="; ".join(rationale_parts),
57
- confidence=confidence, notes="",
58
- )
 
1
+ """
2
+ In-Line Lunge rubric scorer — pure function, no model calls.
3
+
4
+ FMS In-Line Lunge Criteria (bilateral):
5
+ - Score 3: dowel contacts maintained, no torso movement, knee touches behind heel.
6
+ - Score 2: movement completed with compensation (trunk lean, loss of balance).
7
+ - Score 1: loss of balance, inability to maintain foot contact or posture.
8
+ - Score 0: PAIN — never auto-scored.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ from formscout.types import BiomechFeatures, ScoreResult
13
+
14
+
15
+ def score_inline_lunge(features: BiomechFeatures) -> ScoreResult:
16
+ """Pure rubric scorer for in-line lunge."""
17
+ angles = features.angles
18
+ alignments = features.alignments
19
+
20
+ has_knee = "front_knee_flexion_deg" in angles
21
+ if not has_knee:
22
+ return ScoreResult(
23
+ score=1, rationale="Insufficient data: knee flexion not measurable",
24
+ confidence=0.3, notes="missing key measurements",
25
+ )
26
+
27
+ knee_flex = angles.get("front_knee_flexion_deg", 180)
28
+ trunk_upright = alignments.get("trunk_upright", False)
29
+ knee_over_ankle = alignments.get("knee_over_ankle", False)
30
+
31
+ rationale_parts = []
32
+
33
+ # Good lunge: knee flexion < 90° (deep), trunk upright, knee aligned
34
+ deep_enough = knee_flex < 100
35
+ if deep_enough and trunk_upright and knee_over_ankle:
36
+ score = 3
37
+ rationale_parts.append("Deep lunge with trunk upright and knee aligned")
38
+ elif deep_enough or (trunk_upright and knee_over_ankle):
39
+ score = 2
40
+ if not trunk_upright:
41
+ rationale_parts.append(f"trunk lean {angles.get('trunk_lean_from_vertical_deg', '?')}°")
42
+ if not knee_over_ankle:
43
+ rationale_parts.append("knee drifts past ankle")
44
+ if not deep_enough:
45
+ rationale_parts.append(f"knee flexion {knee_flex:.0f}° (insufficient depth)")
46
+ rationale_parts.insert(0, "Completed with compensation")
47
+ else:
48
+ score = 1
49
+ rationale_parts.append("Unable to complete lunge pattern")
50
+ if not deep_enough:
51
+ rationale_parts.append(f"knee flexion only {knee_flex:.0f}°")
52
+
53
+ confidence = features.confidence * 0.85
54
+
55
+ return ScoreResult(
56
+ score=score, rationale="; ".join(rationale_parts),
57
+ confidence=confidence, notes="",
58
+ )
formscout/rubric/rotary_stability.py CHANGED
@@ -1,56 +1,56 @@
1
- """
2
- Rotary Stability rubric scorer — pure function, no model calls.
3
-
4
- FMS Rotary Stability Criteria:
5
- - Score 3: unilateral (same-side) arm/leg extension with trunk stable,
6
- elbow/knee touch performed smoothly.
7
- - Score 2: contralateral (opposite) arm/leg extension performed with trunk stable.
8
- - Score 1: inability to maintain trunk stability during contralateral pattern.
9
- - Score 0: PAIN (spinal flexion clearing test) — never auto-scored.
10
- """
11
- from __future__ import annotations
12
-
13
- from formscout.types import BiomechFeatures, ScoreResult
14
-
15
-
16
- def score_rotary_stability(features: BiomechFeatures) -> ScoreResult:
17
- """Pure rubric scorer for rotary stability."""
18
- angles = features.angles
19
- alignments = features.alignments
20
-
21
- has_data = "trunk_stability_std_px" in angles or "shoulder_level_diff_px" in angles
22
- if not has_data:
23
- return ScoreResult(
24
- score=1, rationale="Insufficient data: trunk stability not measurable",
25
- confidence=0.3, notes="missing key measurements",
26
- )
27
-
28
- trunk_stable = alignments.get("trunk_stable", False)
29
- shoulders_level = alignments.get("shoulders_level", False)
30
- hips_level = alignments.get("hips_level", False)
31
-
32
- rationale_parts = []
33
-
34
- # Without video classification of ipsi vs contra, assume contralateral (safer)
35
- if trunk_stable and shoulders_level and hips_level:
36
- score = 2 # Assume contralateral unless classifier says ipsilateral
37
- rationale_parts.append("Trunk stable during extension, shoulders and hips level")
38
- rationale_parts.append("scored as contralateral pattern (default)")
39
- elif trunk_stable or (shoulders_level and hips_level):
40
- score = 2
41
- if not trunk_stable:
42
- rationale_parts.append("minor trunk instability")
43
- rationale_parts.insert(0, "Contralateral pattern with minor compensation")
44
- else:
45
- score = 1
46
- std = angles.get("trunk_stability_std_px", 0)
47
- rationale_parts.append(f"Trunk instability detected (std {std:.1f}px)")
48
- if not shoulders_level:
49
- rationale_parts.append("shoulder asymmetry during extension")
50
-
51
- confidence = features.confidence * 0.75 # Lower confidence — hard to assess from 2D
52
-
53
- return ScoreResult(
54
- score=score, rationale="; ".join(rationale_parts),
55
- confidence=confidence, notes="ipsi/contra distinction requires VLM classifier",
56
- )
 
1
+ """
2
+ Rotary Stability rubric scorer — pure function, no model calls.
3
+
4
+ FMS Rotary Stability Criteria:
5
+ - Score 3: unilateral (same-side) arm/leg extension with trunk stable,
6
+ elbow/knee touch performed smoothly.
7
+ - Score 2: contralateral (opposite) arm/leg extension performed with trunk stable.
8
+ - Score 1: inability to maintain trunk stability during contralateral pattern.
9
+ - Score 0: PAIN (spinal flexion clearing test) — never auto-scored.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ from formscout.types import BiomechFeatures, ScoreResult
14
+
15
+
16
+ def score_rotary_stability(features: BiomechFeatures) -> ScoreResult:
17
+ """Pure rubric scorer for rotary stability."""
18
+ angles = features.angles
19
+ alignments = features.alignments
20
+
21
+ has_data = "trunk_stability_std_px" in angles or "shoulder_level_diff_px" in angles
22
+ if not has_data:
23
+ return ScoreResult(
24
+ score=1, rationale="Insufficient data: trunk stability not measurable",
25
+ confidence=0.3, notes="missing key measurements",
26
+ )
27
+
28
+ trunk_stable = alignments.get("trunk_stable", False)
29
+ shoulders_level = alignments.get("shoulders_level", False)
30
+ hips_level = alignments.get("hips_level", False)
31
+
32
+ rationale_parts = []
33
+
34
+ # Without video classification of ipsi vs contra, assume contralateral (safer)
35
+ if trunk_stable and shoulders_level and hips_level:
36
+ score = 2 # Assume contralateral unless classifier says ipsilateral
37
+ rationale_parts.append("Trunk stable during extension, shoulders and hips level")
38
+ rationale_parts.append("scored as contralateral pattern (default)")
39
+ elif trunk_stable or (shoulders_level and hips_level):
40
+ score = 2
41
+ if not trunk_stable:
42
+ rationale_parts.append("minor trunk instability")
43
+ rationale_parts.insert(0, "Contralateral pattern with minor compensation")
44
+ else:
45
+ score = 1
46
+ std = angles.get("trunk_stability_std_px", 0)
47
+ rationale_parts.append(f"Trunk instability detected (std {std:.1f}px)")
48
+ if not shoulders_level:
49
+ rationale_parts.append("shoulder asymmetry during extension")
50
+
51
+ confidence = features.confidence * 0.75 # Lower confidence — hard to assess from 2D
52
+
53
+ return ScoreResult(
54
+ score=score, rationale="; ".join(rationale_parts),
55
+ confidence=confidence, notes="ipsi/contra distinction requires VLM classifier",
56
+ )
formscout/rubric/shoulder_mobility.py CHANGED
@@ -1,46 +1,46 @@
1
- """
2
- Shoulder Mobility rubric scorer — pure function, no model calls.
3
-
4
- FMS Shoulder Mobility Criteria (bilateral):
5
- - Score 3: fists within one hand-length of each other.
6
- - Score 2: fists within 1.5 hand-lengths.
7
- - Score 1: fists more than 1.5 hand-lengths apart.
8
- - Score 0: PAIN (clearing test) — never auto-scored.
9
- """
10
- from __future__ import annotations
11
-
12
- from formscout.types import BiomechFeatures, ScoreResult
13
-
14
-
15
- def score_shoulder_mobility(features: BiomechFeatures) -> ScoreResult:
16
- """Pure rubric scorer for shoulder mobility."""
17
- alignments = features.alignments
18
- angles = features.angles
19
-
20
- has_measure = "inter_fist_normalized" in angles
21
- if not has_measure:
22
- return ScoreResult(
23
- score=1, rationale="Insufficient data: inter-fist distance not measurable",
24
- confidence=0.3, notes="missing key measurements",
25
- )
26
-
27
- norm_dist = angles["inter_fist_normalized"]
28
- within_one = alignments.get("fists_within_one_hand", False)
29
- within_1_5 = alignments.get("fists_within_1_5_hand", False)
30
-
31
- if within_one:
32
- score = 3
33
- rationale = f"Fists within one hand-length (normalized distance {norm_dist:.2f})"
34
- elif within_1_5:
35
- score = 2
36
- rationale = f"Fists within 1.5 hand-lengths (normalized distance {norm_dist:.2f})"
37
- else:
38
- score = 1
39
- rationale = f"Fists beyond 1.5 hand-lengths apart (normalized distance {norm_dist:.2f})"
40
-
41
- confidence = features.confidence * 0.9
42
-
43
- return ScoreResult(
44
- score=score, rationale=rationale,
45
- confidence=confidence, notes="",
46
- )
 
1
+ """
2
+ Shoulder Mobility rubric scorer — pure function, no model calls.
3
+
4
+ FMS Shoulder Mobility Criteria (bilateral):
5
+ - Score 3: fists within one hand-length of each other.
6
+ - Score 2: fists within 1.5 hand-lengths.
7
+ - Score 1: fists more than 1.5 hand-lengths apart.
8
+ - Score 0: PAIN (clearing test) — never auto-scored.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ from formscout.types import BiomechFeatures, ScoreResult
13
+
14
+
15
+ def score_shoulder_mobility(features: BiomechFeatures) -> ScoreResult:
16
+ """Pure rubric scorer for shoulder mobility."""
17
+ alignments = features.alignments
18
+ angles = features.angles
19
+
20
+ has_measure = "inter_fist_normalized" in angles
21
+ if not has_measure:
22
+ return ScoreResult(
23
+ score=1, rationale="Insufficient data: inter-fist distance not measurable",
24
+ confidence=0.3, notes="missing key measurements",
25
+ )
26
+
27
+ norm_dist = angles["inter_fist_normalized"]
28
+ within_one = alignments.get("fists_within_one_hand", False)
29
+ within_1_5 = alignments.get("fists_within_1_5_hand", False)
30
+
31
+ if within_one:
32
+ score = 3
33
+ rationale = f"Fists within one hand-length (normalized distance {norm_dist:.2f})"
34
+ elif within_1_5:
35
+ score = 2
36
+ rationale = f"Fists within 1.5 hand-lengths (normalized distance {norm_dist:.2f})"
37
+ else:
38
+ score = 1
39
+ rationale = f"Fists beyond 1.5 hand-lengths apart (normalized distance {norm_dist:.2f})"
40
+
41
+ confidence = features.confidence * 0.9
42
+
43
+ return ScoreResult(
44
+ score=score, rationale=rationale,
45
+ confidence=confidence, notes="",
46
+ )
formscout/rubric/trunk_stability_pushup.py CHANGED
@@ -1,55 +1,55 @@
1
- """
2
- Trunk Stability Push-Up rubric scorer — pure function, no model calls.
3
-
4
- FMS Trunk Stability Push-Up Criteria:
5
- - Score 3: body moves as one unit (rigid) with hands at forehead level (men)
6
- or chin level (women). No sag or segment lag.
7
- - Score 2: body moves as one unit but with hands at chin (men) or clavicle (women).
8
- - Score 1: unable to perform with hands lowered; body sags or segments.
9
- - Score 0: PAIN (spinal extension clearing test) — never auto-scored.
10
- """
11
- from __future__ import annotations
12
-
13
- from formscout.types import BiomechFeatures, ScoreResult
14
-
15
-
16
- def score_trunk_stability_pushup(features: BiomechFeatures) -> ScoreResult:
17
- """Pure rubric scorer for trunk stability push-up."""
18
- angles = features.angles
19
- alignments = features.alignments
20
-
21
- has_data = "max_sag_px" in angles
22
- if not has_data:
23
- return ScoreResult(
24
- score=1, rationale="Insufficient data: trunk rigidity not measurable",
25
- confidence=0.3, notes="missing key measurements",
26
- )
27
-
28
- body_rigid = alignments.get("body_rigid", False)
29
- no_sag = alignments.get("no_sag", False)
30
- hands_high = alignments.get("hands_at_forehead", False)
31
-
32
- rationale_parts = []
33
-
34
- if body_rigid and hands_high:
35
- score = 3
36
- rationale_parts.append("Body rigid as one unit, hands at forehead position")
37
- elif body_rigid or no_sag:
38
- score = 2
39
- if not hands_high:
40
- rationale_parts.append("rigid body but hands in lower position")
41
- else:
42
- rationale_parts.append("minor trunk variance detected")
43
- rationale_parts.insert(0, "Completed with regression")
44
- else:
45
- score = 1
46
- sag = angles.get("max_sag_px", 0)
47
- variance = angles.get("trunk_variance_px", 0)
48
- rationale_parts.append(f"Body sag detected ({sag:.0f}px), variance {variance:.1f}px")
49
-
50
- confidence = features.confidence * 0.8
51
-
52
- return ScoreResult(
53
- score=score, rationale="; ".join(rationale_parts),
54
- confidence=confidence, notes="",
55
- )
 
1
+ """
2
+ Trunk Stability Push-Up rubric scorer — pure function, no model calls.
3
+
4
+ FMS Trunk Stability Push-Up Criteria:
5
+ - Score 3: body moves as one unit (rigid) with hands at forehead level (men)
6
+ or chin level (women). No sag or segment lag.
7
+ - Score 2: body moves as one unit but with hands at chin (men) or clavicle (women).
8
+ - Score 1: unable to perform with hands lowered; body sags or segments.
9
+ - Score 0: PAIN (spinal extension clearing test) — never auto-scored.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ from formscout.types import BiomechFeatures, ScoreResult
14
+
15
+
16
+ def score_trunk_stability_pushup(features: BiomechFeatures) -> ScoreResult:
17
+ """Pure rubric scorer for trunk stability push-up."""
18
+ angles = features.angles
19
+ alignments = features.alignments
20
+
21
+ has_data = "max_sag_px" in angles
22
+ if not has_data:
23
+ return ScoreResult(
24
+ score=1, rationale="Insufficient data: trunk rigidity not measurable",
25
+ confidence=0.3, notes="missing key measurements",
26
+ )
27
+
28
+ body_rigid = alignments.get("body_rigid", False)
29
+ no_sag = alignments.get("no_sag", False)
30
+ hands_high = alignments.get("hands_at_forehead", False)
31
+
32
+ rationale_parts = []
33
+
34
+ if body_rigid and hands_high:
35
+ score = 3
36
+ rationale_parts.append("Body rigid as one unit, hands at forehead position")
37
+ elif body_rigid or no_sag:
38
+ score = 2
39
+ if not hands_high:
40
+ rationale_parts.append("rigid body but hands in lower position")
41
+ else:
42
+ rationale_parts.append("minor trunk variance detected")
43
+ rationale_parts.insert(0, "Completed with regression")
44
+ else:
45
+ score = 1
46
+ sag = angles.get("max_sag_px", 0)
47
+ variance = angles.get("trunk_variance_px", 0)
48
+ rationale_parts.append(f"Body sag detected ({sag:.0f}px), variance {variance:.1f}px")
49
+
50
+ confidence = features.confidence * 0.8
51
+
52
+ return ScoreResult(
53
+ score=score, rationale="; ".join(rationale_parts),
54
+ confidence=confidence, notes="",
55
+ )
formscout/serving/__init__.py CHANGED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Serving backends for the FormScout judge/classifier VLM."""
2
+ from __future__ import annotations
3
+
4
+
5
+ def get_vlm_client():
6
+ """Return the VLM client for the resolved judge backend.
7
+
8
+ transformers (in-process, ZeroGPU) on a Space; llama-server locally. Falls
9
+ back to the llama.cpp client if the transformers backend can't be imported.
10
+ """
11
+ from formscout import config
12
+
13
+ if config.resolve_judge_backend() == "transformers":
14
+ try:
15
+ from formscout.serving.transformers_vlm import TransformersVLMClient
16
+ return TransformersVLMClient()
17
+ except Exception:
18
+ pass
19
+ from formscout.serving.llama_cpp import LlamaCppClient
20
+ return LlamaCppClient(port=config.LLAMA_CPP_PORT_VLM)
formscout/serving/llama_cpp.py CHANGED
@@ -1,174 +1,148 @@
1
- """
2
- llama.cpp HTTP client wrapper for FormScout.
3
-
4
- Wraps the llama.cpp server's /completion and /embedding endpoints.
5
- Falls back gracefully when the server is unavailable.
6
-
7
- Model: Qwen3-VL-8B-Instruct (Q4_K_M GGUF) for VLM inference.
8
- Model: Qwen3-VL-Embedding-8B (Q4_K_M GGUF) for embeddings.
9
- Params: 8B each (shared backbone).
10
- License: Apache-2.0.
11
- """
12
- from __future__ import annotations
13
-
14
- import base64
15
- import json
16
- import logging
17
- from pathlib import Path
18
- from typing import Any
19
-
20
- import requests
21
-
22
- from formscout import config
23
-
24
- logger = logging.getLogger(__name__)
25
-
26
- _TIMEOUT = 120 # seconds — VLM can be slow
27
-
28
-
29
- class LlamaCppClient:
30
- """HTTP client for a llama.cpp server instance."""
31
-
32
- def __init__(self, host: str | None = None, port: int | None = None):
33
- self.host = host or config.LLAMA_CPP_HOST
34
- self.port = port or config.LLAMA_CPP_PORT_VLM
35
- self.base_url = f"http://{self.host}:{self.port}"
36
-
37
- @property
38
- def available(self) -> bool:
39
- """Check if the server is reachable."""
40
- try:
41
- r = requests.get(f"{self.base_url}/health", timeout=5)
42
- return r.status_code == 200
43
- except (requests.ConnectionError, requests.Timeout):
44
- return False
45
-
46
- def complete(
47
- self,
48
- prompt: str,
49
- images: list[str] | None = None,
50
- max_tokens: int = 512,
51
- temperature: float = 0.1,
52
- stop: list[str] | None = None,
53
- ) -> dict[str, Any]:
54
- """
55
- Send a chat-completion request (OpenAI-compatible /v1/chat/completions —
56
- required for multimodal: llama-server routes images through the mmproj
57
- only on this endpoint). Returns parsed JSON if the response is JSON,
58
- otherwise returns {"text": raw_text}.
59
-
60
- Args:
61
- prompt: The text prompt (system + user combined).
62
- images: Optional list of base64-encoded JPEGs or file paths.
63
- max_tokens: Max generation tokens.
64
- temperature: Sampling temperature.
65
- stop: Stop sequences (default: none — JSON output must not be truncated).
66
- """
67
- content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
68
- for img in images or []:
69
- if len(img) < 4096 and Path(img).exists():
70
- with open(img, "rb") as f:
71
- b64 = base64.b64encode(f.read()).decode()
72
- else:
73
- b64 = img # already base64
74
- content.append({
75
- "type": "image_url",
76
- "image_url": {"url": f"data:image/jpeg;base64,{b64}"},
77
- })
78
-
79
- payload: dict[str, Any] = {
80
- "model": config.LLAMA_CPP_MODEL,
81
- "messages": [{"role": "user", "content": content}],
82
- "max_tokens": max_tokens,
83
- "temperature": temperature,
84
- }
85
- if stop:
86
- payload["stop"] = stop
87
-
88
- result = self._post(payload)
89
- if "error" in result and images:
90
- # Multimodal failed — retry text-only so scoring still proceeds.
91
- logger.warning("Multimodal request failed (%s), retrying text-only", result["error"])
92
- text_payload = {
93
- "model": config.LLAMA_CPP_MODEL,
94
- "messages": [{"role": "user", "content": prompt}],
95
- "max_tokens": max_tokens,
96
- "temperature": temperature,
97
- }
98
- if stop:
99
- text_payload["stop"] = stop
100
- result = self._post(text_payload)
101
- return result
102
-
103
- def _post(self, payload: dict[str, Any]) -> dict[str, Any]:
104
- """POST a chat-completion payload, surfacing the response body on errors."""
105
- try:
106
- r = requests.post(
107
- f"{self.base_url}/v1/chat/completions",
108
- json=payload,
109
- timeout=_TIMEOUT,
110
- )
111
- if not r.ok:
112
- # Capture the server's explanation (e.g. "Invalid image ...")
113
- body = ""
114
- try:
115
- body = r.text[:500]
116
- except Exception:
117
- pass
118
- logger.warning("llama-server %s: %s", r.status_code, body)
119
- return {"error": f"HTTP {r.status_code}: {body}", "text": ""}
120
- result = r.json()
121
- text = result["choices"][0]["message"]["content"] or ""
122
- return self._parse_json_reply(text)
123
- except requests.ConnectionError:
124
- return {"error": "llama.cpp server not available", "text": ""}
125
- except requests.Timeout:
126
- return {"error": "llama.cpp server timeout", "text": ""}
127
- except Exception as e:
128
- return {"error": str(e), "text": ""}
129
-
130
- @staticmethod
131
- def _parse_json_reply(text: str) -> dict[str, Any]:
132
- """Parse model output as JSON, tolerating markdown fences."""
133
- stripped = text.strip()
134
- if stripped.startswith("```"):
135
- stripped = stripped.split("\n", 1)[-1]
136
- stripped = stripped.rsplit("```", 1)[0].strip()
137
- try:
138
- parsed = json.loads(stripped)
139
- if isinstance(parsed, dict):
140
- return parsed
141
- except (json.JSONDecodeError, TypeError):
142
- pass
143
- return {"text": text}
144
-
145
-
146
- class EmbeddingClient:
147
- """HTTP client for the llama.cpp embedding server."""
148
-
149
- def __init__(self, host: str | None = None, port: int | None = None):
150
- self.host = host or config.LLAMA_CPP_HOST
151
- self.port = port or config.LLAMA_CPP_PORT_EMBED
152
- self.base_url = f"http://{self.host}:{self.port}"
153
-
154
- @property
155
- def available(self) -> bool:
156
- try:
157
- r = requests.get(f"{self.base_url}/health", timeout=5)
158
- return r.status_code == 200
159
- except (requests.ConnectionError, requests.Timeout):
160
- return False
161
-
162
- def embed(self, text: str) -> list[float] | None:
163
- """Get embedding vector for text. Returns None on failure."""
164
- try:
165
- r = requests.post(
166
- f"{self.base_url}/embedding",
167
- json={"content": text},
168
- timeout=30,
169
- )
170
- r.raise_for_status()
171
- data = r.json()
172
- return data.get("embedding")
173
- except Exception:
174
- return None
 
1
+ """
2
+ llama.cpp HTTP client wrapper for FormScout.
3
+
4
+ Wraps the llama.cpp server's /completion and /embedding endpoints.
5
+ Falls back gracefully when the server is unavailable.
6
+
7
+ Model: Qwen3-VL-8B-Instruct (Q4_K_M GGUF) for VLM inference.
8
+ Model: Qwen3-VL-Embedding-8B (Q4_K_M GGUF) for embeddings.
9
+ Params: 8B each (shared backbone).
10
+ License: Apache-2.0.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import base64
15
+ import json
16
+ import logging
17
+ from pathlib import Path
18
+ from typing import Any
19
+
20
+ import requests
21
+
22
+ from formscout import config
23
+
24
+ logger = logging.getLogger(__name__)
25
+
26
+ _TIMEOUT = 120 # seconds — VLM can be slow
27
+
28
+
29
+ class LlamaCppClient:
30
+ """HTTP client for a llama.cpp server instance."""
31
+
32
+ def __init__(self, host: str | None = None, port: int | None = None):
33
+ self.host = host or config.LLAMA_CPP_HOST
34
+ self.port = port or config.LLAMA_CPP_PORT_VLM
35
+ self.base_url = f"http://{self.host}:{self.port}"
36
+
37
+ @property
38
+ def available(self) -> bool:
39
+ """Check if the server is reachable."""
40
+ try:
41
+ r = requests.get(f"{self.base_url}/health", timeout=5)
42
+ return r.status_code == 200
43
+ except (requests.ConnectionError, requests.Timeout):
44
+ return False
45
+
46
+ def complete(
47
+ self,
48
+ prompt: str,
49
+ images: list[str] | None = None,
50
+ max_tokens: int = 512,
51
+ temperature: float = 0.1,
52
+ stop: list[str] | None = None,
53
+ ) -> dict[str, Any]:
54
+ """
55
+ Send a chat-completion request (OpenAI-compatible /v1/chat/completions —
56
+ required for multimodal: llama-server routes images through the mmproj
57
+ only on this endpoint). Returns parsed JSON if the response is JSON,
58
+ otherwise returns {"text": raw_text}.
59
+
60
+ Args:
61
+ prompt: The text prompt (system + user combined).
62
+ images: Optional list of base64-encoded JPEGs or file paths.
63
+ max_tokens: Max generation tokens.
64
+ temperature: Sampling temperature.
65
+ stop: Stop sequences (default: none — JSON output must not be truncated).
66
+ """
67
+ content: list[dict[str, Any]] = [{"type": "text", "text": prompt}]
68
+ for img in images or []:
69
+ if len(img) < 4096 and Path(img).exists():
70
+ with open(img, "rb") as f:
71
+ b64 = base64.b64encode(f.read()).decode()
72
+ else:
73
+ b64 = img # already base64
74
+ content.append({
75
+ "type": "image_url",
76
+ "image_url": {"url": f"data:image/jpeg;base64,{b64}"},
77
+ })
78
+
79
+ payload: dict[str, Any] = {
80
+ "messages": [{"role": "user", "content": content}],
81
+ "max_tokens": max_tokens,
82
+ "temperature": temperature,
83
+ }
84
+ if stop:
85
+ payload["stop"] = stop
86
+
87
+ try:
88
+ r = requests.post(
89
+ f"{self.base_url}/v1/chat/completions",
90
+ json=payload,
91
+ timeout=_TIMEOUT,
92
+ )
93
+ r.raise_for_status()
94
+ result = r.json()
95
+ text = result["choices"][0]["message"]["content"] or ""
96
+ return self._parse_json_reply(text)
97
+ except requests.ConnectionError:
98
+ return {"error": "llama.cpp server not available", "text": ""}
99
+ except requests.Timeout:
100
+ return {"error": "llama.cpp server timeout", "text": ""}
101
+ except Exception as e:
102
+ return {"error": str(e), "text": ""}
103
+
104
+ @staticmethod
105
+ def _parse_json_reply(text: str) -> dict[str, Any]:
106
+ """Parse model output as JSON, tolerating markdown fences."""
107
+ stripped = text.strip()
108
+ if stripped.startswith("```"):
109
+ stripped = stripped.split("\n", 1)[-1]
110
+ stripped = stripped.rsplit("```", 1)[0].strip()
111
+ try:
112
+ parsed = json.loads(stripped)
113
+ if isinstance(parsed, dict):
114
+ return parsed
115
+ except (json.JSONDecodeError, TypeError):
116
+ pass
117
+ return {"text": text}
118
+
119
+
120
+ class EmbeddingClient:
121
+ """HTTP client for the llama.cpp embedding server."""
122
+
123
+ def __init__(self, host: str | None = None, port: int | None = None):
124
+ self.host = host or config.LLAMA_CPP_HOST
125
+ self.port = port or config.LLAMA_CPP_PORT_EMBED
126
+ self.base_url = f"http://{self.host}:{self.port}"
127
+
128
+ @property
129
+ def available(self) -> bool:
130
+ try:
131
+ r = requests.get(f"{self.base_url}/health", timeout=5)
132
+ return r.status_code == 200
133
+ except (requests.ConnectionError, requests.Timeout):
134
+ return False
135
+
136
+ def embed(self, text: str) -> list[float] | None:
137
+ """Get embedding vector for text. Returns None on failure."""
138
+ try:
139
+ r = requests.post(
140
+ f"{self.base_url}/embedding",
141
+ json={"content": text},
142
+ timeout=30,
143
+ )
144
+ r.raise_for_status()
145
+ data = r.json()
146
+ return data.get("embedding")
147
+ except Exception:
148
+ return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
formscout/serving/transformers_vlm.py ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ In-process Qwen3-VL backend via transformers — the HF Spaces (ZeroGPU) path.
3
+
4
+ Mirrors LlamaCppClient's interface (`available` + `complete(...) -> dict`) so it
5
+ is a drop-in alternative selected by config.resolve_judge_backend(). Inference is
6
+ wrapped in spaces.GPU when the `spaces` package is present (ZeroGPU); otherwise it
7
+ runs on whatever device transformers picks. The model is loaded lazily on first
8
+ use and cached for the process.
9
+
10
+ Any load/inference failure returns {"error": ..., "fallback": True} so JudgeAgent
11
+ falls back to the deterministic rubric score instead of flagging needs_human.
12
+
13
+ Model: Qwen3-VL-8B-Instruct (transformers weights, ~16 GB). License: Apache-2.0.
14
+
15
+ NOTE: requires validation on actual ZeroGPU hardware — it cannot be exercised in
16
+ the CPU test environment (would download 16 GB).
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import base64
21
+ import importlib.util
22
+ import logging
23
+
24
+ from formscout import config
25
+ from formscout.serving.llama_cpp import LlamaCppClient
26
+
27
+ logger = logging.getLogger(__name__)
28
+
29
+ # Module-level model cache (loaded once per process).
30
+ _CACHE: dict = {}
31
+
32
+ # spaces.GPU decorator when on HF infra; a no-op decorator otherwise.
33
+ try: # pragma: no cover - depends on runtime env
34
+ import spaces
35
+
36
+ _gpu = spaces.GPU(duration=120)
37
+ except Exception: # pragma: no cover
38
+ def _gpu(fn):
39
+ return fn
40
+
41
+
42
+ @_gpu
43
+ def _generate(model_id: str, prompt: str, pil_images: list, max_tokens: int,
44
+ temperature: float) -> str: # pragma: no cover - needs GPU + model
45
+ """Load (cached) and run Qwen3-VL; returns the raw decoded string."""
46
+ import torch
47
+ from transformers import AutoModelForImageTextToText, AutoProcessor
48
+
49
+ if "model" not in _CACHE:
50
+ _CACHE["processor"] = AutoProcessor.from_pretrained(model_id)
51
+ _CACHE["model"] = AutoModelForImageTextToText.from_pretrained(
52
+ model_id, torch_dtype="auto", device_map="auto",
53
+ )
54
+ processor, model = _CACHE["processor"], _CACHE["model"]
55
+
56
+ content = [{"type": "image", "image": im} for im in pil_images]
57
+ content.append({"type": "text", "text": prompt})
58
+ messages = [{"role": "user", "content": content}]
59
+
60
+ inputs = processor.apply_chat_template(
61
+ messages, tokenize=True, add_generation_prompt=True,
62
+ return_tensors="pt", return_dict=True,
63
+ ).to(model.device)
64
+
65
+ with torch.no_grad():
66
+ out = model.generate(
67
+ **inputs, max_new_tokens=max_tokens,
68
+ do_sample=temperature > 0, temperature=max(temperature, 1e-2),
69
+ )
70
+ gen = out[:, inputs["input_ids"].shape[1]:]
71
+ return processor.batch_decode(gen, skip_special_tokens=True)[0]
72
+
73
+
74
+ class TransformersVLMClient:
75
+ """In-process Qwen3-VL client (ZeroGPU on Spaces)."""
76
+
77
+ def __init__(self, model_id: str | None = None):
78
+ self.model_id = model_id or config.JUDGE_HF_MODEL
79
+ self._failed = False
80
+
81
+ @property
82
+ def available(self) -> bool:
83
+ """Cheap check — does NOT load the model (so tests stay download-free)."""
84
+ if self._failed:
85
+ return False
86
+ return importlib.util.find_spec("transformers") is not None
87
+
88
+ def complete(self, prompt: str, images: list[str] | None = None,
89
+ max_tokens: int = 512, temperature: float = 0.1,
90
+ stop: list[str] | None = None) -> dict:
91
+ try:
92
+ pil_images = self._decode_images(images)
93
+ text = _generate(self.model_id, prompt, pil_images, max_tokens, temperature)
94
+ return LlamaCppClient._parse_json_reply(text)
95
+ except Exception as e: # pragma: no cover - needs GPU + model
96
+ logger.warning("transformers VLM failed (%s) — falling back to rubric", e)
97
+ self._failed = True
98
+ return {"error": str(e), "fallback": True, "text": ""}
99
+
100
+ @staticmethod
101
+ def _decode_images(images: list[str] | None) -> list:
102
+ """Decode base64 JPEGs (as the JudgeAgent encodes them) into PIL images."""
103
+ if not images:
104
+ return []
105
+ import io
106
+
107
+ from PIL import Image
108
+
109
+ out = []
110
+ for img in images:
111
+ try:
112
+ raw = base64.b64decode(img)
113
+ out.append(Image.open(io.BytesIO(raw)).convert("RGB"))
114
+ except Exception:
115
+ continue
116
+ return out
formscout/session.py CHANGED
@@ -1,194 +1,283 @@
1
- """
2
- Screening-session accumulator.
3
-
4
- Accumulates one SessionEntry per analyzed clip, persists each to a temp session
5
- dir (session.json + analysis.md + key-frame PNGs), and on finish builds a
6
- ReportResult (via ReportAgent) + a PDF (via PdfReportAgent).
7
-
8
- Pure orchestration — no Gradio imports. Disk writes tolerate failure with a
9
- logged warning and never block scoring.
10
- """
11
- from __future__ import annotations
12
-
13
- import json
14
- import logging
15
- import os
16
- import tempfile
17
- import uuid
18
- from dataclasses import dataclass, replace
19
-
20
- from formscout.rubric import score_test
21
- from formscout.types import MovementResult, ReportResult, SessionEntry
22
-
23
- logger = logging.getLogger(__name__)
24
-
25
- # Maps each test to the BiomechFeatures.timing key holding its governing frame.
26
- TIMING_KEY = {
27
- "deep_squat": "deepest_frame",
28
- "hurdle_step": "peak_step_frame",
29
- "inline_lunge": "deepest_lunge_frame",
30
- "shoulder_mobility": "measure_frame",
31
- "active_slr": "peak_raise_frame",
32
- "trunk_stability_pushup": "max_sag_frame",
33
- "rotary_stability": "peak_extension_frame",
34
- }
35
-
36
-
37
- @dataclass
38
- class Session:
39
- """Mutable session: an id, its temp dir, and accumulated entries."""
40
- session_id: str
41
- session_dir: str
42
- entries: list # list[SessionEntry]
43
-
44
-
45
- def new_session() -> Session:
46
- sid = uuid.uuid4().hex[:12]
47
- base = os.path.join(tempfile.gettempdir(), "formscout_sessions", sid)
48
- try:
49
- os.makedirs(os.path.join(base, "keyframes"), exist_ok=True)
50
- except Exception as e:
51
- logger.warning("session dir create failed: %s", e)
52
- return Session(session_id=sid, session_dir=base, entries=[])
53
-
54
-
55
- def governing_frame_index(features) -> int | None:
56
- """Return the governing frame index for this test, or None."""
57
- key = TIMING_KEY.get(features.test_name)
58
- if key is None:
59
- return None
60
- idx = features.timing.get(key)
61
- return int(idx) if isinstance(idx, (int, float)) else None
62
-
63
-
64
- def worst_compensation_caption(judge, features) -> str:
65
- """Short caption naming the worst compensation for the key-frame still."""
66
- if judge and getattr(judge, "compensation_tags", None):
67
- return ", ".join(judge.compensation_tags)
68
- failed = [k.replace("_", " ") for k, v in features.alignments.items() if v is False]
69
- return ("compensation: " + ", ".join(failed)) if failed else "key position"
70
-
71
-
72
- def add_analysis(session, *, ingest, pose2d, features, judge, test_name, side,
73
- draw_trails: bool = False) -> SessionEntry:
74
- """Build a SessionEntry from a completed analysis, render its key-frame,
75
- persist the session, append, and return the entry."""
76
- movement = MovementResult(test_name=test_name, side=side, confidence=1.0)
77
- rubric = score_test(features)
78
-
79
- needs_human = bool((judge and judge.needs_human) or rubric.needs_human)
80
- if needs_human:
81
- score = None
82
- elif judge and judge.score is not None:
83
- score = judge.score
84
- else:
85
- score = rubric.score
86
-
87
- keyframe_path = None
88
- idx = governing_frame_index(features)
89
- if idx is not None and 0 <= idx < len(pose2d.keypoints):
90
- from formscout.agents.visualizer import PoseVisualizer
91
- caption = (f"{test_name.replace('_', ' ').title()} "
92
- f"({side}) — {worst_compensation_caption(judge, features)}")
93
- layers = {"skeleton", "trails"} if draw_trails else {"skeleton"}
94
- out_png = os.path.join(session.session_dir, "keyframes", f"{test_name}_{side}.png")
95
- try:
96
- keyframe_path = PoseVisualizer().render_frame(ingest, pose2d, idx, layers, caption, out_png)
97
- except Exception as e:
98
- logger.warning("keyframe render failed: %s", e)
99
-
100
- measurements = {}
101
- measurements.update(features.angles)
102
- measurements.update(features.alignments)
103
-
104
- entry = SessionEntry(
105
- test_name=test_name, side=side, score=score, needs_human=needs_human,
106
- rationale=(judge.rationale if judge else rubric.rationale),
107
- compensation_tags=list(judge.compensation_tags) if judge else [],
108
- corrective_hint=(judge.corrective_hint if judge else ""),
109
- measurements=measurements,
110
- confidence=(judge.confidence if judge else rubric.confidence),
111
- view=features.view,
112
- keyframe_path=keyframe_path,
113
- movement=movement, features=features, rubric_score=rubric, judge=judge,
114
- )
115
- session.entries.append(entry)
116
- _persist(session)
117
- return entry
118
-
119
-
120
- def finish_session(session) -> tuple[ReportResult | None, str | None]:
121
- """Build the composite report + PDF. Returns (report, pdf_path).
122
- Returns (None, None) for an empty session."""
123
- if not session.entries:
124
- return None, None
125
-
126
- from formscout.agents.report import ReportAgent
127
- report_inputs = [{
128
- "movement": e.movement, "features": e.features,
129
- "rubric_score": e.rubric_score, "judge": e.judge, "side": e.side,
130
- } for e in session.entries]
131
- report = ReportAgent().run(report_inputs)
132
-
133
- pdf_path = None
134
- try:
135
- from formscout.agents.pdf_report import PdfReportAgent
136
- pdf_path = PdfReportAgent().run(report, session.entries, session.session_dir)
137
- except Exception as e:
138
- logger.warning("pdf generation failed: %s", e)
139
-
140
- report = replace(report, pdf_path=pdf_path)
141
- return report, pdf_path
142
-
143
-
144
- # ── Persistence ───────────────────────────────────────────────────────────────
145
-
146
- def _jsonable(d: dict) -> dict:
147
- out = {}
148
- for k, v in d.items():
149
- if isinstance(v, float):
150
- out[k] = round(v, 2)
151
- elif isinstance(v, (int, str, bool)) or v is None:
152
- out[k] = v
153
- else:
154
- out[k] = str(v)
155
- return out
156
-
157
-
158
- def _entry_display(e: SessionEntry) -> dict:
159
- return {
160
- "test_name": e.test_name, "side": e.side, "score": e.score,
161
- "needs_human": e.needs_human, "rationale": e.rationale,
162
- "compensation_tags": list(e.compensation_tags), "corrective_hint": e.corrective_hint,
163
- "measurements": _jsonable(e.measurements), "confidence": round(e.confidence, 2),
164
- "view": e.view, "keyframe_path": e.keyframe_path,
165
- }
166
-
167
-
168
- def _render_markdown(session: Session) -> str:
169
- lines = ["# FormScout — Session Log", ""]
170
- for e in session.entries:
171
- title = e.test_name.replace("_", " ").title()
172
- if e.side in ("left", "right"):
173
- title += f" ({e.side})"
174
- score = "Clinician review required" if e.needs_human else f"{e.score}/3"
175
- lines.append(f"## {title} — {score}")
176
- lines.append(e.rationale or "")
177
- if e.compensation_tags:
178
- lines.append(f"- Compensations: {', '.join(e.compensation_tags)}")
179
- if e.corrective_hint:
180
- lines.append(f"- Corrective: {e.corrective_hint}")
181
- if e.keyframe_path:
182
- lines.append(f"- Key frame: `{e.keyframe_path}`")
183
- lines.append("")
184
- return "\n".join(lines)
185
-
186
-
187
- def _persist(session: Session) -> None:
188
- try:
189
- with open(os.path.join(session.session_dir, "session.json"), "w") as f:
190
- json.dump([_entry_display(e) for e in session.entries], f, indent=2)
191
- with open(os.path.join(session.session_dir, "analysis.md"), "w") as f:
192
- f.write(_render_markdown(session))
193
- except Exception as e:
194
- logger.warning("session persist failed: %s", e)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Screening-session accumulator.
3
+
4
+ Accumulates one SessionEntry per analyzed clip, persists each to a temp session
5
+ dir (session.json + analysis.md + key-frame PNGs), and on finish builds a
6
+ ReportResult (via ReportAgent) + a PDF (via PdfReportAgent).
7
+
8
+ Pure orchestration — no Gradio imports. Disk writes tolerate failure with a
9
+ logged warning and never block scoring.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import json
14
+ import logging
15
+ import os
16
+ import tempfile
17
+ import uuid
18
+ from dataclasses import dataclass, replace
19
+
20
+ from formscout.rubric import score_test
21
+ from formscout.types import MovementResult, ReportResult, SessionEntry
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+ # Maps each test to the BiomechFeatures.timing key holding its governing frame.
26
+ TIMING_KEY = {
27
+ "deep_squat": "deepest_frame",
28
+ "hurdle_step": "peak_step_frame",
29
+ "inline_lunge": "deepest_lunge_frame",
30
+ "shoulder_mobility": "measure_frame",
31
+ "active_slr": "peak_raise_frame",
32
+ "trunk_stability_pushup": "max_sag_frame",
33
+ "rotary_stability": "peak_extension_frame",
34
+ }
35
+
36
+
37
+ @dataclass
38
+ class Session:
39
+ """Mutable session: an id, its temp dir, and accumulated entries."""
40
+ session_id: str
41
+ session_dir: str
42
+ entries: list # list[SessionEntry]
43
+
44
+
45
+ def new_session() -> Session:
46
+ sid = uuid.uuid4().hex[:12]
47
+ base = os.path.join(tempfile.gettempdir(), "formscout_sessions", sid)
48
+ try:
49
+ os.makedirs(os.path.join(base, "keyframes"), exist_ok=True)
50
+ except Exception as e:
51
+ logger.warning("session dir create failed: %s", e)
52
+ return Session(session_id=sid, session_dir=base, entries=[])
53
+
54
+
55
+ def governing_frame_index(features) -> int | None:
56
+ """Return the governing frame index for this test, or None."""
57
+ key = TIMING_KEY.get(features.test_name)
58
+ if key is None:
59
+ return None
60
+ idx = features.timing.get(key)
61
+ return int(idx) if isinstance(idx, (int, float)) else None
62
+
63
+
64
+ def worst_compensation_caption(judge, features) -> str:
65
+ """Short caption naming the worst compensation for the key-frame still."""
66
+ if judge and getattr(judge, "compensation_tags", None):
67
+ return ", ".join(judge.compensation_tags)
68
+ failed = [k.replace("_", " ") for k, v in features.alignments.items() if v is False]
69
+ return ("compensation: " + ", ".join(failed)) if failed else "key position"
70
+
71
+
72
+ def add_analysis(session, *, ingest, pose2d, features, judge, test_name, side,
73
+ draw_trails: bool = False) -> SessionEntry:
74
+ """Build a SessionEntry from a completed analysis, render its key-frame,
75
+ persist the session, append, and return the entry."""
76
+ movement = MovementResult(test_name=test_name, side=side, confidence=1.0)
77
+ rubric = score_test(features)
78
+
79
+ needs_human = bool((judge and judge.needs_human) or rubric.needs_human)
80
+ if needs_human:
81
+ score = None
82
+ elif judge and judge.score is not None:
83
+ score = judge.score
84
+ else:
85
+ score = rubric.score
86
+
87
+ keyframe_path = None
88
+ idx = governing_frame_index(features)
89
+ if idx is not None and 0 <= idx < len(pose2d.keypoints):
90
+ from formscout.agents.visualizer import PoseVisualizer
91
+ caption = (f"{test_name.replace('_', ' ').title()} "
92
+ f"({side}) — {worst_compensation_caption(judge, features)}")
93
+ layers = {"skeleton", "trails"} if draw_trails else {"skeleton"}
94
+ out_png = os.path.join(session.session_dir, "keyframes", f"{test_name}_{side}.png")
95
+ try:
96
+ keyframe_path = PoseVisualizer().render_frame(ingest, pose2d, idx, layers, caption, out_png)
97
+ except Exception as e:
98
+ logger.warning("keyframe render failed: %s", e)
99
+
100
+ measurements = {}
101
+ measurements.update(features.angles)
102
+ measurements.update(features.alignments)
103
+
104
+ laban, flexion, chart_paths = _run_analysis(session, pose2d, test_name, side, idx)
105
+
106
+ entry = SessionEntry(
107
+ test_name=test_name, side=side, score=score, needs_human=needs_human,
108
+ rationale=(judge.rationale if judge else rubric.rationale),
109
+ compensation_tags=list(judge.compensation_tags) if judge else [],
110
+ corrective_hint=(judge.corrective_hint if judge else ""),
111
+ measurements=measurements,
112
+ confidence=(judge.confidence if judge else rubric.confidence),
113
+ view=features.view,
114
+ keyframe_path=keyframe_path,
115
+ movement=movement, features=features, rubric_score=rubric, judge=judge,
116
+ laban=laban, flexion=flexion, chart_paths=chart_paths,
117
+ )
118
+ session.entries.append(entry)
119
+ _persist(session)
120
+ return entry
121
+
122
+
123
+ def _run_analysis(session, pose2d, test_name, side, governing_idx):
124
+ """Compute Laban Effort, relevant-joint flexion, and per-test charts.
125
+
126
+ Returns (laban, flexion, chart_paths); any failure degrades to (None, None, {})
127
+ without blocking the entry."""
128
+ try:
129
+ from formscout.analysis import charts as C
130
+ from formscout.analysis.laban import compute_laban
131
+ from formscout.analysis.relevant_joints import primary_angle, relevant_joints
132
+ from formscout.analysis.timeseries import angle_series, relevant_flexion_at
133
+ except Exception as e:
134
+ logger.warning("analysis engine unavailable: %s", e)
135
+ return None, None, {}
136
+
137
+ try:
138
+ laban = compute_laban(pose2d, test_name, pose2d.fps)
139
+ gidx = governing_idx if governing_idx is not None else 0
140
+ flexion = relevant_flexion_at(pose2d, test_name, gidx)
141
+ series = angle_series(pose2d, test_name)
142
+
143
+ cdir = os.path.join(session.session_dir, "charts")
144
+ os.makedirs(cdir, exist_ok=True)
145
+ tag = f"{test_name}_{side}"
146
+ produced = {
147
+ "angle": C.angle_over_time(series, primary_angle(test_name), governing_idx,
148
+ os.path.join(cdir, f"{tag}_angle.png"),
149
+ title=f"{test_name.replace('_', ' ').title()} — angle over time"),
150
+ "velocity": C.velocity_profile(pose2d.keypoints, pose2d.fps, relevant_joints(test_name),
151
+ os.path.join(cdir, f"{tag}_vel.png")),
152
+ "radar": C.laban_radar(laban["effort"], os.path.join(cdir, f"{tag}_radar.png")),
153
+ "flexion": C.flexion_bars(flexion, os.path.join(cdir, f"{tag}_flex.png")),
154
+ }
155
+ chart_paths = {k: p for k, p in produced.items() if p}
156
+ return laban, flexion, chart_paths
157
+ except Exception as e:
158
+ logger.warning("analysis/charts failed: %s", e)
159
+ return None, None, {}
160
+
161
+
162
+ def finish_session(session) -> tuple[ReportResult | None, str | None]:
163
+ """Build the composite report + PDF. Returns (report, pdf_path).
164
+ Returns (None, None) for an empty session."""
165
+ if not session.entries:
166
+ return None, None
167
+
168
+ from formscout.agents.report import ReportAgent
169
+ report_inputs = [{
170
+ "movement": e.movement, "features": e.features,
171
+ "rubric_score": e.rubric_score, "judge": e.judge, "side": e.side,
172
+ } for e in session.entries]
173
+ report = ReportAgent().run(report_inputs)
174
+
175
+ pdf_path = None
176
+ try:
177
+ from formscout.agents.pdf_report import PdfReportAgent
178
+ pdf_path = PdfReportAgent().run(report, session.entries, session.session_dir)
179
+ except Exception as e:
180
+ logger.warning("pdf generation failed: %s", e)
181
+
182
+ report = replace(report, pdf_path=pdf_path)
183
+ return report, pdf_path
184
+
185
+
186
+ # ���─ Persistence ───────────────────────────────────────────────────────────────
187
+
188
+ def _jsonable(d: dict) -> dict:
189
+ out = {}
190
+ for k, v in d.items():
191
+ if isinstance(v, float):
192
+ out[k] = round(v, 2)
193
+ elif isinstance(v, (int, str, bool)) or v is None:
194
+ out[k] = v
195
+ else:
196
+ out[k] = str(v)
197
+ return out
198
+
199
+
200
+ def _entry_display(e: SessionEntry) -> dict:
201
+ return {
202
+ "test_name": e.test_name, "side": e.side, "score": e.score,
203
+ "needs_human": e.needs_human, "rationale": e.rationale,
204
+ "compensation_tags": list(e.compensation_tags), "corrective_hint": e.corrective_hint,
205
+ "measurements": _jsonable(e.measurements), "confidence": round(e.confidence, 2),
206
+ "view": e.view, "keyframe_path": e.keyframe_path,
207
+ "laban": e.laban, "flexion": _jsonable_flexion(e.flexion), "chart_paths": e.chart_paths,
208
+ }
209
+
210
+
211
+ def _jsonable_flexion(flexion: dict | None) -> dict:
212
+ if not flexion:
213
+ return {}
214
+ return {k: {"deg": round(v["deg"], 1), "openness": v["openness"]} for k, v in flexion.items()}
215
+
216
+
217
+ def _render_markdown(session: Session) -> str:
218
+ lines = ["# FormScout — Session Log", ""]
219
+ for e in session.entries:
220
+ title = e.test_name.replace("_", " ").title()
221
+ if e.side in ("left", "right"):
222
+ title += f" ({e.side})"
223
+ score = "Clinician review required" if e.needs_human else f"{e.score}/3"
224
+ lines.append(f"## {title} — {score}")
225
+ lines.append(f"*view: {e.view} · confidence: {e.confidence:.0%}*")
226
+ lines.append("")
227
+ lines.append(e.rationale or "")
228
+ if e.compensation_tags:
229
+ lines.append(f"- **Compensations:** {', '.join(e.compensation_tags)}")
230
+ if e.corrective_hint:
231
+ lines.append(f"- **Corrective:** {e.corrective_hint}")
232
+
233
+ # Relevant-joint flexion (degrees + open/closed)
234
+ if e.flexion:
235
+ lines.append("\n### Relevant joint flexion (key frame)")
236
+ lines.append("| Joint angle | Degrees | State |")
237
+ lines.append("|---|---|---|")
238
+ for name, v in e.flexion.items():
239
+ lines.append(f"| {name.replace('_', ' ')} | {v['deg']:.1f}° | {v['openness']} |")
240
+
241
+ # Laban Effort
242
+ if e.laban:
243
+ eff, lab = e.laban.get("effort", {}), e.laban.get("labels", {})
244
+ lines.append("\n### Laban Effort (kinematic estimate)")
245
+ lines.append("| Factor | Value | Quality |")
246
+ lines.append("|---|---|---|")
247
+ for k in ("space", "weight", "time", "flow"):
248
+ lines.append(f"| {k.title()} | {eff.get(k, 0):.2f} | {lab.get(k, '')} |")
249
+ if e.laban.get("body_emphasis"):
250
+ emph = ", ".join(f"{n} ({s})" for n, s in e.laban["body_emphasis"])
251
+ lines.append(f"\n- **Body emphasis:** {emph}")
252
+ if e.laban.get("leading_joint"):
253
+ lines.append(f"- **Leading joint:** {e.laban['leading_joint']}")
254
+ lines.append(f"- *{e.laban.get('notes', '')}*")
255
+
256
+ # Full measurement dump
257
+ if e.measurements:
258
+ lines.append("\n### All measurements")
259
+ lines.append("| Measurement | Value |")
260
+ lines.append("|---|---|")
261
+ for k, v in e.measurements.items():
262
+ val = f"{v:.2f}" if isinstance(v, float) else str(v)
263
+ lines.append(f"| {k.replace('_', ' ')} | {val} |")
264
+ if e.features and e.features.symmetry_delta is not None:
265
+ lines.append(f"\n- **L/R symmetry delta:** {e.features.symmetry_delta:.1f}")
266
+
267
+ # Artifacts
268
+ if e.keyframe_path:
269
+ lines.append(f"\n- Key frame: `{e.keyframe_path}`")
270
+ for kind, path in (e.chart_paths or {}).items():
271
+ lines.append(f"- Chart ({kind}): `{path}`")
272
+ lines.append("")
273
+ return "\n".join(lines)
274
+
275
+
276
+ def _persist(session: Session) -> None:
277
+ try:
278
+ with open(os.path.join(session.session_dir, "session.json"), "w") as f:
279
+ json.dump([_entry_display(e) for e in session.entries], f, indent=2)
280
+ with open(os.path.join(session.session_dir, "analysis.md"), "w") as f:
281
+ f.write(_render_markdown(session))
282
+ except Exception as e:
283
+ logger.warning("session persist failed: %s", e)
formscout/startup.py CHANGED
@@ -1,47 +1,47 @@
1
- """
2
- Checkpoint bootstrap — downloads missing model files from HF model repo on first run.
3
- Called once at app startup before build_app(); no-ops if files already present.
4
- """
5
- from __future__ import annotations
6
-
7
- import logging
8
- from pathlib import Path
9
-
10
- logger = logging.getLogger(__name__)
11
-
12
- CHECKPOINT_REPO = "silas-therapy/formscout-checkpoints"
13
- ROOT = Path(__file__).parent.parent
14
-
15
- _CHECKPOINTS = [
16
- "checkpoints/yolo26/yolo26n-pose.pt",
17
- "checkpoints/yolo26/yolo26s-pose.pt",
18
- "checkpoints/yolo26/yolo26m-pose.pt",
19
- "checkpoints/yolo26/yolo26l-pose.pt",
20
- "checkpoints/yolo26/yolo26x-pose.pt",
21
- "checkpoints/mediapipe/pose_landmarker_full.task",
22
- ]
23
-
24
-
25
- def ensure_checkpoints() -> None:
26
- """Download any missing checkpoints from silas-therapy/formscout-checkpoints."""
27
- try:
28
- from huggingface_hub import hf_hub_download
29
- except ImportError:
30
- logger.warning("huggingface_hub not installed — skipping checkpoint download")
31
- return
32
-
33
- for rel_path in _CHECKPOINTS:
34
- local = ROOT / rel_path
35
- if local.exists():
36
- continue
37
- logger.info("Downloading %s ...", rel_path)
38
- try:
39
- local.parent.mkdir(parents=True, exist_ok=True)
40
- hf_hub_download(
41
- repo_id=CHECKPOINT_REPO,
42
- filename=rel_path,
43
- local_dir=str(ROOT),
44
- )
45
- logger.info("Downloaded %s", rel_path)
46
- except Exception as e:
47
- logger.warning("Failed to download %s: %s", rel_path, e)
 
1
+ """
2
+ Checkpoint bootstrap — downloads missing model files from HF model repo on first run.
3
+ Called once at app startup before build_app(); no-ops if files already present.
4
+ """
5
+ from __future__ import annotations
6
+
7
+ import logging
8
+ from pathlib import Path
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+ CHECKPOINT_REPO = "silas-therapy/formscout-checkpoints"
13
+ ROOT = Path(__file__).parent.parent
14
+
15
+ _CHECKPOINTS = [
16
+ "checkpoints/yolo26/yolo26n-pose.pt",
17
+ "checkpoints/yolo26/yolo26s-pose.pt",
18
+ "checkpoints/yolo26/yolo26m-pose.pt",
19
+ "checkpoints/yolo26/yolo26l-pose.pt",
20
+ "checkpoints/yolo26/yolo26x-pose.pt",
21
+ "checkpoints/mediapipe/pose_landmarker_full.task",
22
+ ]
23
+
24
+
25
+ def ensure_checkpoints() -> None:
26
+ """Download any missing checkpoints from silas-therapy/formscout-checkpoints."""
27
+ try:
28
+ from huggingface_hub import hf_hub_download
29
+ except ImportError:
30
+ logger.warning("huggingface_hub not installed — skipping checkpoint download")
31
+ return
32
+
33
+ for rel_path in _CHECKPOINTS:
34
+ local = ROOT / rel_path
35
+ if local.exists():
36
+ continue
37
+ logger.info("Downloading %s ...", rel_path)
38
+ try:
39
+ local.parent.mkdir(parents=True, exist_ok=True)
40
+ hf_hub_download(
41
+ repo_id=CHECKPOINT_REPO,
42
+ filename=rel_path,
43
+ local_dir=str(ROOT),
44
+ )
45
+ logger.info("Downloaded %s", rel_path)
46
+ except Exception as e:
47
+ logger.warning("Failed to download %s: %s", rel_path, e)
formscout/types.py CHANGED
@@ -164,6 +164,9 @@ class SessionEntry:
164
  features: BiomechFeatures
165
  rubric_score: ScoreResult
166
  judge: JudgeResult | None
 
 
 
167
 
168
 
169
  @dataclass
 
164
  features: BiomechFeatures
165
  rubric_score: ScoreResult
166
  judge: JudgeResult | None
167
+ laban: dict | None = None # Laban Effort factors + labels + body emphasis
168
+ flexion: dict | None = None # relevant joint angles at key frame: {name: {deg, openness}}
169
+ chart_paths: dict | None = None # {"angle"|"velocity"|"radar"|"flexion": png path}
170
 
171
 
172
  @dataclass
formscout/ui/theme.py CHANGED
@@ -1,250 +1,272 @@
1
- """
2
- FormScout custom Gradio theme — scout/trail inspired.
3
- Earth tones, topographic accents, sturdy typography.
4
- """
5
- from __future__ import annotations
6
-
7
- import gradio as gr
8
-
9
-
10
- def formscout_theme() -> gr.Theme:
11
- """Create the FormScout scout/trail theme."""
12
- return gr.themes.Soft(
13
- primary_hue=gr.themes.colors.emerald,
14
- secondary_hue=gr.themes.colors.amber,
15
- neutral_hue=gr.themes.colors.stone,
16
- font=[
17
- gr.themes.GoogleFont("Inter"),
18
- "ui-sans-serif",
19
- "system-ui",
20
- "sans-serif",
21
- ],
22
- font_mono=[
23
- gr.themes.GoogleFont("JetBrains Mono"),
24
- "ui-monospace",
25
- "monospace",
26
- ],
27
- ).set(
28
- # Background
29
- body_background_fill="linear-gradient(135deg, #1a1a2e 0%, #16213e 50%, #0f3460 100%)",
30
- body_background_fill_dark="linear-gradient(135deg, #0d1117 0%, #161b22 50%, #1a2332 100%)",
31
- # Blocks
32
- block_background_fill="rgba(30, 41, 59, 0.85)",
33
- block_background_fill_dark="rgba(15, 23, 42, 0.9)",
34
- block_border_width="1px",
35
- block_border_color="rgba(100, 200, 150, 0.2)",
36
- block_shadow="0 4px 20px rgba(0, 0, 0, 0.3)",
37
- block_radius="12px",
38
- # Buttons
39
- button_primary_background_fill="linear-gradient(135deg, #059669 0%, #047857 100%)",
40
- button_primary_background_fill_hover="linear-gradient(135deg, #047857 0%, #065f46 100%)",
41
- button_primary_text_color="white",
42
- button_primary_border_color="rgba(5, 150, 105, 0.5)",
43
- button_secondary_background_fill="rgba(51, 65, 85, 0.8)",
44
- button_secondary_text_color="#e2e8f0",
45
- # Input
46
- input_background_fill="rgba(15, 23, 42, 0.8)",
47
- input_background_fill_dark="rgba(10, 15, 30, 0.9)",
48
- input_border_color="rgba(100, 200, 150, 0.3)",
49
- input_border_color_focus="rgba(5, 150, 105, 0.8)",
50
- # Text
51
- body_text_color="#e2e8f0",
52
- body_text_color_dark="#f1f5f9",
53
- block_title_text_color="#86efac",
54
- block_label_text_color="#94a3b8",
55
- # Spacing
56
- block_padding="16px",
57
- layout_gap="16px",
58
- )
59
-
60
-
61
- FORMSCOUT_CSS = """
62
- /* FormScout Scout/Trail Theme CSS */
63
-
64
- .gradio-container {
65
- max-width: 1400px !important;
66
- margin: 0 auto;
67
- }
68
-
69
- /* Header styling */
70
- .formscout-header {
71
- text-align: center;
72
- padding: 20px 0;
73
- border-bottom: 2px solid rgba(100, 200, 150, 0.3);
74
- margin-bottom: 20px;
75
- }
76
-
77
- .formscout-header h1 {
78
- font-size: 2.2em;
79
- background: linear-gradient(135deg, #86efac, #059669);
80
- -webkit-background-clip: text;
81
- -webkit-text-fill-color: transparent;
82
- background-clip: text;
83
- margin-bottom: 8px;
84
- }
85
-
86
- /* Safety banner */
87
- .safety-banner {
88
- background: linear-gradient(90deg, rgba(245, 158, 11, 0.15), rgba(245, 158, 11, 0.05));
89
- border: 1px solid rgba(245, 158, 11, 0.4);
90
- border-radius: 8px;
91
- padding: 12px 16px;
92
- margin: 12px 0;
93
- font-size: 0.9em;
94
- text-align: center;
95
- color: #fbbf24;
96
- }
97
-
98
- /* Score display */
99
- .score-card {
100
- background: rgba(5, 150, 105, 0.1);
101
- border: 2px solid rgba(5, 150, 105, 0.4);
102
- border-radius: 16px;
103
- padding: 24px;
104
- text-align: center;
105
- }
106
-
107
- .score-value {
108
- font-size: 4em;
109
- font-weight: 800;
110
- background: linear-gradient(135deg, #86efac, #059669);
111
- -webkit-background-clip: text;
112
- -webkit-text-fill-color: transparent;
113
- background-clip: text;
114
- }
115
-
116
- /* Confidence meter */
117
- .confidence-bar {
118
- height: 8px;
119
- border-radius: 4px;
120
- background: rgba(100, 200, 150, 0.2);
121
- overflow: hidden;
122
- margin-top: 8px;
123
- }
124
-
125
- .confidence-fill {
126
- height: 100%;
127
- border-radius: 4px;
128
- background: linear-gradient(90deg, #ef4444, #f59e0b, #059669);
129
- transition: width 0.5s ease;
130
- }
131
-
132
- /* Pipeline steps indicator */
133
- .pipeline-steps {
134
- display: flex;
135
- gap: 4px;
136
- align-items: center;
137
- padding: 8px 0;
138
- }
139
-
140
- .pipeline-step {
141
- flex: 1;
142
- height: 4px;
143
- border-radius: 2px;
144
- background: rgba(100, 200, 150, 0.2);
145
- transition: background 0.3s ease;
146
- }
147
-
148
- .pipeline-step.active {
149
- background: #059669;
150
- }
151
-
152
- .pipeline-step.complete {
153
- background: #86efac;
154
- }
155
-
156
- /* Asymmetry indicator */
157
- .asymmetry-bar {
158
- display: flex;
159
- align-items: center;
160
- gap: 8px;
161
- padding: 8px 12px;
162
- background: rgba(30, 41, 59, 0.6);
163
- border-radius: 8px;
164
- margin: 4px 0;
165
- }
166
-
167
- .asymmetry-label {
168
- min-width: 60px;
169
- font-size: 0.85em;
170
- color: #94a3b8;
171
- }
172
-
173
- .asymmetry-track {
174
- flex: 1;
175
- height: 6px;
176
- background: rgba(100, 200, 150, 0.1);
177
- border-radius: 3px;
178
- position: relative;
179
- }
180
-
181
- .asymmetry-marker {
182
- position: absolute;
183
- top: -3px;
184
- width: 12px;
185
- height: 12px;
186
- border-radius: 50%;
187
- background: #059669;
188
- border: 2px solid #86efac;
189
- }
190
-
191
- /* Topographic pattern accent */
192
- .topo-accent {
193
- background-image:
194
- repeating-linear-gradient(
195
- 0deg,
196
- transparent,
197
- transparent 40px,
198
- rgba(100, 200, 150, 0.03) 40px,
199
- rgba(100, 200, 150, 0.03) 41px
200
- ),
201
- repeating-linear-gradient(
202
- 90deg,
203
- transparent,
204
- transparent 40px,
205
- rgba(100, 200, 150, 0.02) 40px,
206
- rgba(100, 200, 150, 0.02) 41px
207
- );
208
- }
209
-
210
- /* Warning/error states */
211
- .needs-review {
212
- border-color: rgba(245, 158, 11, 0.6) !important;
213
- background: rgba(245, 158, 11, 0.05) !important;
214
- }
215
-
216
- .low-confidence {
217
- opacity: 0.7;
218
- border-style: dashed !important;
219
- }
220
-
221
- /* Rubric drawer */
222
- .rubric-item {
223
- display: flex;
224
- align-items: center;
225
- gap: 8px;
226
- padding: 6px 10px;
227
- border-radius: 6px;
228
- margin: 2px 0;
229
- }
230
-
231
- .rubric-met {
232
- background: rgba(5, 150, 105, 0.1);
233
- border-left: 3px solid #059669;
234
- }
235
-
236
- .rubric-unmet {
237
- background: rgba(239, 68, 68, 0.1);
238
- border-left: 3px solid #ef4444;
239
- }
240
-
241
- /* Responsive */
242
- @media (max-width: 768px) {
243
- .gradio-container {
244
- padding: 8px !important;
245
- }
246
- .score-value {
247
- font-size: 3em;
248
- }
249
- }
250
- """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ FormScout custom Gradio theme — Silas Therapy palette.
3
+
4
+ Warm cream + sage green ground, petrol-teal primary, golden-orange accent,
5
+ dark slate-green text. Matches https://silastherapy.sk branding.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import gradio as gr
10
+
11
+ # ── Silas palette ──────────────────────────────────────────────────────────────
12
+ CREAM = "#f7eedd"
13
+ CREAM_DEEP = "#f1e4cc"
14
+ SAGE = "#bcd3c8"
15
+ SAGE_DEEP = "#9cbcad"
16
+ TEAL = "#2b8a8a"
17
+ TEAL_HOVER = "#1f6e6e"
18
+ TEAL_DEEP = "#175757"
19
+ GOLD = "#e0a43b"
20
+ GOLD_DEEP = "#cf922a"
21
+ INK = "#243a34"
22
+ INK_MUTED = "#4a5f57"
23
+ INK_FAINT = "#6b7d75"
24
+
25
+
26
+ def formscout_theme() -> gr.Theme:
27
+ """Create the FormScout theme in the Silas Therapy palette."""
28
+ return gr.themes.Soft(
29
+ primary_hue=gr.themes.colors.teal,
30
+ secondary_hue=gr.themes.colors.amber,
31
+ neutral_hue=gr.themes.colors.stone,
32
+ font=[
33
+ gr.themes.GoogleFont("Inter"),
34
+ "ui-sans-serif",
35
+ "system-ui",
36
+ "sans-serif",
37
+ ],
38
+ font_mono=[
39
+ gr.themes.GoogleFont("JetBrains Mono"),
40
+ "ui-monospace",
41
+ "monospace",
42
+ ],
43
+ ).set(
44
+ # Background — warm cream into soft sage
45
+ body_background_fill=f"linear-gradient(135deg, {CREAM} 0%, {CREAM_DEEP} 45%, {SAGE} 100%)",
46
+ body_background_fill_dark=f"linear-gradient(135deg, {CREAM} 0%, {CREAM_DEEP} 45%, {SAGE} 100%)",
47
+ # Blocks soft white cards over the sage ground
48
+ block_background_fill="rgba(255, 255, 255, 0.72)",
49
+ block_background_fill_dark="rgba(255, 255, 255, 0.72)",
50
+ block_border_width="1px",
51
+ block_border_color="rgba(43, 138, 138, 0.22)",
52
+ block_shadow="0 6px 22px rgba(36, 58, 52, 0.10)",
53
+ block_radius="14px",
54
+ # Buttons — petrol teal primary, sage secondary
55
+ button_primary_background_fill=f"linear-gradient(135deg, {TEAL} 0%, {TEAL_HOVER} 100%)",
56
+ button_primary_background_fill_hover=f"linear-gradient(135deg, {TEAL_HOVER} 0%, {TEAL_DEEP} 100%)",
57
+ button_primary_text_color="white",
58
+ button_primary_border_color="rgba(43, 138, 138, 0.45)",
59
+ button_secondary_background_fill="rgba(156, 188, 173, 0.55)",
60
+ button_secondary_text_color=INK,
61
+ # Inputs
62
+ input_background_fill="rgba(255, 255, 255, 0.92)",
63
+ input_background_fill_dark="rgba(255, 255, 255, 0.92)",
64
+ input_background_fill_focus="rgba(255, 255, 255, 1.0)",
65
+ input_border_color="rgba(43, 138, 138, 0.30)",
66
+ input_border_color_focus="rgba(43, 138, 138, 0.75)",
67
+ # Labels — pin light in both modes so no dark dropdown header appears
68
+ block_label_background_fill="rgba(188, 211, 200, 0.55)",
69
+ block_label_background_fill_dark="rgba(188, 211, 200, 0.55)",
70
+ block_label_text_color=INK,
71
+ block_label_text_color_dark=INK,
72
+ # Text
73
+ body_text_color=INK,
74
+ body_text_color_dark=INK,
75
+ block_title_text_color=TEAL_DEEP,
76
+ # Spacing
77
+ block_padding="16px",
78
+ layout_gap="16px",
79
+ )
80
+
81
+
82
+ FORMSCOUT_CSS = f"""
83
+ /* FormScout — Silas Therapy theme */
84
+
85
+ .gradio-container {{
86
+ max-width: 1400px !important;
87
+ margin: 0 auto;
88
+ }}
89
+
90
+ /* Header styling */
91
+ .formscout-header {{
92
+ text-align: center;
93
+ padding: 20px 0;
94
+ border-bottom: 2px solid rgba(43, 138, 138, 0.30);
95
+ margin-bottom: 20px;
96
+ }}
97
+
98
+ .formscout-header h1 {{
99
+ font-size: 2.2em;
100
+ background: linear-gradient(135deg, {TEAL} 0%, {GOLD} 100%);
101
+ -webkit-background-clip: text;
102
+ -webkit-text-fill-color: transparent;
103
+ background-clip: text;
104
+ margin-bottom: 8px;
105
+ }}
106
+
107
+ /* Safety banner — golden */
108
+ .safety-banner {{
109
+ background: linear-gradient(90deg, rgba(224, 164, 59, 0.20), rgba(224, 164, 59, 0.08));
110
+ border: 1px solid rgba(224, 164, 59, 0.55);
111
+ border-radius: 8px;
112
+ padding: 12px 16px;
113
+ margin: 12px 0;
114
+ font-size: 0.9em;
115
+ text-align: center;
116
+ color: {GOLD_DEEP};
117
+ }}
118
+
119
+ /* Score display */
120
+ .score-card {{
121
+ background: rgba(43, 138, 138, 0.08);
122
+ border: 2px solid rgba(43, 138, 138, 0.35);
123
+ border-radius: 16px;
124
+ padding: 24px;
125
+ text-align: center;
126
+ }}
127
+
128
+ .score-value {{
129
+ font-size: 4em;
130
+ font-weight: 800;
131
+ background: linear-gradient(135deg, {TEAL} 0%, {GOLD} 100%);
132
+ -webkit-background-clip: text;
133
+ -webkit-text-fill-color: transparent;
134
+ background-clip: text;
135
+ }}
136
+
137
+ /* Confidence meter */
138
+ .confidence-bar {{
139
+ height: 8px;
140
+ border-radius: 4px;
141
+ background: rgba(43, 138, 138, 0.18);
142
+ overflow: hidden;
143
+ margin-top: 8px;
144
+ }}
145
+
146
+ .confidence-fill {{
147
+ height: 100%;
148
+ border-radius: 4px;
149
+ background: linear-gradient(90deg, #d9534f, {GOLD}, {TEAL});
150
+ transition: width 0.5s ease;
151
+ }}
152
+
153
+ /* Pipeline steps indicator */
154
+ .pipeline-steps {{
155
+ display: flex;
156
+ gap: 4px;
157
+ align-items: center;
158
+ padding: 8px 0;
159
+ }}
160
+
161
+ .pipeline-step {{
162
+ flex: 1;
163
+ height: 4px;
164
+ border-radius: 2px;
165
+ background: rgba(43, 138, 138, 0.18);
166
+ transition: background 0.3s ease;
167
+ }}
168
+
169
+ .pipeline-step.active {{
170
+ background: {TEAL};
171
+ }}
172
+
173
+ .pipeline-step.complete {{
174
+ background: {GOLD};
175
+ }}
176
+
177
+ /* Asymmetry indicator */
178
+ .asymmetry-bar {{
179
+ display: flex;
180
+ align-items: center;
181
+ gap: 8px;
182
+ padding: 8px 12px;
183
+ background: rgba(156, 188, 173, 0.30);
184
+ border-radius: 8px;
185
+ margin: 4px 0;
186
+ }}
187
+
188
+ .asymmetry-label {{
189
+ min-width: 60px;
190
+ font-size: 0.85em;
191
+ color: {INK_MUTED};
192
+ }}
193
+
194
+ .asymmetry-track {{
195
+ flex: 1;
196
+ height: 6px;
197
+ background: rgba(43, 138, 138, 0.12);
198
+ border-radius: 3px;
199
+ position: relative;
200
+ }}
201
+
202
+ .asymmetry-marker {{
203
+ position: absolute;
204
+ top: -3px;
205
+ width: 12px;
206
+ height: 12px;
207
+ border-radius: 50%;
208
+ background: {TEAL};
209
+ border: 2px solid {GOLD};
210
+ }}
211
+
212
+ /* Topographic pattern accent */
213
+ .topo-accent {{
214
+ color: {INK_FAINT};
215
+ background-image:
216
+ repeating-linear-gradient(
217
+ 0deg,
218
+ transparent,
219
+ transparent 40px,
220
+ rgba(43, 138, 138, 0.04) 40px,
221
+ rgba(43, 138, 138, 0.04) 41px
222
+ ),
223
+ repeating-linear-gradient(
224
+ 90deg,
225
+ transparent,
226
+ transparent 40px,
227
+ rgba(43, 138, 138, 0.03) 40px,
228
+ rgba(43, 138, 138, 0.03) 41px
229
+ );
230
+ }}
231
+
232
+ /* Warning/error states */
233
+ .needs-review {{
234
+ border-color: rgba(224, 164, 59, 0.65) !important;
235
+ background: rgba(224, 164, 59, 0.10) !important;
236
+ }}
237
+
238
+ .low-confidence {{
239
+ opacity: 0.7;
240
+ border-style: dashed !important;
241
+ }}
242
+
243
+ /* Rubric drawer */
244
+ .rubric-item {{
245
+ display: flex;
246
+ align-items: center;
247
+ gap: 8px;
248
+ padding: 6px 10px;
249
+ border-radius: 6px;
250
+ margin: 2px 0;
251
+ }}
252
+
253
+ .rubric-met {{
254
+ background: rgba(43, 138, 138, 0.10);
255
+ border-left: 3px solid {TEAL};
256
+ }}
257
+
258
+ .rubric-unmet {{
259
+ background: rgba(217, 83, 79, 0.10);
260
+ border-left: 3px solid #d9534f;
261
+ }}
262
+
263
+ /* Responsive */
264
+ @media (max-width: 768px) {{
265
+ .gradio-container {{
266
+ padding: 8px !important;
267
+ }}
268
+ .score-value {{
269
+ font-size: 3em;
270
+ }}
271
+ }}
272
+ """
requirements.txt CHANGED
@@ -2,16 +2,18 @@ gradio>=5.0
2
  ultralytics>=8.3
3
  torch>=2.3
4
  opencv-python>=4.10
5
- imageio-ffmpeg>=0.5
6
  numpy>=1.26
7
  scipy>=1.13
8
  pillow>=10.3
9
  reportlab>=4.0
 
10
  requests>=2.31
11
  pytest>=8.2
12
  ruff>=0.4
13
  black>=24.4
14
  huggingface_hub>=0.23
15
  transformers>=4.44
 
 
16
  onnxruntime>=1.18
17
  mediapipe>=0.10
 
2
  ultralytics>=8.3
3
  torch>=2.3
4
  opencv-python>=4.10
 
5
  numpy>=1.26
6
  scipy>=1.13
7
  pillow>=10.3
8
  reportlab>=4.0
9
+ matplotlib>=3.8
10
  requests>=2.31
11
  pytest>=8.2
12
  ruff>=0.4
13
  black>=24.4
14
  huggingface_hub>=0.23
15
  transformers>=4.44
16
+ accelerate>=0.30
17
+ spaces>=0.30
18
  onnxruntime>=1.18
19
  mediapipe>=0.10
scripts/hf_upload.sh CHANGED
@@ -1,97 +1,97 @@
1
- #!/usr/bin/env bash
2
- # Upload the FormScout source tree to both the model repo and the Space.
3
- #
4
- # Usage:
5
- # ./scripts/hf_upload.sh # message from last git commit
6
- # ./scripts/hf_upload.sh "feat: my change" # custom message
7
- #
8
- # Pushes to:
9
- # silas-therapy/small-functional-movement-screening (model repo)
10
- # spaces/silas-therapy/small-functional-movement-screening (Gradio Space)
11
- #
12
- # `hf upload` does NOT read .hfignore — it only honors .gitignore, and only at
13
- # commit time (after hashing and pre-uploading everything). So we parse
14
- # .hfignore ourselves into --exclude globs and pass them explicitly.
15
- #
16
- # If the filtered file count still exceeds LARGE_THRESHOLD, we fall back to
17
- # `hf upload-large-folder` (resumable, multi-threaded). Caveats of that mode:
18
- # no --create-pr and no custom commit message — it commits directly to main
19
- # in multiple commits.
20
- set -euo pipefail
21
-
22
- cd "$(dirname "$0")/.."
23
-
24
- MODEL_REPO="silas-therapy/small-functional-movement-screening"
25
- SPACE_REPO="spaces/silas-therapy/small-functional-movement-screening"
26
- MSG="${1:-$(git log -1 --pretty=%s)}"
27
- LARGE_THRESHOLD="${FORMSCOUT_HF_LARGE_THRESHOLD:-500}"
28
-
29
- # Belt-and-suspenders extras on top of .hfignore. `.cache/` is the resume
30
- # state upload-large-folder writes into the folder being uploaded.
31
- PATTERNS=(
32
- "*.pdf"
33
- "**/node_modules/**"
34
- ".cache/**"
35
- )
36
-
37
- # Parse .hfignore into fnmatch-style globs. fnmatch's `*` crosses `/`, but a
38
- # bare name like `.DS_Store` or `dir/` only matches at the root, so emit both
39
- # the rooted and `**/`-prefixed forms.
40
- while IFS= read -r line; do
41
- line="${line%%#*}"
42
- line="${line#"${line%%[![:space:]]*}"}"
43
- line="${line%"${line##*[![:space:]]}"}"
44
- [[ -z "$line" ]] && continue
45
- if [[ "$line" == */ ]]; then
46
- PATTERNS+=("${line}**" "**/${line}**")
47
- else
48
- PATTERNS+=("$line" "**/$line")
49
- fi
50
- done < .hfignore
51
-
52
- EXCLUDES=()
53
- for p in "${PATTERNS[@]}"; do
54
- EXCLUDES+=(--exclude="$p")
55
- done
56
-
57
- # Count what would actually be uploaded, using the same filter the hub client
58
- # applies, so the mode decision matches reality.
59
- N_FILES=$(python3 - "${PATTERNS[@]}" <<'EOF'
60
- import sys
61
- from pathlib import Path
62
- from huggingface_hub.utils import filter_repo_objects
63
-
64
- patterns = sys.argv[1:]
65
- files = (
66
- str(p) for p in Path(".").rglob("*")
67
- if p.is_file() and p.parts[0] != ".git"
68
- )
69
- print(len(list(filter_repo_objects(files, ignore_patterns=patterns))))
70
- EOF
71
- )
72
- echo "── $N_FILES files to upload after .hfignore filtering"
73
-
74
- if (( N_FILES == 0 )); then
75
- echo "✗ nothing to upload — check .hfignore" >&2
76
- exit 1
77
- fi
78
-
79
- upload_repo() {
80
- local repo="$1"
81
- if (( N_FILES > LARGE_THRESHOLD )); then
82
- echo "── $repo: $N_FILES files > $LARGE_THRESHOLD, using upload-large-folder"
83
- echo " (resumable; commits directly to main — no PR, no custom message)"
84
- hf upload-large-folder "$repo" . "${EXCLUDES[@]}"
85
- else
86
- echo "── uploading to: $repo"
87
- hf upload "$repo" . . \
88
- "${EXCLUDES[@]}" \
89
- --create-pr \
90
- --commit-message="$MSG"
91
- fi
92
- }
93
-
94
- upload_repo "$MODEL_REPO"
95
- upload_repo "$SPACE_REPO"
96
-
97
- echo "✓ done"
 
1
+ #!/usr/bin/env bash
2
+ # Upload the FormScout source tree to both the model repo and the Space.
3
+ #
4
+ # Usage:
5
+ # ./scripts/hf_upload.sh # message from last git commit
6
+ # ./scripts/hf_upload.sh "feat: my change" # custom message
7
+ #
8
+ # Pushes to:
9
+ # silas-therapy/small-functional-movement-screening (model repo)
10
+ # spaces/silas-therapy/small-functional-movement-screening (Gradio Space)
11
+ #
12
+ # `hf upload` does NOT read .hfignore — it only honors .gitignore, and only at
13
+ # commit time (after hashing and pre-uploading everything). So we parse
14
+ # .hfignore ourselves into --exclude globs and pass them explicitly.
15
+ #
16
+ # If the filtered file count still exceeds LARGE_THRESHOLD, we fall back to
17
+ # `hf upload-large-folder` (resumable, multi-threaded). Caveats of that mode:
18
+ # no --create-pr and no custom commit message — it commits directly to main
19
+ # in multiple commits.
20
+ set -euo pipefail
21
+
22
+ cd "$(dirname "$0")/.."
23
+
24
+ MODEL_REPO="silas-therapy/small-functional-movement-screening"
25
+ SPACE_REPO="spaces/silas-therapy/small-functional-movement-screening"
26
+ MSG="${1:-$(git log -1 --pretty=%s)}"
27
+ LARGE_THRESHOLD="${FORMSCOUT_HF_LARGE_THRESHOLD:-500}"
28
+
29
+ # Belt-and-suspenders extras on top of .hfignore. `.cache/` is the resume
30
+ # state upload-large-folder writes into the folder being uploaded.
31
+ PATTERNS=(
32
+ "*.pdf"
33
+ "**/node_modules/**"
34
+ ".cache/**"
35
+ )
36
+
37
+ # Parse .hfignore into fnmatch-style globs. fnmatch's `*` crosses `/`, but a
38
+ # bare name like `.DS_Store` or `dir/` only matches at the root, so emit both
39
+ # the rooted and `**/`-prefixed forms.
40
+ while IFS= read -r line; do
41
+ line="${line%%#*}"
42
+ line="${line#"${line%%[![:space:]]*}"}"
43
+ line="${line%"${line##*[![:space:]]}"}"
44
+ [[ -z "$line" ]] && continue
45
+ if [[ "$line" == */ ]]; then
46
+ PATTERNS+=("${line}**" "**/${line}**")
47
+ else
48
+ PATTERNS+=("$line" "**/$line")
49
+ fi
50
+ done < .hfignore
51
+
52
+ EXCLUDES=()
53
+ for p in "${PATTERNS[@]}"; do
54
+ EXCLUDES+=(--exclude="$p")
55
+ done
56
+
57
+ # Count what would actually be uploaded, using the same filter the hub client
58
+ # applies, so the mode decision matches reality.
59
+ N_FILES=$(python3 - "${PATTERNS[@]}" <<'EOF'
60
+ import sys
61
+ from pathlib import Path
62
+ from huggingface_hub.utils import filter_repo_objects
63
+
64
+ patterns = sys.argv[1:]
65
+ files = (
66
+ str(p) for p in Path(".").rglob("*")
67
+ if p.is_file() and p.parts[0] != ".git"
68
+ )
69
+ print(len(list(filter_repo_objects(files, ignore_patterns=patterns))))
70
+ EOF
71
+ )
72
+ echo "── $N_FILES files to upload after .hfignore filtering"
73
+
74
+ if (( N_FILES == 0 )); then
75
+ echo "✗ nothing to upload — check .hfignore" >&2
76
+ exit 1
77
+ fi
78
+
79
+ upload_repo() {
80
+ local repo="$1"
81
+ if (( N_FILES > LARGE_THRESHOLD )); then
82
+ echo "── $repo: $N_FILES files > $LARGE_THRESHOLD, using upload-large-folder"
83
+ echo " (resumable; commits directly to main — no PR, no custom message)"
84
+ hf upload-large-folder "$repo" . "${EXCLUDES[@]}"
85
+ else
86
+ echo "── uploading to: $repo"
87
+ hf upload "$repo" . . \
88
+ "${EXCLUDES[@]}" \
89
+ --create-pr \
90
+ --commit-message="$MSG"
91
+ fi
92
+ }
93
+
94
+ upload_repo "$MODEL_REPO"
95
+ upload_repo "$SPACE_REPO"
96
+
97
+ echo "✓ done"
scripts/serve_judge.sh CHANGED
@@ -1,35 +1,35 @@
1
- #!/usr/bin/env bash
2
- # Launch llama-server with the FormScout Judge/Classifier VLM.
3
- #
4
- # Default model: Qwen3-VL-8B-Instruct Q4_K_M (checkpoints/qwen3-vl/).
5
- # To serve a fine-tuned GGUF instead, set:
6
- # FORMSCOUT_JUDGE_GGUF=/path/to/finetuned.gguf
7
- # FORMSCOUT_JUDGE_MMPROJ=/path/to/mmproj.gguf (only if it ships its own)
8
- #
9
- # Requires: brew install llama.cpp
10
- set -euo pipefail
11
-
12
- # Homebrew bin may be missing from non-interactive shells
13
- export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
14
-
15
- ROOT="$(cd "$(dirname "$0")/.." && pwd)"
16
- GGUF="${FORMSCOUT_JUDGE_GGUF:-$ROOT/checkpoints/qwen3-vl/Qwen3VL-8B-Instruct-Q4_K_M.gguf}"
17
- MMPROJ="${FORMSCOUT_JUDGE_MMPROJ:-$ROOT/checkpoints/qwen3-vl/mmproj-Qwen3VL-8B-Instruct-F16.gguf}"
18
- HOST="${FORMSCOUT_LLAMA_HOST:-127.0.0.1}"
19
- PORT="${FORMSCOUT_LLAMA_PORT:-8080}"
20
-
21
- if [[ ! -f "$GGUF" ]]; then
22
- echo "Model not found: $GGUF" >&2
23
- echo "Download it with:" >&2
24
- echo " python3 -c \"from huggingface_hub import hf_hub_download; [hf_hub_download('Qwen/Qwen3-VL-8B-Instruct-GGUF', f, local_dir='$ROOT/checkpoints/qwen3-vl') for f in ['Qwen3VL-8B-Instruct-Q4_K_M.gguf', 'mmproj-Qwen3VL-8B-Instruct-F16.gguf']]\"" >&2
25
- exit 1
26
- fi
27
-
28
- exec llama-server \
29
- --model "$GGUF" \
30
- --mmproj "$MMPROJ" \
31
- --host "$HOST" \
32
- --port "$PORT" \
33
- --ctx-size 16384 \
34
- --n-gpu-layers 99 \
35
- --no-warmup
 
1
+ #!/usr/bin/env bash
2
+ # Launch llama-server with the FormScout Judge/Classifier VLM.
3
+ #
4
+ # Default model: Qwen3-VL-8B-Instruct Q4_K_M (checkpoints/qwen3-vl/).
5
+ # To serve a fine-tuned GGUF instead, set:
6
+ # FORMSCOUT_JUDGE_GGUF=/path/to/finetuned.gguf
7
+ # FORMSCOUT_JUDGE_MMPROJ=/path/to/mmproj.gguf (only if it ships its own)
8
+ #
9
+ # Requires: brew install llama.cpp
10
+ set -euo pipefail
11
+
12
+ # Homebrew bin may be missing from non-interactive shells
13
+ export PATH="/opt/homebrew/bin:/usr/local/bin:$PATH"
14
+
15
+ ROOT="$(cd "$(dirname "$0")/.." && pwd)"
16
+ GGUF="${FORMSCOUT_JUDGE_GGUF:-$ROOT/checkpoints/qwen3-vl/Qwen3VL-8B-Instruct-Q4_K_M.gguf}"
17
+ MMPROJ="${FORMSCOUT_JUDGE_MMPROJ:-$ROOT/checkpoints/qwen3-vl/mmproj-Qwen3VL-8B-Instruct-F16.gguf}"
18
+ HOST="${FORMSCOUT_LLAMA_HOST:-127.0.0.1}"
19
+ PORT="${FORMSCOUT_LLAMA_PORT:-8080}"
20
+
21
+ if [[ ! -f "$GGUF" ]]; then
22
+ echo "Model not found: $GGUF" >&2
23
+ echo "Download it with:" >&2
24
+ echo " python3 -c \"from huggingface_hub import hf_hub_download; [hf_hub_download('Qwen/Qwen3-VL-8B-Instruct-GGUF', f, local_dir='$ROOT/checkpoints/qwen3-vl') for f in ['Qwen3VL-8B-Instruct-Q4_K_M.gguf', 'mmproj-Qwen3VL-8B-Instruct-F16.gguf']]\"" >&2
25
+ exit 1
26
+ fi
27
+
28
+ exec llama-server \
29
+ --model "$GGUF" \
30
+ --mmproj "$MMPROJ" \
31
+ --host "$HOST" \
32
+ --port "$PORT" \
33
+ --ctx-size 16384 \
34
+ --n-gpu-layers 99 \
35
+ --no-warmup
tests/test_analysis.py ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tests for the movement-analysis engine — no GPU, no model downloads."""
2
+ import math
3
+ import numpy as np
4
+
5
+ from formscout.types import Pose2DResult
6
+
7
+
8
+ def _pose_from(traj_by_joint, n, base_conf=0.9):
9
+ """Build a Pose2DResult; traj_by_joint maps joint->callable(i)->(x,y)."""
10
+ kps = []
11
+ for i in range(n):
12
+ frame = {}
13
+ for j in range(17):
14
+ if j in traj_by_joint:
15
+ x, y = traj_by_joint[j](i)
16
+ else:
17
+ x, y = float(100 + j * 5), float(100 + j * 5)
18
+ frame[j] = {"x": float(x), "y": float(y), "conf": base_conf}
19
+ kps.append(frame)
20
+ return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
21
+
22
+
23
+ # ── relevant_joints ────────────────────────────────────────────────────────────
24
+
25
+ def test_relevant_joints_and_primary_consistent():
26
+ from formscout.analysis import relevant_joints as RJ
27
+ for test in RJ.RELEVANT:
28
+ joints = RJ.relevant_joints(test)
29
+ angles = RJ.relevant_angles(test)
30
+ prim = RJ.primary_angle(test)
31
+ assert joints, f"{test} has no relevant joints"
32
+ assert prim in angles, f"{test} primary angle not in angles"
33
+
34
+
35
+ def test_openness_label_monotonic():
36
+ from formscout.analysis.relevant_joints import openness_label
37
+ assert "open" in openness_label(175)
38
+ assert openness_label(30).endswith("closed")
39
+
40
+
41
+ # ── timeseries ──────────────────────────────────────────────────────────────────
42
+
43
+ def test_angle_series_length_matches_frames():
44
+ from formscout.analysis.timeseries import angle_series
45
+ pose = _pose_from({}, n=8)
46
+ series = angle_series(pose, "deep_squat")
47
+ assert "left_knee_flexion" in series
48
+ for name, vals in series.items():
49
+ assert len(vals) == 8
50
+
51
+
52
+ def test_relevant_flexion_reports_degrees_and_openness():
53
+ from formscout.analysis.timeseries import relevant_flexion_at
54
+ # Straight leg: hip, knee, ankle collinear vertical → ~180°
55
+ straight = {
56
+ 11: lambda i: (200, 100), 13: lambda i: (200, 200), 15: lambda i: (200, 300),
57
+ 12: lambda i: (260, 100), 14: lambda i: (260, 200), 16: lambda i: (260, 300),
58
+ 5: lambda i: (200, 40), 6: lambda i: (260, 40),
59
+ }
60
+ pose = _pose_from(straight, n=5)
61
+ flex = relevant_flexion_at(pose, "deep_squat", 2)
62
+ assert "left_knee_flexion" in flex
63
+ assert flex["left_knee_flexion"]["deg"] > 160
64
+ assert "open" in flex["left_knee_flexion"]["openness"]
65
+
66
+
67
+ # ── laban ───────────────────────────────────────────────────────────────────────
68
+
69
+ def test_laban_factors_in_unit_range():
70
+ from formscout.analysis.laban import compute_laban
71
+ pose = _pose_from({13: lambda i: (100 + i * 8, 200)}, n=20)
72
+ res = compute_laban(pose, "deep_squat", fps=30.0)
73
+ for k, v in res["effort"].items():
74
+ assert 0.0 <= v <= 1.0, f"{k}={v} out of range"
75
+ assert set(res["labels"]) == {"space", "weight", "time", "flow"}
76
+
77
+
78
+ def test_laban_straight_line_is_direct():
79
+ from formscout.analysis.laban import compute_laban
80
+ # Knee travels in a straight horizontal line → high directness (Space)
81
+ pose = _pose_from({13: lambda i: (100 + i * 10, 200)}, n=20)
82
+ res = compute_laban(pose, "deep_squat", fps=30.0)
83
+ assert res["effort"]["space"] > 0.8
84
+
85
+
86
+ def test_laban_static_clip_low_energy():
87
+ from formscout.analysis.laban import compute_laban
88
+ pose = _pose_from({13: lambda i: (200, 200)}, n=20) # no motion
89
+ res = compute_laban(pose, "deep_squat", fps=30.0)
90
+ assert res["effort"]["weight"] < 0.2
91
+
92
+
93
+ # ── charts ──────────────────────────────────────────────────────────────────────
94
+
95
+ def test_angle_over_time_chart(tmp_path):
96
+ from formscout.analysis import charts
97
+ out = str(tmp_path / "angle.png")
98
+ series = {"left_knee_flexion": [90, 100, 110, 95], "right_knee_flexion": [88, 99, 108, 94]}
99
+ p = charts.angle_over_time(series, "left_knee_flexion", 2, out)
100
+ assert p == out
101
+ import os
102
+ assert os.path.getsize(out) > 0
103
+
104
+
105
+ def test_velocity_profile_chart(tmp_path):
106
+ from formscout.analysis import charts
107
+ out = str(tmp_path / "vel.png")
108
+ kps = [{j: {"x": 100 + j + i * 3, "y": 100 + j, "conf": 0.9} for j in range(17)} for i in range(10)]
109
+ p = charts.velocity_profile(kps, 30.0, [13, 14, 11, 12], out)
110
+ import os
111
+ assert p == out and os.path.getsize(out) > 0
112
+
113
+
114
+ def test_laban_radar_chart(tmp_path):
115
+ from formscout.analysis import charts
116
+ out = str(tmp_path / "radar.png")
117
+ p = charts.laban_radar({"space": 0.8, "weight": 0.4, "time": 0.6, "flow": 0.3}, out)
118
+ import os
119
+ assert p == out and os.path.getsize(out) > 0
120
+
121
+
122
+ def test_flexion_bars_chart(tmp_path):
123
+ from formscout.analysis import charts
124
+ out = str(tmp_path / "flex.png")
125
+ flex = {"left_knee_flexion": {"deg": 95.0, "openness": "flexed"},
126
+ "left_hip_flexion": {"deg": 120.0, "openness": "mid-range"}}
127
+ p = charts.flexion_bars(flex, out)
128
+ import os
129
+ assert p == out and os.path.getsize(out) > 0
130
+
131
+
132
+ def test_symmetry_bars_chart(tmp_path):
133
+ from formscout.analysis import charts
134
+ out = str(tmp_path / "sym.png")
135
+ asym = [{"test": "hurdle_step", "left_score": 2, "right_score": 3, "delta": 1}]
136
+ p = charts.symmetry_bars(asym, out)
137
+ import os
138
+ assert p == out and os.path.getsize(out) > 0
139
+
140
+
141
+ def test_charts_return_none_on_empty(tmp_path):
142
+ from formscout.analysis import charts
143
+ assert charts.angle_over_time({}, None, None, str(tmp_path / "a.png")) is None
144
+ assert charts.flexion_bars({}, str(tmp_path / "f.png")) is None
145
+ assert charts.symmetry_bars([], str(tmp_path / "s.png")) is None
tests/test_judge_backend.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Judge backend selection + transformers-fallback — no GPU, no downloads."""
2
+ import importlib
3
+
4
+ import formscout.config as config
5
+ from formscout.serving import get_vlm_client
6
+ from formscout.serving.llama_cpp import LlamaCppClient
7
+
8
+
9
+ def _reload_config(monkeypatch, **env):
10
+ for k, v in env.items():
11
+ if v is None:
12
+ monkeypatch.delenv(k, raising=False)
13
+ else:
14
+ monkeypatch.setenv(k, v)
15
+ importlib.reload(config)
16
+ return config
17
+
18
+
19
+ def test_resolve_backend_default_local(monkeypatch):
20
+ cfg = _reload_config(monkeypatch, FORMSCOUT_JUDGE_BACKEND=None, SPACE_ID=None)
21
+ assert cfg.resolve_judge_backend() == "llama_cpp"
22
+
23
+
24
+ def test_resolve_backend_auto_on_space(monkeypatch):
25
+ cfg = _reload_config(monkeypatch, FORMSCOUT_JUDGE_BACKEND="auto", SPACE_ID="me/space")
26
+ assert cfg.resolve_judge_backend() == "transformers"
27
+
28
+
29
+ def test_resolve_backend_explicit(monkeypatch):
30
+ cfg = _reload_config(monkeypatch, FORMSCOUT_JUDGE_BACKEND="llama_cpp", SPACE_ID="me/space")
31
+ assert cfg.resolve_judge_backend() == "llama_cpp"
32
+ importlib.reload(config) # restore
33
+
34
+
35
+ def test_factory_returns_llama_cpp_locally(monkeypatch):
36
+ _reload_config(monkeypatch, FORMSCOUT_JUDGE_BACKEND="llama_cpp", SPACE_ID=None)
37
+ client = get_vlm_client()
38
+ assert isinstance(client, LlamaCppClient)
39
+ importlib.reload(config)
40
+
41
+
42
+ def test_transformers_client_available_is_cheap_bool():
43
+ # available must not load/download the model — just report a bool
44
+ from formscout.serving.transformers_vlm import TransformersVLMClient
45
+ c = TransformersVLMClient()
46
+ assert isinstance(c.available, bool)
47
+ c._failed = True
48
+ assert c.available is False
49
+
50
+
51
+ def test_judge_uses_rubric_on_fallback_sentinel():
52
+ from formscout.agents.judge import JudgeAgent
53
+ from formscout.types import BiomechFeatures, ScoreResult, MovementResult, IngestResult
54
+ import numpy as np
55
+
56
+ agent = JudgeAgent()
57
+
58
+ class _FakeClient:
59
+ available = True
60
+
61
+ def complete(self, *a, **k):
62
+ return {"error": "no gpu", "fallback": True, "text": ""}
63
+
64
+ agent._client = _FakeClient()
65
+ features = BiomechFeatures(test_name="deep_squat", view="2d", side="na",
66
+ angles={}, alignments={}, symmetry_delta=None,
67
+ timing={}, confidence=0.9)
68
+ rubric = ScoreResult(score=2, rationale="rubric says 2", confidence=0.8)
69
+ movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
70
+ ingest = IngestResult(frames=[np.zeros((10, 10, 3), np.uint8)], fps=30.0,
71
+ duration=0.03, n_people=1, width=10, height=10)
72
+
73
+ res = agent.run(features, rubric, movement, ingest)
74
+ assert res.score == 2
75
+ assert "rubric-only" in res.rationale
tests/test_keyframe.py CHANGED
@@ -1,37 +1,37 @@
1
- """Tests for PoseVisualizer.render_frame — single annotated still."""
2
- import os
3
- import numpy as np
4
-
5
- from formscout.types import IngestResult, Pose2DResult
6
-
7
-
8
- def _ingest(n=5, h=480, w=640):
9
- frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
10
- return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
11
-
12
-
13
- def _pose(n=5):
14
- kps = []
15
- for i in range(n):
16
- kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
17
- for j in range(17)})
18
- return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
19
-
20
-
21
- def test_render_frame_writes_png(tmp_path):
22
- from formscout.agents.visualizer import PoseVisualizer
23
- out = str(tmp_path / "key.png")
24
- path = PoseVisualizer().render_frame(_ingest(), _pose(), frame_idx=2,
25
- layers={"skeleton"}, caption="Deep Squat — heels elevated",
26
- out_png=out)
27
- assert path == out
28
- assert os.path.exists(out)
29
- assert os.path.getsize(out) > 0
30
-
31
-
32
- def test_render_frame_bad_index_returns_none(tmp_path):
33
- from formscout.agents.visualizer import PoseVisualizer
34
- out = str(tmp_path / "key.png")
35
- path = PoseVisualizer().render_frame(_ingest(n=3), _pose(n=3), frame_idx=99,
36
- layers={"skeleton"}, caption="", out_png=out)
37
- assert path is None
 
1
+ """Tests for PoseVisualizer.render_frame — single annotated still."""
2
+ import os
3
+ import numpy as np
4
+
5
+ from formscout.types import IngestResult, Pose2DResult
6
+
7
+
8
+ def _ingest(n=5, h=480, w=640):
9
+ frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
10
+ return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
11
+
12
+
13
+ def _pose(n=5):
14
+ kps = []
15
+ for i in range(n):
16
+ kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
17
+ for j in range(17)})
18
+ return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
19
+
20
+
21
+ def test_render_frame_writes_png(tmp_path):
22
+ from formscout.agents.visualizer import PoseVisualizer
23
+ out = str(tmp_path / "key.png")
24
+ path = PoseVisualizer().render_frame(_ingest(), _pose(), frame_idx=2,
25
+ layers={"skeleton"}, caption="Deep Squat — heels elevated",
26
+ out_png=out)
27
+ assert path == out
28
+ assert os.path.exists(out)
29
+ assert os.path.getsize(out) > 0
30
+
31
+
32
+ def test_render_frame_bad_index_returns_none(tmp_path):
33
+ from formscout.agents.visualizer import PoseVisualizer
34
+ out = str(tmp_path / "key.png")
35
+ path = PoseVisualizer().render_frame(_ingest(n=3), _pose(n=3), frame_idx=99,
36
+ layers={"skeleton"}, caption="", out_png=out)
37
+ assert path is None
tests/test_pdf_report.py CHANGED
@@ -1,51 +1,51 @@
1
- """Tests for PdfReportAgent — no GPU, no model downloads."""
2
- import os
3
-
4
- from formscout.types import (
5
- ReportResult, SessionEntry, MovementResult, BiomechFeatures, ScoreResult, JudgeResult,
6
- )
7
-
8
-
9
- def _entry(test_name="deep_squat", score=2, needs_human=False):
10
- movement = MovementResult(test_name=test_name, side="na", confidence=1.0)
11
- features = BiomechFeatures(
12
- test_name=test_name, view="2d", side="na",
13
- angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": False},
14
- symmetry_delta=None, timing={"deepest_frame": 1}, confidence=0.9,
15
- )
16
- rubric = ScoreResult(score=2, rationale="rubric ok", confidence=0.8)
17
- judge = JudgeResult(score=None if needs_human else score, rationale="judge rationale",
18
- compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
19
- confidence=0.85, needs_human=needs_human)
20
- return SessionEntry(
21
- test_name=test_name, side="na", score=None if needs_human else score,
22
- needs_human=needs_human, rationale="judge rationale",
23
- compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
24
- measurements={"left_knee_flexion_deg": 95.0, "knees_tracking_over_feet": False},
25
- confidence=0.85, view="2d", keyframe_path=None,
26
- movement=movement, features=features, rubric_score=rubric, judge=judge,
27
- )
28
-
29
-
30
- def _report(composite=2):
31
- return ReportResult(
32
- per_test=[], composite=composite, asymmetries=[],
33
- overlay_video_path=None, pdf_path=None,
34
- low_confidence_flags=[], disagreement_flags=[],
35
- )
36
-
37
-
38
- def test_pdf_is_created(tmp_path):
39
- from formscout.agents.pdf_report import PdfReportAgent
40
- path = PdfReportAgent().run(_report(2), [_entry()], str(tmp_path))
41
- assert path is not None
42
- assert os.path.exists(path)
43
- assert os.path.getsize(path) > 1000 # a real PDF, not an empty file
44
- with open(path, "rb") as f:
45
- assert f.read(5) == b"%PDF-"
46
-
47
-
48
- def test_pdf_handles_incomplete_composite(tmp_path):
49
- from formscout.agents.pdf_report import PdfReportAgent
50
- path = PdfReportAgent().run(_report(None), [_entry(needs_human=True)], str(tmp_path))
51
- assert path is not None and os.path.exists(path)
 
1
+ """Tests for PdfReportAgent — no GPU, no model downloads."""
2
+ import os
3
+
4
+ from formscout.types import (
5
+ ReportResult, SessionEntry, MovementResult, BiomechFeatures, ScoreResult, JudgeResult,
6
+ )
7
+
8
+
9
+ def _entry(test_name="deep_squat", score=2, needs_human=False):
10
+ movement = MovementResult(test_name=test_name, side="na", confidence=1.0)
11
+ features = BiomechFeatures(
12
+ test_name=test_name, view="2d", side="na",
13
+ angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": False},
14
+ symmetry_delta=None, timing={"deepest_frame": 1}, confidence=0.9,
15
+ )
16
+ rubric = ScoreResult(score=2, rationale="rubric ok", confidence=0.8)
17
+ judge = JudgeResult(score=None if needs_human else score, rationale="judge rationale",
18
+ compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
19
+ confidence=0.85, needs_human=needs_human)
20
+ return SessionEntry(
21
+ test_name=test_name, side="na", score=None if needs_human else score,
22
+ needs_human=needs_human, rationale="judge rationale",
23
+ compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
24
+ measurements={"left_knee_flexion_deg": 95.0, "knees_tracking_over_feet": False},
25
+ confidence=0.85, view="2d", keyframe_path=None,
26
+ movement=movement, features=features, rubric_score=rubric, judge=judge,
27
+ )
28
+
29
+
30
+ def _report(composite=2):
31
+ return ReportResult(
32
+ per_test=[], composite=composite, asymmetries=[],
33
+ overlay_video_path=None, pdf_path=None,
34
+ low_confidence_flags=[], disagreement_flags=[],
35
+ )
36
+
37
+
38
+ def test_pdf_is_created(tmp_path):
39
+ from formscout.agents.pdf_report import PdfReportAgent
40
+ path = PdfReportAgent().run(_report(2), [_entry()], str(tmp_path))
41
+ assert path is not None
42
+ assert os.path.exists(path)
43
+ assert os.path.getsize(path) > 1000 # a real PDF, not an empty file
44
+ with open(path, "rb") as f:
45
+ assert f.read(5) == b"%PDF-"
46
+
47
+
48
+ def test_pdf_handles_incomplete_composite(tmp_path):
49
+ from formscout.agents.pdf_report import PdfReportAgent
50
+ path = PdfReportAgent().run(_report(None), [_entry(needs_human=True)], str(tmp_path))
51
+ assert path is not None and os.path.exists(path)
tests/test_phase2.py CHANGED
@@ -1,354 +1,354 @@
1
- """Tests for all rubric scorers and Phase 2 agents."""
2
-
3
- from formscout.types import (
4
- BiomechFeatures, ScoreResult, MovementResult, JudgeResult, ReportResult,
5
- )
6
- from formscout.rubric import score_test, SCORERS
7
- from formscout.rubric.hurdle_step import score_hurdle_step
8
- from formscout.rubric.inline_lunge import score_inline_lunge
9
- from formscout.rubric.shoulder_mobility import score_shoulder_mobility
10
- from formscout.rubric.active_slr import score_active_slr
11
- from formscout.rubric.trunk_stability_pushup import score_trunk_stability_pushup
12
- from formscout.rubric.rotary_stability import score_rotary_stability
13
- from formscout.agents.judge import JudgeAgent
14
- from formscout.agents.report import ReportAgent
15
-
16
-
17
- def _make_features(test_name, angles=None, alignments=None, side="na", sym_delta=None):
18
- return BiomechFeatures(
19
- test_name=test_name, view="2d", side=side,
20
- angles=angles or {}, alignments=alignments or {},
21
- symmetry_delta=sym_delta, timing={}, confidence=0.8,
22
- )
23
-
24
-
25
- # ─── Rubric dispatch ─────────────────────────────────────────────────────────
26
-
27
- class TestRubricDispatch:
28
- def test_all_tests_have_scorers(self):
29
- expected = {"deep_squat", "hurdle_step", "inline_lunge", "shoulder_mobility",
30
- "active_slr", "trunk_stability_pushup", "rotary_stability"}
31
- assert set(SCORERS.keys()) == expected
32
-
33
- def test_dispatch_unknown_test(self):
34
- f = _make_features("unknown_test")
35
- r = score_test(f)
36
- assert r.confidence == 0.0
37
-
38
-
39
- # ─── Hurdle Step ──────────────────────────────────────────────────────────────
40
-
41
- class TestHurdleStep:
42
- def test_score_3_good_form(self):
43
- f = _make_features("hurdle_step", angles={
44
- "step_hip_flexion_deg": 100.0, "stance_knee_angle_deg": 175.0,
45
- "shoulder_tilt_deg": 5.0,
46
- }, alignments={"trunk_stable": True, "stance_knee_extended": True})
47
- r = score_hurdle_step(f)
48
- assert r.score == 3
49
-
50
- def test_score_2_compensation(self):
51
- f = _make_features("hurdle_step", angles={
52
- "step_hip_flexion_deg": 80.0, "stance_knee_angle_deg": 170.0,
53
- }, alignments={"trunk_stable": True, "stance_knee_extended": True})
54
- r = score_hurdle_step(f)
55
- assert r.score == 2
56
-
57
- def test_score_1_poor(self):
58
- f = _make_features("hurdle_step", angles={
59
- "step_hip_flexion_deg": 50.0, "stance_knee_angle_deg": 140.0,
60
- }, alignments={"trunk_stable": False, "stance_knee_extended": False})
61
- r = score_hurdle_step(f)
62
- assert r.score == 1
63
-
64
- def test_never_scores_zero(self):
65
- f = _make_features("hurdle_step", angles={
66
- "step_hip_flexion_deg": 30.0,
67
- }, alignments={"trunk_stable": False, "stance_knee_extended": False})
68
- r = score_hurdle_step(f)
69
- assert r.score >= 1
70
-
71
-
72
- # ─── Inline Lunge ─────────────────────────────────────────────────────────────
73
-
74
- class TestInlineLunge:
75
- def test_score_3_deep_and_aligned(self):
76
- f = _make_features("inline_lunge", angles={
77
- "front_knee_flexion_deg": 85.0, "trunk_lean_from_vertical_deg": 5.0,
78
- }, alignments={"trunk_upright": True, "knee_over_ankle": True})
79
- r = score_inline_lunge(f)
80
- assert r.score == 3
81
-
82
- def test_score_1_shallow(self):
83
- f = _make_features("inline_lunge", angles={
84
- "front_knee_flexion_deg": 140.0,
85
- }, alignments={"trunk_upright": False, "knee_over_ankle": False})
86
- r = score_inline_lunge(f)
87
- assert r.score == 1
88
-
89
-
90
- # ─── Shoulder Mobility ────────────────────────────────────────────────────────
91
-
92
- class TestShoulderMobility:
93
- def test_score_3_close(self):
94
- f = _make_features("shoulder_mobility", angles={
95
- "inter_fist_normalized": 0.25,
96
- }, alignments={"fists_within_one_hand": True, "fists_within_1_5_hand": True})
97
- r = score_shoulder_mobility(f)
98
- assert r.score == 3
99
-
100
- def test_score_2_moderate(self):
101
- f = _make_features("shoulder_mobility", angles={
102
- "inter_fist_normalized": 0.45,
103
- }, alignments={"fists_within_one_hand": False, "fists_within_1_5_hand": True})
104
- r = score_shoulder_mobility(f)
105
- assert r.score == 2
106
-
107
- def test_score_1_far(self):
108
- f = _make_features("shoulder_mobility", angles={
109
- "inter_fist_normalized": 0.8,
110
- }, alignments={"fists_within_one_hand": False, "fists_within_1_5_hand": False})
111
- r = score_shoulder_mobility(f)
112
- assert r.score == 1
113
-
114
-
115
- # ─── Active SLR ───────────────────────────────────────────────────────────────
116
-
117
- class TestActiveSLR:
118
- def test_score_3_high_raise(self):
119
- f = _make_features("active_slr", angles={
120
- "raised_leg_angle_deg": 80.0,
121
- }, alignments={"past_contralateral_knee": True, "past_mid_thigh": True, "down_leg_flat": True})
122
- r = score_active_slr(f)
123
- assert r.score == 3
124
-
125
- def test_score_2_moderate_raise(self):
126
- f = _make_features("active_slr", angles={
127
- "raised_leg_angle_deg": 55.0,
128
- }, alignments={"past_contralateral_knee": False, "past_mid_thigh": True, "down_leg_flat": True})
129
- r = score_active_slr(f)
130
- assert r.score == 2
131
-
132
- def test_score_1_low_raise(self):
133
- f = _make_features("active_slr", angles={
134
- "raised_leg_angle_deg": 30.0,
135
- }, alignments={"past_contralateral_knee": False, "past_mid_thigh": False, "down_leg_flat": True})
136
- r = score_active_slr(f)
137
- assert r.score == 1
138
-
139
-
140
- # ─── Trunk Stability Push-Up ─────────────────────────────────────────────────
141
-
142
- class TestTrunkStabilityPushup:
143
- def test_score_3_rigid_hands_high(self):
144
- f = _make_features("trunk_stability_pushup", angles={
145
- "max_sag_px": 10.0, "trunk_variance_px": 5.0,
146
- }, alignments={"body_rigid": True, "no_sag": True, "hands_at_forehead": True})
147
- r = score_trunk_stability_pushup(f)
148
- assert r.score == 3
149
-
150
- def test_score_1_sag(self):
151
- f = _make_features("trunk_stability_pushup", angles={
152
- "max_sag_px": 50.0, "trunk_variance_px": 25.0,
153
- }, alignments={"body_rigid": False, "no_sag": False, "hands_at_forehead": True})
154
- r = score_trunk_stability_pushup(f)
155
- assert r.score == 1
156
-
157
-
158
- # ─── Rotary Stability ────────────────────────────────────────────────────────
159
-
160
- class TestRotaryStability:
161
- def test_score_2_stable(self):
162
- f = _make_features("rotary_stability", angles={
163
- "trunk_stability_std_px": 8.0, "shoulder_level_diff_px": 10.0, "hip_level_diff_px": 12.0,
164
- }, alignments={"trunk_stable": True, "shoulders_level": True, "hips_level": True})
165
- r = score_rotary_stability(f)
166
- assert r.score == 2 # Default to 2 (contralateral assumption)
167
-
168
- def test_score_1_unstable(self):
169
- f = _make_features("rotary_stability", angles={
170
- "trunk_stability_std_px": 30.0, "shoulder_level_diff_px": 35.0, "hip_level_diff_px": 30.0,
171
- }, alignments={"trunk_stable": False, "shoulders_level": False, "hips_level": False})
172
- r = score_rotary_stability(f)
173
- assert r.score == 1
174
-
175
-
176
- # ─── JudgeAgent fallback ─────────────────────────────────────────────────────
177
-
178
- class TestJudgeAgent:
179
- def test_fallback_when_judge_disabled(self, monkeypatch):
180
- """When ENABLE_JUDGE=False, judge promotes rubric score."""
181
- from formscout import config
182
- monkeypatch.setattr(config, "ENABLE_JUDGE", False)
183
- agent = JudgeAgent()
184
- features = _make_features("deep_squat", angles={"left_femur_from_horizontal_deg": 70.0})
185
- rubric = ScoreResult(score=3, rationale="all good", confidence=0.9)
186
- movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
187
- result = agent.run(features, rubric, movement)
188
- assert isinstance(result, JudgeResult)
189
- assert result.score == 3
190
- assert "[rubric-only]" in result.rationale
191
-
192
- def test_fallback_when_server_unavailable(self, monkeypatch):
193
- """ENABLE_JUDGE=True but llama-server down → rubric fallback, never a crash."""
194
- from unittest.mock import PropertyMock, patch
195
- from formscout import config
196
- monkeypatch.setattr(config, "ENABLE_JUDGE", True)
197
- agent = JudgeAgent()
198
- with patch.object(type(agent._client), "available", new_callable=PropertyMock, return_value=False):
199
- features = _make_features("deep_squat")
200
- rubric = ScoreResult(score=2, rationale="heels up", confidence=0.8)
201
- movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
202
- result = agent.run(features, rubric, movement)
203
- assert result.score == 2
204
- assert "[rubric-only]" in result.rationale
205
-
206
- def test_vlm_response_parsed_into_judge_result(self, monkeypatch):
207
- """ENABLE_JUDGE=True with live client → VLM JSON becomes JudgeResult."""
208
- from unittest.mock import PropertyMock, patch
209
- from formscout import config
210
- monkeypatch.setattr(config, "ENABLE_JUDGE", True)
211
- agent = JudgeAgent()
212
- vlm_json = {
213
- "test": "deep_squat", "side": "na", "score": 2, "needs_human": False,
214
- "rationale": "Femur 5° above horizontal; 2D estimate.",
215
- "compensation_tags": ["forward_lean"], "corrective_hint": "Sit back into heels.",
216
- "confidence": 0.78,
217
- }
218
- with patch.object(type(agent._client), "available", new_callable=PropertyMock, return_value=True), \
219
- patch.object(agent._client, "complete", return_value=vlm_json):
220
- features = _make_features("deep_squat")
221
- rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
222
- movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
223
- result = agent.run(features, rubric, movement)
224
- assert result.score == 2
225
- assert result.compensation_tags == ["forward_lean"]
226
- assert result.needs_human is False
227
-
228
- def test_vlm_needs_human_yields_no_score(self, monkeypatch):
229
- """needs_human=True from the VLM must produce score=None."""
230
- from unittest.mock import PropertyMock, patch
231
- from formscout import config
232
- monkeypatch.setattr(config, "ENABLE_JUDGE", True)
233
- agent = JudgeAgent()
234
- vlm_json = {"score": 1, "needs_human": True, "rationale": "Possible pain.", "confidence": 0.9}
235
- with patch.object(type(agent._client), "available", new_callable=PropertyMock, return_value=True), \
236
- patch.object(agent._client, "complete", return_value=vlm_json):
237
- result = agent.run(
238
- _make_features("deep_squat"),
239
- ScoreResult(score=1, rationale="x", confidence=0.5),
240
- MovementResult(test_name="deep_squat", side="na", confidence=1.0),
241
- )
242
- assert result.needs_human is True
243
- assert result.score is None
244
-
245
-
246
- # ─── LlamaCppClient (chat-completions endpoint) ──────────────────────────────
247
-
248
- class TestLlamaCppClient:
249
- def test_parse_plain_json(self):
250
- from formscout.serving.llama_cpp import LlamaCppClient
251
- assert LlamaCppClient._parse_json_reply('{"score": 3}') == {"score": 3}
252
-
253
- def test_parse_fenced_json(self):
254
- from formscout.serving.llama_cpp import LlamaCppClient
255
- fenced = '```json\n{"score": 2, "needs_human": false}\n```'
256
- assert LlamaCppClient._parse_json_reply(fenced) == {"score": 2, "needs_human": False}
257
-
258
- def test_parse_non_json_returns_text(self):
259
- from formscout.serving.llama_cpp import LlamaCppClient
260
- assert LlamaCppClient._parse_json_reply("not json") == {"text": "not json"}
261
-
262
- def test_complete_posts_chat_endpoint_with_images(self):
263
- from unittest.mock import MagicMock, patch
264
- from formscout.serving.llama_cpp import LlamaCppClient
265
-
266
- client = LlamaCppClient(port=8080)
267
- resp = MagicMock()
268
- resp.json.return_value = {"choices": [{"message": {"content": '{"ok": true}'}}]}
269
- resp.raise_for_status.return_value = None
270
- with patch("formscout.serving.llama_cpp.requests.post", return_value=resp) as mock_post:
271
- result = client.complete("score this", images=["aGVsbG8=" * 600])
272
- assert result == {"ok": True}
273
- url = mock_post.call_args.args[0] if mock_post.call_args.args else mock_post.call_args.kwargs.get("url")
274
- assert url.endswith("/v1/chat/completions")
275
- payload = mock_post.call_args.kwargs["json"]
276
- content = payload["messages"][0]["content"]
277
- assert content[0] == {"type": "text", "text": "score this"}
278
- assert content[1]["type"] == "image_url"
279
- assert content[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
280
-
281
- def test_complete_connection_error_returns_safe_dict(self):
282
- from unittest.mock import patch
283
- import requests as _requests
284
- from formscout.serving.llama_cpp import LlamaCppClient
285
-
286
- client = LlamaCppClient(port=8080)
287
- with patch("formscout.serving.llama_cpp.requests.post", side_effect=_requests.ConnectionError):
288
- result = client.complete("hello")
289
- assert "error" in result
290
-
291
-
292
- # ─── ReportAgent ──────────────────────────────────────────────────────────────
293
-
294
- class TestReportAgent:
295
- def test_single_test_report(self):
296
- agent = ReportAgent()
297
- entries = [{
298
- "movement": MovementResult(test_name="deep_squat", side="na", confidence=1.0),
299
- "features": _make_features("deep_squat"),
300
- "rubric_score": ScoreResult(score=3, rationale="ok", confidence=0.9),
301
- "judge": JudgeResult(
302
- score=3, rationale="good", compensation_tags=[], corrective_hint="",
303
- confidence=0.9,
304
- ),
305
- "side": "na",
306
- }]
307
- result = agent.run(entries)
308
- assert isinstance(result, ReportResult)
309
- assert len(result.per_test) == 1
310
- assert result.per_test[0]["score"] == 3
311
-
312
- def test_bilateral_reports_lower_score(self):
313
- agent = ReportAgent()
314
- entries = [
315
- {
316
- "movement": MovementResult(test_name="hurdle_step", side="left", confidence=1.0),
317
- "features": _make_features("hurdle_step", side="left"),
318
- "rubric_score": ScoreResult(score=3, rationale="ok", confidence=0.9),
319
- "judge": JudgeResult(
320
- score=3, rationale="", compensation_tags=[], corrective_hint="",
321
- confidence=0.9,
322
- ),
323
- "side": "left",
324
- },
325
- {
326
- "movement": MovementResult(test_name="hurdle_step", side="right", confidence=1.0),
327
- "features": _make_features("hurdle_step", side="right"),
328
- "rubric_score": ScoreResult(score=2, rationale="comp", confidence=0.8),
329
- "judge": JudgeResult(
330
- score=2, rationale="", compensation_tags=[], corrective_hint="",
331
- confidence=0.8,
332
- ),
333
- "side": "right",
334
- },
335
- ]
336
- result = agent.run(entries)
337
- assert result.per_test[0]["score"] == 2 # lower of 3 and 2
338
- assert len(result.asymmetries) == 1
339
- assert result.asymmetries[0]["delta"] == 1
340
-
341
- def test_composite_none_when_unscored(self):
342
- agent = ReportAgent()
343
- entries = [{
344
- "movement": MovementResult(test_name="deep_squat", side="na", confidence=1.0),
345
- "features": _make_features("deep_squat"),
346
- "rubric_score": ScoreResult(score=1, rationale="", confidence=0.5),
347
- "judge": JudgeResult(
348
- score=None, rationale="pain", compensation_tags=[], corrective_hint="",
349
- confidence=0.0, needs_human=True,
350
- ),
351
- "side": "na",
352
- }]
353
- result = agent.run(entries)
354
- assert result.composite is None
 
1
+ """Tests for all rubric scorers and Phase 2 agents."""
2
+
3
+ from formscout.types import (
4
+ BiomechFeatures, ScoreResult, MovementResult, JudgeResult, ReportResult,
5
+ )
6
+ from formscout.rubric import score_test, SCORERS
7
+ from formscout.rubric.hurdle_step import score_hurdle_step
8
+ from formscout.rubric.inline_lunge import score_inline_lunge
9
+ from formscout.rubric.shoulder_mobility import score_shoulder_mobility
10
+ from formscout.rubric.active_slr import score_active_slr
11
+ from formscout.rubric.trunk_stability_pushup import score_trunk_stability_pushup
12
+ from formscout.rubric.rotary_stability import score_rotary_stability
13
+ from formscout.agents.judge import JudgeAgent
14
+ from formscout.agents.report import ReportAgent
15
+
16
+
17
+ def _make_features(test_name, angles=None, alignments=None, side="na", sym_delta=None):
18
+ return BiomechFeatures(
19
+ test_name=test_name, view="2d", side=side,
20
+ angles=angles or {}, alignments=alignments or {},
21
+ symmetry_delta=sym_delta, timing={}, confidence=0.8,
22
+ )
23
+
24
+
25
+ # ─── Rubric dispatch ─────────────────────────────────────────────────────────
26
+
27
+ class TestRubricDispatch:
28
+ def test_all_tests_have_scorers(self):
29
+ expected = {"deep_squat", "hurdle_step", "inline_lunge", "shoulder_mobility",
30
+ "active_slr", "trunk_stability_pushup", "rotary_stability"}
31
+ assert set(SCORERS.keys()) == expected
32
+
33
+ def test_dispatch_unknown_test(self):
34
+ f = _make_features("unknown_test")
35
+ r = score_test(f)
36
+ assert r.confidence == 0.0
37
+
38
+
39
+ # ─── Hurdle Step ──────────────────────────────────────────────────────────────
40
+
41
+ class TestHurdleStep:
42
+ def test_score_3_good_form(self):
43
+ f = _make_features("hurdle_step", angles={
44
+ "step_hip_flexion_deg": 100.0, "stance_knee_angle_deg": 175.0,
45
+ "shoulder_tilt_deg": 5.0,
46
+ }, alignments={"trunk_stable": True, "stance_knee_extended": True})
47
+ r = score_hurdle_step(f)
48
+ assert r.score == 3
49
+
50
+ def test_score_2_compensation(self):
51
+ f = _make_features("hurdle_step", angles={
52
+ "step_hip_flexion_deg": 80.0, "stance_knee_angle_deg": 170.0,
53
+ }, alignments={"trunk_stable": True, "stance_knee_extended": True})
54
+ r = score_hurdle_step(f)
55
+ assert r.score == 2
56
+
57
+ def test_score_1_poor(self):
58
+ f = _make_features("hurdle_step", angles={
59
+ "step_hip_flexion_deg": 50.0, "stance_knee_angle_deg": 140.0,
60
+ }, alignments={"trunk_stable": False, "stance_knee_extended": False})
61
+ r = score_hurdle_step(f)
62
+ assert r.score == 1
63
+
64
+ def test_never_scores_zero(self):
65
+ f = _make_features("hurdle_step", angles={
66
+ "step_hip_flexion_deg": 30.0,
67
+ }, alignments={"trunk_stable": False, "stance_knee_extended": False})
68
+ r = score_hurdle_step(f)
69
+ assert r.score >= 1
70
+
71
+
72
+ # ─── Inline Lunge ─────────────────────────────────────────────────────────────
73
+
74
+ class TestInlineLunge:
75
+ def test_score_3_deep_and_aligned(self):
76
+ f = _make_features("inline_lunge", angles={
77
+ "front_knee_flexion_deg": 85.0, "trunk_lean_from_vertical_deg": 5.0,
78
+ }, alignments={"trunk_upright": True, "knee_over_ankle": True})
79
+ r = score_inline_lunge(f)
80
+ assert r.score == 3
81
+
82
+ def test_score_1_shallow(self):
83
+ f = _make_features("inline_lunge", angles={
84
+ "front_knee_flexion_deg": 140.0,
85
+ }, alignments={"trunk_upright": False, "knee_over_ankle": False})
86
+ r = score_inline_lunge(f)
87
+ assert r.score == 1
88
+
89
+
90
+ # ─── Shoulder Mobility ────────────────────────────────────────────────────────
91
+
92
+ class TestShoulderMobility:
93
+ def test_score_3_close(self):
94
+ f = _make_features("shoulder_mobility", angles={
95
+ "inter_fist_normalized": 0.25,
96
+ }, alignments={"fists_within_one_hand": True, "fists_within_1_5_hand": True})
97
+ r = score_shoulder_mobility(f)
98
+ assert r.score == 3
99
+
100
+ def test_score_2_moderate(self):
101
+ f = _make_features("shoulder_mobility", angles={
102
+ "inter_fist_normalized": 0.45,
103
+ }, alignments={"fists_within_one_hand": False, "fists_within_1_5_hand": True})
104
+ r = score_shoulder_mobility(f)
105
+ assert r.score == 2
106
+
107
+ def test_score_1_far(self):
108
+ f = _make_features("shoulder_mobility", angles={
109
+ "inter_fist_normalized": 0.8,
110
+ }, alignments={"fists_within_one_hand": False, "fists_within_1_5_hand": False})
111
+ r = score_shoulder_mobility(f)
112
+ assert r.score == 1
113
+
114
+
115
+ # ─── Active SLR ───────────────────────────────────────────────────────────────
116
+
117
+ class TestActiveSLR:
118
+ def test_score_3_high_raise(self):
119
+ f = _make_features("active_slr", angles={
120
+ "raised_leg_angle_deg": 80.0,
121
+ }, alignments={"past_contralateral_knee": True, "past_mid_thigh": True, "down_leg_flat": True})
122
+ r = score_active_slr(f)
123
+ assert r.score == 3
124
+
125
+ def test_score_2_moderate_raise(self):
126
+ f = _make_features("active_slr", angles={
127
+ "raised_leg_angle_deg": 55.0,
128
+ }, alignments={"past_contralateral_knee": False, "past_mid_thigh": True, "down_leg_flat": True})
129
+ r = score_active_slr(f)
130
+ assert r.score == 2
131
+
132
+ def test_score_1_low_raise(self):
133
+ f = _make_features("active_slr", angles={
134
+ "raised_leg_angle_deg": 30.0,
135
+ }, alignments={"past_contralateral_knee": False, "past_mid_thigh": False, "down_leg_flat": True})
136
+ r = score_active_slr(f)
137
+ assert r.score == 1
138
+
139
+
140
+ # ─── Trunk Stability Push-Up ─────────────────────────────────────────────────
141
+
142
+ class TestTrunkStabilityPushup:
143
+ def test_score_3_rigid_hands_high(self):
144
+ f = _make_features("trunk_stability_pushup", angles={
145
+ "max_sag_px": 10.0, "trunk_variance_px": 5.0,
146
+ }, alignments={"body_rigid": True, "no_sag": True, "hands_at_forehead": True})
147
+ r = score_trunk_stability_pushup(f)
148
+ assert r.score == 3
149
+
150
+ def test_score_1_sag(self):
151
+ f = _make_features("trunk_stability_pushup", angles={
152
+ "max_sag_px": 50.0, "trunk_variance_px": 25.0,
153
+ }, alignments={"body_rigid": False, "no_sag": False, "hands_at_forehead": True})
154
+ r = score_trunk_stability_pushup(f)
155
+ assert r.score == 1
156
+
157
+
158
+ # ─── Rotary Stability ────────────────────────────────────────────────────────
159
+
160
+ class TestRotaryStability:
161
+ def test_score_2_stable(self):
162
+ f = _make_features("rotary_stability", angles={
163
+ "trunk_stability_std_px": 8.0, "shoulder_level_diff_px": 10.0, "hip_level_diff_px": 12.0,
164
+ }, alignments={"trunk_stable": True, "shoulders_level": True, "hips_level": True})
165
+ r = score_rotary_stability(f)
166
+ assert r.score == 2 # Default to 2 (contralateral assumption)
167
+
168
+ def test_score_1_unstable(self):
169
+ f = _make_features("rotary_stability", angles={
170
+ "trunk_stability_std_px": 30.0, "shoulder_level_diff_px": 35.0, "hip_level_diff_px": 30.0,
171
+ }, alignments={"trunk_stable": False, "shoulders_level": False, "hips_level": False})
172
+ r = score_rotary_stability(f)
173
+ assert r.score == 1
174
+
175
+
176
+ # ─── JudgeAgent fallback ─────────────────────────────────────────────────────
177
+
178
+ class TestJudgeAgent:
179
+ def test_fallback_when_judge_disabled(self, monkeypatch):
180
+ """When ENABLE_JUDGE=False, judge promotes rubric score."""
181
+ from formscout import config
182
+ monkeypatch.setattr(config, "ENABLE_JUDGE", False)
183
+ agent = JudgeAgent()
184
+ features = _make_features("deep_squat", angles={"left_femur_from_horizontal_deg": 70.0})
185
+ rubric = ScoreResult(score=3, rationale="all good", confidence=0.9)
186
+ movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
187
+ result = agent.run(features, rubric, movement)
188
+ assert isinstance(result, JudgeResult)
189
+ assert result.score == 3
190
+ assert "[rubric-only]" in result.rationale
191
+
192
+ def test_fallback_when_server_unavailable(self, monkeypatch):
193
+ """ENABLE_JUDGE=True but llama-server down → rubric fallback, never a crash."""
194
+ from unittest.mock import PropertyMock, patch
195
+ from formscout import config
196
+ monkeypatch.setattr(config, "ENABLE_JUDGE", True)
197
+ agent = JudgeAgent()
198
+ with patch.object(type(agent._client), "available", new_callable=PropertyMock, return_value=False):
199
+ features = _make_features("deep_squat")
200
+ rubric = ScoreResult(score=2, rationale="heels up", confidence=0.8)
201
+ movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
202
+ result = agent.run(features, rubric, movement)
203
+ assert result.score == 2
204
+ assert "[rubric-only]" in result.rationale
205
+
206
+ def test_vlm_response_parsed_into_judge_result(self, monkeypatch):
207
+ """ENABLE_JUDGE=True with live client → VLM JSON becomes JudgeResult."""
208
+ from unittest.mock import PropertyMock, patch
209
+ from formscout import config
210
+ monkeypatch.setattr(config, "ENABLE_JUDGE", True)
211
+ agent = JudgeAgent()
212
+ vlm_json = {
213
+ "test": "deep_squat", "side": "na", "score": 2, "needs_human": False,
214
+ "rationale": "Femur 5° above horizontal; 2D estimate.",
215
+ "compensation_tags": ["forward_lean"], "corrective_hint": "Sit back into heels.",
216
+ "confidence": 0.78,
217
+ }
218
+ with patch.object(type(agent._client), "available", new_callable=PropertyMock, return_value=True), \
219
+ patch.object(agent._client, "complete", return_value=vlm_json):
220
+ features = _make_features("deep_squat")
221
+ rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
222
+ movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
223
+ result = agent.run(features, rubric, movement)
224
+ assert result.score == 2
225
+ assert result.compensation_tags == ["forward_lean"]
226
+ assert result.needs_human is False
227
+
228
+ def test_vlm_needs_human_yields_no_score(self, monkeypatch):
229
+ """needs_human=True from the VLM must produce score=None."""
230
+ from unittest.mock import PropertyMock, patch
231
+ from formscout import config
232
+ monkeypatch.setattr(config, "ENABLE_JUDGE", True)
233
+ agent = JudgeAgent()
234
+ vlm_json = {"score": 1, "needs_human": True, "rationale": "Possible pain.", "confidence": 0.9}
235
+ with patch.object(type(agent._client), "available", new_callable=PropertyMock, return_value=True), \
236
+ patch.object(agent._client, "complete", return_value=vlm_json):
237
+ result = agent.run(
238
+ _make_features("deep_squat"),
239
+ ScoreResult(score=1, rationale="x", confidence=0.5),
240
+ MovementResult(test_name="deep_squat", side="na", confidence=1.0),
241
+ )
242
+ assert result.needs_human is True
243
+ assert result.score is None
244
+
245
+
246
+ # ─── LlamaCppClient (chat-completions endpoint) ──────────────────────────────
247
+
248
+ class TestLlamaCppClient:
249
+ def test_parse_plain_json(self):
250
+ from formscout.serving.llama_cpp import LlamaCppClient
251
+ assert LlamaCppClient._parse_json_reply('{"score": 3}') == {"score": 3}
252
+
253
+ def test_parse_fenced_json(self):
254
+ from formscout.serving.llama_cpp import LlamaCppClient
255
+ fenced = '```json\n{"score": 2, "needs_human": false}\n```'
256
+ assert LlamaCppClient._parse_json_reply(fenced) == {"score": 2, "needs_human": False}
257
+
258
+ def test_parse_non_json_returns_text(self):
259
+ from formscout.serving.llama_cpp import LlamaCppClient
260
+ assert LlamaCppClient._parse_json_reply("not json") == {"text": "not json"}
261
+
262
+ def test_complete_posts_chat_endpoint_with_images(self):
263
+ from unittest.mock import MagicMock, patch
264
+ from formscout.serving.llama_cpp import LlamaCppClient
265
+
266
+ client = LlamaCppClient(port=8080)
267
+ resp = MagicMock()
268
+ resp.json.return_value = {"choices": [{"message": {"content": '{"ok": true}'}}]}
269
+ resp.raise_for_status.return_value = None
270
+ with patch("formscout.serving.llama_cpp.requests.post", return_value=resp) as mock_post:
271
+ result = client.complete("score this", images=["aGVsbG8=" * 600])
272
+ assert result == {"ok": True}
273
+ url = mock_post.call_args.args[0] if mock_post.call_args.args else mock_post.call_args.kwargs.get("url")
274
+ assert url.endswith("/v1/chat/completions")
275
+ payload = mock_post.call_args.kwargs["json"]
276
+ content = payload["messages"][0]["content"]
277
+ assert content[0] == {"type": "text", "text": "score this"}
278
+ assert content[1]["type"] == "image_url"
279
+ assert content[1]["image_url"]["url"].startswith("data:image/jpeg;base64,")
280
+
281
+ def test_complete_connection_error_returns_safe_dict(self):
282
+ from unittest.mock import patch
283
+ import requests as _requests
284
+ from formscout.serving.llama_cpp import LlamaCppClient
285
+
286
+ client = LlamaCppClient(port=8080)
287
+ with patch("formscout.serving.llama_cpp.requests.post", side_effect=_requests.ConnectionError):
288
+ result = client.complete("hello")
289
+ assert "error" in result
290
+
291
+
292
+ # ─── ReportAgent ──────────────────────────────────────────────────────────────
293
+
294
+ class TestReportAgent:
295
+ def test_single_test_report(self):
296
+ agent = ReportAgent()
297
+ entries = [{
298
+ "movement": MovementResult(test_name="deep_squat", side="na", confidence=1.0),
299
+ "features": _make_features("deep_squat"),
300
+ "rubric_score": ScoreResult(score=3, rationale="ok", confidence=0.9),
301
+ "judge": JudgeResult(
302
+ score=3, rationale="good", compensation_tags=[], corrective_hint="",
303
+ confidence=0.9,
304
+ ),
305
+ "side": "na",
306
+ }]
307
+ result = agent.run(entries)
308
+ assert isinstance(result, ReportResult)
309
+ assert len(result.per_test) == 1
310
+ assert result.per_test[0]["score"] == 3
311
+
312
+ def test_bilateral_reports_lower_score(self):
313
+ agent = ReportAgent()
314
+ entries = [
315
+ {
316
+ "movement": MovementResult(test_name="hurdle_step", side="left", confidence=1.0),
317
+ "features": _make_features("hurdle_step", side="left"),
318
+ "rubric_score": ScoreResult(score=3, rationale="ok", confidence=0.9),
319
+ "judge": JudgeResult(
320
+ score=3, rationale="", compensation_tags=[], corrective_hint="",
321
+ confidence=0.9,
322
+ ),
323
+ "side": "left",
324
+ },
325
+ {
326
+ "movement": MovementResult(test_name="hurdle_step", side="right", confidence=1.0),
327
+ "features": _make_features("hurdle_step", side="right"),
328
+ "rubric_score": ScoreResult(score=2, rationale="comp", confidence=0.8),
329
+ "judge": JudgeResult(
330
+ score=2, rationale="", compensation_tags=[], corrective_hint="",
331
+ confidence=0.8,
332
+ ),
333
+ "side": "right",
334
+ },
335
+ ]
336
+ result = agent.run(entries)
337
+ assert result.per_test[0]["score"] == 2 # lower of 3 and 2
338
+ assert len(result.asymmetries) == 1
339
+ assert result.asymmetries[0]["delta"] == 1
340
+
341
+ def test_composite_none_when_unscored(self):
342
+ agent = ReportAgent()
343
+ entries = [{
344
+ "movement": MovementResult(test_name="deep_squat", side="na", confidence=1.0),
345
+ "features": _make_features("deep_squat"),
346
+ "rubric_score": ScoreResult(score=1, rationale="", confidence=0.5),
347
+ "judge": JudgeResult(
348
+ score=None, rationale="pain", compensation_tags=[], corrective_hint="",
349
+ confidence=0.0, needs_human=True,
350
+ ),
351
+ "side": "na",
352
+ }]
353
+ result = agent.run(entries)
354
+ assert result.composite is None
tests/test_session.py CHANGED
@@ -1,94 +1,94 @@
1
- """Tests for the FMS session accumulator — no GPU, no model downloads."""
2
- import numpy as np
3
-
4
- from formscout.types import (
5
- IngestResult, Pose2DResult, BiomechFeatures, ScoreResult, JudgeResult,
6
- MovementResult, SessionEntry,
7
- )
8
-
9
-
10
- def test_session_entry_holds_typed_objects():
11
- movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
12
- features = BiomechFeatures(
13
- test_name="deep_squat", view="2d", side="na",
14
- angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": True},
15
- symmetry_delta=None, timing={"deepest_frame": 2}, confidence=0.9,
16
- )
17
- rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
18
- judge = JudgeResult(score=2, rationale="ok", compensation_tags=["heels elevated"],
19
- corrective_hint="ankle mobility", confidence=0.85)
20
- entry = SessionEntry(
21
- test_name="deep_squat", side="na", score=2, needs_human=False,
22
- rationale="ok", compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
23
- measurements={"left_knee_flexion_deg": 95.0}, confidence=0.85, view="2d",
24
- keyframe_path=None, movement=movement, features=features,
25
- rubric_score=rubric, judge=judge,
26
- )
27
- assert entry.score == 2
28
- assert entry.movement.test_name == "deep_squat"
29
- assert entry.rubric_score.score == 2
30
- assert entry.judge.compensation_tags == ["heels elevated"]
31
-
32
-
33
- def _ingest(n=5, h=480, w=640):
34
- frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
35
- return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
36
-
37
-
38
- def _pose(n=5):
39
- kps = []
40
- for i in range(n):
41
- kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
42
- for j in range(17)})
43
- return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
44
-
45
-
46
- def _features(test_name="deep_squat", side="na", frame_key="deepest_frame"):
47
- return BiomechFeatures(
48
- test_name=test_name, view="2d", side=side,
49
- angles={"left_knee_flexion_deg": 95.0},
50
- alignments={"knees_tracking_over_feet": False},
51
- symmetry_delta=None, timing={frame_key: 2}, confidence=0.9,
52
- )
53
-
54
-
55
- def _judge(score=2, needs_human=False):
56
- return JudgeResult(
57
- score=None if needs_human else score, rationale="r",
58
- compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
59
- confidence=0.85, needs_human=needs_human,
60
- )
61
-
62
-
63
- def test_add_analysis_appends_entry_and_writes_files():
64
- import os
65
- from formscout import session as S
66
- sess = S.new_session()
67
- entry = S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
68
- features=_features(), judge=_judge(), test_name="deep_squat", side="na")
69
- assert len(sess.entries) == 1
70
- assert entry.score == 2
71
- assert os.path.exists(os.path.join(sess.session_dir, "session.json"))
72
- assert os.path.exists(os.path.join(sess.session_dir, "analysis.md"))
73
- # key-frame still written (deepest_frame=2 is valid)
74
- assert entry.keyframe_path and os.path.exists(entry.keyframe_path)
75
-
76
-
77
- def test_finish_composite_null_when_needs_human():
78
- from formscout import session as S
79
- sess = S.new_session()
80
- S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(), features=_features(),
81
- judge=_judge(score=3), test_name="deep_squat", side="na")
82
- S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
83
- features=_features("trunk_stability_pushup", frame_key="max_sag_frame"),
84
- judge=_judge(needs_human=True), test_name="trunk_stability_pushup", side="na")
85
- report, pdf_path = S.finish_session(sess)
86
- assert report is not None
87
- assert report.composite is None # one test needs_human
88
-
89
-
90
- def test_finish_empty_session_returns_none():
91
- from formscout import session as S
92
- sess = S.new_session()
93
- report, pdf_path = S.finish_session(sess)
94
- assert report is None and pdf_path is None
 
1
+ """Tests for the FMS session accumulator — no GPU, no model downloads."""
2
+ import numpy as np
3
+
4
+ from formscout.types import (
5
+ IngestResult, Pose2DResult, BiomechFeatures, ScoreResult, JudgeResult,
6
+ MovementResult, SessionEntry,
7
+ )
8
+
9
+
10
+ def test_session_entry_holds_typed_objects():
11
+ movement = MovementResult(test_name="deep_squat", side="na", confidence=1.0)
12
+ features = BiomechFeatures(
13
+ test_name="deep_squat", view="2d", side="na",
14
+ angles={"left_knee_flexion_deg": 95.0}, alignments={"knees_tracking_over_feet": True},
15
+ symmetry_delta=None, timing={"deepest_frame": 2}, confidence=0.9,
16
+ )
17
+ rubric = ScoreResult(score=2, rationale="ok", confidence=0.8)
18
+ judge = JudgeResult(score=2, rationale="ok", compensation_tags=["heels elevated"],
19
+ corrective_hint="ankle mobility", confidence=0.85)
20
+ entry = SessionEntry(
21
+ test_name="deep_squat", side="na", score=2, needs_human=False,
22
+ rationale="ok", compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
23
+ measurements={"left_knee_flexion_deg": 95.0}, confidence=0.85, view="2d",
24
+ keyframe_path=None, movement=movement, features=features,
25
+ rubric_score=rubric, judge=judge,
26
+ )
27
+ assert entry.score == 2
28
+ assert entry.movement.test_name == "deep_squat"
29
+ assert entry.rubric_score.score == 2
30
+ assert entry.judge.compensation_tags == ["heels elevated"]
31
+
32
+
33
+ def _ingest(n=5, h=480, w=640):
34
+ frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
35
+ return IngestResult(frames=frames, fps=30.0, duration=n / 30.0, n_people=1, width=w, height=h)
36
+
37
+
38
+ def _pose(n=5):
39
+ kps = []
40
+ for i in range(n):
41
+ kps.append({j: {"x": float(50 + j * 25), "y": float(80 + j * 18), "conf": 0.9}
42
+ for j in range(17)})
43
+ return Pose2DResult(keypoints=kps, fps=30.0, confidence=0.9)
44
+
45
+
46
+ def _features(test_name="deep_squat", side="na", frame_key="deepest_frame"):
47
+ return BiomechFeatures(
48
+ test_name=test_name, view="2d", side=side,
49
+ angles={"left_knee_flexion_deg": 95.0},
50
+ alignments={"knees_tracking_over_feet": False},
51
+ symmetry_delta=None, timing={frame_key: 2}, confidence=0.9,
52
+ )
53
+
54
+
55
+ def _judge(score=2, needs_human=False):
56
+ return JudgeResult(
57
+ score=None if needs_human else score, rationale="r",
58
+ compensation_tags=["heels elevated"], corrective_hint="ankle mobility",
59
+ confidence=0.85, needs_human=needs_human,
60
+ )
61
+
62
+
63
+ def test_add_analysis_appends_entry_and_writes_files():
64
+ import os
65
+ from formscout import session as S
66
+ sess = S.new_session()
67
+ entry = S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
68
+ features=_features(), judge=_judge(), test_name="deep_squat", side="na")
69
+ assert len(sess.entries) == 1
70
+ assert entry.score == 2
71
+ assert os.path.exists(os.path.join(sess.session_dir, "session.json"))
72
+ assert os.path.exists(os.path.join(sess.session_dir, "analysis.md"))
73
+ # key-frame still written (deepest_frame=2 is valid)
74
+ assert entry.keyframe_path and os.path.exists(entry.keyframe_path)
75
+
76
+
77
+ def test_finish_composite_null_when_needs_human():
78
+ from formscout import session as S
79
+ sess = S.new_session()
80
+ S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(), features=_features(),
81
+ judge=_judge(score=3), test_name="deep_squat", side="na")
82
+ S.add_analysis(sess, ingest=_ingest(), pose2d=_pose(),
83
+ features=_features("trunk_stability_pushup", frame_key="max_sag_frame"),
84
+ judge=_judge(needs_human=True), test_name="trunk_stability_pushup", side="na")
85
+ report, pdf_path = S.finish_session(sess)
86
+ assert report is not None
87
+ assert report.composite is None # one test needs_human
88
+
89
+
90
+ def test_finish_empty_session_returns_none():
91
+ from formscout import session as S
92
+ sess = S.new_session()
93
+ report, pdf_path = S.finish_session(sess)
94
+ assert report is None and pdf_path is None
tests/test_visualizer.py CHANGED
@@ -1,176 +1,176 @@
1
- """Tests for PoseVisualizer — no GPU, no model downloads."""
2
- import numpy as np
3
- import pytest
4
- from formscout.types import IngestResult, Pose2DResult
5
-
6
-
7
- def _make_ingest(n=5, h=480, w=640, fps=30.0):
8
- frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
9
- return IngestResult(frames=frames, fps=fps, duration=n / fps, n_people=1, width=w, height=h)
10
-
11
-
12
- def _make_pose(n=5, w=640, h=480):
13
- """Synthetic Pose2DResult: 17 joints at fixed pixel positions, conf=0.9."""
14
- kps_per_frame = []
15
- for i in range(n):
16
- frame_kps = {}
17
- for j in range(17):
18
- frame_kps[j] = {
19
- "x": float(50 + j * 30 + i * 2),
20
- "y": float(100 + j * 20),
21
- "conf": 0.9,
22
- }
23
- kps_per_frame.append(frame_kps)
24
- return Pose2DResult(keypoints=kps_per_frame, fps=30.0, confidence=0.9, notes="")
25
-
26
-
27
- class TestComputeJointVelocity:
28
- def test_returns_17_joints(self):
29
- from formscout.agents.visualizer import compute_joint_velocity
30
- pose = _make_pose(n=5)
31
- result = compute_joint_velocity(pose.keypoints, fps=30.0)
32
- assert len(result) == 17
33
-
34
- def test_each_list_has_n_frames(self):
35
- from formscout.agents.visualizer import compute_joint_velocity
36
- pose = _make_pose(n=5)
37
- result = compute_joint_velocity(pose.keypoints, fps=30.0)
38
- for joint_idx, speeds in result.items():
39
- assert len(speeds) == 5, f"joint {joint_idx} has {len(speeds)} speeds, expected 5"
40
-
41
- def test_speeds_are_non_negative(self):
42
- from formscout.agents.visualizer import compute_joint_velocity
43
- pose = _make_pose(n=5)
44
- result = compute_joint_velocity(pose.keypoints, fps=30.0)
45
- for speeds in result.values():
46
- assert all(s >= 0.0 for s in speeds)
47
-
48
- def test_missing_keypoints_give_zero_speed(self):
49
- from formscout.agents.visualizer import compute_joint_velocity
50
- empty_kps = [{} for _ in range(5)]
51
- result = compute_joint_velocity(empty_kps, fps=30.0)
52
- for speeds in result.values():
53
- assert all(s == 0.0 for s in speeds)
54
-
55
-
56
- class TestDrawSkeleton:
57
- def test_skeleton_draws_without_error(self):
58
- from formscout.agents.visualizer import PoseVisualizer
59
- vis = PoseVisualizer()
60
- frame = np.zeros((480, 640, 3), dtype=np.uint8)
61
- kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
62
- for j in range(17)}
63
- result = vis._draw_skeleton(frame.copy(), kps)
64
- assert result.shape == frame.shape
65
- assert not np.array_equal(result, frame)
66
-
67
- def test_low_confidence_keypoints_not_drawn(self):
68
- from formscout.agents.visualizer import PoseVisualizer
69
- vis = PoseVisualizer()
70
- frame = np.zeros((480, 640, 3), dtype=np.uint8)
71
- kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.1} for j in range(17)}
72
- result = vis._draw_skeleton(frame.copy(), kps)
73
- assert np.array_equal(result, frame)
74
-
75
-
76
- class TestDrawTrails:
77
- def test_trails_draw_without_error(self):
78
- from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
79
- from collections import deque
80
- vis = PoseVisualizer()
81
- frame = np.zeros((480, 640, 3), dtype=np.uint8)
82
- trail_history = {
83
- 0: deque([(100 + i * 5, 200 + i * 3) for i in range(5)], maxlen=TRAIL_LENGTH)
84
- }
85
- result = vis._draw_trails(frame.copy(), trail_history)
86
- assert result.shape == frame.shape
87
- assert not np.array_equal(result, frame)
88
-
89
- def test_short_trail_no_crash(self):
90
- from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
91
- from collections import deque
92
- vis = PoseVisualizer()
93
- frame = np.zeros((480, 640, 3), dtype=np.uint8)
94
- trail_history = {0: deque([(100, 200)], maxlen=TRAIL_LENGTH)}
95
- result = vis._draw_trails(frame.copy(), trail_history)
96
- assert np.array_equal(result, frame)
97
-
98
-
99
- class TestDrawVelocityArrows:
100
- def test_arrows_draw_without_error(self):
101
- from formscout.agents.visualizer import PoseVisualizer
102
- vis = PoseVisualizer()
103
- frame = np.zeros((480, 640, 3), dtype=np.uint8)
104
- kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
105
- for j in range(17)}
106
- prev_kps = {j: {"x": float(48 + j * 30), "y": float(98 + j * 20), "conf": 0.9}
107
- for j in range(17)}
108
- velocities = {j: [0.0] * 5 for j in range(17)}
109
- velocities[5] = [0.0, 10.0, 50.0, 80.0, 120.0]
110
- result = vis._draw_velocity_arrows(frame.copy(), kps, prev_kps, velocities, frame_idx=4)
111
- assert result.shape == frame.shape
112
-
113
- def test_no_prev_kps_no_crash(self):
114
- from formscout.agents.visualizer import PoseVisualizer
115
- vis = PoseVisualizer()
116
- frame = np.zeros((480, 640, 3), dtype=np.uint8)
117
- kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.9} for j in range(17)}
118
- velocities = {j: [50.0] * 5 for j in range(17)}
119
- result = vis._draw_velocity_arrows(frame.copy(), kps, None, velocities, frame_idx=0)
120
- assert result.shape == frame.shape
121
-
122
-
123
- class TestRenderVideo:
124
- def test_creates_mp4_file(self, tmp_path):
125
- from formscout.agents.visualizer import PoseVisualizer
126
- vis = PoseVisualizer()
127
- ingest = _make_ingest(n=5)
128
- pose = _make_pose(n=5)
129
- out = str(tmp_path / "out.mp4")
130
- result = vis.render_video(ingest, pose, {"skeleton"}, out)
131
- assert result is not None
132
- import os
133
- assert os.path.exists(result)
134
- assert os.path.getsize(result) > 0
135
-
136
- def test_empty_layers_returns_none(self, tmp_path):
137
- from formscout.agents.visualizer import PoseVisualizer
138
- vis = PoseVisualizer()
139
- out = str(tmp_path / "out.mp4")
140
- result = vis.render_video(_make_ingest(), _make_pose(), set(), out)
141
- assert result is None
142
-
143
- def test_no_detections_returns_none(self, tmp_path):
144
- from formscout.agents.visualizer import PoseVisualizer
145
- vis = PoseVisualizer()
146
- ingest = _make_ingest(n=5)
147
- empty_pose = Pose2DResult(
148
- keypoints=[{} for _ in range(5)], fps=30.0, confidence=0.0, notes=""
149
- )
150
- out = str(tmp_path / "out.mp4")
151
- result = vis.render_video(ingest, empty_pose, {"skeleton"}, out)
152
- assert result is None
153
-
154
- def test_last_velocities_set_after_render(self, tmp_path):
155
- from formscout.agents.visualizer import PoseVisualizer
156
- vis = PoseVisualizer()
157
- out = str(tmp_path / "out.mp4")
158
- vis.render_video(_make_ingest(n=5), _make_pose(n=5), {"skeleton"}, out)
159
- assert len(vis.last_velocities) == 17
160
-
161
-
162
- class TestBuildVelocitySummary:
163
- def test_returns_markdown_table(self):
164
- from formscout.agents.visualizer import build_velocity_summary, compute_joint_velocity
165
- pose = _make_pose(n=10)
166
- vels = compute_joint_velocity(pose.keypoints, fps=30.0)
167
- result = build_velocity_summary(pose.keypoints, vels)
168
- assert "|" in result
169
- assert any(name in result for name in ["knee", "shoulder", "hip", "ankle"])
170
-
171
- def test_empty_keypoints_returns_empty_string(self):
172
- from formscout.agents.visualizer import build_velocity_summary
173
- empty_kps = [{} for _ in range(5)]
174
- vels = {j: [0.0] * 5 for j in range(17)}
175
- result = build_velocity_summary(empty_kps, vels)
176
- assert result == ""
 
1
+ """Tests for PoseVisualizer — no GPU, no model downloads."""
2
+ import numpy as np
3
+ import pytest
4
+ from formscout.types import IngestResult, Pose2DResult
5
+
6
+
7
+ def _make_ingest(n=5, h=480, w=640, fps=30.0):
8
+ frames = [np.zeros((h, w, 3), dtype=np.uint8) for _ in range(n)]
9
+ return IngestResult(frames=frames, fps=fps, duration=n / fps, n_people=1, width=w, height=h)
10
+
11
+
12
+ def _make_pose(n=5, w=640, h=480):
13
+ """Synthetic Pose2DResult: 17 joints at fixed pixel positions, conf=0.9."""
14
+ kps_per_frame = []
15
+ for i in range(n):
16
+ frame_kps = {}
17
+ for j in range(17):
18
+ frame_kps[j] = {
19
+ "x": float(50 + j * 30 + i * 2),
20
+ "y": float(100 + j * 20),
21
+ "conf": 0.9,
22
+ }
23
+ kps_per_frame.append(frame_kps)
24
+ return Pose2DResult(keypoints=kps_per_frame, fps=30.0, confidence=0.9, notes="")
25
+
26
+
27
+ class TestComputeJointVelocity:
28
+ def test_returns_17_joints(self):
29
+ from formscout.agents.visualizer import compute_joint_velocity
30
+ pose = _make_pose(n=5)
31
+ result = compute_joint_velocity(pose.keypoints, fps=30.0)
32
+ assert len(result) == 17
33
+
34
+ def test_each_list_has_n_frames(self):
35
+ from formscout.agents.visualizer import compute_joint_velocity
36
+ pose = _make_pose(n=5)
37
+ result = compute_joint_velocity(pose.keypoints, fps=30.0)
38
+ for joint_idx, speeds in result.items():
39
+ assert len(speeds) == 5, f"joint {joint_idx} has {len(speeds)} speeds, expected 5"
40
+
41
+ def test_speeds_are_non_negative(self):
42
+ from formscout.agents.visualizer import compute_joint_velocity
43
+ pose = _make_pose(n=5)
44
+ result = compute_joint_velocity(pose.keypoints, fps=30.0)
45
+ for speeds in result.values():
46
+ assert all(s >= 0.0 for s in speeds)
47
+
48
+ def test_missing_keypoints_give_zero_speed(self):
49
+ from formscout.agents.visualizer import compute_joint_velocity
50
+ empty_kps = [{} for _ in range(5)]
51
+ result = compute_joint_velocity(empty_kps, fps=30.0)
52
+ for speeds in result.values():
53
+ assert all(s == 0.0 for s in speeds)
54
+
55
+
56
+ class TestDrawSkeleton:
57
+ def test_skeleton_draws_without_error(self):
58
+ from formscout.agents.visualizer import PoseVisualizer
59
+ vis = PoseVisualizer()
60
+ frame = np.zeros((480, 640, 3), dtype=np.uint8)
61
+ kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
62
+ for j in range(17)}
63
+ result = vis._draw_skeleton(frame.copy(), kps)
64
+ assert result.shape == frame.shape
65
+ assert not np.array_equal(result, frame)
66
+
67
+ def test_low_confidence_keypoints_not_drawn(self):
68
+ from formscout.agents.visualizer import PoseVisualizer
69
+ vis = PoseVisualizer()
70
+ frame = np.zeros((480, 640, 3), dtype=np.uint8)
71
+ kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.1} for j in range(17)}
72
+ result = vis._draw_skeleton(frame.copy(), kps)
73
+ assert np.array_equal(result, frame)
74
+
75
+
76
+ class TestDrawTrails:
77
+ def test_trails_draw_without_error(self):
78
+ from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
79
+ from collections import deque
80
+ vis = PoseVisualizer()
81
+ frame = np.zeros((480, 640, 3), dtype=np.uint8)
82
+ trail_history = {
83
+ 0: deque([(100 + i * 5, 200 + i * 3) for i in range(5)], maxlen=TRAIL_LENGTH)
84
+ }
85
+ result = vis._draw_trails(frame.copy(), trail_history)
86
+ assert result.shape == frame.shape
87
+ assert not np.array_equal(result, frame)
88
+
89
+ def test_short_trail_no_crash(self):
90
+ from formscout.agents.visualizer import PoseVisualizer, TRAIL_LENGTH
91
+ from collections import deque
92
+ vis = PoseVisualizer()
93
+ frame = np.zeros((480, 640, 3), dtype=np.uint8)
94
+ trail_history = {0: deque([(100, 200)], maxlen=TRAIL_LENGTH)}
95
+ result = vis._draw_trails(frame.copy(), trail_history)
96
+ assert np.array_equal(result, frame)
97
+
98
+
99
+ class TestDrawVelocityArrows:
100
+ def test_arrows_draw_without_error(self):
101
+ from formscout.agents.visualizer import PoseVisualizer
102
+ vis = PoseVisualizer()
103
+ frame = np.zeros((480, 640, 3), dtype=np.uint8)
104
+ kps = {j: {"x": float(50 + j * 30), "y": float(100 + j * 20), "conf": 0.9}
105
+ for j in range(17)}
106
+ prev_kps = {j: {"x": float(48 + j * 30), "y": float(98 + j * 20), "conf": 0.9}
107
+ for j in range(17)}
108
+ velocities = {j: [0.0] * 5 for j in range(17)}
109
+ velocities[5] = [0.0, 10.0, 50.0, 80.0, 120.0]
110
+ result = vis._draw_velocity_arrows(frame.copy(), kps, prev_kps, velocities, frame_idx=4)
111
+ assert result.shape == frame.shape
112
+
113
+ def test_no_prev_kps_no_crash(self):
114
+ from formscout.agents.visualizer import PoseVisualizer
115
+ vis = PoseVisualizer()
116
+ frame = np.zeros((480, 640, 3), dtype=np.uint8)
117
+ kps = {j: {"x": float(50 + j * 30), "y": 100.0, "conf": 0.9} for j in range(17)}
118
+ velocities = {j: [50.0] * 5 for j in range(17)}
119
+ result = vis._draw_velocity_arrows(frame.copy(), kps, None, velocities, frame_idx=0)
120
+ assert result.shape == frame.shape
121
+
122
+
123
+ class TestRenderVideo:
124
+ def test_creates_mp4_file(self, tmp_path):
125
+ from formscout.agents.visualizer import PoseVisualizer
126
+ vis = PoseVisualizer()
127
+ ingest = _make_ingest(n=5)
128
+ pose = _make_pose(n=5)
129
+ out = str(tmp_path / "out.mp4")
130
+ result = vis.render_video(ingest, pose, {"skeleton"}, out)
131
+ assert result is not None
132
+ import os
133
+ assert os.path.exists(result)
134
+ assert os.path.getsize(result) > 0
135
+
136
+ def test_empty_layers_returns_none(self, tmp_path):
137
+ from formscout.agents.visualizer import PoseVisualizer
138
+ vis = PoseVisualizer()
139
+ out = str(tmp_path / "out.mp4")
140
+ result = vis.render_video(_make_ingest(), _make_pose(), set(), out)
141
+ assert result is None
142
+
143
+ def test_no_detections_returns_none(self, tmp_path):
144
+ from formscout.agents.visualizer import PoseVisualizer
145
+ vis = PoseVisualizer()
146
+ ingest = _make_ingest(n=5)
147
+ empty_pose = Pose2DResult(
148
+ keypoints=[{} for _ in range(5)], fps=30.0, confidence=0.0, notes=""
149
+ )
150
+ out = str(tmp_path / "out.mp4")
151
+ result = vis.render_video(ingest, empty_pose, {"skeleton"}, out)
152
+ assert result is None
153
+
154
+ def test_last_velocities_set_after_render(self, tmp_path):
155
+ from formscout.agents.visualizer import PoseVisualizer
156
+ vis = PoseVisualizer()
157
+ out = str(tmp_path / "out.mp4")
158
+ vis.render_video(_make_ingest(n=5), _make_pose(n=5), {"skeleton"}, out)
159
+ assert len(vis.last_velocities) == 17
160
+
161
+
162
+ class TestBuildVelocitySummary:
163
+ def test_returns_markdown_table(self):
164
+ from formscout.agents.visualizer import build_velocity_summary, compute_joint_velocity
165
+ pose = _make_pose(n=10)
166
+ vels = compute_joint_velocity(pose.keypoints, fps=30.0)
167
+ result = build_velocity_summary(pose.keypoints, vels)
168
+ assert "|" in result
169
+ assert any(name in result for name in ["knee", "shoulder", "hip", "ankle"])
170
+
171
+ def test_empty_keypoints_returns_empty_string(self):
172
+ from formscout.agents.visualizer import build_velocity_summary
173
+ empty_kps = [{} for _ in range(5)]
174
+ vels = {j: [0.0] * 5 for j in range(17)}
175
+ result = build_velocity_summary(empty_kps, vels)
176
+ assert result == ""