Spaces:
Running on Zero
Running on Zero
Publish services/inference
Browse filesThis view is limited to 50 files because it contains too many changes. Β See raw diff
- .gitattributes +1 -34
- README.md +144 -7
- THIRD_PARTY_NOTICES.md +68 -0
- app/__init__.py +0 -0
- app/adapters/__init__.py +43 -0
- app/adapters/audio/__init__.py +197 -0
- app/adapters/audio/decode.py +250 -0
- app/adapters/audio/events.py +491 -0
- app/adapters/audio/features.py +350 -0
- app/adapters/base.py +418 -0
- app/adapters/claims.py +0 -0
- app/adapters/deterministic.py +210 -0
- app/adapters/embedding/__init__.py +58 -0
- app/adapters/embedding/backbones.py +397 -0
- app/adapters/embedding/identity.py +657 -0
- app/adapters/fingerprints.py +891 -0
- app/adapters/geometry/__init__.py +128 -0
- app/adapters/geometry/body.py +196 -0
- app/adapters/geometry/equations.py +268 -0
- app/adapters/geometry/weight.py +334 -0
- app/adapters/licence_policy.py +188 -0
- app/adapters/licences.py +639 -0
- app/adapters/multimodal.py +322 -0
- app/adapters/pose/__init__.py +142 -0
- app/adapters/pose/gait.py +566 -0
- app/adapters/pose/tracks.py +212 -0
- app/adapters/pose/vocabulary.py +136 -0
- app/adapters/registry.py +133 -0
- app/adapters/signal/__init__.py +3 -0
- app/adapters/signal/flow.py +138 -0
- app/adapters/signal/geometry.py +173 -0
- app/adapters/signal/periodicity.py +434 -0
- app/adapters/signal/respiration.py +168 -0
- app/adapters/tiled/__init__.py +197 -0
- app/adapters/tiled/blobs.py +245 -0
- app/adapters/tiled/exemplar.py +230 -0
- app/adapters/tiled/openvocab.py +253 -0
- app/adapters/tiled/tiles.py +291 -0
- app/adapters/transports/__init__.py +115 -0
- app/adapters/transports/anthropic_transport.py +342 -0
- app/adapters/unavailable.py +298 -0
- app/capabilities.py +0 -0
- app/counting.py +526 -0
- app/detectors/__init__.py +68 -0
- app/detectors/base.py +95 -0
- app/detectors/ultralytics_yolo.py +127 -0
- app/detectors/yolox_onnx.py +194 -0
- app/dispositions.py +1178 -0
- app/identification.py +699 -0
- app/main.py +597 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,2 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.jpg filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
README.md
CHANGED
|
@@ -1,13 +1,150 @@
|
|
| 1 |
---
|
| 2 |
-
title: Animap
|
| 3 |
-
emoji:
|
| 4 |
colorFrom: green
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version:
|
| 8 |
-
|
| 9 |
-
|
| 10 |
pinned: false
|
|
|
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Animap Inference
|
| 3 |
+
emoji: π
|
| 4 |
colorFrom: green
|
| 5 |
+
colorTo: gray
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 5.49.1
|
| 8 |
+
app_file: space_app.py
|
| 9 |
+
python_version: "3.12.12"
|
| 10 |
pinned: false
|
| 11 |
+
license: apache-2.0
|
| 12 |
+
short_description: Livestock models that refuse to invent a result.
|
| 13 |
---
|
| 14 |
|
| 15 |
+
# Animap inference
|
| 16 |
+
|
| 17 |
+
This runs validated livestock models against farm photographs and returns what
|
| 18 |
+
the model actually produced. Its most important property is what it **refuses**:
|
| 19 |
+
a capability with no checksummed artefact behind it answers `unavailable`, not a
|
| 20 |
+
placeholder and not a plausible-looking number.
|
| 21 |
+
|
| 22 |
+
Twenty-eight capabilities are registered. **Two run.** `cattle_detection` and
|
| 23 |
+
`poultry_count` execute a YOLOX-m ONNX artefact whose sha256 is verified against
|
| 24 |
+
its model card at start-up. The other twenty-six say so plainly, which is the
|
| 25 |
+
honest answer rather than a gap.
|
| 26 |
+
|
| 27 |
+
The source is `services/inference` in the Animap repository. `app.py` mounts
|
| 28 |
+
`app.main:app` β the same FastAPI service the Azure Container App runs β under a
|
| 29 |
+
Gradio page, so `/health`, `/capabilities` and `/jobs` behave exactly as they do
|
| 30 |
+
in production and the page is a demonstration of them.
|
| 31 |
+
|
| 32 |
+
**It was a Docker Space until 2026-08-24 and is a Gradio one now**, for one
|
| 33 |
+
reason: ZeroGPU is Gradio-SDK only. The Docker image ran the identical Azure
|
| 34 |
+
bytes, which was the better provenance story, and it could not be given a GPU at
|
| 35 |
+
any price β which left CountGD, the one capability a GPU actually unblocks,
|
| 36 |
+
unmeasurable. See **What this Space is not** for what the change cost.
|
| 37 |
+
|
| 38 |
+
## What is open and what is not
|
| 39 |
+
|
| 40 |
+
| Endpoint | Auth | Why |
|
| 41 |
+
|---|---|---|
|
| 42 |
+
| `GET /health` | none | Carries no farm data, and a platform probe has to reach it |
|
| 43 |
+
| `GET /capabilities` | none | The published contract: what may be claimed, and what may not |
|
| 44 |
+
| `POST /jobs` | **bearer token** | Runs a model against a farm's photographs |
|
| 45 |
+
| `GET /jobs/{id}` | **bearer token** | Returns a farm's result |
|
| 46 |
+
|
| 47 |
+
`space/publish.py --set-secret` mints `ANIMAP_INFERENCE_TOKEN` and sets it as a
|
| 48 |
+
Space secret. **Check it rather than assuming it**: `GET /health` reports
|
| 49 |
+
`"authenticated": false` when no token is configured, so a deployment that
|
| 50 |
+
reached the internet without one says so to anyone who asks.
|
| 51 |
+
|
| 52 |
+
curl -s https://bluman1-animap-inference.hf.space/health
|
| 53 |
+
|
| 54 |
+
## Running a model
|
| 55 |
+
|
| 56 |
+
Two public-domain captures are baked in, so a real detection can be obtained
|
| 57 |
+
without an Azure account. `space/FIXTURES.md` in the repository lists their ids,
|
| 58 |
+
their sources and the human count on record for each.
|
| 59 |
+
|
| 60 |
+
```bash
|
| 61 |
+
curl -s -X POST https://bluman1-animap-inference.hf.space/jobs \
|
| 62 |
+
-H "authorization: Bearer $ANIMAP_INFERENCE_TOKEN" \
|
| 63 |
+
-H 'content-type: application/json' \
|
| 64 |
+
-d '{"capability_key":"cattle_detection",
|
| 65 |
+
"subject_type":"herd",
|
| 66 |
+
"subject_id":"00000000-0000-0000-0000-000000000001",
|
| 67 |
+
"farm_id":"00000000-0000-0000-0000-000000000002",
|
| 68 |
+
"media_ids":["aa5e8481-8be6-509d-b1fa-f1a178c7cda0"],
|
| 69 |
+
"captured_at":"2026-08-22T10:00:00Z"}'
|
| 70 |
+
```
|
| 71 |
+
|
| 72 |
+
A frame that settles at the first grid answers in well under a second. A dense
|
| 73 |
+
one runs all three grids and takes a few seconds; there is no queue, because no
|
| 74 |
+
capability yet takes tens of seconds.
|
| 75 |
+
|
| 76 |
+
Read `warnings` before you read the number. A count is of the animals **visible
|
| 77 |
+
in one frame** β never the herd size, never a flock population, and never a
|
| 78 |
+
house reconciliation. When the count keeps rising as the frame is read more
|
| 79 |
+
finely, or exceeds twenty, the service publishes `count_withheld` and no number
|
| 80 |
+
at all. That refusal is a feature and it is measured: see the `known_limits` and
|
| 81 |
+
`validation_notes` on each model card.
|
| 82 |
+
|
| 83 |
+
## What the SDK change cost, and what it did not
|
| 84 |
+
|
| 85 |
+
**Lost: the image is no longer byte-identical to Azure's.** A Gradio Space has
|
| 86 |
+
no Dockerfile, so the claim *"this Space builds from the same Dockerfile"* is
|
| 87 |
+
gone and cannot be got back while ZeroGPU is Gradio-only. What runs is the same
|
| 88 |
+
`app/` tree with the same `requirements.txt`, which is close and is not the same
|
| 89 |
+
thing, and this file says so rather than letting the old sentence stand.
|
| 90 |
+
|
| 91 |
+
**Lost: a build-time licence gate.** The Docker build failed if an AGPL runtime
|
| 92 |
+
arrived. There is no build to fail now.
|
| 93 |
+
|
| 94 |
+
**Kept: every gate that actually protects a result.** `app.py` runs
|
| 95 |
+
`scripts/install_models.py --check` before it imports the service, so an
|
| 96 |
+
artefact that disagrees with its model card stops the Space at start-up rather
|
| 97 |
+
than being found by a farm's job. `providers.discover()` still refuses to serve
|
| 98 |
+
a capability whose artefact fingerprints as a copyleft runtime, and `/health`
|
| 99 |
+
still publishes `artefact_licenses` so a deployment in breach is visible from
|
| 100 |
+
outside.
|
| 101 |
+
|
| 102 |
+
**Gained: the ability to be given a GPU.** Nothing here reaches for CUDA yet β
|
| 103 |
+
YOLOX-m and DINOv3 are both ONNX on CPU β so this buys no speed-up today. It is
|
| 104 |
+
the prerequisite for CountGD, which gets MAE 14.84 on broiler houses against the
|
| 105 |
+
deployed detector's 156.80 and has never been runnable anywhere in this project.
|
| 106 |
+
|
| 107 |
+
## What this Space is not
|
| 108 |
+
|
| 109 |
+
**It is not the production media path, and it must not be read as evidence for
|
| 110 |
+
one.** This was true of the Docker Space and the SDK change did nothing to it. Production reads captures out of an Azure Blob container using the
|
| 111 |
+
Container App's managed identity β no key, no SAS, nothing stored. A Space is
|
| 112 |
+
not inside Azure and has no managed identity, so that credential is unavailable
|
| 113 |
+
to it. The alternatives a Space *could* use are a storage account key or a SAS
|
| 114 |
+
token in a secret, and neither is the production posture: one hands a public
|
| 115 |
+
Space full access to every farm's evidence, and the other expires.
|
| 116 |
+
|
| 117 |
+
So this Space serves `ANIMAP_MEDIA_PROVIDER=local` against the two baked-in
|
| 118 |
+
frames. Everything downstream of the pixels β the quality gate, the detection
|
| 119 |
+
pyramid, the counting guard, the observation vocabulary β is the production code
|
| 120 |
+
path exactly. Everything upstream of them is not.
|
| 121 |
+
|
| 122 |
+
**No farm data reaches this Space.** It cannot read `animapmedia`, and the only
|
| 123 |
+
captures it holds are two public-domain photographs from Wikimedia Commons.
|
| 124 |
+
|
| 125 |
+
## Weights
|
| 126 |
+
|
| 127 |
+
YOLOX-m, Apache-2.0, from the Megvii `0.1.1rc0` release, **vendored into this
|
| 128 |
+
repository under Git LFS rather than fetched at build time**. The service
|
| 129 |
+
verifies its sha256 against `models/cattle_detection/model_card.json` at
|
| 130 |
+
start-up and refuses to load an artefact that does not match. Vendoring is what
|
| 131 |
+
lets the build step stay `install_models.py --check` β verify, never fetch β
|
| 132 |
+
which is the posture ADR 0005 asks for and the same command the Azure build
|
| 133 |
+
runs.
|
| 134 |
+
|
| 135 |
+
`models/cattle_identity` is deliberately absent. Its artefact is DINOv3 under a
|
| 136 |
+
bespoke Meta licence whose two published texts disagree about an attribution
|
| 137 |
+
obligation, and publishing a copy into a public Space is redistribution. That
|
| 138 |
+
capability answers `unavailable` here, and correctly.
|
| 139 |
+
|
| 140 |
+
No AGPL-3.0 software is installed and none may be. `requirements.txt` omits
|
| 141 |
+
`ultralytics`, `.dockerignore` excludes `*.pt`, the build fails if one arrives
|
| 142 |
+
anyway, and `providers.discover()` refuses to serve a capability whose artefact
|
| 143 |
+
fingerprints as a copyleft runtime. `GET /health` publishes
|
| 144 |
+
`artefact_licenses`, so a deployment in breach is visible from outside.
|
| 145 |
+
|
| 146 |
+
## Attribution
|
| 147 |
+
|
| 148 |
+
Third-party notices travel with the image in `THIRD_PARTY_NOTICES.md`. The two
|
| 149 |
+
demonstration captures are CC0 and public domain; their sources are in
|
| 150 |
+
`space/FIXTURES.md`.
|
THIRD_PARTY_NOTICES.md
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Third-party assets
|
| 2 |
+
|
| 3 |
+
## Models
|
| 4 |
+
|
| 5 |
+
### DINOv3
|
| 6 |
+
|
| 7 |
+
Built with DINOv3
|
| 8 |
+
|
| 9 |
+
Animap's cattle identity embedding is a local ONNX export of
|
| 10 |
+
`timm/vit_small_patch16_dinov3.lvd1689m`. The weights are covered by Meta's
|
| 11 |
+
bespoke DINOv3 licence, not by Apache-2.0.
|
| 12 |
+
|
| 13 |
+
Two published texts of that licence differ. The `LICENSE.md` shipped with the
|
| 14 |
+
weights, dated 19 August 2025, ends clause 1.b.i at providing a copy of the
|
| 15 |
+
agreement. The text at
|
| 16 |
+
`ai.meta.com/resources/models-and-libraries/dinov3-license`, dated 14 August
|
| 17 |
+
2025 and linked from the Hugging Face model card, additionally requires that you
|
| 18 |
+
prominently display "Built with DINOv3". Section 8 lets Meta amend the licence
|
| 19 |
+
unilaterally with immediate effect.
|
| 20 |
+
|
| 21 |
+
Displaying the attribution satisfies both readings, which is why it appears
|
| 22 |
+
here. **This file is not yet enough.** "Prominently display" points at a surface
|
| 23 |
+
a person using Animap can see, and the app has no about screen. The outstanding
|
| 24 |
+
surfaces are listed in `attribution_outstanding` in
|
| 25 |
+
`services/inference/app/adapters/licences.py`, and `tests/test_attribution.py`
|
| 26 |
+
asserts that this file carries the string and that those surfaces still do not.
|
| 27 |
+
|
| 28 |
+
- Source: https://huggingface.co/timm/vit_small_patch16_dinov3.lvd1689m
|
| 29 |
+
- Licence: https://github.com/facebookresearch/dinov3/blob/main/LICENSE.md
|
| 30 |
+
|
| 31 |
+
### DINOv2
|
| 32 |
+
|
| 33 |
+
The alternate embedding backbone, `facebook/dinov2-small`, is licensed
|
| 34 |
+
Apache-2.0 for both code and weights. It requires no attribution beyond the
|
| 35 |
+
licence text travelling with any redistributed copy, and Animap redistributes
|
| 36 |
+
none.
|
| 37 |
+
|
| 38 |
+
- Source: https://huggingface.co/facebook/dinov2-small
|
| 39 |
+
- Licence: https://github.com/facebookresearch/dinov2/blob/main/LICENSE
|
| 40 |
+
|
| 41 |
+
### YOLOX
|
| 42 |
+
|
| 43 |
+
The shipped detector is YOLOX-m, from Megvii's YOLOX repository, under
|
| 44 |
+
Apache-2.0. Megvii publishes no separate licence for the released ONNX weights;
|
| 45 |
+
the repository's licence is read as covering the artefacts it distributes, and
|
| 46 |
+
that inference is recorded in `docs/adr/0017-ultralytics-licence.md`.
|
| 47 |
+
|
| 48 |
+
- Source: https://github.com/Megvii-BaseDetection/YOLOX
|
| 49 |
+
- Licence: https://github.com/Megvii-BaseDetection/YOLOX/blob/main/LICENSE
|
| 50 |
+
|
| 51 |
+
## Material Design Icons
|
| 52 |
+
|
| 53 |
+
Animap includes the following icons from Material Design Icons by
|
| 54 |
+
Pictogrammers:
|
| 55 |
+
|
| 56 |
+
- `cow`
|
| 57 |
+
- `turkey`
|
| 58 |
+
- `arrow-left`
|
| 59 |
+
- `home-outline`
|
| 60 |
+
- `bell-outline`
|
| 61 |
+
- `dots-horizontal`
|
| 62 |
+
- `camera-outline`
|
| 63 |
+
|
| 64 |
+
Material Design Icons is licensed under the Apache License 2.0.
|
| 65 |
+
|
| 66 |
+
- Project: https://pictogrammers.com/library/mdi/
|
| 67 |
+
- License: https://pictogrammers.com/docs/general/license/
|
| 68 |
+
- Source: https://github.com/Templarian/MaterialDesign
|
app/__init__.py
ADDED
|
File without changes
|
app/adapters/__init__.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Adapters for the zero-training model stack (directive Β§3, Β§4, Β§40.1).
|
| 2 |
+
|
| 3 |
+
One interface over things with nothing else in common: a segmenter, a frozen
|
| 4 |
+
backbone, an open-vocabulary detector, a hosted reasoner, and a pair of
|
| 5 |
+
deterministic methods that need no weights at all. What they share is
|
| 6 |
+
governance, and that is what `base.py` unifies β what a thing costs, what
|
| 7 |
+
licence it really carries, where it should run, and whether it can run at all.
|
| 8 |
+
|
| 9 |
+
Read `base.py` first. The two properties it exists to hold are that an adapter
|
| 10 |
+
cannot produce a result with no model behind it, and that a cost is either
|
| 11 |
+
measured or reported as unmeasured.
|
| 12 |
+
|
| 13 |
+
`licences.py` is the control ADR 0017 asked for, generalised: what each runtime
|
| 14 |
+
actually loads under, rather than what a card says about itself.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from app.adapters.base import (
|
| 18 |
+
Adapter,
|
| 19 |
+
AdapterError,
|
| 20 |
+
AdapterSpec,
|
| 21 |
+
AdapterUnavailable,
|
| 22 |
+
Availability,
|
| 23 |
+
MeasuredCost,
|
| 24 |
+
Measurement,
|
| 25 |
+
Modality,
|
| 26 |
+
Placement,
|
| 27 |
+
Region,
|
| 28 |
+
Task,
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
__all__ = [
|
| 32 |
+
"Adapter",
|
| 33 |
+
"AdapterError",
|
| 34 |
+
"AdapterSpec",
|
| 35 |
+
"AdapterUnavailable",
|
| 36 |
+
"Availability",
|
| 37 |
+
"MeasuredCost",
|
| 38 |
+
"Measurement",
|
| 39 |
+
"Modality",
|
| 40 |
+
"Placement",
|
| 41 |
+
"Region",
|
| 42 |
+
"Task",
|
| 43 |
+
]
|
app/adapters/audio/__init__.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Audio, as a first-class adapter with no weights in it.
|
| 2 |
+
|
| 3 |
+
Directive Β§26 names four things to test for poultry respiratory audio:
|
| 4 |
+
|
| 5 |
+
1. SAM Audio;
|
| 6 |
+
2. Perception Encoder audio / AV embeddings;
|
| 7 |
+
3. hosted multimodal audio reasoning;
|
| 8 |
+
4. classical audio features.
|
| 9 |
+
|
| 10 |
+
**This package is the fourth, and it is first because Β§4 says so** β *"Do not
|
| 11 |
+
use a neural model when deterministic signal processing is better"* β and
|
| 12 |
+
because this repository already has one clean result from taking that rule
|
| 13 |
+
seriously: `adapters/signal/periodicity.py` recovers a metronome's stated 96
|
| 14 |
+
beats per minute as 96.48 from an FFT and nothing else.
|
| 15 |
+
|
| 16 |
+
Whether it is *better* here than the three neural options is not something this
|
| 17 |
+
package can settle on its own. `experiments/poultry_respiratory/README.md`
|
| 18 |
+
records what each of the other three would take and which of them was reachable
|
| 19 |
+
from this machine, which is what Β§36 asks for before anything is called
|
| 20 |
+
unavailable.
|
| 21 |
+
|
| 22 |
+
## Layout
|
| 23 |
+
|
| 24 |
+
`decode.py` β a phone recording into a NumPy array, via an `ffmpeg` binary,
|
| 25 |
+
because the service has no audio library at all.
|
| 26 |
+
|
| 27 |
+
`features.py` β STFT, spectral flux, machinery and speech screens. NumPy only.
|
| 28 |
+
|
| 29 |
+
`events.py` β the Β§26 screen: cough- or sneeze-like events, or a refusal
|
| 30 |
+
carrying one of the registry's own rejection codes.
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
from __future__ import annotations
|
| 34 |
+
|
| 35 |
+
from pathlib import Path
|
| 36 |
+
|
| 37 |
+
from app.adapters.audio.decode import (
|
| 38 |
+
MAX_SECONDS,
|
| 39 |
+
SAMPLE_RATE_HZ,
|
| 40 |
+
AudioUnreadable,
|
| 41 |
+
DecoderMissing,
|
| 42 |
+
Recording,
|
| 43 |
+
available,
|
| 44 |
+
ffmpeg_path,
|
| 45 |
+
from_samples,
|
| 46 |
+
read_audio,
|
| 47 |
+
)
|
| 48 |
+
from app.adapters.audio.events import (
|
| 49 |
+
MIN_CAPTURE_SECONDS,
|
| 50 |
+
Event,
|
| 51 |
+
RespiratoryScreen,
|
| 52 |
+
screen,
|
| 53 |
+
)
|
| 54 |
+
from app.adapters.base import (
|
| 55 |
+
Adapter,
|
| 56 |
+
AdapterSpec,
|
| 57 |
+
AdapterUnavailable,
|
| 58 |
+
Availability,
|
| 59 |
+
MeasuredCost,
|
| 60 |
+
Modality,
|
| 61 |
+
Placement,
|
| 62 |
+
Task,
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
RESPIRATORY_AUDIO_SPEC = AdapterSpec(
|
| 66 |
+
adapter_id="respiratory-audio-flux",
|
| 67 |
+
runtime="ffmpeg-numpy",
|
| 68 |
+
tasks=(Task.MEASURE,),
|
| 69 |
+
modalities=(Modality.AUDIO,),
|
| 70 |
+
directive_role=(
|
| 71 |
+
"Β§26 poultry respiratory audio β 'stand quietly in the house and record "
|
| 72 |
+
"30 seconds', reported as cough/sneeze-like events and a spot screen. "
|
| 73 |
+
"Β§4's classical route: spectral flux, band energies and an "
|
| 74 |
+
"autocorrelation pitch test, with no weights. Β§27 keeps continuous "
|
| 75 |
+
"monitoring a separate capability that needs a fixed microphone."
|
| 76 |
+
),
|
| 77 |
+
#: There is no artefact β but unlike the other deterministic adapters this
|
| 78 |
+
#: one is not unconditionally available, because it needs an external
|
| 79 |
+
#: binary. That distinction is the reason `availability()` below is not
|
| 80 |
+
#: simply `Availability(True)`.
|
| 81 |
+
requires_artefact=False,
|
| 82 |
+
placement=Placement.ON_DEVICE,
|
| 83 |
+
placement_reason=(
|
| 84 |
+
"An STFT over 60 seconds at 16 kHz is 7,500 frames of 512-point FFT β "
|
| 85 |
+
"milliseconds of arithmetic, and a phone has both the CPU and the "
|
| 86 |
+
"recording already. Running it on the device keeps ADR 0002's "
|
| 87 |
+
"offline-first promise for a capability a farmer uses standing in a "
|
| 88 |
+
"shed, and means the audio never leaves the phone. The obstacle is not "
|
| 89 |
+
"compute, it is the decoder: Android supplies its own, so a port would "
|
| 90 |
+
"replace `decode.py` and nothing else."
|
| 91 |
+
),
|
| 92 |
+
measured=MeasuredCost(
|
| 93 |
+
hardware=(
|
| 94 |
+
"Apple M-series laptop (NOT the target container), OMP_NUM_THREADS=1"
|
| 95 |
+
),
|
| 96 |
+
threads=1,
|
| 97 |
+
sample=(
|
| 98 |
+
"60.0 s of 16 kHz mono holding 12 injected transients β decode "
|
| 99 |
+
"excluded, since that is FFmpeg's cost and not this module's. "
|
| 100 |
+
"Spectrogram, onset envelope, adaptive threshold, machinery and "
|
| 101 |
+
"speech screens, and the per-event gates"
|
| 102 |
+
),
|
| 103 |
+
runs=9,
|
| 104 |
+
median_seconds=0.094,
|
| 105 |
+
peak_rss_mb=263.4,
|
| 106 |
+
measured_on="2026-08-22",
|
| 107 |
+
),
|
| 108 |
+
notes=(
|
| 109 |
+
"**This detector does not work, and that is measured rather than "
|
| 110 |
+
"suspected.** Over 6,346 real poultry-house clips from two CC BY 4.0 "
|
| 111 |
+
"datasets it separates Sick from Healthy at AUC 0.4141 β below chance β "
|
| 112 |
+
"because a healthy house is a noisy one and spectral flux counts "
|
| 113 |
+
"activity. Frozen CLAP embeddings over the identical clips do better, "
|
| 114 |
+
"so the signal is there and the failure is the method's. "
|
| 115 |
+
"`experiments/poultry_respiratory/` has the tables. No figure from it "
|
| 116 |
+
"may be shown to a farm as an accuracy.\n\n"
|
| 117 |
+
"**No poultry-house recording with event-level cough annotation exists "
|
| 118 |
+
"under a free licence**, so the thing this adapter claims to do β count "
|
| 119 |
+
"events β has never been scored by anybody. An earlier version of this "
|
| 120 |
+
"note said the benchmark measured detector behaviour on audio mixed "
|
| 121 |
+
"with events at known times and signal-to-noise ratios. No such mixture "
|
| 122 |
+
"was ever built; every threshold in events.py is chosen rather than "
|
| 123 |
+
"derived, and each one now says so.\n\n"
|
| 124 |
+
"**It needs an `ffmpeg` binary and the one on a laptop is not the one "
|
| 125 |
+
"to ship.** See `adapters/licences.py:ffmpeg-numpy`: the build this was "
|
| 126 |
+
"developed against is `--enable-gpl --enable-nonfree`, which is the one "
|
| 127 |
+
"configuration FFmpeg may not be redistributed under at all."
|
| 128 |
+
),
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
class RespiratoryAudioAdapter(Adapter):
|
| 133 |
+
"""Β§26's spot screen. No weights, one external binary.
|
| 134 |
+
|
| 135 |
+
Deliberately not a subclass of `deterministic.DeterministicAdapter`. That
|
| 136 |
+
class answers `Availability(True)` unconditionally, and the sentence
|
| 137 |
+
justifying it β *"there is no artefact to be absent"* β is true of optical
|
| 138 |
+
flow and false here. Inheriting it to save six lines would be inheriting a
|
| 139 |
+
claim that does not hold.
|
| 140 |
+
"""
|
| 141 |
+
|
| 142 |
+
spec = RESPIRATORY_AUDIO_SPEC
|
| 143 |
+
|
| 144 |
+
def availability(self) -> Availability:
|
| 145 |
+
# Must not decode anything: `/health` calls this often enough that
|
| 146 |
+
# spawning a subprocess to answer it would be its own outage.
|
| 147 |
+
if not available():
|
| 148 |
+
return Availability(
|
| 149 |
+
False,
|
| 150 |
+
"No `ffmpeg` binary on PATH, and this service has no audio "
|
| 151 |
+
"decoding library β no soundfile, no librosa, no av. A phone "
|
| 152 |
+
"recording cannot be read at all without it.",
|
| 153 |
+
"Install FFmpeg, and read adapters/licences.py:ffmpeg-numpy "
|
| 154 |
+
"before choosing a build β the common Homebrew and static "
|
| 155 |
+
"builds are GPL or non-free.",
|
| 156 |
+
)
|
| 157 |
+
return Availability(True)
|
| 158 |
+
|
| 159 |
+
def load(self) -> "RespiratoryAudioAdapter":
|
| 160 |
+
state = self.availability()
|
| 161 |
+
if not state.ready:
|
| 162 |
+
raise AdapterUnavailable(state)
|
| 163 |
+
return self
|
| 164 |
+
|
| 165 |
+
def measure(self, audio_path: Path | str, **kwargs) -> RespiratoryScreen:
|
| 166 |
+
"""Decode and screen one recording.
|
| 167 |
+
|
| 168 |
+
Goes through `load()` rather than assuming it: a caller who skipped it
|
| 169 |
+
would get a `FileNotFoundError` from deep inside `subprocess` instead of
|
| 170 |
+
the availability answer this adapter exists to give.
|
| 171 |
+
"""
|
| 172 |
+
self.load()
|
| 173 |
+
return screen(read_audio(audio_path), **kwargs)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def audio_adapters() -> list[Adapter]:
|
| 177 |
+
return [RespiratoryAudioAdapter()]
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
__all__ = [
|
| 181 |
+
"MAX_SECONDS",
|
| 182 |
+
"MIN_CAPTURE_SECONDS",
|
| 183 |
+
"RESPIRATORY_AUDIO_SPEC",
|
| 184 |
+
"SAMPLE_RATE_HZ",
|
| 185 |
+
"AudioUnreadable",
|
| 186 |
+
"DecoderMissing",
|
| 187 |
+
"Event",
|
| 188 |
+
"Recording",
|
| 189 |
+
"RespiratoryAudioAdapter",
|
| 190 |
+
"RespiratoryScreen",
|
| 191 |
+
"audio_adapters",
|
| 192 |
+
"available",
|
| 193 |
+
"ffmpeg_path",
|
| 194 |
+
"from_samples",
|
| 195 |
+
"read_audio",
|
| 196 |
+
"screen",
|
| 197 |
+
]
|
app/adapters/audio/decode.py
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Getting a phone recording into a NumPy array, and refusing when it cannot.
|
| 2 |
+
|
| 3 |
+
Directive Β§26 asks the farmer to "stand quietly in the house and record 30
|
| 4 |
+
seconds". Whatever the phone hands over β `.m4a`, `.opus`, `.ogg`, `.3gp` β has
|
| 5 |
+
to become mono float samples at one known rate before any of the arithmetic in
|
| 6 |
+
`features` or `events` means anything.
|
| 7 |
+
|
| 8 |
+
**The service has no audio library.** There is no `soundfile`, no `librosa`, no
|
| 9 |
+
`av`; `cv2` bundles FFmpeg but exposes no audio path, and `wave` in the standard
|
| 10 |
+
library reads WAV and nothing else. So this module shells out to an `ffmpeg`
|
| 11 |
+
binary, which is a real external dependency and is treated as one: `available()`
|
| 12 |
+
looks for it, and the adapter that wraps this reports unavailable rather than
|
| 13 |
+
raising from the middle of a request.
|
| 14 |
+
|
| 15 |
+
**Two things about that binary are recorded rather than assumed.**
|
| 16 |
+
|
| 17 |
+
*It writes a file, not a pipe.* The obvious form is `-f s16le -` into
|
| 18 |
+
`subprocess`. That fails on any FFmpeg configured with a muxer whitelist, and
|
| 19 |
+
the build on this machine is one β `--disable-muxers --enable-muxer='webm,opus,
|
| 20 |
+
mp4,wav,...'` has no `s16le` in it, so the pipe form exits 234 with *"Requested
|
| 21 |
+
output format 's16le' is not known"*. Writing a temporary `.wav` and reading it
|
| 22 |
+
back with `wave` costs one file and works against every build, including the
|
| 23 |
+
minimal ones.
|
| 24 |
+
|
| 25 |
+
*Its licence is not this repository's to assume.* See
|
| 26 |
+
`adapters/licences.py:ffmpeg-cli`. The build here is `--enable-gpl
|
| 27 |
+
--enable-nonfree`, which is the one configuration FFmpeg may not be
|
| 28 |
+
redistributed under at all. That does not reach Animap's own code β calling a
|
| 29 |
+
separate program over a pipe is not linking, and `app/adapters/licences.py`
|
| 30 |
+
records the reasoning β but it does mean **the binary on a developer's laptop is
|
| 31 |
+
not the binary a deployment may ship**, and that is a deployment finding rather
|
| 32 |
+
than a footnote.
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
from __future__ import annotations
|
| 36 |
+
|
| 37 |
+
import shutil
|
| 38 |
+
import subprocess
|
| 39 |
+
import tempfile
|
| 40 |
+
import wave
|
| 41 |
+
from dataclasses import dataclass
|
| 42 |
+
from pathlib import Path
|
| 43 |
+
|
| 44 |
+
import numpy as np
|
| 45 |
+
|
| 46 |
+
#: What everything downstream assumes, in hertz.
|
| 47 |
+
#:
|
| 48 |
+
#: 16 kHz resolves to 8 kHz, and a chicken snick's energy is in the low
|
| 49 |
+
#: kilohertz β Mahdavian et al. and the broader poultry-audio literature place
|
| 50 |
+
#: sneeze and rale energy under 6 kHz. 22.05 or 44.1 kHz would carry more of the
|
| 51 |
+
#: transient's top edge and would quadruple the STFT cost for a band nothing
|
| 52 |
+
#: here measures.
|
| 53 |
+
#:
|
| 54 |
+
#: **It is fixed rather than passed through** because every threshold in
|
| 55 |
+
#: `events.py` is a property of the analysis band, and the periodicity module
|
| 56 |
+
#: next door is a standing lesson in what happens when a constant derived at one
|
| 57 |
+
#: band is applied at another.
|
| 58 |
+
SAMPLE_RATE_HZ = 16_000
|
| 59 |
+
|
| 60 |
+
#: Longest clip this will decode, in seconds. Β§26 asks for 30 and prefers 60;
|
| 61 |
+
#: Β§27 is explicit that continuous monitoring is a different capability with a
|
| 62 |
+
#: fixed microphone. Ten minutes is far past a spot check and is here to stop a
|
| 63 |
+
#: mis-sent file eating the container's memory, not to express a product limit.
|
| 64 |
+
MAX_SECONDS = 600.0
|
| 65 |
+
|
| 66 |
+
#: How long to let FFmpeg run. A 60-second clip transcodes in well under a
|
| 67 |
+
#: second; anything near this is a malformed file FFmpeg is chewing on.
|
| 68 |
+
DECODE_TIMEOUT_SECONDS = 120
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
class AudioUnreadable(RuntimeError):
|
| 72 |
+
"""The recording could not be decoded, so there is nothing to measure."""
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class DecoderMissing(RuntimeError):
|
| 76 |
+
"""No `ffmpeg` on PATH. A missing tool, not a broken recording."""
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
@dataclass(frozen=True)
|
| 80 |
+
class Recording:
|
| 81 |
+
"""Mono float samples in [-1, 1], plus what it took to get them."""
|
| 82 |
+
|
| 83 |
+
samples: np.ndarray
|
| 84 |
+
sample_rate_hz: int
|
| 85 |
+
#: The file it came from, for a result that has to be traced back.
|
| 86 |
+
source: str
|
| 87 |
+
#: Before resampling, so a clip recorded at 8 kHz is diagnosable later β it
|
| 88 |
+
#: has no content above 4 kHz however it is resampled, and half the band
|
| 89 |
+
#: this module analyses is empty for it.
|
| 90 |
+
source_sample_rate_hz: int
|
| 91 |
+
source_channels: int
|
| 92 |
+
|
| 93 |
+
@property
|
| 94 |
+
def duration_seconds(self) -> float:
|
| 95 |
+
return len(self.samples) / self.sample_rate_hz
|
| 96 |
+
|
| 97 |
+
@property
|
| 98 |
+
def is_silent(self) -> bool:
|
| 99 |
+
"""No signal at all, as distinct from no events.
|
| 100 |
+
|
| 101 |
+
A muted microphone and a quiet house are different findings and only
|
| 102 |
+
one of them is about the birds.
|
| 103 |
+
"""
|
| 104 |
+
return float(np.max(np.abs(self.samples), initial=0.0)) < 1e-6
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def ffmpeg_path() -> str | None:
|
| 108 |
+
return shutil.which("ffmpeg")
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def ffprobe_path() -> str | None:
|
| 112 |
+
return shutil.which("ffprobe")
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def available() -> bool:
|
| 116 |
+
return ffmpeg_path() is not None
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _probe(path: Path) -> tuple[int, int]:
|
| 120 |
+
"""The source rate and channel count, or `(0, 0)` when ffprobe is absent.
|
| 121 |
+
|
| 122 |
+
Recorded rather than required. The decode does not need it β FFmpeg
|
| 123 |
+
resamples whatever it finds β but a rate of 8,000 explains an empty upper
|
| 124 |
+
band better than any later measurement can, and losing that to a missing
|
| 125 |
+
optional tool would be worse than reporting it as unknown.
|
| 126 |
+
"""
|
| 127 |
+
probe = ffprobe_path()
|
| 128 |
+
if probe is None:
|
| 129 |
+
return 0, 0
|
| 130 |
+
try:
|
| 131 |
+
result = subprocess.run(
|
| 132 |
+
[probe, "-v", "error", "-select_streams", "a:0", "-show_entries",
|
| 133 |
+
"stream=sample_rate,channels", "-of", "csv=p=0", str(path)],
|
| 134 |
+
capture_output=True, text=True, timeout=30, check=True,
|
| 135 |
+
)
|
| 136 |
+
except (subprocess.SubprocessError, OSError):
|
| 137 |
+
return 0, 0
|
| 138 |
+
parts = result.stdout.strip().split(",")
|
| 139 |
+
try:
|
| 140 |
+
return int(parts[0]), int(parts[1])
|
| 141 |
+
except (IndexError, ValueError):
|
| 142 |
+
return 0, 0
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def read_audio(
|
| 146 |
+
path: Path | str,
|
| 147 |
+
*,
|
| 148 |
+
sample_rate_hz: int = SAMPLE_RATE_HZ,
|
| 149 |
+
max_seconds: float = MAX_SECONDS,
|
| 150 |
+
) -> Recording:
|
| 151 |
+
"""Decode to mono float32 at `sample_rate_hz`.
|
| 152 |
+
|
| 153 |
+
Raises `DecoderMissing` when there is no FFmpeg and `AudioUnreadable` when
|
| 154 |
+
there is one and the file defeats it. The two are separate exceptions
|
| 155 |
+
because they need different answers: install a tool, or ask for a different
|
| 156 |
+
recording.
|
| 157 |
+
"""
|
| 158 |
+
path = Path(path)
|
| 159 |
+
binary = ffmpeg_path()
|
| 160 |
+
if binary is None:
|
| 161 |
+
raise DecoderMissing(
|
| 162 |
+
"No `ffmpeg` on PATH. This service has no audio decoding library β "
|
| 163 |
+
"no soundfile, no librosa, no av β so a phone recording cannot be "
|
| 164 |
+
"read at all without it. Install FFmpeg, and read "
|
| 165 |
+
"adapters/licences.py:ffmpeg-cli before choosing a build."
|
| 166 |
+
)
|
| 167 |
+
if not path.is_file():
|
| 168 |
+
raise AudioUnreadable(f"{path} is not a file.")
|
| 169 |
+
|
| 170 |
+
source_rate, channels = _probe(path)
|
| 171 |
+
|
| 172 |
+
with tempfile.TemporaryDirectory(prefix="animap-audio-") as workspace:
|
| 173 |
+
decoded = Path(workspace) / "mono.wav"
|
| 174 |
+
command = [
|
| 175 |
+
binary, "-v", "error", "-nostdin", "-y",
|
| 176 |
+
"-i", str(path),
|
| 177 |
+
# `-t` before the output rather than `-ss`: the cap is on how much
|
| 178 |
+
# is decoded, and a spot check has no reason to start late.
|
| 179 |
+
"-t", f"{max_seconds:.3f}",
|
| 180 |
+
"-map", "a:0?",
|
| 181 |
+
"-ac", "1",
|
| 182 |
+
"-ar", str(sample_rate_hz),
|
| 183 |
+
"-acodec", "pcm_s16le",
|
| 184 |
+
"-f", "wav",
|
| 185 |
+
str(decoded),
|
| 186 |
+
]
|
| 187 |
+
try:
|
| 188 |
+
result = subprocess.run(
|
| 189 |
+
command, capture_output=True, text=True,
|
| 190 |
+
timeout=DECODE_TIMEOUT_SECONDS,
|
| 191 |
+
)
|
| 192 |
+
except subprocess.TimeoutExpired as expired:
|
| 193 |
+
raise AudioUnreadable(
|
| 194 |
+
f"FFmpeg did not finish decoding {path.name} within "
|
| 195 |
+
f"{DECODE_TIMEOUT_SECONDS} s."
|
| 196 |
+
) from expired
|
| 197 |
+
except OSError as failure:
|
| 198 |
+
raise AudioUnreadable(f"Could not run ffmpeg: {failure}") from failure
|
| 199 |
+
|
| 200 |
+
if result.returncode != 0 or not decoded.is_file():
|
| 201 |
+
raise AudioUnreadable(
|
| 202 |
+
f"FFmpeg could not decode {path.name} "
|
| 203 |
+
f"(exit {result.returncode}): {result.stderr.strip()[:300]}"
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
with wave.open(str(decoded)) as handle:
|
| 207 |
+
frames = handle.getnframes()
|
| 208 |
+
width = handle.getsampwidth()
|
| 209 |
+
raw = handle.readframes(frames)
|
| 210 |
+
|
| 211 |
+
if width != 2:
|
| 212 |
+
# Only reachable if a future edit changes `-acodec`; asserted rather
|
| 213 |
+
# than assumed because reading 16-bit as 32-bit is silent and produces
|
| 214 |
+
# a plausible-looking waveform of noise.
|
| 215 |
+
raise AudioUnreadable(
|
| 216 |
+
f"Expected 16-bit samples from the decode step, got {width * 8}-bit."
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
samples = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
|
| 220 |
+
if samples.size < 2:
|
| 221 |
+
raise AudioUnreadable(
|
| 222 |
+
f"{path.name} yielded {samples.size} samples. There is no audio in it."
|
| 223 |
+
)
|
| 224 |
+
|
| 225 |
+
return Recording(
|
| 226 |
+
samples=samples,
|
| 227 |
+
sample_rate_hz=sample_rate_hz,
|
| 228 |
+
source=str(path),
|
| 229 |
+
source_sample_rate_hz=source_rate,
|
| 230 |
+
source_channels=channels,
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def from_samples(
|
| 235 |
+
samples: np.ndarray, sample_rate_hz: int = SAMPLE_RATE_HZ, *, source: str = "memory",
|
| 236 |
+
) -> Recording:
|
| 237 |
+
"""A `Recording` over samples that are already in hand.
|
| 238 |
+
|
| 239 |
+
The mixer in the respiratory experiment builds its composites in memory, and
|
| 240 |
+
routing them through a temporary file to get a `Recording` would mean the
|
| 241 |
+
measured pipeline and the tested pipeline differed by an encode.
|
| 242 |
+
"""
|
| 243 |
+
samples = np.asarray(samples, dtype=np.float32).ravel()
|
| 244 |
+
return Recording(
|
| 245 |
+
samples=samples,
|
| 246 |
+
sample_rate_hz=int(sample_rate_hz),
|
| 247 |
+
source=source,
|
| 248 |
+
source_sample_rate_hz=int(sample_rate_hz),
|
| 249 |
+
source_channels=1,
|
| 250 |
+
)
|
app/adapters/audio/events.py
ADDED
|
@@ -0,0 +1,491 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Cough- and sneeze-like events in a 30-second recording, and the refusals.
|
| 2 |
+
|
| 3 |
+
Directive Β§26 in full:
|
| 4 |
+
|
| 5 |
+
Stand quietly in the house and record 30 seconds.
|
| 6 |
+
β "Cough/sneeze-like events detected"
|
| 7 |
+
β "Spot respiratory screen only"
|
| 8 |
+
Do not present as continuous surveillance.
|
| 9 |
+
|
| 10 |
+
**The output is deliberately weaker than the thing farmers want.** Β§27 makes
|
| 11 |
+
continuous cough monitoring a separate capability that needs a fixed
|
| 12 |
+
microphone, and Β§30 lists *"24/7 respiratory surveillance from one 30-second
|
| 13 |
+
recording"* among the claims that may never be made. Nothing here counts
|
| 14 |
+
coughs per bird, per hour, or per house. It counts *events that look like a
|
| 15 |
+
cough or a sneeze* in one recording, and the word "like" is load-bearing.
|
| 16 |
+
|
| 17 |
+
## The shape this module borrows
|
| 18 |
+
|
| 19 |
+
`adapters/signal/periodicity.py` is the pattern, and its opening line is the
|
| 20 |
+
one that matters here too: *the hard part is not finding a peak β every
|
| 21 |
+
spectrum has a peak.* Every recording has transients. Slamming doors, feeders,
|
| 22 |
+
a boot on litter, a bird landing, a microphone rubbing a coat. So this module
|
| 23 |
+
is mostly gates, and a refusal is a first-class result: `count` is `None`
|
| 24 |
+
whenever `usable` is False, and there is no way to read a number out of a
|
| 25 |
+
screen that refused.
|
| 26 |
+
|
| 27 |
+
## The three refusals, and where they come from
|
| 28 |
+
|
| 29 |
+
The capability registry names them. `app/capabilities.py` gives
|
| 30 |
+
`poultry_respiratory` a `reject_if` of exactly
|
| 31 |
+
`("recording_too_short", "machinery_dominates", "speech_dominates")`, and each
|
| 32 |
+
one is implemented here under that name so a rejection the product declares is
|
| 33 |
+
a rejection the code can actually produce.
|
| 34 |
+
|
| 35 |
+
## What is measured, and what is not
|
| 36 |
+
|
| 37 |
+
`experiments/poultry_respiratory/` holds the benchmark, and the honest summary
|
| 38 |
+
is short: **this detector does not work.** Over 6,346 real poultry-house clips
|
| 39 |
+
from two CC BY 4.0 datasets it separates Sick from Healthy at AUC 0.4141 β
|
| 40 |
+
below chance β because a healthy house is a noisy one and spectral flux counts
|
| 41 |
+
activity. Frozen CLAP embeddings over the identical clips do better, so the
|
| 42 |
+
signal is there and this is a method failure.
|
| 43 |
+
|
| 44 |
+
**No recording of a poultry house with event-level cough annotation exists under
|
| 45 |
+
a free licence**, so the thing this module actually claims to do β count events
|
| 46 |
+
β has never been scored by anybody, here or elsewhere. An earlier version of
|
| 47 |
+
this paragraph said the benchmark measured "detector behaviour on real farm
|
| 48 |
+
noise mixed with real transient events at known times and known
|
| 49 |
+
signal-to-noise ratios". No such mixture was ever built. Every threshold below
|
| 50 |
+
is chosen rather than derived, and each one now says so.
|
| 51 |
+
"""
|
| 52 |
+
|
| 53 |
+
from __future__ import annotations
|
| 54 |
+
|
| 55 |
+
from dataclasses import dataclass, field
|
| 56 |
+
|
| 57 |
+
import numpy as np
|
| 58 |
+
|
| 59 |
+
from app.adapters.audio.decode import Recording
|
| 60 |
+
from app.adapters.audio.features import (
|
| 61 |
+
FLUX_BAND_HZ,
|
| 62 |
+
Spectrogram,
|
| 63 |
+
adaptive_threshold,
|
| 64 |
+
harmonicity,
|
| 65 |
+
machinery_dominance,
|
| 66 |
+
onset_strength,
|
| 67 |
+
spectrogram,
|
| 68 |
+
speech_dominance,
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
#: Β§26's capture, and `app/capabilities.py`'s `minimum_capture_seconds` for
|
| 72 |
+
#: `poultry_respiratory`. Kept equal to the registry's number on purpose: a
|
| 73 |
+
#: capture minimum the app advertises and a capture minimum the code enforces
|
| 74 |
+
#: that differ by a second is a bug report nobody can reproduce.
|
| 75 |
+
MIN_CAPTURE_SECONDS = 30.0
|
| 76 |
+
|
| 77 |
+
#: Amplitude below which the microphone is considered dead rather than the house
|
| 78 |
+
#: quiet. Full scale is 1.0, so this is roughly β80 dBFS β beneath the
|
| 79 |
+
#: self-noise of any phone microphone.
|
| 80 |
+
SILENCE_PEAK = 1e-4
|
| 81 |
+
|
| 82 |
+
#: Window the onset threshold adapts over, in seconds. Long enough to hold
|
| 83 |
+
#: several seconds of background between events; short enough to follow a fan
|
| 84 |
+
#: cycling. Two seconds at an 8 ms hop is 250 frames, and the median of 250
|
| 85 |
+
#: frames is unmoved by the handful an event occupies.
|
| 86 |
+
ADAPTIVE_WINDOW_SECONDS = 2.0
|
| 87 |
+
|
| 88 |
+
#: How many local median-absolute-deviations above the local median a frame's
|
| 89 |
+
#: spectral flux must reach to be a candidate.
|
| 90 |
+
#:
|
| 91 |
+
#: **CHOSEN, NOT DERIVED, and an earlier version of this comment said the
|
| 92 |
+
#: opposite.** It claimed the experiment swept this value against composites of
|
| 93 |
+
#: real farm noise and real transient events at known times. No such composites
|
| 94 |
+
#: exist, no such sweep was run, and
|
| 95 |
+
#: `experiments/poultry_respiratory/config.yaml` says so in its own
|
| 96 |
+
#: `thresholds` block. A hostile audit found the two files contradicting each
|
| 97 |
+
#: other and it was the shipped code that was lying.
|
| 98 |
+
#:
|
| 99 |
+
#: 6.0 is a robust-statistics default β roughly four standard deviations for
|
| 100 |
+
#: Gaussian noise, via the 1.4826 MAD-to-sigma factor. What IS measured is what
|
| 101 |
+
#: it does: over 6,346 real poultry-house clips it fires on 22.1% of Healthy
|
| 102 |
+
#: clips and 5.2% of Sick ones, which is the wrong way round. That is a fact
|
| 103 |
+
#: about the method rather than about the constant, and a sweep would move the
|
| 104 |
+
#: rate without moving the ordering β but nobody has run one, so that sentence
|
| 105 |
+
#: is an argument and not a measurement.
|
| 106 |
+
ONSET_K = 6.0
|
| 107 |
+
|
| 108 |
+
#: Two onsets closer together than this are one event. A double sneeze exists
|
| 109 |
+
#: and this will merge it, which biases the count *down* β the safe direction
|
| 110 |
+
#: for a screen whose failure mode is alarming a farm about birds that are fine.
|
| 111 |
+
REFRACTORY_SECONDS = 0.12
|
| 112 |
+
|
| 113 |
+
#: An event's duration must fall inside this, in seconds. Below the floor is a
|
| 114 |
+
#: click or a sample dropout; above the ceiling is a door, a vehicle or a bird
|
| 115 |
+
#: landing on the microphone. Poultry snicks and sneezes are reported in the
|
| 116 |
+
#: literature at roughly 50β250 ms, and the band is widened either side because
|
| 117 |
+
#: nothing here has measured a Nigerian house.
|
| 118 |
+
EVENT_SECONDS = (0.02, 0.60)
|
| 119 |
+
|
| 120 |
+
#: Share of an event's energy that must sit above 1 kHz.
|
| 121 |
+
#:
|
| 122 |
+
#: The gate that removes thumps. A boot on litter, a feeder chain and a slammed
|
| 123 |
+
#: door are loud and low; a snick is not.
|
| 124 |
+
#:
|
| 125 |
+
#: **Chosen, not derived**, like `ONSET_K` above and for the same reason: the
|
| 126 |
+
#: sweep an earlier comment credited it to was never run. It belongs to the
|
| 127 |
+
#: analysis band in `features.FLUX_BAND_HZ` and would not transfer to another.
|
| 128 |
+
MIN_HIGH_BAND_FRACTION = 0.30
|
| 129 |
+
|
| 130 |
+
#: Where "high band" starts, in hertz.
|
| 131 |
+
HIGH_BAND_HZ = (1_000.0, 8_000.0)
|
| 132 |
+
|
| 133 |
+
#: Normalised autocorrelation above which an event is pitched, and therefore a
|
| 134 |
+
#: vocalisation rather than a respiratory transient. A cluck, a crow and a
|
| 135 |
+
#: spoken vowel repeat; a sneeze does not.
|
| 136 |
+
MAX_HARMONICITY = 0.55
|
| 137 |
+
|
| 138 |
+
#: `machinery_dominance` above which no count is published. A recording this
|
| 139 |
+
#: bottom-heavy and this stationary is a fan, and the birds under it are not
|
| 140 |
+
#: being heard.
|
| 141 |
+
#:
|
| 142 |
+
#: **Chosen, and measured to be inert.** Across all 6,346 clips of
|
| 143 |
+
#: `experiments/poultry_respiratory/` the statistic peaks at 0.4999 and never
|
| 144 |
+
#: reaches this, so `machinery_dominates` is a rejection the product declares
|
| 145 |
+
#: and that set never triggers. Either those recordings are not fan-dominated,
|
| 146 |
+
#: or the threshold is too high to be useful; nothing separates the two, and
|
| 147 |
+
#: the synthetic drone in `tests/test_audio.py` scores well above it, which is
|
| 148 |
+
#: how a value this high came to look reasonable.
|
| 149 |
+
MAX_MACHINERY_DOMINANCE = 0.55
|
| 150 |
+
|
| 151 |
+
#: `speech_dominance` above which no count is published.
|
| 152 |
+
#:
|
| 153 |
+
#: **Chosen, and an earlier comment claimed it came from composites of real
|
| 154 |
+
#: speech over real farm noise. Those do not exist.** What is known about it is
|
| 155 |
+
#: one measurement: over 6,346 real poultry-house clips it fires 26 times, which
|
| 156 |
+
#: is the only one of the registry's three rejection codes that set exercises.
|
| 157 |
+
#: Nobody has listened to those 26 clips to check whether anybody is talking in
|
| 158 |
+
#: them.
|
| 159 |
+
MAX_SPEECH_DOMINANCE = 0.20
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
@dataclass(frozen=True)
|
| 163 |
+
class Event:
|
| 164 |
+
"""One candidate, with the evidence that made it one.
|
| 165 |
+
|
| 166 |
+
Every field a gate looked at is kept, whether the event passed or not, so a
|
| 167 |
+
threshold can be re-derived from stored results rather than by going back to
|
| 168 |
+
recordings nobody kept. `periodicity.Periodicity` keeps its diagnostics for
|
| 169 |
+
the same reason and it is the thing that made re-deriving its constants
|
| 170 |
+
possible a month later.
|
| 171 |
+
"""
|
| 172 |
+
|
| 173 |
+
start_seconds: float
|
| 174 |
+
peak_seconds: float
|
| 175 |
+
end_seconds: float
|
| 176 |
+
onset_strength: float
|
| 177 |
+
#: Multiples of the local MAD above the local median, at the peak frame.
|
| 178 |
+
#: This is the closest thing the method has to a per-event score, and Β§37
|
| 179 |
+
#: forbids it reaching a farm: nothing has calibrated it.
|
| 180 |
+
prominence: float
|
| 181 |
+
high_band_fraction: float
|
| 182 |
+
harmonicity: float
|
| 183 |
+
peak_level_dbfs: float
|
| 184 |
+
#: Empty when the event was kept. Otherwise the gate that removed it.
|
| 185 |
+
rejected_because: str = ""
|
| 186 |
+
|
| 187 |
+
@property
|
| 188 |
+
def duration_seconds(self) -> float:
|
| 189 |
+
return self.end_seconds - self.start_seconds
|
| 190 |
+
|
| 191 |
+
@property
|
| 192 |
+
def kept(self) -> bool:
|
| 193 |
+
return not self.rejected_because
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
@dataclass(frozen=True)
|
| 197 |
+
class RespiratoryScreen:
|
| 198 |
+
"""A count of cough- or sneeze-like events, or an account of why not.
|
| 199 |
+
|
| 200 |
+
`count` is `None` whenever `usable` is False. There is deliberately no way
|
| 201 |
+
to read a number out of a refused screen β the property `app/counting.py`
|
| 202 |
+
and `periodicity.Periodicity` both have, where a withheld number is not a
|
| 203 |
+
number of zero.
|
| 204 |
+
|
| 205 |
+
**A usable screen reporting zero is a real result and a different one.** It
|
| 206 |
+
says the recording was analysable and held nothing cough-like, which is what
|
| 207 |
+
a healthy house sounds like.
|
| 208 |
+
"""
|
| 209 |
+
|
| 210 |
+
usable: bool
|
| 211 |
+
count: int | None
|
| 212 |
+
events: tuple[Event, ...]
|
| 213 |
+
#: Everything the gates rejected, kept rather than dropped: a screen that
|
| 214 |
+
#: found forty transients and passed none of them is a different situation
|
| 215 |
+
#: from one that found none, and only this field distinguishes them.
|
| 216 |
+
rejected: tuple[Event, ...]
|
| 217 |
+
reason: str = ""
|
| 218 |
+
#: The registry's own rejection name, when one applies. Matches
|
| 219 |
+
#: `app/capabilities.py`'s `reject_if` for `poultry_respiratory` so a caller
|
| 220 |
+
#: can route on it rather than parsing prose.
|
| 221 |
+
rejection_code: str = ""
|
| 222 |
+
diagnostics: dict[str, float] = field(default_factory=dict)
|
| 223 |
+
|
| 224 |
+
@property
|
| 225 |
+
def statement(self) -> str:
|
| 226 |
+
"""The Β§26 wording, and nothing stronger.
|
| 227 |
+
|
| 228 |
+
Authored here rather than in a caller because Β§26 gives the sentence
|
| 229 |
+
and Β§27 gives the qualifier, and separating them is how the qualifier
|
| 230 |
+
gets lost between a service and a screen.
|
| 231 |
+
"""
|
| 232 |
+
if not self.usable:
|
| 233 |
+
return self.reason
|
| 234 |
+
if self.count == 0:
|
| 235 |
+
return (
|
| 236 |
+
"No cough- or sneeze-like events detected in this recording. "
|
| 237 |
+
"Spot respiratory screen only β a 30-second sample is not "
|
| 238 |
+
"continuous monitoring."
|
| 239 |
+
)
|
| 240 |
+
plural = "" if self.count == 1 else "s"
|
| 241 |
+
return (
|
| 242 |
+
f"{self.count} cough/sneeze-like event{plural} detected. "
|
| 243 |
+
f"Spot respiratory screen only β a 30-second sample is not "
|
| 244 |
+
f"continuous monitoring."
|
| 245 |
+
)
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def _segment(strength: np.ndarray, threshold: np.ndarray, peak: int) -> tuple[int, int]:
|
| 249 |
+
"""Where the event around a peak frame starts and stops.
|
| 250 |
+
|
| 251 |
+
Walks outwards to the first frame at or below the local threshold. Bounding
|
| 252 |
+
an event by its own threshold rather than by a fixed width is what lets the
|
| 253 |
+
duration gate mean something: a fixed window would give every event the same
|
| 254 |
+
duration and the gate would never fire.
|
| 255 |
+
"""
|
| 256 |
+
start = peak
|
| 257 |
+
while start > 0 and strength[start - 1] > threshold[start - 1]:
|
| 258 |
+
start -= 1
|
| 259 |
+
end = peak
|
| 260 |
+
last = len(strength) - 1
|
| 261 |
+
while end < last and strength[end + 1] > threshold[end + 1]:
|
| 262 |
+
end += 1
|
| 263 |
+
return start, end
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def _candidates(
|
| 267 |
+
spec: Spectrogram, strength: np.ndarray, threshold: np.ndarray,
|
| 268 |
+
refractory_frames: int,
|
| 269 |
+
) -> list[tuple[int, int, int]]:
|
| 270 |
+
"""`(start, peak, end)` frame indices, strongest peak first, non-overlapping.
|
| 271 |
+
|
| 272 |
+
Strongest-first rather than left-to-right: when two onsets fall inside one
|
| 273 |
+
refractory period the louder one should be the event, not whichever happened
|
| 274 |
+
to come first.
|
| 275 |
+
"""
|
| 276 |
+
above = np.flatnonzero(strength > threshold)
|
| 277 |
+
if above.size == 0:
|
| 278 |
+
return []
|
| 279 |
+
|
| 280 |
+
taken = np.zeros(spec.frames, dtype=bool)
|
| 281 |
+
found: list[tuple[int, int, int]] = []
|
| 282 |
+
for peak in above[np.argsort(-strength[above])]:
|
| 283 |
+
peak = int(peak)
|
| 284 |
+
if taken[peak]:
|
| 285 |
+
continue
|
| 286 |
+
start, end = _segment(strength, threshold, peak)
|
| 287 |
+
low = max(0, peak - refractory_frames)
|
| 288 |
+
high = min(spec.frames, peak + refractory_frames + 1)
|
| 289 |
+
if taken[low:high].any():
|
| 290 |
+
continue
|
| 291 |
+
taken[min(low, start):max(high, end + 1)] = True
|
| 292 |
+
found.append((start, peak, end))
|
| 293 |
+
return sorted(found, key=lambda triple: triple[1])
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def _describe(
|
| 297 |
+
spec: Spectrogram, recording: Recording, strength: np.ndarray,
|
| 298 |
+
threshold: np.ndarray, span: tuple[int, int, int],
|
| 299 |
+
) -> Event:
|
| 300 |
+
"""Measure one candidate against every gate, without applying any of them."""
|
| 301 |
+
start, peak, end = span
|
| 302 |
+
magnitude = spec.magnitude[start:end + 1]
|
| 303 |
+
total = float(np.sum(magnitude ** 2))
|
| 304 |
+
high_bins = spec.band(*HIGH_BAND_HZ)
|
| 305 |
+
high = float(np.sum(magnitude[:, high_bins] ** 2)) if high_bins.size else 0.0
|
| 306 |
+
|
| 307 |
+
start_sample = int(spec.times[start] * recording.sample_rate_hz
|
| 308 |
+
- spec.window_seconds * recording.sample_rate_hz / 2)
|
| 309 |
+
end_sample = int(spec.times[end] * recording.sample_rate_hz
|
| 310 |
+
+ spec.window_seconds * recording.sample_rate_hz / 2)
|
| 311 |
+
window = recording.samples[max(0, start_sample):max(0, end_sample)]
|
| 312 |
+
peak_amplitude = float(np.max(np.abs(window), initial=0.0))
|
| 313 |
+
|
| 314 |
+
local_spread = float(threshold[peak] - np.median(strength))
|
| 315 |
+
return Event(
|
| 316 |
+
start_seconds=round(float(spec.times[start]) - spec.window_seconds / 2, 4),
|
| 317 |
+
peak_seconds=round(float(spec.times[peak]), 4),
|
| 318 |
+
end_seconds=round(float(spec.times[end]) + spec.window_seconds / 2, 4),
|
| 319 |
+
onset_strength=round(float(strength[peak]), 4),
|
| 320 |
+
prominence=round(
|
| 321 |
+
float((strength[peak] - threshold[peak]) / max(abs(local_spread), 1e-9)), 4
|
| 322 |
+
),
|
| 323 |
+
high_band_fraction=round(high / max(total, 1e-30), 4),
|
| 324 |
+
harmonicity=round(harmonicity(window, recording.sample_rate_hz), 4),
|
| 325 |
+
peak_level_dbfs=round(
|
| 326 |
+
float(20.0 * np.log10(max(peak_amplitude, 1e-10))), 2
|
| 327 |
+
),
|
| 328 |
+
)
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def _gate(event: Event) -> str:
|
| 332 |
+
"""The first gate this candidate fails, or an empty string.
|
| 333 |
+
|
| 334 |
+
A candidate usually fails more than one, so the **order decides what the
|
| 335 |
+
rejection is called**, and the reason is read by a person diagnosing a
|
| 336 |
+
capture rather than by a machine. So the order is by how informative the
|
| 337 |
+
answer is, not by how cheap the check is.
|
| 338 |
+
|
| 339 |
+
**Pitch is tested before frequency band, and a test is why.** A synthetic
|
| 340 |
+
cluck β a 400 Hz fundamental with harmonics to 4 kHz β measures harmonicity
|
| 341 |
+
0.83 and puts 0.20 of its energy above 1 kHz. Both gates reject it and both
|
| 342 |
+
are telling the truth, but band-first labels it *"low-frequency"*, which
|
| 343 |
+
describes a slamming door. Pitch-first labels it *"pitched β a vocalisation,
|
| 344 |
+
not a respiratory transient"*, which describes what it is. Vocalisation is
|
| 345 |
+
the commonest confounder in a poultry house by an enormous margin, and
|
| 346 |
+
mislabelling the common case to save an autocorrelation on a bounded number
|
| 347 |
+
of candidates is the wrong trade.
|
| 348 |
+
|
| 349 |
+
Duration stays first because a 5 ms click and a 2-second vehicle are not
|
| 350 |
+
usefully described by either of the other two.
|
| 351 |
+
"""
|
| 352 |
+
low, high = EVENT_SECONDS
|
| 353 |
+
if event.duration_seconds < low:
|
| 354 |
+
return f"too short ({event.duration_seconds * 1000:.0f} ms)"
|
| 355 |
+
if event.duration_seconds > high:
|
| 356 |
+
return f"too long ({event.duration_seconds * 1000:.0f} ms)"
|
| 357 |
+
if event.harmonicity > MAX_HARMONICITY:
|
| 358 |
+
return (
|
| 359 |
+
f"pitched ({event.harmonicity:.2f} autocorrelation, against "
|
| 360 |
+
f"{MAX_HARMONICITY:.2f}) β a vocalisation, not a respiratory transient"
|
| 361 |
+
)
|
| 362 |
+
if event.high_band_fraction < MIN_HIGH_BAND_FRACTION:
|
| 363 |
+
return (
|
| 364 |
+
f"low-frequency ({event.high_band_fraction:.2f} of its energy above "
|
| 365 |
+
f"1 kHz, against {MIN_HIGH_BAND_FRACTION:.2f})"
|
| 366 |
+
)
|
| 367 |
+
return ""
|
| 368 |
+
|
| 369 |
+
|
| 370 |
+
def screen(
|
| 371 |
+
recording: Recording,
|
| 372 |
+
*,
|
| 373 |
+
min_capture_seconds: float = MIN_CAPTURE_SECONDS,
|
| 374 |
+
onset_k: float = ONSET_K,
|
| 375 |
+
flux_band_hz: tuple[float, float] = FLUX_BAND_HZ,
|
| 376 |
+
) -> RespiratoryScreen:
|
| 377 |
+
"""Β§26's spot screen, end to end.
|
| 378 |
+
|
| 379 |
+
The parameters exist so the experiment can sweep them. Every default is the
|
| 380 |
+
value `experiments/poultry_respiratory/` derived, and a caller passing a
|
| 381 |
+
different one is running a different method with different accuracy.
|
| 382 |
+
"""
|
| 383 |
+
duration = recording.duration_seconds
|
| 384 |
+
base = {"duration_seconds": round(duration, 2)}
|
| 385 |
+
|
| 386 |
+
if duration < min_capture_seconds:
|
| 387 |
+
return RespiratoryScreen(
|
| 388 |
+
False, None, (), (),
|
| 389 |
+
reason=(
|
| 390 |
+
f"The recording is {duration:.0f} seconds. Stand quietly and "
|
| 391 |
+
f"record for at least {min_capture_seconds:.0f} β a house that "
|
| 392 |
+
f"is coughing does not do it on cue, and a shorter sample is a "
|
| 393 |
+
f"count of whatever happened to be in it."
|
| 394 |
+
),
|
| 395 |
+
rejection_code="recording_too_short",
|
| 396 |
+
diagnostics=base,
|
| 397 |
+
)
|
| 398 |
+
|
| 399 |
+
if recording.is_silent or float(
|
| 400 |
+
np.max(np.abs(recording.samples), initial=0.0)
|
| 401 |
+
) < SILENCE_PEAK:
|
| 402 |
+
return RespiratoryScreen(
|
| 403 |
+
False, None, (), (),
|
| 404 |
+
reason=(
|
| 405 |
+
"The recording holds no audible signal. Check that the "
|
| 406 |
+
"microphone was not covered."
|
| 407 |
+
),
|
| 408 |
+
# Not one of the registry's three codes, and deliberately not
|
| 409 |
+
# forced into one: a dead microphone is not a house full of
|
| 410 |
+
# machinery. The registry should grow a fourth, and until it does
|
| 411 |
+
# the honest thing is to leave this blank rather than mislabel it.
|
| 412 |
+
rejection_code="",
|
| 413 |
+
diagnostics=base,
|
| 414 |
+
)
|
| 415 |
+
|
| 416 |
+
spec = spectrogram(recording.samples, recording.sample_rate_hz)
|
| 417 |
+
machinery = machinery_dominance(spec)
|
| 418 |
+
speech = speech_dominance(spec)
|
| 419 |
+
diagnostics = dict(
|
| 420 |
+
base,
|
| 421 |
+
machinery_dominance=round(machinery, 4),
|
| 422 |
+
speech_dominance=round(speech, 4),
|
| 423 |
+
frames=spec.frames,
|
| 424 |
+
peak_dbfs=round(float(20.0 * np.log10(
|
| 425 |
+
max(float(np.max(np.abs(recording.samples), initial=0.0)), 1e-10)
|
| 426 |
+
)), 2),
|
| 427 |
+
rms_dbfs=round(float(20.0 * np.log10(
|
| 428 |
+
max(float(np.sqrt(np.mean(recording.samples.astype(np.float64) ** 2))), 1e-10)
|
| 429 |
+
)), 2),
|
| 430 |
+
onset_k=onset_k,
|
| 431 |
+
flux_band_low_hz=flux_band_hz[0],
|
| 432 |
+
flux_band_high_hz=flux_band_hz[1],
|
| 433 |
+
)
|
| 434 |
+
|
| 435 |
+
if machinery > MAX_MACHINERY_DOMINANCE:
|
| 436 |
+
return RespiratoryScreen(
|
| 437 |
+
False, None, (), (),
|
| 438 |
+
reason=(
|
| 439 |
+
f"Machinery drowns this recording: {machinery:.0%} of it is "
|
| 440 |
+
f"steady low-frequency noise, against a "
|
| 441 |
+
f"{MAX_MACHINERY_DOMINANCE:.0%} limit. Record again away from "
|
| 442 |
+
f"the fans, or when they cycle off."
|
| 443 |
+
),
|
| 444 |
+
rejection_code="machinery_dominates",
|
| 445 |
+
diagnostics=diagnostics,
|
| 446 |
+
)
|
| 447 |
+
|
| 448 |
+
if speech > MAX_SPEECH_DOMINANCE:
|
| 449 |
+
return RespiratoryScreen(
|
| 450 |
+
False, None, (), (),
|
| 451 |
+
reason=(
|
| 452 |
+
f"Somebody is talking through this recording. Record again "
|
| 453 |
+
f"without speaking β Β§26 asks you to stand quietly, and a voice "
|
| 454 |
+
f"close to the microphone hides every bird in the house."
|
| 455 |
+
),
|
| 456 |
+
rejection_code="speech_dominates",
|
| 457 |
+
diagnostics=diagnostics,
|
| 458 |
+
)
|
| 459 |
+
|
| 460 |
+
strength = onset_strength(spec, band_hz=flux_band_hz)
|
| 461 |
+
threshold = adaptive_threshold(
|
| 462 |
+
strength,
|
| 463 |
+
window_frames=int(round(ADAPTIVE_WINDOW_SECONDS / spec.hop_seconds)),
|
| 464 |
+
k=onset_k,
|
| 465 |
+
)
|
| 466 |
+
refractory = int(round(REFRACTORY_SECONDS / spec.hop_seconds))
|
| 467 |
+
|
| 468 |
+
kept: list[Event] = []
|
| 469 |
+
rejected: list[Event] = []
|
| 470 |
+
for span in _candidates(spec, strength, threshold, refractory):
|
| 471 |
+
event = _describe(spec, recording, strength, threshold, span)
|
| 472 |
+
failure = _gate(event)
|
| 473 |
+
if failure:
|
| 474 |
+
rejected.append(
|
| 475 |
+
Event(**{**event.__dict__, "rejected_because": failure})
|
| 476 |
+
)
|
| 477 |
+
else:
|
| 478 |
+
kept.append(event)
|
| 479 |
+
|
| 480 |
+
diagnostics["candidates"] = len(kept) + len(rejected)
|
| 481 |
+
diagnostics["events_per_minute"] = round(
|
| 482 |
+
len(kept) / (duration / 60.0), 3
|
| 483 |
+
) if duration > 0 else 0.0
|
| 484 |
+
|
| 485 |
+
return RespiratoryScreen(
|
| 486 |
+
usable=True,
|
| 487 |
+
count=len(kept),
|
| 488 |
+
events=tuple(kept),
|
| 489 |
+
rejected=tuple(rejected),
|
| 490 |
+
diagnostics=diagnostics,
|
| 491 |
+
)
|
app/adapters/audio/features.py
ADDED
|
@@ -0,0 +1,350 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Classical audio features, in NumPy, with nothing learned.
|
| 2 |
+
|
| 3 |
+
Directive Β§26 lists four things to test for poultry respiratory audio and
|
| 4 |
+
"classical audio features" is the last of them. Directive Β§4 is why it is the
|
| 5 |
+
one built first: *"Do not use a neural model when deterministic signal
|
| 6 |
+
processing is better."* The sibling module `adapters/signal/periodicity.py` is
|
| 7 |
+
this project's evidence that the rule pays β a metronome's stated 96 beats per
|
| 8 |
+
minute comes back as 96.48 from an FFT and no weights at all.
|
| 9 |
+
|
| 10 |
+
**A snick is a transient, and transients are what spectral flux is for.** A
|
| 11 |
+
chicken's sneeze or snick is a short broadband burst with a sharp attack. A
|
| 12 |
+
ventilation fan is the opposite: loud, broadband, and unchanging. Neither an
|
| 13 |
+
absolute level nor a spectrum tells them apart, and the *rate of change* of the
|
| 14 |
+
spectrum separates them at a glance. Everything here exists to compute that
|
| 15 |
+
difference and the three or four quantities needed to know whether it means
|
| 16 |
+
anything.
|
| 17 |
+
|
| 18 |
+
**No mel filterbank, and that is deliberate.** Mel spacing exists to model human
|
| 19 |
+
pitch perception; nothing here is about a human ear, and a linear STFT keeps
|
| 20 |
+
every threshold in this file expressible in hertz, which is the unit the
|
| 21 |
+
poultry-audio literature states its bands in.
|
| 22 |
+
|
| 23 |
+
Nothing in this module decides anything. It returns numbers; `events.py` applies
|
| 24 |
+
the thresholds and, more often, refuses.
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
from __future__ import annotations
|
| 28 |
+
|
| 29 |
+
from dataclasses import dataclass
|
| 30 |
+
|
| 31 |
+
import numpy as np
|
| 32 |
+
|
| 33 |
+
#: STFT window, in seconds. 32 ms at 16 kHz is 512 samples, giving 31.25 Hz
|
| 34 |
+
#: bins. Short enough that a 60 ms snick spans several frames rather than being
|
| 35 |
+
#: smeared into one; long enough to resolve the low-frequency fan energy that
|
| 36 |
+
#: `machinery_dominance` has to measure.
|
| 37 |
+
WINDOW_SECONDS = 0.032
|
| 38 |
+
|
| 39 |
+
#: Hop, in seconds. 8 ms at 16 kHz is 128 samples β a quarter of the window, so
|
| 40 |
+
#: an onset is localised to about a hundredth of a second. Β§26 counts events; it
|
| 41 |
+
#: does not need better timing than that, and a finer hop is linear cost for
|
| 42 |
+
#: nothing.
|
| 43 |
+
HOP_SECONDS = 0.008
|
| 44 |
+
|
| 45 |
+
#: The band spectral flux is summed over, in hertz.
|
| 46 |
+
#:
|
| 47 |
+
#: **The low edge is the load-bearing one.** Ventilation fans, extractor
|
| 48 |
+
#: machinery and wind on a microphone put most of their power below a few
|
| 49 |
+
#: hundred hertz, and a broadband flux measure that includes them tracks the
|
| 50 |
+
#: machinery instead of the birds. 400 Hz is above the fundamental of a fan and
|
| 51 |
+
#: below where a snick's energy starts.
|
| 52 |
+
#:
|
| 53 |
+
#: The high edge is the Nyquist of the analysis rate, so the band is "everything
|
| 54 |
+
#: above the machinery" rather than a claim about where a snick stops.
|
| 55 |
+
FLUX_BAND_HZ = (400.0, 8_000.0)
|
| 56 |
+
|
| 57 |
+
#: The band `machinery_dominance` measures, in hertz. Below `FLUX_BAND_HZ`'s low
|
| 58 |
+
#: edge on purpose: the two are meant to be disjoint, so a recording can be
|
| 59 |
+
#: loud in one and quiet in the other and the pair of numbers says which.
|
| 60 |
+
MACHINERY_BAND_HZ = (20.0, 400.0)
|
| 61 |
+
|
| 62 |
+
#: Voiced speech puts its fundamental here. An adult male's fundamental sits
|
| 63 |
+
#: around 85β180 Hz and a female's around 165β255; the band is widened at both
|
| 64 |
+
#: ends because a farmer's voice reaches a microphone through a shed.
|
| 65 |
+
VOICE_BAND_HZ = (80.0, 300.0)
|
| 66 |
+
|
| 67 |
+
#: Syllable rate, in hertz. The modulation spectrum of running speech peaks
|
| 68 |
+
#: between 3 and 5 Hz across languages and speakers β one of the most stable
|
| 69 |
+
#: facts in speech acoustics. The band is widened to 2β8 to cover fast and slow
|
| 70 |
+
#: talkers without reaching the 10β20 Hz region where a flock's own chatter
|
| 71 |
+
#: sits.
|
| 72 |
+
SPEECH_MODULATION_BAND_HZ = (2.0, 8.0)
|
| 73 |
+
|
| 74 |
+
#: Percentile taken over time to estimate the stationary noise floor of each
|
| 75 |
+
#: frequency bin. The 10th rather than the minimum, because a single quiet frame
|
| 76 |
+
#: β a dropout, a gap between fan blades β would otherwise set the floor for the
|
| 77 |
+
#: whole recording.
|
| 78 |
+
FLOOR_PERCENTILE = 10.0
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
@dataclass(frozen=True)
|
| 82 |
+
class Spectrogram:
|
| 83 |
+
"""Magnitude STFT, with the axes needed to say what a number means."""
|
| 84 |
+
|
| 85 |
+
#: `(frames, bins)`, magnitude β not power. Flux is computed on log
|
| 86 |
+
#: magnitude, and squaring first only doubles it.
|
| 87 |
+
magnitude: np.ndarray
|
| 88 |
+
#: Bin centre frequencies, in hertz.
|
| 89 |
+
frequencies: np.ndarray
|
| 90 |
+
#: Frame centre times, in seconds.
|
| 91 |
+
times: np.ndarray
|
| 92 |
+
hop_seconds: float
|
| 93 |
+
window_seconds: float
|
| 94 |
+
|
| 95 |
+
@property
|
| 96 |
+
def frames(self) -> int:
|
| 97 |
+
return int(self.magnitude.shape[0])
|
| 98 |
+
|
| 99 |
+
def band(self, low_hz: float, high_hz: float) -> np.ndarray:
|
| 100 |
+
"""Indices of the bins inside a band. Empty when the band is above
|
| 101 |
+
Nyquist, which a caller must handle rather than divide by."""
|
| 102 |
+
return np.flatnonzero(
|
| 103 |
+
(self.frequencies >= low_hz) & (self.frequencies <= high_hz)
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
def band_energy(self, low_hz: float, high_hz: float) -> np.ndarray:
|
| 107 |
+
"""Per-frame energy in a band. Zeros when the band is empty."""
|
| 108 |
+
bins = self.band(low_hz, high_hz)
|
| 109 |
+
if bins.size == 0:
|
| 110 |
+
return np.zeros(self.frames, dtype=np.float64)
|
| 111 |
+
return np.sum(self.magnitude[:, bins] ** 2, axis=1)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def spectrogram(
|
| 115 |
+
samples: np.ndarray,
|
| 116 |
+
sample_rate_hz: float,
|
| 117 |
+
*,
|
| 118 |
+
window_seconds: float = WINDOW_SECONDS,
|
| 119 |
+
hop_seconds: float = HOP_SECONDS,
|
| 120 |
+
) -> Spectrogram:
|
| 121 |
+
"""A Hann-windowed magnitude STFT, framed with `np.lib.stride_tricks`.
|
| 122 |
+
|
| 123 |
+
Written out rather than imported because the service has neither `scipy` nor
|
| 124 |
+
`librosa`, and adding either for one function would be a dependency a
|
| 125 |
+
2 vCPU container carries forever.
|
| 126 |
+
"""
|
| 127 |
+
samples = np.asarray(samples, dtype=np.float64).ravel()
|
| 128 |
+
window_length = max(8, int(round(window_seconds * sample_rate_hz)))
|
| 129 |
+
hop = max(1, int(round(hop_seconds * sample_rate_hz)))
|
| 130 |
+
|
| 131 |
+
if samples.size < window_length:
|
| 132 |
+
return Spectrogram(
|
| 133 |
+
magnitude=np.zeros((0, window_length // 2 + 1)),
|
| 134 |
+
frequencies=np.fft.rfftfreq(window_length, d=1.0 / sample_rate_hz),
|
| 135 |
+
times=np.zeros(0),
|
| 136 |
+
hop_seconds=hop / sample_rate_hz,
|
| 137 |
+
window_seconds=window_length / sample_rate_hz,
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
frame_count = 1 + (samples.size - window_length) // hop
|
| 141 |
+
frames = np.lib.stride_tricks.as_strided(
|
| 142 |
+
samples,
|
| 143 |
+
shape=(frame_count, window_length),
|
| 144 |
+
strides=(samples.strides[0] * hop, samples.strides[0]),
|
| 145 |
+
writeable=False,
|
| 146 |
+
)
|
| 147 |
+
magnitude = np.abs(np.fft.rfft(frames * np.hanning(window_length), axis=1))
|
| 148 |
+
|
| 149 |
+
return Spectrogram(
|
| 150 |
+
magnitude=magnitude,
|
| 151 |
+
frequencies=np.fft.rfftfreq(window_length, d=1.0 / sample_rate_hz),
|
| 152 |
+
# Frame *centres*, so an onset time is the middle of the window that
|
| 153 |
+
# holds it rather than its leading edge. An 8 ms hop makes the
|
| 154 |
+
# difference small and a systematic 16 ms offset in every published
|
| 155 |
+
# event time would still be wrong.
|
| 156 |
+
times=(np.arange(frame_count) * hop + window_length / 2.0) / sample_rate_hz,
|
| 157 |
+
hop_seconds=hop / sample_rate_hz,
|
| 158 |
+
window_seconds=window_length / sample_rate_hz,
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def onset_strength(
|
| 163 |
+
spec: Spectrogram, *, band_hz: tuple[float, float] = FLUX_BAND_HZ
|
| 164 |
+
) -> np.ndarray:
|
| 165 |
+
"""Half-wave rectified spectral flux over a band, one value per frame.
|
| 166 |
+
|
| 167 |
+
Log magnitude rather than linear, so the measure is a *relative* change and
|
| 168 |
+
a quiet snick between fan cycles counts as much as a loud one next to the
|
| 169 |
+
microphone. Rectified, because a spectrum falling away is the tail of an
|
| 170 |
+
event and only its arrival is an onset.
|
| 171 |
+
|
| 172 |
+
The first frame is zero by construction: there is no frame before it to
|
| 173 |
+
differ from, and a large opening value is the artefact that makes a
|
| 174 |
+
recording's first moment look like an event.
|
| 175 |
+
"""
|
| 176 |
+
bins = spec.band(*band_hz)
|
| 177 |
+
if spec.frames < 2 or bins.size == 0:
|
| 178 |
+
return np.zeros(spec.frames, dtype=np.float64)
|
| 179 |
+
|
| 180 |
+
# +1e-10 rather than a smaller floor: below about 1e-12 the log of a silent
|
| 181 |
+
# bin dominates the difference and every silence boundary reads as an onset.
|
| 182 |
+
logs = np.log(spec.magnitude[:, bins] + 1e-10)
|
| 183 |
+
flux = np.maximum(np.diff(logs, axis=0), 0.0).sum(axis=1)
|
| 184 |
+
return np.concatenate([[0.0], flux])
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def adaptive_threshold(
|
| 188 |
+
strength: np.ndarray, *, window_frames: int, k: float
|
| 189 |
+
) -> np.ndarray:
|
| 190 |
+
"""Local median plus `k` local median-absolute-deviations.
|
| 191 |
+
|
| 192 |
+
**Median and MAD, never mean and standard deviation.** The events being
|
| 193 |
+
looked for are exactly the outliers, so a mean threshold is pulled up by the
|
| 194 |
+
events it is meant to find, and a recording with many of them raises its own
|
| 195 |
+
bar until it reports few. The median is unmoved by anything under half the
|
| 196 |
+
window.
|
| 197 |
+
|
| 198 |
+
Computed by sorting a strided view β O(nΒ·w log w) and a few milliseconds for
|
| 199 |
+
a 60-second clip, against a rolling-median implementation this repository
|
| 200 |
+
would have to own.
|
| 201 |
+
"""
|
| 202 |
+
strength = np.asarray(strength, dtype=np.float64)
|
| 203 |
+
n = strength.size
|
| 204 |
+
if n == 0:
|
| 205 |
+
return np.zeros(0)
|
| 206 |
+
window = max(3, min(int(window_frames) | 1, n if n % 2 else n - 1))
|
| 207 |
+
if window < 3:
|
| 208 |
+
centre = float(np.median(strength))
|
| 209 |
+
spread = float(np.median(np.abs(strength - centre)))
|
| 210 |
+
return np.full(n, centre + k * spread)
|
| 211 |
+
|
| 212 |
+
half = window // 2
|
| 213 |
+
padded = np.pad(strength, half, mode="reflect")
|
| 214 |
+
view = np.lib.stride_tricks.sliding_window_view(padded, window)
|
| 215 |
+
centre = np.median(view, axis=1)
|
| 216 |
+
spread = np.median(np.abs(view - centre[:, None]), axis=1)
|
| 217 |
+
return centre + k * spread
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def stationary_floor(spec: Spectrogram) -> np.ndarray:
|
| 221 |
+
"""Per-bin noise floor: the `FLOOR_PERCENTILE`-th percentile over time.
|
| 222 |
+
|
| 223 |
+
What a fan leaves behind. A bin carrying only machinery has a floor close to
|
| 224 |
+
its mean; a bin carrying only events has a floor near zero.
|
| 225 |
+
"""
|
| 226 |
+
if spec.frames == 0:
|
| 227 |
+
return np.zeros(spec.magnitude.shape[1])
|
| 228 |
+
return np.percentile(spec.magnitude, FLOOR_PERCENTILE, axis=0)
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def machinery_dominance(spec: Spectrogram) -> float:
|
| 232 |
+
"""How much of the recording is unchanging low-frequency noise, in [0, 1].
|
| 233 |
+
|
| 234 |
+
Two factors multiplied, because either alone is wrong:
|
| 235 |
+
|
| 236 |
+
**How much energy is below 400 Hz.** A shed with the fans on is bottom-heavy
|
| 237 |
+
and a shed with them off is not.
|
| 238 |
+
|
| 239 |
+
**How much of that low-band energy is stationary** β the per-bin floor over
|
| 240 |
+
the per-bin mean. A fan is nearly all floor. A door slamming is loud, low,
|
| 241 |
+
and not floor at all, and a measure that only looked at the band would call
|
| 242 |
+
it machinery.
|
| 243 |
+
|
| 244 |
+
The product is what `events.py` compares against a threshold, and a
|
| 245 |
+
recording can be very loud below 400 Hz without tripping it as long as the
|
| 246 |
+
low band is *changing*.
|
| 247 |
+
"""
|
| 248 |
+
if spec.frames == 0:
|
| 249 |
+
return 0.0
|
| 250 |
+
total = float(np.sum(spec.magnitude ** 2))
|
| 251 |
+
if total <= 0.0:
|
| 252 |
+
return 0.0
|
| 253 |
+
|
| 254 |
+
low = spec.band(*MACHINERY_BAND_HZ)
|
| 255 |
+
if low.size == 0:
|
| 256 |
+
return 0.0
|
| 257 |
+
|
| 258 |
+
low_energy = float(np.sum(spec.magnitude[:, low] ** 2))
|
| 259 |
+
share_of_total = low_energy / total
|
| 260 |
+
|
| 261 |
+
floor = np.percentile(spec.magnitude[:, low], FLOOR_PERCENTILE, axis=0)
|
| 262 |
+
mean = np.mean(spec.magnitude[:, low], axis=0)
|
| 263 |
+
stationarity = float(np.mean(floor / np.maximum(mean, 1e-12)))
|
| 264 |
+
|
| 265 |
+
return float(share_of_total * stationarity)
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def speech_dominance(spec: Spectrogram) -> float:
|
| 269 |
+
"""How much the recording looks like somebody talking, in [0, 1].
|
| 270 |
+
|
| 271 |
+
Also two factors, and again both are needed:
|
| 272 |
+
|
| 273 |
+
**A 2β8 Hz modulation peak.** Running speech opens and closes the vocal
|
| 274 |
+
tract at the syllable rate, which puts a peak in the modulation spectrum of
|
| 275 |
+
its energy envelope between 3 and 5 Hz. Fans have no modulation; a flock's
|
| 276 |
+
chatter modulates faster and less regularly.
|
| 277 |
+
|
| 278 |
+
**Energy in the voicing band.** 80β300 Hz is where a human fundamental
|
| 279 |
+
lives. On its own it is useless β a fan is louder there β which is why it is
|
| 280 |
+
a factor rather than a test.
|
| 281 |
+
|
| 282 |
+
**This is a screen, not a speech detector, and it has never been measured
|
| 283 |
+
against real speech in a poultry house.** It is here because the capability
|
| 284 |
+
registry names `speech_dominates` as a rejection reason, and a rejection
|
| 285 |
+
reason with no implementation behind it is the shape of thing this project
|
| 286 |
+
keeps finding in its own past.
|
| 287 |
+
"""
|
| 288 |
+
if spec.frames < 8:
|
| 289 |
+
return 0.0
|
| 290 |
+
|
| 291 |
+
envelope = np.sqrt(spec.band_energy(20.0, 8_000.0))
|
| 292 |
+
if float(np.max(envelope, initial=0.0)) <= 0.0:
|
| 293 |
+
return 0.0
|
| 294 |
+
|
| 295 |
+
envelope = envelope - envelope.mean()
|
| 296 |
+
frame_rate = 1.0 / spec.hop_seconds
|
| 297 |
+
power = np.abs(np.fft.rfft(envelope * np.hanning(envelope.size))) ** 2
|
| 298 |
+
modulation = np.fft.rfftfreq(envelope.size, d=1.0 / frame_rate)
|
| 299 |
+
|
| 300 |
+
# Bin 0 is the mean, already removed; including it would make every
|
| 301 |
+
# recording's modulation peak its own DC.
|
| 302 |
+
band = np.flatnonzero(
|
| 303 |
+
(modulation >= SPEECH_MODULATION_BAND_HZ[0])
|
| 304 |
+
& (modulation <= SPEECH_MODULATION_BAND_HZ[1])
|
| 305 |
+
)
|
| 306 |
+
rest = np.flatnonzero((modulation > 0.0) & (modulation < 40.0))
|
| 307 |
+
if band.size == 0 or rest.size == 0:
|
| 308 |
+
return 0.0
|
| 309 |
+
modulation_share = float(np.sum(power[band]) / max(float(np.sum(power[rest])), 1e-30))
|
| 310 |
+
|
| 311 |
+
total = float(np.sum(spec.magnitude ** 2))
|
| 312 |
+
voice = float(np.sum(spec.magnitude[:, spec.band(*VOICE_BAND_HZ)] ** 2))
|
| 313 |
+
voice_share = voice / max(total, 1e-30)
|
| 314 |
+
|
| 315 |
+
return float(min(1.0, modulation_share) * voice_share)
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def harmonicity(frame_samples: np.ndarray, sample_rate_hz: float,
|
| 319 |
+
*, min_hz: float = 80.0, max_hz: float = 2_000.0) -> float:
|
| 320 |
+
"""Strength of the strongest autocorrelation peak in a pitch range, in [0, 1].
|
| 321 |
+
|
| 322 |
+
A cluck, a crow and a spoken vowel are pitched: the waveform repeats, so its
|
| 323 |
+
normalised autocorrelation has a tall peak at the period. A snick, a sneeze
|
| 324 |
+
and a fan are not, and theirs does not.
|
| 325 |
+
|
| 326 |
+
Returns 0 when the segment is too short to hold a full period at `min_hz`,
|
| 327 |
+
which is a refusal rather than a low score β a 40 ms segment cannot be asked
|
| 328 |
+
whether it repeats at 80 Hz.
|
| 329 |
+
"""
|
| 330 |
+
x = np.asarray(frame_samples, dtype=np.float64).ravel()
|
| 331 |
+
if x.size < 8:
|
| 332 |
+
return 0.0
|
| 333 |
+
x = x - x.mean()
|
| 334 |
+
energy = float(np.dot(x, x))
|
| 335 |
+
if energy <= 0.0:
|
| 336 |
+
return 0.0
|
| 337 |
+
|
| 338 |
+
min_lag = max(1, int(sample_rate_hz / max_hz))
|
| 339 |
+
max_lag = int(sample_rate_hz / min_hz)
|
| 340 |
+
if max_lag >= x.size:
|
| 341 |
+
return 0.0
|
| 342 |
+
|
| 343 |
+
# Full autocorrelation via FFT; the direct form is O(nΒ²) and this is called
|
| 344 |
+
# once per candidate event.
|
| 345 |
+
size = 1 << int(np.ceil(np.log2(2 * x.size)))
|
| 346 |
+
spectrum = np.fft.rfft(x, size)
|
| 347 |
+
correlation = np.fft.irfft(spectrum * np.conj(spectrum), size)[: x.size]
|
| 348 |
+
if min_lag >= max_lag:
|
| 349 |
+
return 0.0
|
| 350 |
+
return float(np.clip(np.max(correlation[min_lag:max_lag]) / energy, 0.0, 1.0))
|
app/adapters/base.py
ADDED
|
@@ -0,0 +1,418 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""What every adapter agrees to, and the two things none of them may do.
|
| 2 |
+
|
| 3 |
+
The zero-training directive (Β§3, Β§4, Β§40.1) names a stack of pretrained models β
|
| 4 |
+
SAM, DINOv3, MegaDescriptor, Grounding DINO, CountGD, a hosted reasoner β plus a
|
| 5 |
+
deterministic OpenCV/NumPy path that Β§4 says to prefer whenever it is better.
|
| 6 |
+
Those have almost nothing in common at the point of use: one returns masks, one
|
| 7 |
+
returns a 384-dimensional vector, one returns a breath rate. So this file does
|
| 8 |
+
**not** try to give them a single `run`.
|
| 9 |
+
|
| 10 |
+
What they do have in common is governance, and that is what is unified here:
|
| 11 |
+
|
| 12 |
+
**An adapter says what it costs, and `None` means nobody measured it.**
|
| 13 |
+
`MeasuredCost` has no defaults and no published-figure fallback. A latency copied
|
| 14 |
+
from a paper is a claim about somebody else's GPU, and the farms this serves run
|
| 15 |
+
a 2 vCPU / 4 GiB container.
|
| 16 |
+
|
| 17 |
+
**An adapter cannot produce a result it has no model for.** There is deliberately
|
| 18 |
+
no `run` on this class β the same reason `providers.InferenceProvider` has none.
|
| 19 |
+
A base implementation would be a way to return something plausible with nothing
|
| 20 |
+
behind it, and that is the single failure this service exists to prevent. Work
|
| 21 |
+
happens on the object `load()` returns, and `load()` raises when the artefact is
|
| 22 |
+
absent.
|
| 23 |
+
|
| 24 |
+
The task protocols below are the narrow interfaces callers actually use. They are
|
| 25 |
+
kept as small as `detectors.Detector` is, for the same reason: everything added
|
| 26 |
+
here is something the replacement adapter has to reimplement on the day a licence
|
| 27 |
+
forces a swap, and ADR 0017 is the record of that day arriving.
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
from __future__ import annotations
|
| 31 |
+
|
| 32 |
+
from dataclasses import dataclass, field
|
| 33 |
+
from enum import Enum
|
| 34 |
+
from typing import Protocol, runtime_checkable
|
| 35 |
+
|
| 36 |
+
import numpy as np
|
| 37 |
+
from PIL import Image
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class Task(str, Enum):
|
| 41 |
+
"""What an adapter produces. A model may do several."""
|
| 42 |
+
|
| 43 |
+
DETECT = "detect"
|
| 44 |
+
SEGMENT = "segment"
|
| 45 |
+
EMBED = "embed"
|
| 46 |
+
COUNT = "count"
|
| 47 |
+
TRACK = "track"
|
| 48 |
+
POSE = "pose"
|
| 49 |
+
#: Structured reasoning from a hosted multimodal model. Named apart from the
|
| 50 |
+
#: rest because Β§4 is explicit that it is "an experimental visual reasoner,
|
| 51 |
+
#: not an authority", and a caller should have to type the difference.
|
| 52 |
+
REASON = "reason"
|
| 53 |
+
#: Deterministic signal processing β optical flow, FFT, contour geometry.
|
| 54 |
+
#: No weights, no licence question, and Β§4 says to prefer it where it wins.
|
| 55 |
+
MEASURE = "measure"
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class Modality(str, Enum):
|
| 59 |
+
IMAGE = "image"
|
| 60 |
+
VIDEO = "video"
|
| 61 |
+
AUDIO = "audio"
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
@dataclass(frozen=True)
|
| 65 |
+
class MeasuredCost:
|
| 66 |
+
"""Latency and memory from a run that actually happened.
|
| 67 |
+
|
| 68 |
+
Every field is required. There is no `estimated` variant and no default,
|
| 69 |
+
because the only thing worse than not knowing what an adapter costs on a
|
| 70 |
+
2 vCPU box is believing a number nobody produced there.
|
| 71 |
+
|
| 72 |
+
`hardware` is free text on purpose: it has to be able to say "MacBook, 8
|
| 73 |
+
performance cores, not the target" as easily as it says the container SKU,
|
| 74 |
+
and a reader needs to see which one they are looking at.
|
| 75 |
+
"""
|
| 76 |
+
|
| 77 |
+
hardware: str
|
| 78 |
+
threads: int
|
| 79 |
+
#: What it ran on, specifically enough to re-run. A slug from
|
| 80 |
+
#: `evaluation/dataset.json`, or a count of frames from a named set.
|
| 81 |
+
sample: str
|
| 82 |
+
runs: int
|
| 83 |
+
median_seconds: float
|
| 84 |
+
peak_rss_mb: float
|
| 85 |
+
measured_on: str
|
| 86 |
+
|
| 87 |
+
@property
|
| 88 |
+
def fits_cpu_service(self) -> bool:
|
| 89 |
+
"""Whether this would survive the 2 vCPU / 4 GiB CPU worker.
|
| 90 |
+
|
| 91 |
+
**This is a placement hint, not a verdict on the model.** A model that
|
| 92 |
+
returns False here belongs on a GPU host, and that is a deployment
|
| 93 |
+
decision rather than a reason to drop a capability. The distinction is
|
| 94 |
+
recorded because the earlier version of this file got it wrong and would
|
| 95 |
+
have excluded most of directive Β§3 on the strength of a container size.
|
| 96 |
+
"""
|
| 97 |
+
return (
|
| 98 |
+
self.peak_rss_mb <= CPU_SERVICE_MEMORY_CEILING_MB
|
| 99 |
+
and self.median_seconds <= INLINE_LATENCY_CEILING_SECONDS
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
#: Peak RSS above which an adapter will not sit comfortably beside the API on the
|
| 104 |
+
#: existing CPU worker. ADR 0018 measured YOLOX-m at 591 MB and YOLOX-x at
|
| 105 |
+
#: 1,003 MB. 2,000 MB leaves the Python process, onnxruntime's arenas and
|
| 106 |
+
#: Pillow's decode buffers room inside 4 GiB.
|
| 107 |
+
CPU_SERVICE_MEMORY_CEILING_MB = 2000.0
|
| 108 |
+
|
| 109 |
+
#: Wall-clock above which a capability cannot run inline on a request, wherever
|
| 110 |
+
#: it is hosted. ADR 0018's phrasing: "A 16-second inline request is not a
|
| 111 |
+
#: request; it is a timeout with a result attached." Past this a capability needs
|
| 112 |
+
#: a queue, not a bigger box.
|
| 113 |
+
INLINE_LATENCY_CEILING_SECONDS = 8.0
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
class Placement(str, Enum):
|
| 117 |
+
"""Where a leg of the stack should run.
|
| 118 |
+
|
| 119 |
+
Three tiers, and the choice between them is a product decision as much as an
|
| 120 |
+
engineering one. Animap is offline-first (ADR 0002): a capability that needs
|
| 121 |
+
a round trip is one a farm cannot use in a shed with no signal, so pushing
|
| 122 |
+
work off the phone has a cost that a latency table does not show.
|
| 123 |
+
"""
|
| 124 |
+
|
| 125 |
+
#: On the phone. The only tier that works with no signal at all.
|
| 126 |
+
ON_DEVICE = "on_device"
|
| 127 |
+
#: The existing 2 vCPU / 4 GiB CPU container, beside the API.
|
| 128 |
+
CPU_SERVICE = "cpu_service"
|
| 129 |
+
#: A GPU host. Available, and the right answer for most of directive Β§3 β
|
| 130 |
+
#: the models it names are GPU-class work and it was written knowing that.
|
| 131 |
+
GPU_SERVICE = "gpu_service"
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
@dataclass(frozen=True)
|
| 135 |
+
class AdapterSpec:
|
| 136 |
+
"""An adapter's identity, licence position and measured cost.
|
| 137 |
+
|
| 138 |
+
This is committed code rather than a JSON card, and that is the point.
|
| 139 |
+
`providers.ModelArtefact` reads a card, and a card is written by whoever
|
| 140 |
+
writes the card β ADR 0017 records a watchdog defeating the licence gate by
|
| 141 |
+
declaring `Apache-2.0` over a path to AGPL weights. The `runtime` here names
|
| 142 |
+
which loader runs, which is a fact about the code and not a claim about
|
| 143 |
+
terms, and `adapters.licences` holds what that runtime's weights are really
|
| 144 |
+
licensed under.
|
| 145 |
+
"""
|
| 146 |
+
|
| 147 |
+
adapter_id: str
|
| 148 |
+
#: Which loader runs. The key into `licences.RUNTIME_LICENCES`, and the only
|
| 149 |
+
#: field the licence gate trusts.
|
| 150 |
+
runtime: str
|
| 151 |
+
tasks: tuple[Task, ...]
|
| 152 |
+
modalities: tuple[Modality, ...]
|
| 153 |
+
#: What the zero-training directive asks this model for, quoted closely
|
| 154 |
+
#: enough that a reader can find the section.
|
| 155 |
+
directive_role: str
|
| 156 |
+
#: False for the deterministic methods β optical flow, FFT, contour
|
| 157 |
+
#: geometry. They need no weights, so they have no artefact to be absent and
|
| 158 |
+
#: no licence to refuse, which is most of why Β§4 prefers them.
|
| 159 |
+
requires_artefact: bool = True
|
| 160 |
+
#: `None` until somebody runs it and writes the number down. Reported as
|
| 161 |
+
#: "not measured", never filled in from a paper.
|
| 162 |
+
measured: MeasuredCost | None = None
|
| 163 |
+
#: Where this leg should run. A recommendation with a reason, not a
|
| 164 |
+
#: constraint β see `Placement`.
|
| 165 |
+
placement: Placement = Placement.CPU_SERVICE
|
| 166 |
+
#: Whether a GPU is needed for this to be usable at all, as opposed to
|
| 167 |
+
#: merely faster. Recorded separately from `placement` because "runs on CPU
|
| 168 |
+
#: but slowly" and "does not run on CPU" are different facts and only the
|
| 169 |
+
#: second one closes a door.
|
| 170 |
+
requires_gpu: bool = False
|
| 171 |
+
placement_reason: str = ""
|
| 172 |
+
notes: str = ""
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
@dataclass(frozen=True)
|
| 176 |
+
class Availability:
|
| 177 |
+
"""Whether an adapter can run, and if not, what would change that.
|
| 178 |
+
|
| 179 |
+
`remedy` exists because "unavailable" without it is the answer that gets
|
| 180 |
+
read as "broken". The service already distinguishes *"no validated model
|
| 181 |
+
exists"* from *"this is not planned"* in `main._unavailable_reason`, and an
|
| 182 |
+
adapter that cannot say which of those it is has lost the distinction.
|
| 183 |
+
"""
|
| 184 |
+
|
| 185 |
+
ready: bool
|
| 186 |
+
#: Empty when ready. Otherwise says what is missing, not what went wrong.
|
| 187 |
+
reason: str = ""
|
| 188 |
+
remedy: str = ""
|
| 189 |
+
|
| 190 |
+
def __post_init__(self) -> None:
|
| 191 |
+
if not self.ready and not self.reason:
|
| 192 |
+
raise ValueError(
|
| 193 |
+
"An unavailable adapter must say why. A bare False is what a "
|
| 194 |
+
"caller renders as a silent failure."
|
| 195 |
+
)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
class AdapterError(RuntimeError):
|
| 199 |
+
"""The adapter is present but could not do the work."""
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
class AdapterUnavailable(AdapterError):
|
| 203 |
+
"""No model behind this adapter, so there is nothing to run.
|
| 204 |
+
|
| 205 |
+
Raised by `load()`, never returned as a result. A caller that catches this
|
| 206 |
+
reports `unavailable` β the state `JobState.UNAVAILABLE` already exists for,
|
| 207 |
+
and which is the honest answer for most of the stack today.
|
| 208 |
+
"""
|
| 209 |
+
|
| 210 |
+
def __init__(self, availability: Availability) -> None:
|
| 211 |
+
self.availability = availability
|
| 212 |
+
message = availability.reason
|
| 213 |
+
if availability.remedy:
|
| 214 |
+
message = f"{message} {availability.remedy}"
|
| 215 |
+
super().__init__(message)
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
class Adapter:
|
| 219 |
+
"""A pretrained model, or a deterministic method, behind one interface.
|
| 220 |
+
|
| 221 |
+
**There is no `run` here, and adding one would be the bug.** Subclasses
|
| 222 |
+
expose whichever task protocol they satisfy β `Embedder`, `Segmenter`,
|
| 223 |
+
`Reasoner` β and only after `load()` has succeeded against a real artefact.
|
| 224 |
+
A default implementation on this class would be a way to answer a farmer
|
| 225 |
+
with no model in the loop.
|
| 226 |
+
"""
|
| 227 |
+
|
| 228 |
+
spec: AdapterSpec
|
| 229 |
+
|
| 230 |
+
def availability(self) -> Availability:
|
| 231 |
+
"""Whether this adapter could run right now.
|
| 232 |
+
|
| 233 |
+
Must not load anything. Called on `/health` and `/capabilities`, which
|
| 234 |
+
a platform probe hits often enough that reading a hundred megabytes of
|
| 235 |
+
weights to answer it would be its own outage.
|
| 236 |
+
"""
|
| 237 |
+
raise NotImplementedError
|
| 238 |
+
|
| 239 |
+
def load(self) -> "Adapter":
|
| 240 |
+
"""Prepare the runtime, or raise `AdapterUnavailable`.
|
| 241 |
+
|
| 242 |
+
Returns self so a caller can write `adapter.load().embed(image)` and
|
| 243 |
+
have no path to `embed` that skipped the check.
|
| 244 |
+
"""
|
| 245 |
+
raise NotImplementedError
|
| 246 |
+
|
| 247 |
+
def describe(self) -> dict[str, object]:
|
| 248 |
+
"""Everything a governance reader needs, including what is unmeasured."""
|
| 249 |
+
from app.adapters import licences
|
| 250 |
+
|
| 251 |
+
availability = self.availability()
|
| 252 |
+
licence = licences.RUNTIME_LICENCES.get(self.spec.runtime)
|
| 253 |
+
cost = self.spec.measured
|
| 254 |
+
return {
|
| 255 |
+
"adapter_id": self.spec.adapter_id,
|
| 256 |
+
"runtime": self.spec.runtime,
|
| 257 |
+
"tasks": [t.value for t in self.spec.tasks],
|
| 258 |
+
"modalities": [m.value for m in self.spec.modalities],
|
| 259 |
+
"directive_role": self.spec.directive_role,
|
| 260 |
+
"ready": availability.ready,
|
| 261 |
+
"reason": availability.reason,
|
| 262 |
+
"remedy": availability.remedy,
|
| 263 |
+
"licence": licence.licence if licence else "unknown runtime",
|
| 264 |
+
"licence_source": licence.source_url if licence else "",
|
| 265 |
+
"servable": bool(licence and licence.servable),
|
| 266 |
+
"placement": self.spec.placement.value,
|
| 267 |
+
"requires_gpu": self.spec.requires_gpu,
|
| 268 |
+
"placement_reason": self.spec.placement_reason,
|
| 269 |
+
# The absence is the finding, so it is spelled rather than nulled.
|
| 270 |
+
"measured": (
|
| 271 |
+
{
|
| 272 |
+
"hardware": cost.hardware,
|
| 273 |
+
"threads": cost.threads,
|
| 274 |
+
"sample": cost.sample,
|
| 275 |
+
"runs": cost.runs,
|
| 276 |
+
"median_seconds": cost.median_seconds,
|
| 277 |
+
"peak_rss_mb": cost.peak_rss_mb,
|
| 278 |
+
"measured_on": cost.measured_on,
|
| 279 |
+
"fits_cpu_service": cost.fits_cpu_service,
|
| 280 |
+
}
|
| 281 |
+
if cost is not None
|
| 282 |
+
else "not measured"
|
| 283 |
+
),
|
| 284 |
+
"notes": self.spec.notes,
|
| 285 |
+
}
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
# --- Task protocols. Narrow on purpose. --------------------------------------
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
@dataclass(frozen=True)
|
| 292 |
+
class Region:
|
| 293 |
+
"""A box, a mask, or both. The common currency of detection and segmentation.
|
| 294 |
+
|
| 295 |
+
`box` is in source-image pixels, matching `detectors.Detection`, so a caller
|
| 296 |
+
that already knows how to read a YOLOX box does not learn a second convention.
|
| 297 |
+
`mask` is a boolean array at source-image resolution, or `None` when the
|
| 298 |
+
adapter only localises.
|
| 299 |
+
"""
|
| 300 |
+
|
| 301 |
+
label: str
|
| 302 |
+
score: float
|
| 303 |
+
box: tuple[float, float, float, float]
|
| 304 |
+
mask: np.ndarray | None = None
|
| 305 |
+
area_fraction: float = 0.0
|
| 306 |
+
|
| 307 |
+
@property
|
| 308 |
+
def has_mask(self) -> bool:
|
| 309 |
+
return self.mask is not None
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
@runtime_checkable
|
| 313 |
+
class Embedder(Protocol):
|
| 314 |
+
"""Frozen features. Β§3's instruction for DINOv3 is explicit that this comes
|
| 315 |
+
before any fine-tuning: "Start with frozen embeddings + nearest-neighbor
|
| 316 |
+
retrieval."
|
| 317 |
+
"""
|
| 318 |
+
|
| 319 |
+
#: Length of the vector `embed` returns. Recorded because a retrieval index
|
| 320 |
+
#: built at one dimension and queried at another fails silently.
|
| 321 |
+
dimensions: int
|
| 322 |
+
|
| 323 |
+
def embed(self, image: Image.Image) -> np.ndarray:
|
| 324 |
+
"""One L2-normalised float32 vector.
|
| 325 |
+
|
| 326 |
+
Normalised by the adapter rather than the caller, so cosine similarity
|
| 327 |
+
is a dot product everywhere and no index has to remember which
|
| 328 |
+
convention it was built under.
|
| 329 |
+
"""
|
| 330 |
+
...
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
@runtime_checkable
|
| 334 |
+
class Segmenter(Protocol):
|
| 335 |
+
def segment(
|
| 336 |
+
self, image: Image.Image, *, concepts: tuple[str, ...] = ()
|
| 337 |
+
) -> list[Region]:
|
| 338 |
+
...
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
@runtime_checkable
|
| 342 |
+
class OpenVocabularyDetector(Protocol):
|
| 343 |
+
"""Text-prompted detection. Β§4 names Grounding DINO as the fallback for when
|
| 344 |
+
SAM's concept prompting is weak."""
|
| 345 |
+
|
| 346 |
+
def detect_text(
|
| 347 |
+
self, image: Image.Image, prompts: tuple[str, ...]
|
| 348 |
+
) -> list[Region]:
|
| 349 |
+
...
|
| 350 |
+
|
| 351 |
+
|
| 352 |
+
@runtime_checkable
|
| 353 |
+
class ExemplarCounter(Protocol):
|
| 354 |
+
"""Zero-shot counting, optionally guided by example boxes (Β§4, CountGD)."""
|
| 355 |
+
|
| 356 |
+
def count(
|
| 357 |
+
self,
|
| 358 |
+
image: Image.Image,
|
| 359 |
+
*,
|
| 360 |
+
text: str = "",
|
| 361 |
+
exemplars: tuple[tuple[float, float, float, float], ...] = (),
|
| 362 |
+
) -> "CountEstimate":
|
| 363 |
+
...
|
| 364 |
+
|
| 365 |
+
|
| 366 |
+
@dataclass(frozen=True)
|
| 367 |
+
class CountEstimate:
|
| 368 |
+
"""A count, or an honest refusal to publish one.
|
| 369 |
+
|
| 370 |
+
`value` is `None` when the method ran and the result should not be shown β
|
| 371 |
+
the same shape `app/counting.py` already uses, where withholding is a first
|
| 372 |
+
class outcome rather than an exception. Β§6.3 requires the distinction
|
| 373 |
+
between *visible count*, *unique birds observed* and *reconciled population*
|
| 374 |
+
to survive to the UI, so `kind` carries it.
|
| 375 |
+
"""
|
| 376 |
+
|
| 377 |
+
value: float | None
|
| 378 |
+
kind: str
|
| 379 |
+
withheld_reason: str = ""
|
| 380 |
+
confidence: float | None = None
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
@runtime_checkable
|
| 384 |
+
class Reasoner(Protocol):
|
| 385 |
+
"""A hosted multimodal model.
|
| 386 |
+
|
| 387 |
+
Β§4: "All calls must return structured JSON" and "The multimodal model is an
|
| 388 |
+
**experimental visual reasoner**, not an authority." Both are enforced in
|
| 389 |
+
the adapter rather than left to the prompt β see `adapters/multimodal.py`.
|
| 390 |
+
"""
|
| 391 |
+
|
| 392 |
+
def reason(
|
| 393 |
+
self,
|
| 394 |
+
images: list[Image.Image],
|
| 395 |
+
*,
|
| 396 |
+
schema: dict,
|
| 397 |
+
rubric: str,
|
| 398 |
+
) -> dict:
|
| 399 |
+
...
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
@dataclass(frozen=True)
|
| 403 |
+
class Measurement:
|
| 404 |
+
"""A number a deterministic method produced, with its own quality verdict.
|
| 405 |
+
|
| 406 |
+
Signal processing fails differently from a model: it does not become
|
| 407 |
+
uncertain, it becomes wrong in a way that still returns a float. So a
|
| 408 |
+
measurement carries the evidence that the signal was there at all β
|
| 409 |
+
`support` is whatever the method uses to know it measured something rather
|
| 410 |
+
than measuring noise, and `usable` is its own judgement about that.
|
| 411 |
+
"""
|
| 412 |
+
|
| 413 |
+
kind: str
|
| 414 |
+
value: float | None
|
| 415 |
+
unit: str
|
| 416 |
+
usable: bool
|
| 417 |
+
support: dict[str, float] = field(default_factory=dict)
|
| 418 |
+
detail: str = ""
|
app/adapters/claims.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
app/adapters/deterministic.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The deterministic methods, as first-class adapters.
|
| 2 |
+
|
| 3 |
+
Directive Β§4, and it is the sentence most easily skipped in the whole document:
|
| 4 |
+
|
| 5 |
+
> Do not use a neural model when deterministic signal processing is better.
|
| 6 |
+
|
| 7 |
+
So optical flow, FFT periodicity, contour measurement and reference-marker
|
| 8 |
+
calibration are registered here alongside SAM and DINOv3 rather than living in a
|
| 9 |
+
utilities module. They sit in the same registry, answer the same
|
| 10 |
+
`availability()`, and appear in the same listing, because a reader comparing the
|
| 11 |
+
stack should see that two of the capabilities with the clearest path forward
|
| 12 |
+
need no weights at all.
|
| 13 |
+
|
| 14 |
+
**These adapters are always available**, which no other adapter in this package
|
| 15 |
+
can say. There is no artefact to be absent, no card to be checksummed, no
|
| 16 |
+
licence to be refused and no gate to fail β which is most of the argument for
|
| 17 |
+
preferring them. `availability()` still exists and still answers, because a
|
| 18 |
+
caller should not have to know which kind of adapter it is holding.
|
| 19 |
+
|
| 20 |
+
The honesty property is not weaker here, it is only located differently. A
|
| 21 |
+
neural adapter refuses by having no model; these refuse by measuring whether the
|
| 22 |
+
signal was present, in `periodicity`'s two gates and in `geometry`'s refusal to
|
| 23 |
+
invent a scale.
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
from __future__ import annotations
|
| 27 |
+
|
| 28 |
+
from pathlib import Path
|
| 29 |
+
|
| 30 |
+
from app.adapters.base import (
|
| 31 |
+
Adapter,
|
| 32 |
+
AdapterSpec,
|
| 33 |
+
Availability,
|
| 34 |
+
MeasuredCost,
|
| 35 |
+
Measurement,
|
| 36 |
+
Modality,
|
| 37 |
+
Placement,
|
| 38 |
+
Task,
|
| 39 |
+
)
|
| 40 |
+
from app.adapters.signal.geometry import (
|
| 41 |
+
NoReference,
|
| 42 |
+
Scale,
|
| 43 |
+
measure_region,
|
| 44 |
+
scale_from_marker,
|
| 45 |
+
)
|
| 46 |
+
from app.adapters.signal.respiration import RespirationResult, respiratory_rate
|
| 47 |
+
|
| 48 |
+
RESPIRATION_SPEC = AdapterSpec(
|
| 49 |
+
adapter_id="respiration-flow-fft",
|
| 50 |
+
runtime="opencv-numpy",
|
| 51 |
+
tasks=(Task.MEASURE,),
|
| 52 |
+
modalities=(Modality.VIDEO,),
|
| 53 |
+
directive_role=(
|
| 54 |
+
"Β§14 cattle respiratory rate β video, flank region, optical flow, "
|
| 55 |
+
"periodicity, FFT, breaths per minute. Β§4 names optical flow, FFT and "
|
| 56 |
+
"periodic motion analysis as OpenCV work rather than model work."
|
| 57 |
+
),
|
| 58 |
+
requires_artefact=False,
|
| 59 |
+
placement=Placement.CPU_SERVICE,
|
| 60 |
+
placement_reason=(
|
| 61 |
+
"No weights, and the arithmetic is cheap β but the clip is long. Dense "
|
| 62 |
+
"flow costs 4.3 ms per frame pair at 320 px, so Β§14's 30β60 second "
|
| 63 |
+
"capture is 4β8 seconds of flow plus decode, measured at 9.7 s median "
|
| 64 |
+
"for a 31-second clip. **That is past the inline ceiling**, so this "
|
| 65 |
+
"capability needs a queue rather than a bigger box. It is also the "
|
| 66 |
+
"strongest on-device candidate in the stack: OpenCV is on the phone "
|
| 67 |
+
"already, the video never has to leave it, and ADR 0002's offline-first "
|
| 68 |
+
"promise is kept for free."
|
| 69 |
+
),
|
| 70 |
+
measured=MeasuredCost(
|
| 71 |
+
hardware=(
|
| 72 |
+
"Apple M-series laptop (NOT the target container). OpenCV 5.0.0 "
|
| 73 |
+
"reports 11 threads and ignores setNumThreads(), so a "
|
| 74 |
+
"single-threaded figure could not be taken on this build"
|
| 75 |
+
),
|
| 76 |
+
threads=11,
|
| 77 |
+
sample=(
|
| 78 |
+
"Cow_crosses_cattle_grid.webm, 925 frames, 30.86 s at 29.97 fps, "
|
| 79 |
+
"whole frame, decode plus flow plus spectrum"
|
| 80 |
+
),
|
| 81 |
+
runs=7,
|
| 82 |
+
median_seconds=9.68,
|
| 83 |
+
peak_rss_mb=239.0,
|
| 84 |
+
measured_on="2026-08-21",
|
| 85 |
+
),
|
| 86 |
+
notes=(
|
| 87 |
+
"**The latency figure is load-sensitive and should be read as a band, "
|
| 88 |
+
"not a point.** Seven runs give a 9.68 s median over a 9.29β12.58 s "
|
| 89 |
+
"spread, and separate sessions on the same machine and the same clip "
|
| 90 |
+
"produced medians of 12.15 s and 14.17 s. The previously recorded "
|
| 91 |
+
"9.07 s / 331 MB does not reproduce in any configuration tried: memory "
|
| 92 |
+
"is consistently around 239 MB, and no threading setting moves the "
|
| 93 |
+
"latency, because this OpenCV build does not honour setNumThreads. "
|
| 94 |
+
"What survives all of it is the conclusion β every measurement is past "
|
| 95 |
+
"the 8 s inline ceiling, so this capability needs a queue.\n\n"
|
| 96 |
+
"**What the metronome validates is the extractor, not this adapter.** "
|
| 97 |
+
"On footage whose Commons description states 96 beats per minute, "
|
| 98 |
+
"`signal.dominant_rate` returns 96.48, and 48.38 on a crop of the "
|
| 99 |
+
"pendulum alone, the swing being half the tick rate β a 0.5% error "
|
| 100 |
+
"against a stated rate on real video. But `Metronome.webm` is 11.71 "
|
| 101 |
+
"seconds, and `respiration.MIN_CAPTURE_SECONDS` is 20, so "
|
| 102 |
+
"`measure()` refuses all three of those regions before any signal "
|
| 103 |
+
"processing runs. The only clip with a ground truth cannot reach the "
|
| 104 |
+
"code path this adapter exposes, and an earlier version of this note "
|
| 105 |
+
"read as though it had. `tests/test_adapters.py` asserts the gap so it "
|
| 106 |
+
"cannot be quietly re-closed in prose.\n\n"
|
| 107 |
+
"**No cattle rate is validated** β all three real cattle clips are "
|
| 108 |
+
"refused, two for being shorter than the capture protocol and one for "
|
| 109 |
+
"having no clear rhythm. What is missing is not model work: it is a "
|
| 110 |
+
"thirty-second clip of a cow's flank with somebody's counted breath "
|
| 111 |
+
"rate beside it."
|
| 112 |
+
),
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
GEOMETRY_SPEC = AdapterSpec(
|
| 116 |
+
adapter_id="marker-geometry",
|
| 117 |
+
runtime="opencv-numpy",
|
| 118 |
+
tasks=(Task.MEASURE,),
|
| 119 |
+
modalities=(Modality.IMAGE,),
|
| 120 |
+
directive_role=(
|
| 121 |
+
"Β§4 geometry, contour measurement and reference-marker calibration; "
|
| 122 |
+
"Β§9's 'approximate visible area: 12β16 cmΒ²' for a wound, and Β§22's "
|
| 123 |
+
"fallback scale when metric depth is unreliable."
|
| 124 |
+
),
|
| 125 |
+
requires_artefact=False,
|
| 126 |
+
placement=Placement.ON_DEVICE,
|
| 127 |
+
placement_reason=(
|
| 128 |
+
"Marker detection and a contour area are microseconds of arithmetic on "
|
| 129 |
+
"a phone. Running it on the device means the farmer learns the card was "
|
| 130 |
+
"not in shot while still standing next to the animal, which is the "
|
| 131 |
+
"difference between a re-capture and a lost record."
|
| 132 |
+
),
|
| 133 |
+
notes=(
|
| 134 |
+
"**Unmeasured, and unexercised on a real photograph.** No image "
|
| 135 |
+
"available to this project contains an Animap reference marker, so the "
|
| 136 |
+
"marker-detection half has never run on anything real. The arithmetic "
|
| 137 |
+
"either side of it is exercised by unit tests. Do not quote an area "
|
| 138 |
+
"from this until somebody has photographed a printed card beside a "
|
| 139 |
+
"ruler."
|
| 140 |
+
),
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
class DeterministicAdapter(Adapter):
|
| 145 |
+
"""Signal processing and geometry. Always available, never guessing."""
|
| 146 |
+
|
| 147 |
+
def __init__(self, spec: AdapterSpec) -> None:
|
| 148 |
+
self.spec = spec
|
| 149 |
+
|
| 150 |
+
def availability(self) -> Availability:
|
| 151 |
+
# OpenCV and NumPy are production dependencies, so there is genuinely
|
| 152 |
+
# nothing to check. Importing cv2 here to prove it would make a health
|
| 153 |
+
# probe pay for a 60 MB import.
|
| 154 |
+
return Availability(True)
|
| 155 |
+
|
| 156 |
+
def load(self) -> "DeterministicAdapter":
|
| 157 |
+
return self
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
class RespirationAdapter(DeterministicAdapter):
|
| 161 |
+
"""Β§14, end to end."""
|
| 162 |
+
|
| 163 |
+
def __init__(self) -> None:
|
| 164 |
+
super().__init__(RESPIRATION_SPEC)
|
| 165 |
+
|
| 166 |
+
def measure(
|
| 167 |
+
self,
|
| 168 |
+
video_path: Path | str,
|
| 169 |
+
*,
|
| 170 |
+
region: tuple[float, float, float, float] | None = None,
|
| 171 |
+
) -> RespirationResult:
|
| 172 |
+
return respiratory_rate(video_path, region=region)
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
class GeometryAdapter(DeterministicAdapter):
|
| 176 |
+
"""Β§9 and Β§4, once something in the frame has a known size."""
|
| 177 |
+
|
| 178 |
+
def __init__(self) -> None:
|
| 179 |
+
super().__init__(GEOMETRY_SPEC)
|
| 180 |
+
|
| 181 |
+
def scale(self, image, marker_side_mm: float) -> Scale:
|
| 182 |
+
"""Pixels per millimetre from a printed marker.
|
| 183 |
+
|
| 184 |
+
Propagates `NoReference` rather than returning a default. A frame with
|
| 185 |
+
no marker has no scale, and the honest answer is a re-capture prompt.
|
| 186 |
+
"""
|
| 187 |
+
return scale_from_marker(image, marker_side_mm)
|
| 188 |
+
|
| 189 |
+
def region_size(self, mask, scale: Scale) -> dict:
|
| 190 |
+
return measure_region(mask, scale)
|
| 191 |
+
|
| 192 |
+
def try_scale(self, image, marker_side_mm: float) -> Measurement:
|
| 193 |
+
"""The same thing, as a `Measurement` a runner can put in a result."""
|
| 194 |
+
try:
|
| 195 |
+
found = self.scale(image, marker_side_mm)
|
| 196 |
+
except NoReference as absent:
|
| 197 |
+
return Measurement(
|
| 198 |
+
kind="scale", value=None, unit="px/mm", usable=False,
|
| 199 |
+
detail=str(absent),
|
| 200 |
+
)
|
| 201 |
+
return Measurement(
|
| 202 |
+
kind="scale", value=round(found.pixels_per_mm, 4), unit="px/mm",
|
| 203 |
+
usable=True,
|
| 204 |
+
support={"relative_error": round(found.relative_error, 4)},
|
| 205 |
+
detail=found.source,
|
| 206 |
+
)
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def deterministic_adapters() -> list[Adapter]:
|
| 210 |
+
return [RespirationAdapter(), GeometryAdapter()]
|
app/adapters/embedding/__init__.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Frozen visual embeddings, and the identity index built on top of them.
|
| 2 |
+
|
| 3 |
+
Two modules, split along the line Β§6.4 draws:
|
| 4 |
+
|
| 5 |
+
- `backbones` β the ONNX adapters that turn an image into one unit vector, and
|
| 6 |
+
the four specs Β§40.2 asks to be compared. Nothing in it knows what an animal
|
| 7 |
+
is.
|
| 8 |
+
- `identity` β enrolment, matching, and the open-set decision. Nothing in it
|
| 9 |
+
knows what ONNX is.
|
| 10 |
+
|
| 11 |
+
The split is not tidiness. The benchmark that chooses a backbone and the index
|
| 12 |
+
that serves a farm fail in different ways and are audited by different people,
|
| 13 |
+
and keeping the retrieval logic testable without a 837 MB artefact on disk is
|
| 14 |
+
what lets the open-set rules have unit tests at all.
|
| 15 |
+
|
| 16 |
+
Everything `adapters.embedding` exported before the split is re-exported here, so
|
| 17 |
+
`from app.adapters.embedding import DINOV3_SPEC` still resolves.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
from app.adapters.embedding.backbones import (
|
| 23 |
+
DINOV2_SPEC,
|
| 24 |
+
DINOV3_SPEC,
|
| 25 |
+
MEGADESCRIPTOR_SPEC,
|
| 26 |
+
MIEWID_SPEC,
|
| 27 |
+
OnnxEmbeddingAdapter,
|
| 28 |
+
preprocess,
|
| 29 |
+
)
|
| 30 |
+
from app.adapters.embedding.identity import (
|
| 31 |
+
ENROLMENT_VIEWS,
|
| 32 |
+
Candidate,
|
| 33 |
+
Embedding,
|
| 34 |
+
EnrolledView,
|
| 35 |
+
IdentityIndex,
|
| 36 |
+
IdentityResult,
|
| 37 |
+
IndexMismatch,
|
| 38 |
+
OpenSetPolicy,
|
| 39 |
+
UnmeasuredThreshold,
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
__all__ = [
|
| 43 |
+
"DINOV2_SPEC",
|
| 44 |
+
"DINOV3_SPEC",
|
| 45 |
+
"ENROLMENT_VIEWS",
|
| 46 |
+
"MEGADESCRIPTOR_SPEC",
|
| 47 |
+
"MIEWID_SPEC",
|
| 48 |
+
"Candidate",
|
| 49 |
+
"Embedding",
|
| 50 |
+
"EnrolledView",
|
| 51 |
+
"IdentityIndex",
|
| 52 |
+
"IdentityResult",
|
| 53 |
+
"IndexMismatch",
|
| 54 |
+
"OnnxEmbeddingAdapter",
|
| 55 |
+
"OpenSetPolicy",
|
| 56 |
+
"UnmeasuredThreshold",
|
| 57 |
+
"preprocess",
|
| 58 |
+
]
|
app/adapters/embedding/backbones.py
ADDED
|
@@ -0,0 +1,397 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Frozen visual embeddings, through onnxruntime.
|
| 2 |
+
|
| 3 |
+
Directive Β§3 on DINOv3: "Do not assume fine-tuning is required. Start with frozen
|
| 4 |
+
embeddings + nearest-neighbor retrieval." That is what this is β a backbone with
|
| 5 |
+
no head, one vector per image, and every capability that wants it (identity,
|
| 6 |
+
breed, BCS reference, fecal reference, footpad reference) built as retrieval on
|
| 7 |
+
top rather than as a trained classifier.
|
| 8 |
+
|
| 9 |
+
**Why ONNX and not `transformers`.** ADR 0017 took roughly a gigabyte of torch
|
| 10 |
+
out of the serving image and cut start-up from about 28 seconds to under one.
|
| 11 |
+
Reaching for `transformers` at serve time hands all of that back for a model
|
| 12 |
+
whose forward pass is a fixed graph with no control flow. So torch is a
|
| 13 |
+
*build-time* tool: `scripts/export_embedding.py` runs it once on a developer's
|
| 14 |
+
machine, and the service ships an `.onnx` that `onnxruntime` β already a
|
| 15 |
+
production dependency β loads in about a fifth of a second.
|
| 16 |
+
|
| 17 |
+
The export puts the pooling and the L2 normalisation inside the graph, so there
|
| 18 |
+
is no post-processing convention that can drift between whoever exported the
|
| 19 |
+
artefact and whoever serves it. A vector out of this adapter is always unit
|
| 20 |
+
length and cosine similarity is always a dot product.
|
| 21 |
+
|
| 22 |
+
**The artefact is still governed by a card.** This is not a second way to load a
|
| 23 |
+
model β `providers.load_card` checksums it and `adapters.licences.gate` checks
|
| 24 |
+
what the runtime really loads under, which is the check ADR 0017 added after a
|
| 25 |
+
watchdog defeated the card's own licence field.
|
| 26 |
+
"""
|
| 27 |
+
|
| 28 |
+
from __future__ import annotations
|
| 29 |
+
|
| 30 |
+
import threading
|
| 31 |
+
from functools import lru_cache
|
| 32 |
+
|
| 33 |
+
import numpy as np
|
| 34 |
+
from PIL import Image
|
| 35 |
+
|
| 36 |
+
from app.adapters.base import (
|
| 37 |
+
Adapter,
|
| 38 |
+
AdapterError,
|
| 39 |
+
AdapterSpec,
|
| 40 |
+
AdapterUnavailable,
|
| 41 |
+
Availability,
|
| 42 |
+
MeasuredCost,
|
| 43 |
+
Modality,
|
| 44 |
+
Placement,
|
| 45 |
+
Task,
|
| 46 |
+
)
|
| 47 |
+
from app.adapters.licences import LicenceRefused, gate
|
| 48 |
+
from app.providers import ModelArtefact
|
| 49 |
+
|
| 50 |
+
_session_lock = threading.Lock()
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@lru_cache(maxsize=4)
|
| 54 |
+
def _session(artefact_path: str):
|
| 55 |
+
"""One session per artefact. Building one costs about as much as an
|
| 56 |
+
inference, so a request that rebuilds it doubles its own latency."""
|
| 57 |
+
import onnxruntime as ort
|
| 58 |
+
|
| 59 |
+
return ort.InferenceSession(
|
| 60 |
+
artefact_path, providers=["CPUExecutionProvider"]
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def preprocess(
|
| 65 |
+
image: Image.Image,
|
| 66 |
+
size: int,
|
| 67 |
+
mean: tuple[float, float, float],
|
| 68 |
+
std: tuple[float, float, float],
|
| 69 |
+
) -> np.ndarray:
|
| 70 |
+
"""Resize the short side, centre crop, normalise. RGB, NCHW, float32.
|
| 71 |
+
|
| 72 |
+
This is torchvision's standard eval transform written out, for the same
|
| 73 |
+
reason `detectors/yolox_onnx.py` writes out its own NMS: pulling in a
|
| 74 |
+
training framework for one resize is what the ONNX path exists to avoid.
|
| 75 |
+
Getting it wrong does not raise β it quietly returns worse vectors β so the
|
| 76 |
+
numbers come from the card rather than from a constant here.
|
| 77 |
+
"""
|
| 78 |
+
rgb = image.convert("RGB")
|
| 79 |
+
width, height = rgb.size
|
| 80 |
+
scale = size / min(width, height)
|
| 81 |
+
resized = rgb.resize(
|
| 82 |
+
(max(size, round(width * scale)), max(size, round(height * scale))),
|
| 83 |
+
Image.BICUBIC,
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
new_width, new_height = resized.size
|
| 87 |
+
left = (new_width - size) // 2
|
| 88 |
+
top = (new_height - size) // 2
|
| 89 |
+
cropped = resized.crop((left, top, left + size, top + size))
|
| 90 |
+
|
| 91 |
+
array = np.asarray(cropped, dtype=np.float32) / 255.0
|
| 92 |
+
array = (array - np.asarray(mean, dtype=np.float32)) / np.asarray(
|
| 93 |
+
std, dtype=np.float32
|
| 94 |
+
)
|
| 95 |
+
return np.ascontiguousarray(array.transpose(2, 0, 1)[None])
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
class OnnxEmbeddingAdapter(Adapter):
|
| 99 |
+
"""A frozen backbone that turns an image into one unit vector.
|
| 100 |
+
|
| 101 |
+
Constructed from a `ModelArtefact` that `providers.load_card` has already
|
| 102 |
+
checksummed. It reads nothing else and downloads nothing.
|
| 103 |
+
"""
|
| 104 |
+
|
| 105 |
+
def __init__(
|
| 106 |
+
self,
|
| 107 |
+
artefact: ModelArtefact | None,
|
| 108 |
+
spec: AdapterSpec,
|
| 109 |
+
*,
|
| 110 |
+
input_size: int = 224,
|
| 111 |
+
mean: tuple[float, float, float] = (0.485, 0.456, 0.406),
|
| 112 |
+
std: tuple[float, float, float] = (0.229, 0.224, 0.225),
|
| 113 |
+
dimensions: int = 768,
|
| 114 |
+
) -> None:
|
| 115 |
+
self.artefact = artefact
|
| 116 |
+
self.spec = spec
|
| 117 |
+
self.input_size = input_size
|
| 118 |
+
self.mean = mean
|
| 119 |
+
self.std = std
|
| 120 |
+
self.dimensions = dimensions
|
| 121 |
+
self._session = None
|
| 122 |
+
self._input_name = ""
|
| 123 |
+
|
| 124 |
+
def availability(self) -> Availability:
|
| 125 |
+
if self.artefact is None:
|
| 126 |
+
return Availability(
|
| 127 |
+
False,
|
| 128 |
+
f"No artefact is installed for {self.spec.adapter_id}.",
|
| 129 |
+
"Run scripts/install_models.py, which fetches what the "
|
| 130 |
+
"committed card names and refuses anything whose checksum "
|
| 131 |
+
"does not match.",
|
| 132 |
+
)
|
| 133 |
+
try:
|
| 134 |
+
gate(self.spec.runtime, self.artefact.license)
|
| 135 |
+
except LicenceRefused as refusal:
|
| 136 |
+
return Availability(
|
| 137 |
+
False,
|
| 138 |
+
str(refusal),
|
| 139 |
+
"Move the capability to a permissively licensed backbone.",
|
| 140 |
+
)
|
| 141 |
+
if not self.artefact.is_validated:
|
| 142 |
+
return Availability(
|
| 143 |
+
False,
|
| 144 |
+
f"{self.artefact.model_id} has empty validation notes, so "
|
| 145 |
+
f"nothing attests that it works.",
|
| 146 |
+
"Fill in what was tested, on what data, with what result.",
|
| 147 |
+
)
|
| 148 |
+
return Availability(True)
|
| 149 |
+
|
| 150 |
+
def load(self) -> "OnnxEmbeddingAdapter":
|
| 151 |
+
availability = self.availability()
|
| 152 |
+
if not availability.ready:
|
| 153 |
+
raise AdapterUnavailable(availability)
|
| 154 |
+
|
| 155 |
+
assert self.artefact is not None # availability() proved it
|
| 156 |
+
with _session_lock:
|
| 157 |
+
session = _session(str(self.artefact.path))
|
| 158 |
+
|
| 159 |
+
inputs = session.get_inputs()
|
| 160 |
+
if len(inputs) != 1:
|
| 161 |
+
raise AdapterError(
|
| 162 |
+
f"{self.artefact.path.name} takes {len(inputs)} inputs; this "
|
| 163 |
+
f"adapter was written for a single image tensor."
|
| 164 |
+
)
|
| 165 |
+
# Shape-checked rather than trusted, because an export at a different
|
| 166 |
+
# resolution produces vectors that are the right length and the wrong
|
| 167 |
+
# thing, and nothing downstream would notice.
|
| 168 |
+
expected = [3, self.input_size, self.input_size]
|
| 169 |
+
actual = list(inputs[0].shape[1:])
|
| 170 |
+
if actual != expected:
|
| 171 |
+
raise AdapterError(
|
| 172 |
+
f"{self.artefact.path.name} takes {actual}, but the card "
|
| 173 |
+
f"describes a {expected} input. The artefact and its card "
|
| 174 |
+
f"disagree about what was exported."
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
self._session = session
|
| 178 |
+
self._input_name = inputs[0].name
|
| 179 |
+
return self
|
| 180 |
+
|
| 181 |
+
def embed(self, image: Image.Image) -> np.ndarray:
|
| 182 |
+
if self._session is None:
|
| 183 |
+
raise AdapterError(
|
| 184 |
+
"embed() called before load(). There is no path to a vector "
|
| 185 |
+
"that skipped the artefact check, and this is it refusing."
|
| 186 |
+
)
|
| 187 |
+
blob = preprocess(image, self.input_size, self.mean, self.std)
|
| 188 |
+
vector = self._session.run(None, {self._input_name: blob})[0][0]
|
| 189 |
+
|
| 190 |
+
if vector.shape[0] != self.dimensions:
|
| 191 |
+
raise AdapterError(
|
| 192 |
+
f"The graph returned {vector.shape[0]} dimensions; the card "
|
| 193 |
+
f"says {self.dimensions}. An index built at one and queried at "
|
| 194 |
+
f"the other fails silently, so this fails loudly."
|
| 195 |
+
)
|
| 196 |
+
return vector.astype(np.float32)
|
| 197 |
+
|
| 198 |
+
def embed_many(self, images: list[Image.Image]) -> np.ndarray:
|
| 199 |
+
"""One row per image. Kept separate because building a retrieval index
|
| 200 |
+
is the batch case and a request is the single case, and batching a
|
| 201 |
+
request would only add latency."""
|
| 202 |
+
return np.stack([self.embed(image) for image in images])
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
#: Β§3's first choice. Bespoke Meta licence with a live ambiguity about
|
| 206 |
+
#: attribution β see `licences.RUNTIME_LICENCES["dinov3-onnx"]`.
|
| 207 |
+
DINOV3_SPEC = AdapterSpec(
|
| 208 |
+
adapter_id="dinov3-vits16",
|
| 209 |
+
runtime="dinov3-onnx",
|
| 210 |
+
tasks=(Task.EMBED,),
|
| 211 |
+
modalities=(Modality.IMAGE,),
|
| 212 |
+
directive_role=(
|
| 213 |
+
"Β§3 DINOv3 β general visual embeddings, image similarity, retrieval, "
|
| 214 |
+
"cattle identity experiments, breed similarity, BCS and fecal and "
|
| 215 |
+
"footpad reference retrieval. Frozen, with nearest-neighbour on top."
|
| 216 |
+
),
|
| 217 |
+
placement=Placement.CPU_SERVICE,
|
| 218 |
+
placement_reason=(
|
| 219 |
+
"86.6 MB of ONNX, 59 ms a frame and a 470 MB peak single-threaded. It "
|
| 220 |
+
"belongs beside the API, and it is small enough that an on-device "
|
| 221 |
+
"build is worth investigating β ADR 0002 makes Animap offline-first, "
|
| 222 |
+
"and identity is exactly the capability a worker wants in a pen with "
|
| 223 |
+
"no signal."
|
| 224 |
+
),
|
| 225 |
+
measured=MeasuredCost(
|
| 226 |
+
hardware="Apple M-series laptop (NOT the target container)",
|
| 227 |
+
threads=1,
|
| 228 |
+
sample="61 Commons frames, evaluation/dataset.json",
|
| 229 |
+
runs=61,
|
| 230 |
+
median_seconds=0.059,
|
| 231 |
+
peak_rss_mb=470.0,
|
| 232 |
+
measured_on="2026-08-21",
|
| 233 |
+
),
|
| 234 |
+
notes=(
|
| 235 |
+
"**Measured at re-identification, which is what it is registered for.** "
|
| 236 |
+
"On 169 enrolled cattle from the CC BY 4.0 Zenodo 6324361 muzzle "
|
| 237 |
+
"database, five enrolment images each: closed-set top-1 0.977, top-3 "
|
| 238 |
+
"0.994, MRR 0.985, against a 0.0059 chance rate. Best of the two "
|
| 239 |
+
"servable backbones; the two unservable ones were only run on a "
|
| 240 |
+
"30-animal set, where all four saturate. "
|
| 241 |
+
"**And the open-set result is the one that shapes the product**: with no "
|
| 242 |
+
"threshold it names an unenrolled animal 100% of the time, because every "
|
| 243 |
+
"query has a nearest neighbour. The similarity cutoff that admits no "
|
| 244 |
+
"impostor accepts only 24.1% of the correct matches β the two "
|
| 245 |
+
"distributions overlap badly, enrolled probes median 0.971 against "
|
| 246 |
+
"unenrolled median 0.904 with an unenrolled maximum of 0.978. A margin "
|
| 247 |
+
"rule does not rescue it. That is why the confirm step in Β§6.4 is "
|
| 248 |
+
"load-bearing rather than decorative. "
|
| 249 |
+
"No Nigerian and no zebu animal has been through this; the database is "
|
| 250 |
+
"US beef breeds. "
|
| 251 |
+
"Exported at 224 px so the comparison against DINOv2-small is "
|
| 252 |
+
"like-for-like β timm resolves this checkpoint's native config to 256 px, "
|
| 253 |
+
"so these figures understate it slightly. On the older Commons proxy it "
|
| 254 |
+
"measured 1.000 species 1-NN and 1.000 Nigerian-cattle 1-NN against a "
|
| 255 |
+
"0.357 base rate, better than every other backbone and faster than all "
|
| 256 |
+
"but DINOv2."
|
| 257 |
+
),
|
| 258 |
+
)
|
| 259 |
+
|
| 260 |
+
#: The same interface over Apache-2.0 weights of the same size and embedding
|
| 261 |
+
#: width. Not a downgrade chosen for convenience β Β§4 asks for the benchmark,
|
| 262 |
+
#: and `experiments/cattle_identity/` is where the two are compared.
|
| 263 |
+
DINOV2_SPEC = AdapterSpec(
|
| 264 |
+
adapter_id="dinov2-small",
|
| 265 |
+
runtime="dinov2-onnx",
|
| 266 |
+
tasks=(Task.EMBED,),
|
| 267 |
+
modalities=(Modality.IMAGE,),
|
| 268 |
+
directive_role=(
|
| 269 |
+
"Β§3 DINOv3's role, served from the Apache-2.0 generation. 22.06M "
|
| 270 |
+
"parameters against DINOv3 ViT-S/16's 21.60M, and the same 384-wide "
|
| 271 |
+
"patch embedding."
|
| 272 |
+
),
|
| 273 |
+
placement=Placement.CPU_SERVICE,
|
| 274 |
+
placement_reason=(
|
| 275 |
+
"88.4 MB of ONNX, 81 ms a frame, 390 MB peak. The registered artefact, "
|
| 276 |
+
"because it is the one with no licence question attached."
|
| 277 |
+
),
|
| 278 |
+
measured=MeasuredCost(
|
| 279 |
+
hardware="Apple M-series laptop (NOT the target container)",
|
| 280 |
+
threads=1,
|
| 281 |
+
sample="61 Commons frames, evaluation/dataset.json",
|
| 282 |
+
runs=61,
|
| 283 |
+
median_seconds=0.081,
|
| 284 |
+
peak_rss_mb=390.0,
|
| 285 |
+
measured_on="2026-08-21",
|
| 286 |
+
),
|
| 287 |
+
notes=(
|
| 288 |
+
"Re-identification on the same 169 enrolled cattle as DINOv3: closed-set "
|
| 289 |
+
"top-1 0.957 against DINOv3's 0.977, top-3 0.986 against 0.994, MRR "
|
| 290 |
+
"0.971 against 0.985. Its "
|
| 291 |
+
"open-set behaviour is worse in the same shape β 21.3% true accepts at "
|
| 292 |
+
"the 1% false-accept point against DINOv3's 24.1%. "
|
| 293 |
+
"On the Commons proxy, 0.984 species 1-NN and 0.727 Nigerian-cattle 1-NN "
|
| 294 |
+
"against a 0.357 base rate. **Measurably worse than DINOv3 on every "
|
| 295 |
+
"figure taken on both arms**, which is what makes DINOv3's licence "
|
| 296 |
+
"ambiguity worth somebody's time rather than an academic point: the "
|
| 297 |
+
"permissive fallback costs about two points of top-1 on a task where "
|
| 298 |
+
"the errors are somebody's cow."
|
| 299 |
+
),
|
| 300 |
+
)
|
| 301 |
+
|
| 302 |
+
#: Β§40.2's head-to-head, and the reason it can exist at all.
|
| 303 |
+
#:
|
| 304 |
+
#: **This runs and it is not servable, and both halves are deliberate.** The
|
| 305 |
+
#: weights are CC-BY-NC-4.0, which a commercial product cannot satisfy at any
|
| 306 |
+
#: size; what the founder lifted was the rule that a licence like that stops the
|
| 307 |
+
#: model being *measured*. So the artefact is installed, the adapter is built,
|
| 308 |
+
#: and `licences.gate` refuses it under the default `enforce` policy and records
|
| 309 |
+
#: it under `record`. `describe()` reports `servable: False` either way.
|
| 310 |
+
MEGADESCRIPTOR_SPEC = AdapterSpec(
|
| 311 |
+
adapter_id="megadescriptor",
|
| 312 |
+
runtime="megadescriptor-timm",
|
| 313 |
+
tasks=(Task.EMBED,),
|
| 314 |
+
modalities=(Modality.IMAGE,),
|
| 315 |
+
directive_role=(
|
| 316 |
+
"Β§4 and Β§40.2 MegaDescriptor β wildlife re-ID embeddings for cattle "
|
| 317 |
+
"identity, benchmarked head-to-head against DINOv3."
|
| 318 |
+
),
|
| 319 |
+
placement=Placement.CPU_SERVICE,
|
| 320 |
+
placement_reason=(
|
| 321 |
+
"837 MB of ONNX and a measured 1,296 MB peak, which is inside the "
|
| 322 |
+
"2,000 MB the CPU worker is judged against but 3.7x DINOv3's peak on "
|
| 323 |
+
"the same run. The 0.723 s median is also inside the inline ceiling. "
|
| 324 |
+
"It fits; it is simply not worth the room, because it lost the "
|
| 325 |
+
"benchmark it was installed to win."
|
| 326 |
+
),
|
| 327 |
+
measured=MeasuredCost(
|
| 328 |
+
hardware="Apple M-series laptop (NOT the target container)",
|
| 329 |
+
threads=1,
|
| 330 |
+
sample="61 Commons frames, evaluation/dataset.json",
|
| 331 |
+
runs=61,
|
| 332 |
+
median_seconds=0.723,
|
| 333 |
+
peak_rss_mb=1295.8,
|
| 334 |
+
measured_on="2026-08-21",
|
| 335 |
+
),
|
| 336 |
+
notes=(
|
| 337 |
+
"**Β§40.2 answered: DINOv3 wins, and not narrowly.** MegaDescriptor-L-384 "
|
| 338 |
+
"measured 0.934 species 1-NN and 0.636 Nigerian-cattle 1-NN against a "
|
| 339 |
+
"0.357 base rate; DINOv3 measured 1.000 and 1.000 on the same 61 frames "
|
| 340 |
+
"in the same run. It is beaten by Apache-2.0 DINOv2-small on both "
|
| 341 |
+
"accuracy figures as well, at roughly 10x the artefact size and 7x the "
|
| 342 |
+
"latency. Its nearest-neighbour cosines are much flatter β 0.257 median "
|
| 343 |
+
"against DINOv3's 0.672. "
|
| 344 |
+
"**None of that measures re-identification**, which is what "
|
| 345 |
+
"MegaDescriptor is for: no available image set has the same animal "
|
| 346 |
+
"twice, so this says the space is worse *organised* for cattle and "
|
| 347 |
+
"geography, not that it cannot tell two White Fulani apart. "
|
| 348 |
+
"L-384 was chosen over the smaller variants because Β§40.2 names it and "
|
| 349 |
+
"because 837 MB of ONNX exports cleanly under the 2 GB protobuf limit; "
|
| 350 |
+
"T-224, S-224, B-224 and L-224 publish checkpoints of 204, 290, 473 and "
|
| 351 |
+
"1,922 MB and none was exported."
|
| 352 |
+
),
|
| 353 |
+
)
|
| 354 |
+
|
| 355 |
+
#: The other non-commercial contender, installed for the same reason and to no
|
| 356 |
+
#: better end. Its licence problem is quieter than MegaDescriptor's: nothing was
|
| 357 |
+
#: granted at all, and silence defaults to all rights reserved.
|
| 358 |
+
MIEWID_SPEC = AdapterSpec(
|
| 359 |
+
adapter_id="miewid-msv3",
|
| 360 |
+
runtime="miewid",
|
| 361 |
+
tasks=(Task.EMBED,),
|
| 362 |
+
modalities=(Modality.IMAGE,),
|
| 363 |
+
directive_role=(
|
| 364 |
+
"Β§4's 'Wildlife ReID embeddings' β the alternative to MegaDescriptor, "
|
| 365 |
+
"benchmarked alongside it under Β§40.2."
|
| 366 |
+
),
|
| 367 |
+
placement=Placement.CPU_SERVICE,
|
| 368 |
+
placement_reason=(
|
| 369 |
+
"206 MB of ONNX, 0.213 s a frame, 502 MB peak. Comfortably the cheapest "
|
| 370 |
+
"of the two wildlife re-ID models and still 2.9x DINOv3's latency for "
|
| 371 |
+
"the worst Nigerian retrieval of the four."
|
| 372 |
+
),
|
| 373 |
+
measured=MeasuredCost(
|
| 374 |
+
hardware="Apple M-series laptop (NOT the target container)",
|
| 375 |
+
threads=1,
|
| 376 |
+
sample="61 Commons frames, evaluation/dataset.json",
|
| 377 |
+
runs=61,
|
| 378 |
+
median_seconds=0.213,
|
| 379 |
+
peak_rss_mb=502.2,
|
| 380 |
+
measured_on="2026-08-21",
|
| 381 |
+
),
|
| 382 |
+
notes=(
|
| 383 |
+
"Measured 0.951 species 1-NN and 0.455 Nigerian-cattle 1-NN against a "
|
| 384 |
+
"0.357 base rate β a lift of 1.27x on 11 frames, which is a failure to "
|
| 385 |
+
"show anything rather than a measured floor. Last of the four on the "
|
| 386 |
+
"figure that matters most for Nigerian farms. "
|
| 387 |
+
"**Building this artefact meant running a third party's Python.** The "
|
| 388 |
+
"upstream repo ships `modeling_miewid.py` instead of a `transformers` "
|
| 389 |
+
"architecture, so `scripts/export_embedding.py` loads it with "
|
| 390 |
+
"`trust_remote_code=True`. That is a build-step supply-chain exposure, "
|
| 391 |
+
"not a serving one β the service loads a fixed ONNX graph with no "
|
| 392 |
+
"Python in it β and the three modules were read before they were run. "
|
| 393 |
+
"Preprocessing deviates from the published transform: the model card "
|
| 394 |
+
"specifies `Resize((440, 440))` and this pipeline centre-crops, so its "
|
| 395 |
+
"figures here may understate it."
|
| 396 |
+
),
|
| 397 |
+
)
|
app/adapters/embedding/identity.py
ADDED
|
@@ -0,0 +1,657 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Enrolment, matching, and the open-set decision for `cattle_identity`.
|
| 2 |
+
|
| 3 |
+
Directive Β§6.4. An animal is enrolled from five views β front face, left face,
|
| 4 |
+
right face, muzzle, side body β each stored as one unit vector. A new photograph
|
| 5 |
+
is embedded, compared against every enrolled animal, and the farm is shown a
|
| 6 |
+
ranked list with *"This looks like Kofi"* on top and a confirm button under it.
|
| 7 |
+
|
| 8 |
+
**Why there is no model here.** The exemplar is the animal. `poultry_house_count`
|
| 9 |
+
proved the shape on a different capability: CountGD reached MAE 14.84 on frames a
|
| 10 |
+
trained detector found 8.5% of the birds in, because it was shown three example
|
| 11 |
+
boxes instead of being retrained. An enrolled animal is the same trick β five
|
| 12 |
+
photographs of Kofi are what teaches the system Kofi, and nothing is fitted.
|
| 13 |
+
That is also why this module imports no model code and takes vectors rather than
|
| 14 |
+
images: the backbone is chosen by `experiments/cattle_identity/`, and this file
|
| 15 |
+
must keep working when that choice changes.
|
| 16 |
+
|
| 17 |
+
## The two rules that are load-bearing
|
| 18 |
+
|
| 19 |
+
**An index belongs to one backbone.** DINOv3's 768 dimensions and MegaDescriptor's
|
| 20 |
+
1536 are not the only difference between them; two exports of the *same* backbone
|
| 21 |
+
with different pooling produce vectors of identical length and incompatible
|
| 22 |
+
meaning, which the model cards in `models/alternates/` each warn about
|
| 23 |
+
separately. A silent mismatch does not error β it returns confident nonsense,
|
| 24 |
+
which is the failure this project can least afford on the capability every other
|
| 25 |
+
record hangs off.
|
| 26 |
+
|
| 27 |
+
So the index pins the backbone id and the artefact digest it was built under, and
|
| 28 |
+
checks both against any `Embedding` handed to it. **A bare `ndarray` can only be
|
| 29 |
+
checked for width**, which is not provenance, and the count of those is kept on
|
| 30 |
+
`unverified_queries` rather than being waved through silently. This paragraph
|
| 31 |
+
previously claimed the index "refuses a query that arrives from anything else"
|
| 32 |
+
while the code compared only dimensions; a watchdog built an index tagged
|
| 33 |
+
`dinov3-vits16`, scored a foreign vector against it at similarity 1.0, and was
|
| 34 |
+
right to call the sentence false. Prefer `Embedding` at every call site that can
|
| 35 |
+
produce one.
|
| 36 |
+
|
| 37 |
+
**A threshold is measured or it is absent.** Closed-set accuracy β *given that
|
| 38 |
+
this animal is enrolled, is the top candidate right* β is the easy half. The half
|
| 39 |
+
that decides whether the product is safe is open-set: an animal nobody enrolled
|
| 40 |
+
must not come back as Kofi. That boundary is a number, and a number nobody
|
| 41 |
+
measured is a guess with a decimal point on it (Β§37). `OpenSetPolicy.measured`
|
| 42 |
+
is the only constructor that produces thresholds and it demands the run id that
|
| 43 |
+
produced them; `OpenSetPolicy.unmeasured` produces none, still returns the ranked
|
| 44 |
+
candidates Β§6.4 asks for, and marks the result `open_set_verified=False` with a
|
| 45 |
+
warning the caller has to carry. It does not invent a cutoff, and it does not
|
| 46 |
+
suppress the capability either.
|
| 47 |
+
|
| 48 |
+
## What a result is allowed to say
|
| 49 |
+
|
| 50 |
+
`app/capabilities.py` already fixes the vocabulary: allowed claims are
|
| 51 |
+
`identity_candidate` and `no_confident_match`, the forbidden one is
|
| 52 |
+
`identity_without_confirmation`, and the four confirmation options are
|
| 53 |
+
`confirm`, `not_this_animal`, `choose_another_animal`, `register_new_animal`.
|
| 54 |
+
This module emits exactly those and nothing else.
|
| 55 |
+
|
| 56 |
+
**A candidate carries a similarity and never a confidence.** Nothing calibrated
|
| 57 |
+
cosine similarity into a probability that the animal is Kofi, and two White
|
| 58 |
+
Fulani photographed in the same light score high because the light is the same.
|
| 59 |
+
`Candidate.confidence` is therefore `None` until something measures a mapping,
|
| 60 |
+
and `Interpretation.confidence` downstream stays `None` with it.
|
| 61 |
+
|
| 62 |
+
## Why the ranked list is the shape, rather than one answer
|
| 63 |
+
|
| 64 |
+
`evidence_correction.selected_interpretation` β a foreign key at
|
| 65 |
+
`services/api/apps/evidence/models/correction.py` β exists so that a farmer
|
| 66 |
+
picking the second name in the list records *"rank 2 was right"* rather than
|
| 67 |
+
*"the model was wrong"*. Its own docstring calls it the highest-value training
|
| 68 |
+
signal in the table. Returning one name would throw that gradient away at the
|
| 69 |
+
moment it is collected, which is the same mistake Β§32 records having been made
|
| 70 |
+
with confirmations. So `match` returns a list with ranks on it even when the top
|
| 71 |
+
candidate is obvious.
|
| 72 |
+
"""
|
| 73 |
+
|
| 74 |
+
from __future__ import annotations
|
| 75 |
+
|
| 76 |
+
from dataclasses import dataclass, field
|
| 77 |
+
from typing import Iterable, Mapping, Sequence
|
| 78 |
+
|
| 79 |
+
import numpy as np
|
| 80 |
+
|
| 81 |
+
#: Β§6.4's enrolment protocol, in the order the directive lists it. The registry
|
| 82 |
+
#: entry for `cattle_identity` declares the same five as `required_views`, and
|
| 83 |
+
#: `tests/test_identity.py` asserts the two agree β a capture flow and an index
|
| 84 |
+
#: that disagree about what a view is called produce an enrolment that silently
|
| 85 |
+
#: stores nothing under the name the matcher looks for.
|
| 86 |
+
ENROLMENT_VIEWS: tuple[str, ...] = (
|
| 87 |
+
"front_face",
|
| 88 |
+
"left_face",
|
| 89 |
+
"right_face",
|
| 90 |
+
"muzzle",
|
| 91 |
+
"side_body",
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
#: The view Β§6.4 says to lead with, and the one the identity literature is about.
|
| 95 |
+
#: Coat pattern is not an option for Nigerian herds β a White Fulani is white all
|
| 96 |
+
#: over, so the Holstein re-identification results that dominate the published
|
| 97 |
+
#: work transfer to the method and not to the signal.
|
| 98 |
+
PRIMARY_VIEW = "muzzle"
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class IndexMismatch(ValueError):
|
| 102 |
+
"""A query vector that did not come from the backbone this index was built on."""
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
class UnmeasuredThreshold(ValueError):
|
| 106 |
+
"""A threshold was asserted without the measurement that produced it."""
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def _unit(vector: np.ndarray) -> np.ndarray:
|
| 110 |
+
"""Length-check and normalise.
|
| 111 |
+
|
| 112 |
+
The ONNX exports normalise inside the graph, so this is usually a no-op β and
|
| 113 |
+
it runs anyway, because an index is also built in tests and in notebooks from
|
| 114 |
+
vectors that did not come through the graph, and cosine similarity computed
|
| 115 |
+
as a dot product over a non-unit vector is wrong without being an error.
|
| 116 |
+
"""
|
| 117 |
+
array = np.asarray(vector, dtype=np.float32).reshape(-1)
|
| 118 |
+
norm = float(np.linalg.norm(array))
|
| 119 |
+
if norm == 0.0:
|
| 120 |
+
raise ValueError(
|
| 121 |
+
"A zero vector has no direction, so it is nearest to everything and "
|
| 122 |
+
"to nothing. Something upstream returned an empty embedding."
|
| 123 |
+
)
|
| 124 |
+
return array / norm
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
@dataclass(frozen=True)
|
| 128 |
+
class Embedding:
|
| 129 |
+
"""A vector that says where it came from.
|
| 130 |
+
|
| 131 |
+
**This exists because the docstring above used to be false.** It claimed the
|
| 132 |
+
index "refuses a query that arrives from anything else", and the code checked
|
| 133 |
+
only the *width* β so a vector from a different pooling of the same backbone,
|
| 134 |
+
which is the exact case the model cards warn about and which produces the
|
| 135 |
+
right number of dimensions and the wrong meaning, sailed through. A watchdog
|
| 136 |
+
built an index tagged `dinov3-vits16` and scored a foreign vector against it
|
| 137 |
+
at similarity 1.0.
|
| 138 |
+
|
| 139 |
+
Passing one of these makes the check real. A bare `ndarray` is still
|
| 140 |
+
accepted, because tests and notebooks legitimately have vectors with no
|
| 141 |
+
provenance, and `IdentityIndex.unverified_queries` counts how many arrived
|
| 142 |
+
that way so the gap is visible rather than assumed away.
|
| 143 |
+
"""
|
| 144 |
+
|
| 145 |
+
vector: np.ndarray
|
| 146 |
+
#: The adapter that produced it, e.g. `dinov3-vits16`.
|
| 147 |
+
backbone_id: str
|
| 148 |
+
#: The sha256 of the ONNX artefact. Empty means the producer did not say,
|
| 149 |
+
#: which is checked as far as it can be and no further.
|
| 150 |
+
artefact_sha256: str = ""
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
#: What `enrol` and `candidates` accept: a vector, or a vector that can prove
|
| 154 |
+
#: where it came from.
|
| 155 |
+
Vector = "np.ndarray | Embedding"
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
@dataclass(frozen=True)
|
| 159 |
+
class EnrolledView:
|
| 160 |
+
"""One photograph of one animal, as a unit vector.
|
| 161 |
+
|
| 162 |
+
The image itself is not held. Β§33 requires the raw media to be preserved so a
|
| 163 |
+
later correction can be trained on, and that is the API's job β `MediaAsset`
|
| 164 |
+
with a retention hold written by the correction. An index that also kept the
|
| 165 |
+
bytes would be a second copy nobody was pinning.
|
| 166 |
+
"""
|
| 167 |
+
|
| 168 |
+
animal_id: str
|
| 169 |
+
view: str
|
| 170 |
+
vector: np.ndarray
|
| 171 |
+
#: The media id the vector came from, so a candidate can be traced back to a
|
| 172 |
+
#: photograph a person can look at. Β§6.4's *"Not Kofi"* is far more useful
|
| 173 |
+
#: when the farm can see which picture of Kofi the system thought matched.
|
| 174 |
+
media_id: str = ""
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
@dataclass(frozen=True)
|
| 178 |
+
class OpenSetPolicy:
|
| 179 |
+
"""When a top candidate is good enough to show as a name.
|
| 180 |
+
|
| 181 |
+
Two thresholds rather than one, because they fail differently.
|
| 182 |
+
`accept_similarity` catches the animal nobody enrolled: every candidate is
|
| 183 |
+
poor, and the best of them is still poor. `accept_margin` catches the case
|
| 184 |
+
that matters more on a real farm β a herd of animals that genuinely look
|
| 185 |
+
alike, where the top two candidates are both strong and the ordering between
|
| 186 |
+
them is noise. A margin rule refuses to pick a name out of a tie, which is
|
| 187 |
+
exactly the situation *"This looks like Kofi"* is most likely to be wrong in
|
| 188 |
+
and most likely to be believed in.
|
| 189 |
+
"""
|
| 190 |
+
|
| 191 |
+
#: `None` when nothing has been measured. Not a permissive default and not a
|
| 192 |
+
#: strict one; the absence is carried into the result instead.
|
| 193 |
+
accept_similarity: float | None
|
| 194 |
+
accept_margin: float | None
|
| 195 |
+
#: Where the numbers came from. Empty only for `unmeasured`.
|
| 196 |
+
measured_by_run_id: str = ""
|
| 197 |
+
measured_on: str = ""
|
| 198 |
+
#: Why there is no threshold. Empty only for `measured`.
|
| 199 |
+
unmeasured_reason: str = ""
|
| 200 |
+
|
| 201 |
+
def __post_init__(self) -> None:
|
| 202 |
+
has_thresholds = (
|
| 203 |
+
self.accept_similarity is not None or self.accept_margin is not None
|
| 204 |
+
)
|
| 205 |
+
if has_thresholds and not self.measured_by_run_id:
|
| 206 |
+
raise UnmeasuredThreshold(
|
| 207 |
+
"A threshold decides whether a farm is shown an animal's name, "
|
| 208 |
+
"so it names the run that measured it. Use "
|
| 209 |
+
"OpenSetPolicy.unmeasured() to say honestly that nothing has."
|
| 210 |
+
)
|
| 211 |
+
if not has_thresholds and not self.unmeasured_reason:
|
| 212 |
+
raise UnmeasuredThreshold(
|
| 213 |
+
"A policy with no thresholds has to say why it has none."
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
@classmethod
|
| 217 |
+
def measured(cls, *, accept_similarity: float, accept_margin: float,
|
| 218 |
+
run_id: str, measured_on: str) -> "OpenSetPolicy":
|
| 219 |
+
"""Thresholds from a benchmark, citing it.
|
| 220 |
+
|
| 221 |
+
`run_id` is a `results/run-*.json` in `experiments/cattle_identity/`, and
|
| 222 |
+
`measured_on` names the image set. Both travel onto every result the
|
| 223 |
+
policy decides, so a threshold that came from the wrong species or the
|
| 224 |
+
wrong farm is visible in the record rather than only in somebody's memory.
|
| 225 |
+
"""
|
| 226 |
+
if not run_id:
|
| 227 |
+
raise UnmeasuredThreshold("A measured policy cites the run that measured it.")
|
| 228 |
+
return cls(
|
| 229 |
+
accept_similarity=float(accept_similarity),
|
| 230 |
+
accept_margin=float(accept_margin),
|
| 231 |
+
measured_by_run_id=run_id,
|
| 232 |
+
measured_on=measured_on,
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
+
@classmethod
|
| 236 |
+
def unmeasured(cls, reason: str) -> "OpenSetPolicy":
|
| 237 |
+
"""No thresholds, and the reason recorded.
|
| 238 |
+
|
| 239 |
+
This is what ships until an open-set benchmark exists. It does **not**
|
| 240 |
+
disable matching β Β§6.4 says to build immediately and the ranked list is
|
| 241 |
+
useful with a confirm button under it β it records that the boundary
|
| 242 |
+
between *"this looks like Kofi"* and *"I do not know this animal"* has
|
| 243 |
+
never been measured, so the product must not lean on it.
|
| 244 |
+
"""
|
| 245 |
+
return cls(
|
| 246 |
+
accept_similarity=None, accept_margin=None, unmeasured_reason=reason
|
| 247 |
+
)
|
| 248 |
+
|
| 249 |
+
@property
|
| 250 |
+
def is_measured(self) -> bool:
|
| 251 |
+
return self.accept_similarity is not None or self.accept_margin is not None
|
| 252 |
+
|
| 253 |
+
def to_json(self) -> dict:
|
| 254 |
+
return {
|
| 255 |
+
"accept_similarity": self.accept_similarity,
|
| 256 |
+
"accept_margin": self.accept_margin,
|
| 257 |
+
"measured_by_run_id": self.measured_by_run_id,
|
| 258 |
+
"measured_on": self.measured_on,
|
| 259 |
+
"unmeasured_reason": self.unmeasured_reason,
|
| 260 |
+
"is_measured": self.is_measured,
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
#: The default until an open-set benchmark exists. Named rather than constructed
|
| 265 |
+
#: at each call site so there is one place to change when one does, and so a grep
|
| 266 |
+
#: for it finds every caller that is still running without a measured boundary.
|
| 267 |
+
NO_MEASURED_THRESHOLD = OpenSetPolicy.unmeasured(
|
| 268 |
+
"No open-set benchmark has been run for cattle identity. Nothing has "
|
| 269 |
+
"measured how similar an unenrolled animal looks to the nearest enrolled "
|
| 270 |
+
"one, so there is no cutoff to apply and every candidate needs a person."
|
| 271 |
+
)
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
@dataclass(frozen=True)
|
| 275 |
+
class Candidate:
|
| 276 |
+
"""One enrolled animal, ranked, with the view that matched it.
|
| 277 |
+
|
| 278 |
+
`similarity` is a cosine in [-1, 1] and `confidence` is `None`. The two are
|
| 279 |
+
separate fields rather than one so that nothing downstream can quietly
|
| 280 |
+
promote the first into the second β Β§37's whole complaint is that an
|
| 281 |
+
uncalibrated model score reaches a person as a promise.
|
| 282 |
+
"""
|
| 283 |
+
|
| 284 |
+
rank: int
|
| 285 |
+
animal_id: str
|
| 286 |
+
#: The name a screen shows. Supplied by the caller from the farm's own
|
| 287 |
+
#: records, never derived from `animal_id`, for the same reason
|
| 288 |
+
#: `saveConfirmation` takes `displayText` separately from
|
| 289 |
+
#: `confirmationOption`: rendering a key as copy is how an id reaches a
|
| 290 |
+
#: person as a word.
|
| 291 |
+
display_name: str
|
| 292 |
+
similarity: float
|
| 293 |
+
#: Which of the five enrolled views scored highest, and the photograph it
|
| 294 |
+
#: came from.
|
| 295 |
+
matched_view: str
|
| 296 |
+
matched_media_id: str = ""
|
| 297 |
+
#: Never set by this module. Present because `Interpretation.confidence`
|
| 298 |
+
#: exists and something may one day calibrate it against confirmations.
|
| 299 |
+
confidence: float | None = None
|
| 300 |
+
|
| 301 |
+
def to_json(self) -> dict:
|
| 302 |
+
return {
|
| 303 |
+
"rank": self.rank,
|
| 304 |
+
"animal_id": self.animal_id,
|
| 305 |
+
"display_name": self.display_name,
|
| 306 |
+
"similarity": round(self.similarity, 4),
|
| 307 |
+
"matched_view": self.matched_view,
|
| 308 |
+
"matched_media_id": self.matched_media_id,
|
| 309 |
+
"confidence": self.confidence,
|
| 310 |
+
}
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
@dataclass(frozen=True)
|
| 314 |
+
class IdentityResult:
|
| 315 |
+
"""What Β§6.4 shows a farm, and what Β§32 stores when they answer.
|
| 316 |
+
|
| 317 |
+
`claim` is one of the two `app/capabilities.py` allows for this capability.
|
| 318 |
+
`confirmation_options` is the registry's list verbatim, because the option a
|
| 319 |
+
person taps is written into `evidence_correction.confirmation_option` and
|
| 320 |
+
compared against the registry there; a list assembled independently here
|
| 321 |
+
would drift.
|
| 322 |
+
"""
|
| 323 |
+
|
| 324 |
+
claim: str
|
| 325 |
+
candidates: tuple[Candidate, ...]
|
| 326 |
+
policy: OpenSetPolicy
|
| 327 |
+
enrolled_animals: int
|
| 328 |
+
#: True only when a measured policy accepted the top candidate. False both
|
| 329 |
+
#: when a measured policy rejected it and when no policy has been measured β
|
| 330 |
+
#: which are different states, and `warnings` says which.
|
| 331 |
+
#:
|
| 332 |
+
#: **Both rejection branches returned `True` until this was corrected**, so
|
| 333 |
+
#: a refusal reached the API as `open_set_verified 1.0` beside
|
| 334 |
+
#: `no_confident_match` and a similarity under the cutoff β a stored row
|
| 335 |
+
#: saying, of one run, both that nothing was verified and that something
|
| 336 |
+
#: was. Whoever re-derives the threshold from stored results reads this
|
| 337 |
+
#: column, and it is the one that has to mean a verdict.
|
| 338 |
+
#:
|
| 339 |
+
#: *Was the check performed* is a different question and has its own answer:
|
| 340 |
+
#: `policy.is_measured`, which travels in the same result.
|
| 341 |
+
open_set_verified: bool
|
| 342 |
+
warnings: tuple[str, ...] = ()
|
| 343 |
+
#: Β§6.4's four buttons. Order matters: it is the order the directive lists.
|
| 344 |
+
confirmation_options: tuple[str, ...] = (
|
| 345 |
+
"confirm", "not_this_animal", "choose_another_animal", "register_new_animal",
|
| 346 |
+
)
|
| 347 |
+
|
| 348 |
+
@property
|
| 349 |
+
def top(self) -> Candidate | None:
|
| 350 |
+
return self.candidates[0] if self.candidates else None
|
| 351 |
+
|
| 352 |
+
def to_json(self) -> dict:
|
| 353 |
+
return {
|
| 354 |
+
"claim": self.claim,
|
| 355 |
+
"candidates": [c.to_json() for c in self.candidates],
|
| 356 |
+
"open_set_verified": self.open_set_verified,
|
| 357 |
+
"enrolled_animals": self.enrolled_animals,
|
| 358 |
+
"policy": self.policy.to_json(),
|
| 359 |
+
"warnings": list(self.warnings),
|
| 360 |
+
"confirmation_options": list(self.confirmation_options),
|
| 361 |
+
# Stated on the result rather than left to a UI, because the one
|
| 362 |
+
# claim Β§6.4 forbids is an identity asserted without a person, and
|
| 363 |
+
# the surface that would forget is the one furthest from this file.
|
| 364 |
+
"requires_confirmation": True,
|
| 365 |
+
}
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
@dataclass
|
| 369 |
+
class IdentityIndex:
|
| 370 |
+
"""Every enrolled animal on one farm, as vectors.
|
| 371 |
+
|
| 372 |
+
**`farm_id` is a label, not a mechanism, and saying otherwise was an
|
| 373 |
+
overstatement a watchdog caught.** Nothing in `enrol` or `candidates` reads
|
| 374 |
+
it. What actually keeps one farm's animals away from another's is that the
|
| 375 |
+
caller builds one index per farm and never enrols across them β a discipline
|
| 376 |
+
this class records but does not enforce. The field is here so that a
|
| 377 |
+
mismatch is *detectable*: an index can say which farm it believes it holds,
|
| 378 |
+
and a caller that cached the wrong one can be caught by comparing.
|
| 379 |
+
|
| 380 |
+
It matters because every other table in this product is `FarmScopedModel`,
|
| 381 |
+
and an index holding two farms' animals would answer *"which animal is
|
| 382 |
+
this"* with a neighbour's cow, which is both wrong and a disclosure. If this
|
| 383 |
+
ever moves behind an API that takes a farm id from a request, the check
|
| 384 |
+
belongs there and not here.
|
| 385 |
+
|
| 386 |
+
Small by design: a farm has tens to low hundreds of animals and five views
|
| 387 |
+
each, so a brute-force matrix multiply over a few hundred rows is
|
| 388 |
+
microseconds and an approximate-nearest-neighbour structure would add a
|
| 389 |
+
dependency, an index-build step and a recall question in exchange for
|
| 390 |
+
nothing. The moment that stops being true is when one farm passes a few
|
| 391 |
+
thousand animals, and `match` is where it would change.
|
| 392 |
+
"""
|
| 393 |
+
|
| 394 |
+
farm_id: str
|
| 395 |
+
#: The backbone that produced every vector in here, and the artefact digest
|
| 396 |
+
#: it was produced by. Both are checked against any `Embedding` that arrives;
|
| 397 |
+
#: a bare `ndarray` can only be checked for width, and each one that arrives
|
| 398 |
+
#: increments `unverified_queries`.
|
| 399 |
+
backbone_id: str
|
| 400 |
+
dimensions: int
|
| 401 |
+
artefact_sha256: str = ""
|
| 402 |
+
views: list[EnrolledView] = field(default_factory=list)
|
| 403 |
+
#: How many vectors were accepted without being able to prove their origin.
|
| 404 |
+
#: Not an error and not zero in practice β it is the size of the gap between
|
| 405 |
+
#: what this class checks and what it would like to.
|
| 406 |
+
unverified_queries: int = 0
|
| 407 |
+
|
| 408 |
+
def _accept(self, vector: "np.ndarray | Embedding") -> np.ndarray:
|
| 409 |
+
"""Check where a vector came from, then normalise it.
|
| 410 |
+
|
| 411 |
+
Width alone is not provenance. Two exports of the same backbone with
|
| 412 |
+
different pooling produce vectors of identical length and incompatible
|
| 413 |
+
meaning β every model card in `models/alternates/` warns about it
|
| 414 |
+
separately β and the comparison would succeed rather than fail, which on
|
| 415 |
+
this capability means a confident wrong name.
|
| 416 |
+
"""
|
| 417 |
+
if isinstance(vector, Embedding):
|
| 418 |
+
if vector.backbone_id != self.backbone_id:
|
| 419 |
+
raise IndexMismatch(
|
| 420 |
+
f"This index was built on {self.backbone_id!r} and the "
|
| 421 |
+
f"vector came from {vector.backbone_id!r}. Two backbones' "
|
| 422 |
+
f"vectors are not comparable, and comparing them succeeds "
|
| 423 |
+
f"rather than fails."
|
| 424 |
+
)
|
| 425 |
+
if (
|
| 426 |
+
vector.artefact_sha256 and self.artefact_sha256
|
| 427 |
+
and vector.artefact_sha256 != self.artefact_sha256
|
| 428 |
+
):
|
| 429 |
+
raise IndexMismatch(
|
| 430 |
+
f"Same backbone name, different artefact: the index was "
|
| 431 |
+
f"built under {self.artefact_sha256[:12]}β¦ and the query "
|
| 432 |
+
f"came from {vector.artefact_sha256[:12]}β¦. A re-export "
|
| 433 |
+
f"with different pooling is the case this catches."
|
| 434 |
+
)
|
| 435 |
+
array = vector.vector
|
| 436 |
+
else:
|
| 437 |
+
# Counted rather than refused. A test or a notebook has a bare array
|
| 438 |
+
# and no way to prove anything about it, and refusing would make the
|
| 439 |
+
# provenance check the reason nobody uses the class.
|
| 440 |
+
self.unverified_queries += 1
|
| 441 |
+
array = vector
|
| 442 |
+
|
| 443 |
+
unit = _unit(array)
|
| 444 |
+
if unit.shape[0] != self.dimensions:
|
| 445 |
+
raise IndexMismatch(
|
| 446 |
+
f"The vector is {unit.shape[0]}-dimensional and this index is "
|
| 447 |
+
f"{self.dimensions}. It was built on {self.backbone_id!r}."
|
| 448 |
+
)
|
| 449 |
+
return unit
|
| 450 |
+
|
| 451 |
+
# ---- enrolment ----------------------------------------------------------
|
| 452 |
+
|
| 453 |
+
def enrol(self, animal_id: str, vectors: Mapping[str, np.ndarray],
|
| 454 |
+
media_ids: Mapping[str, str] | None = None) -> tuple[str, ...]:
|
| 455 |
+
"""Store one animal's views. Returns the Β§6.4 views still missing.
|
| 456 |
+
|
| 457 |
+
Unknown view names are refused rather than stored: a typo'd `"muzzel"`
|
| 458 |
+
would enrol cleanly, never be queried by a muzzle capture, and leave a
|
| 459 |
+
farm wondering why one animal never matches.
|
| 460 |
+
|
| 461 |
+
A partial enrolment is allowed and reported. Requiring all five would
|
| 462 |
+
mean a worker who cannot get a cow to hold still for a front-face shot
|
| 463 |
+
enrols nothing at all, and one good muzzle photograph is worth more than
|
| 464 |
+
a refused enrolment β but the caller is told what is missing so a capture
|
| 465 |
+
flow can ask for the rest later.
|
| 466 |
+
"""
|
| 467 |
+
unknown = sorted(set(vectors) - set(ENROLMENT_VIEWS))
|
| 468 |
+
if unknown:
|
| 469 |
+
raise ValueError(
|
| 470 |
+
f"{unknown} are not enrolment views. Β§6.4 names exactly "
|
| 471 |
+
f"{list(ENROLMENT_VIEWS)}, and a view stored under any other "
|
| 472 |
+
f"name is a view nothing will ever query."
|
| 473 |
+
)
|
| 474 |
+
media_ids = media_ids or {}
|
| 475 |
+
for view, vector in vectors.items():
|
| 476 |
+
try:
|
| 477 |
+
unit = self._accept(vector)
|
| 478 |
+
except IndexMismatch as mismatch:
|
| 479 |
+
raise IndexMismatch(
|
| 480 |
+
f"{view} of {animal_id}: {mismatch} Mixing backbones in one "
|
| 481 |
+
f"index produces matches that are arithmetic rather than "
|
| 482 |
+
f"evidence."
|
| 483 |
+
) from mismatch
|
| 484 |
+
self.views.append(
|
| 485 |
+
EnrolledView(
|
| 486 |
+
animal_id=animal_id, view=view, vector=unit,
|
| 487 |
+
media_id=media_ids.get(view, ""),
|
| 488 |
+
)
|
| 489 |
+
)
|
| 490 |
+
return tuple(v for v in ENROLMENT_VIEWS if v not in vectors)
|
| 491 |
+
|
| 492 |
+
@property
|
| 493 |
+
def animal_ids(self) -> tuple[str, ...]:
|
| 494 |
+
"""Enrolled animals, in enrolment order and without repeats."""
|
| 495 |
+
seen: dict[str, None] = {}
|
| 496 |
+
for view in self.views:
|
| 497 |
+
seen.setdefault(view.animal_id, None)
|
| 498 |
+
return tuple(seen)
|
| 499 |
+
|
| 500 |
+
def views_for(self, animal_id: str) -> tuple[str, ...]:
|
| 501 |
+
return tuple(v.view for v in self.views if v.animal_id == animal_id)
|
| 502 |
+
|
| 503 |
+
def missing_views(self, animal_id: str) -> tuple[str, ...]:
|
| 504 |
+
held = set(self.views_for(animal_id))
|
| 505 |
+
return tuple(v for v in ENROLMENT_VIEWS if v not in held)
|
| 506 |
+
|
| 507 |
+
# ---- matching -----------------------------------------------------------
|
| 508 |
+
|
| 509 |
+
def candidates(self, query: np.ndarray, *, top_k: int = 3,
|
| 510 |
+
restrict_to_views: Sequence[str] | None = None,
|
| 511 |
+
names: Mapping[str, str] | None = None,
|
| 512 |
+
exclude: Iterable[str] = ()) -> tuple[Candidate, ...]:
|
| 513 |
+
"""The ranked animals, best first.
|
| 514 |
+
|
| 515 |
+
An animal scores its **best** view, not its average. The query is one
|
| 516 |
+
photograph of one part of an animal, and averaging a muzzle close-up
|
| 517 |
+
against a side-body shot dilutes the view that actually matched with four
|
| 518 |
+
that could not have. Maximum over views is the standard multi-shot
|
| 519 |
+
retrieval rule and it is the right one here for a reason specific to
|
| 520 |
+
Β§6.4: the five enrolment views are deliberately *different pictures*, not
|
| 521 |
+
five samples of one distribution.
|
| 522 |
+
|
| 523 |
+
`restrict_to_views` is how a muzzle capture asks to be compared against
|
| 524 |
+
muzzles only. Worth using when the capture flow knows what it took β
|
| 525 |
+
comparing a muzzle print against a side-body vector contributes nothing
|
| 526 |
+
but a chance of a spurious high score.
|
| 527 |
+
|
| 528 |
+
`exclude` drops named animals, which is what makes leave-one-out
|
| 529 |
+
evaluation possible without building a second index per query.
|
| 530 |
+
"""
|
| 531 |
+
names = names or {}
|
| 532 |
+
excluded = set(exclude)
|
| 533 |
+
wanted = set(restrict_to_views) if restrict_to_views else None
|
| 534 |
+
|
| 535 |
+
unit = self._accept(query)
|
| 536 |
+
|
| 537 |
+
pool = [
|
| 538 |
+
v for v in self.views
|
| 539 |
+
if v.animal_id not in excluded
|
| 540 |
+
and (wanted is None or v.view in wanted)
|
| 541 |
+
]
|
| 542 |
+
if not pool:
|
| 543 |
+
return ()
|
| 544 |
+
|
| 545 |
+
scores = np.stack([v.vector for v in pool]) @ unit
|
| 546 |
+
|
| 547 |
+
best: dict[str, tuple[float, EnrolledView]] = {}
|
| 548 |
+
for view, score in zip(pool, scores):
|
| 549 |
+
current = best.get(view.animal_id)
|
| 550 |
+
if current is None or score > current[0]:
|
| 551 |
+
best[view.animal_id] = (float(score), view)
|
| 552 |
+
|
| 553 |
+
ordered = sorted(
|
| 554 |
+
best.items(),
|
| 555 |
+
# Ties broken by animal id rather than left to dict order, so the
|
| 556 |
+
# same index and the same query always produce the same list. A
|
| 557 |
+
# ranked list that reorders between runs makes `selected_
|
| 558 |
+
# interpretation` mean two different things on two devices.
|
| 559 |
+
key=lambda item: (-item[1][0], item[0]),
|
| 560 |
+
)[:top_k]
|
| 561 |
+
|
| 562 |
+
return tuple(
|
| 563 |
+
Candidate(
|
| 564 |
+
rank=position,
|
| 565 |
+
animal_id=animal_id,
|
| 566 |
+
display_name=names.get(animal_id, animal_id),
|
| 567 |
+
similarity=score,
|
| 568 |
+
matched_view=view.view,
|
| 569 |
+
matched_media_id=view.media_id,
|
| 570 |
+
)
|
| 571 |
+
for position, (animal_id, (score, view)) in enumerate(ordered, start=1)
|
| 572 |
+
)
|
| 573 |
+
|
| 574 |
+
def match(self, query: np.ndarray, *, policy: OpenSetPolicy = NO_MEASURED_THRESHOLD,
|
| 575 |
+
top_k: int = 3, restrict_to_views: Sequence[str] | None = None,
|
| 576 |
+
names: Mapping[str, str] | None = None,
|
| 577 |
+
exclude: Iterable[str] = ()) -> IdentityResult:
|
| 578 |
+
"""The Β§6.4 answer: a ranked list, a claim, and four buttons.
|
| 579 |
+
|
| 580 |
+
The claim is `no_confident_match` in three situations, and they are worth
|
| 581 |
+
distinguishing because only one of them is a model result: nothing is
|
| 582 |
+
enrolled yet, a measured policy rejected the top candidate on similarity,
|
| 583 |
+
or a measured policy rejected it on margin. All three are honest answers
|
| 584 |
+
and the third is the one that protects a farm from a confident wrong name.
|
| 585 |
+
"""
|
| 586 |
+
ranked = self.candidates(
|
| 587 |
+
query, top_k=top_k, restrict_to_views=restrict_to_views,
|
| 588 |
+
names=names, exclude=exclude,
|
| 589 |
+
)
|
| 590 |
+
enrolled = len({v.animal_id for v in self.views} - set(exclude))
|
| 591 |
+
warnings: list[str] = []
|
| 592 |
+
|
| 593 |
+
if not ranked:
|
| 594 |
+
return IdentityResult(
|
| 595 |
+
claim="no_confident_match",
|
| 596 |
+
candidates=(), policy=policy, enrolled_animals=enrolled,
|
| 597 |
+
open_set_verified=False,
|
| 598 |
+
warnings=(
|
| 599 |
+
"Nothing to compare against. No animal on this farm has an "
|
| 600 |
+
"enrolled view matching this capture.",
|
| 601 |
+
),
|
| 602 |
+
)
|
| 603 |
+
|
| 604 |
+
if not policy.is_measured:
|
| 605 |
+
warnings.append(
|
| 606 |
+
"The open-set boundary is unmeasured, so this candidate has not "
|
| 607 |
+
"been checked against the possibility that the animal is not "
|
| 608 |
+
"enrolled at all. " + policy.unmeasured_reason
|
| 609 |
+
)
|
| 610 |
+
return IdentityResult(
|
| 611 |
+
claim="identity_candidate", candidates=ranked, policy=policy,
|
| 612 |
+
enrolled_animals=enrolled, open_set_verified=False,
|
| 613 |
+
warnings=tuple(warnings),
|
| 614 |
+
)
|
| 615 |
+
|
| 616 |
+
top = ranked[0]
|
| 617 |
+
runner_up = ranked[1].similarity if len(ranked) > 1 else None
|
| 618 |
+
margin = None if runner_up is None else top.similarity - runner_up
|
| 619 |
+
|
| 620 |
+
if policy.accept_similarity is not None and top.similarity < policy.accept_similarity:
|
| 621 |
+
warnings.append(
|
| 622 |
+
f"Best match scored {top.similarity:.3f}, under the "
|
| 623 |
+
f"{policy.accept_similarity:.3f} measured on {policy.measured_on} "
|
| 624 |
+
f"in run {policy.measured_by_run_id}. On that evidence this is "
|
| 625 |
+
f"more likely an animal nobody has enrolled."
|
| 626 |
+
)
|
| 627 |
+
return IdentityResult(
|
| 628 |
+
claim="no_confident_match", candidates=ranked, policy=policy,
|
| 629 |
+
# False, because nothing was verified. The check ran and it
|
| 630 |
+
# said no; `policy.is_measured` is what records that it ran.
|
| 631 |
+
open_set_verified=False,
|
| 632 |
+
enrolled_animals=enrolled,
|
| 633 |
+
warnings=tuple(warnings),
|
| 634 |
+
)
|
| 635 |
+
|
| 636 |
+
if (policy.accept_margin is not None and margin is not None
|
| 637 |
+
and margin < policy.accept_margin):
|
| 638 |
+
warnings.append(
|
| 639 |
+
f"{ranked[0].display_name} and {ranked[1].display_name} are "
|
| 640 |
+
f"{margin:.3f} apart, under the {policy.accept_margin:.3f} "
|
| 641 |
+
f"measured in run {policy.measured_by_run_id}. Two animals look "
|
| 642 |
+
f"this alike; picking between them is not something this "
|
| 643 |
+
f"photograph supports."
|
| 644 |
+
)
|
| 645 |
+
return IdentityResult(
|
| 646 |
+
claim="no_confident_match", candidates=ranked, policy=policy,
|
| 647 |
+
# False for the same reason as the branch above: two animals
|
| 648 |
+
# too alike to separate is a refusal, not a verified match.
|
| 649 |
+
open_set_verified=False,
|
| 650 |
+
enrolled_animals=enrolled,
|
| 651 |
+
warnings=tuple(warnings),
|
| 652 |
+
)
|
| 653 |
+
|
| 654 |
+
return IdentityResult(
|
| 655 |
+
claim="identity_candidate", candidates=ranked, policy=policy,
|
| 656 |
+
enrolled_animals=enrolled, open_set_verified=True,
|
| 657 |
+
)
|
app/adapters/fingerprints.py
ADDED
|
@@ -0,0 +1,891 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""What an artefact *is*, read from its bytes rather than from its card.
|
| 2 |
+
|
| 3 |
+
**Nothing else in this service binds a model's identity to its contents.**
|
| 4 |
+
`providers.load_card` checksums the file, so the card and the bytes cannot drift
|
| 5 |
+
apart β but a sha256 only says *"these are the bytes somebody wrote this card
|
| 6 |
+
about"*. It says nothing about what the bytes are. Change the `runtime` field
|
| 7 |
+
and the same checksum now describes AGPL Ultralytics weights served as
|
| 8 |
+
`cattle_identity` and reported as Apache-2.0, because every licence decision in
|
| 9 |
+
this package keys off a string the card supplies.
|
| 10 |
+
|
| 11 |
+
ADR 0017's fix was a hard-coded `DISALLOWED_RUNTIMES = {"ultralytics"}`, and
|
| 12 |
+
`app/adapters/licences.py` generalised it into a table of what each runtime's
|
| 13 |
+
weights are really licensed under. Both improve on trusting the `license` field
|
| 14 |
+
and neither closes the hole, because both still start from the card's claim
|
| 15 |
+
about which loader runs. This module supplies the missing fact.
|
| 16 |
+
|
| 17 |
+
## What it reads
|
| 18 |
+
|
| 19 |
+
An ONNX file is a protobuf. Its framing carries the graph's structure β input
|
| 20 |
+
and output names and shapes, every node's op type, every initializer's name and
|
| 21 |
+
dimensions β and that structure is a fingerprint no card can edit without
|
| 22 |
+
editing the model. A DINOv3 export has a 16-pixel patch convolution, a
|
| 23 |
+
`reg_token` of shape `[1, 4, 384]`, twenty-four `gamma_1`/`gamma_2` LayerScale
|
| 24 |
+
vectors and a pair of RoPE `Sin`/`Cos` nodes. A DINOv2 export has a 14-pixel
|
| 25 |
+
patch convolution, Hugging Face's `encoder.layer.N.β¦` module-path naming, and no
|
| 26 |
+
`Sin` or `Cos` anywhere. YOLOX has 83 or more convolutions, purely numeric
|
| 27 |
+
initializer names, and a `[1, 8400, 85]` output.
|
| 28 |
+
|
| 29 |
+
**The reader is written against the protobuf wire format directly, using only
|
| 30 |
+
the standard library.** That is a deliberate constraint rather than an
|
| 31 |
+
affectation: `onnx` lives in `requirements-export.txt`, which says *"Build-time
|
| 32 |
+
only. Never install this into the serving image."* A control that only runs
|
| 33 |
+
where its dependency is installed is a control that is absent in production, and
|
| 34 |
+
`onnxruntime` β which is a production dependency β cannot substitute. Its
|
| 35 |
+
`get_modelmeta()` exposes `producer_name`, `graph_name`, `domain`, `version` and
|
| 36 |
+
a metadata map, and no nodes, no initializers and no attributes at all. Through
|
| 37 |
+
onnxruntime alone, DINOv2 and DINOv3 both report `producer_name='pytorch'`,
|
| 38 |
+
`graph_name='main_graph'` and an output of `['batch', 768]`: indistinguishable.
|
| 39 |
+
|
| 40 |
+
The scan never materialises `TensorProto.raw_data`. It walks the framing and
|
| 41 |
+
skips those byte ranges, so reading an 87 MB artefact costs about 22 ms and
|
| 42 |
+
68 MB of peak RSS β roughly three times faster than the sha256 the loader
|
| 43 |
+
already pays for, and twenty-six times faster on the 396 MB YOLOX-x.
|
| 44 |
+
|
| 45 |
+
A torch `.pt` is a zip. Its pickle is read with `pickletools.genops`, which
|
| 46 |
+
decodes opcodes without executing any of them, so identifying an Ultralytics
|
| 47 |
+
checkpoint never runs a line of its author's code.
|
| 48 |
+
|
| 49 |
+
## What it does not do
|
| 50 |
+
|
| 51 |
+
**It does not decide what may be served.** The founder's standing instruction is
|
| 52 |
+
that no model is dropped for its licence right now, so a mismatch between the
|
| 53 |
+
bytes and the card records itself and warns by default. What changes is that the
|
| 54 |
+
record is *true*: an exception logged under `ANIMAP_LICENCE_POLICY=record` names
|
| 55 |
+
the licence the bytes actually arrive under instead of the one the card claimed,
|
| 56 |
+
which is the whole reason for keeping a ledger. `ANIMAP_ARTEFACT_IDENTITY=refuse`
|
| 57 |
+
turns the same finding into a refusal, one variable on one deployment.
|
| 58 |
+
|
| 59 |
+
**An unidentified artefact is not a mismatch.** A file this module has no
|
| 60 |
+
fingerprint for is recorded as unverified and loads. Treating "nobody has
|
| 61 |
+
written a fingerprint for MegaDescriptor yet" as "this file is lying" would make
|
| 62 |
+
every new model an incident, and the honest state of an unfingerprinted artefact
|
| 63 |
+
is that its identity rests on the card β exactly where it rested before.
|
| 64 |
+
|
| 65 |
+
**A refuted claim is a mismatch, and used not to be one.** `Identification.
|
| 66 |
+
refutes` draws the line the paragraph above was missing: a card naming a runtime
|
| 67 |
+
that *has* a fingerprint, over bytes that fail it, is a claim this module
|
| 68 |
+
disproved, not a file it has nothing to say about. Renaming an Ultralytics
|
| 69 |
+
export's initializers is enough to stop `ULTRALYTICS_ONNX` matching and was
|
| 70 |
+
enough to serve AGPL weights as `cattle_detection`; it is not enough to make
|
| 71 |
+
those bytes YOLOX, and `YOLOX_ONNX` is right there to say so. `providers.py`
|
| 72 |
+
reads it, so nothing loads on a card claim its own artefact contradicts.
|
| 73 |
+
|
| 74 |
+
**A fingerprint identifies a family, not an export.** It matches on structure
|
| 75 |
+
that survives a re-export, so re-running `scripts/export_embedding.py` under a
|
| 76 |
+
different torch does not turn DINOv3 into an impostor. Pinning the exact bytes
|
| 77 |
+
is what sha256 is for, and doing it twice would only produce a check that fails
|
| 78 |
+
for the wrong reason.
|
| 79 |
+
"""
|
| 80 |
+
|
| 81 |
+
from __future__ import annotations
|
| 82 |
+
|
| 83 |
+
import logging
|
| 84 |
+
import mmap
|
| 85 |
+
import pickletools
|
| 86 |
+
import re
|
| 87 |
+
import zipfile
|
| 88 |
+
from dataclasses import dataclass, field
|
| 89 |
+
from pathlib import Path
|
| 90 |
+
from typing import Any, Iterator
|
| 91 |
+
|
| 92 |
+
logger = logging.getLogger(__name__)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
class NotReadable(RuntimeError):
|
| 96 |
+
"""The file is not in a format this module can read the structure of."""
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
# --- The ONNX protobuf reader -------------------------------------------------
|
| 100 |
+
#
|
| 101 |
+
# Field numbers below come from onnx/onnx.proto. They are quoted in comments at
|
| 102 |
+
# each use because a bare integer is unreviewable, and one of them is a trap:
|
| 103 |
+
# `AttributeProto.ints` is field 8, while field 7 is `floats`. Reading 7 returns
|
| 104 |
+
# an empty kernel shape and no error.
|
| 105 |
+
|
| 106 |
+
_WIRE_VARINT, _WIRE_64BIT, _WIRE_LENGTH, _WIRE_32BIT = 0, 1, 2, 5
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def _varint(buffer, index: int, end: int) -> tuple[int, int]:
|
| 110 |
+
result = shift = 0
|
| 111 |
+
while True:
|
| 112 |
+
if index >= end:
|
| 113 |
+
raise NotReadable("a protobuf varint runs past the end of the file")
|
| 114 |
+
byte = buffer[index]
|
| 115 |
+
index += 1
|
| 116 |
+
result |= (byte & 0x7F) << shift
|
| 117 |
+
if not byte & 0x80:
|
| 118 |
+
return result, index
|
| 119 |
+
shift += 7
|
| 120 |
+
if shift > 63:
|
| 121 |
+
raise NotReadable("a protobuf varint is longer than 64 bits")
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _fields(buffer, start: int, end: int) -> Iterator[tuple[int, int, int, int, int]]:
|
| 125 |
+
"""Walk one protobuf message, yielding `(number, wire_type, from, to, value)`.
|
| 126 |
+
|
| 127 |
+
Length-delimited payloads are yielded as a byte range and never copied,
|
| 128 |
+
which is what keeps a 396 MB artefact off the heap.
|
| 129 |
+
"""
|
| 130 |
+
index = start
|
| 131 |
+
while index < end:
|
| 132 |
+
key, index = _varint(buffer, index, end)
|
| 133 |
+
number, wire = key >> 3, key & 7
|
| 134 |
+
if number == 0:
|
| 135 |
+
raise NotReadable("protobuf field number 0 is not legal")
|
| 136 |
+
if wire == _WIRE_VARINT:
|
| 137 |
+
value, after = _varint(buffer, index, end)
|
| 138 |
+
yield number, wire, index, after, value
|
| 139 |
+
index = after
|
| 140 |
+
elif wire == _WIRE_64BIT:
|
| 141 |
+
yield number, wire, index, index + 8, 0
|
| 142 |
+
index += 8
|
| 143 |
+
elif wire == _WIRE_LENGTH:
|
| 144 |
+
length, after = _varint(buffer, index, end)
|
| 145 |
+
if after + length > end:
|
| 146 |
+
raise NotReadable("a protobuf field runs past its own message")
|
| 147 |
+
yield number, wire, after, after + length, 0
|
| 148 |
+
index = after + length
|
| 149 |
+
elif wire == _WIRE_32BIT:
|
| 150 |
+
yield number, wire, index, index + 4, 0
|
| 151 |
+
index += 4
|
| 152 |
+
else:
|
| 153 |
+
raise NotReadable(f"protobuf wire type {wire} is not legal")
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def _text(buffer, start: int, end: int) -> str:
|
| 157 |
+
return bytes(buffer[start:end]).decode("utf-8", "replace")
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def _signed(value: int) -> int:
|
| 161 |
+
"""Protobuf stores int64 as an unsigned varint, so negatives arrive huge."""
|
| 162 |
+
return value - (1 << 64) if value >= (1 << 63) else value
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def _packed(buffer, start: int, end: int) -> list[int]:
|
| 166 |
+
values: list[int] = []
|
| 167 |
+
index = start
|
| 168 |
+
while index < end:
|
| 169 |
+
value, index = _varint(buffer, index, end)
|
| 170 |
+
values.append(_signed(value))
|
| 171 |
+
return values
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
@dataclass
|
| 175 |
+
class OnnxStructure:
|
| 176 |
+
"""Everything the framing of an ONNX file says about the model in it."""
|
| 177 |
+
|
| 178 |
+
ir_version: int = 0
|
| 179 |
+
producer_name: str = ""
|
| 180 |
+
producer_version: str = ""
|
| 181 |
+
opset: int = 0
|
| 182 |
+
graph_name: str = ""
|
| 183 |
+
#: `ModelProto.metadata_props`. Exporters write provenance here β Ultralytics
|
| 184 |
+
#: writes `author`, `license`, `docs`, `task` and `stride` β and it is the
|
| 185 |
+
#: cheapest way to recognise a toolchain that has otherwise been renamed.
|
| 186 |
+
#: **Read as evidence, never as authority**; see `_self_declared_refusal`.
|
| 187 |
+
metadata: dict[str, str] = field(default_factory=dict)
|
| 188 |
+
inputs: list[tuple[str, list[Any]]] = field(default_factory=list)
|
| 189 |
+
outputs: list[tuple[str, list[Any]]] = field(default_factory=list)
|
| 190 |
+
op_counts: dict[str, int] = field(default_factory=dict)
|
| 191 |
+
node_count: int = 0
|
| 192 |
+
#: `(name, dims)` for every initializer. Names are the discriminator that
|
| 193 |
+
#: matters most β a module path is a statement about which implementation
|
| 194 |
+
#: exported the graph.
|
| 195 |
+
initializers: list[tuple[str, list[int]]] = field(default_factory=list)
|
| 196 |
+
total_params: int = 0
|
| 197 |
+
#: Kernel shape and strides of the first convolution. For a vision
|
| 198 |
+
#: transformer this is the patch size, which differs between every backbone
|
| 199 |
+
#: worth telling apart.
|
| 200 |
+
first_conv_kernel: list[int] = field(default_factory=list)
|
| 201 |
+
first_conv_strides: list[int] = field(default_factory=list)
|
| 202 |
+
first_conv_weight_dims: list[int] = field(default_factory=list)
|
| 203 |
+
#: Name of the first convolution's weight tensor. Kept because `GraphProto`
|
| 204 |
+
#: lists every node (field 1) before every initializer (field 5), so the
|
| 205 |
+
#: tensor a node refers to has not been read yet when the node is.
|
| 206 |
+
_first_conv_weight_name: str = ""
|
| 207 |
+
|
| 208 |
+
def initializer(self, name: str) -> list[int] | None:
|
| 209 |
+
"""Dimensions of one initializer, matched on the tail of its path.
|
| 210 |
+
|
| 211 |
+
The tail rather than the whole name, because `scripts/export_embedding.py`
|
| 212 |
+
wraps the backbone twice and every path arrives prefixed `inner.inner.`.
|
| 213 |
+
A fingerprint written against the wrapper would break the day the
|
| 214 |
+
wrapper is renamed, which would be a change to this repository's code
|
| 215 |
+
and not to the model's identity.
|
| 216 |
+
"""
|
| 217 |
+
for path, dims in self.initializers:
|
| 218 |
+
if path == name or path.endswith("." + name):
|
| 219 |
+
return dims
|
| 220 |
+
return None
|
| 221 |
+
|
| 222 |
+
def initializers_matching(self, pattern: str) -> int:
|
| 223 |
+
expression = re.compile(pattern)
|
| 224 |
+
return sum(1 for name, _ in self.initializers if expression.search(name))
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
def scan_onnx(path: Path | str) -> OnnxStructure:
|
| 228 |
+
"""Read an ONNX file's structure without loading its weights."""
|
| 229 |
+
path = Path(path)
|
| 230 |
+
if path.stat().st_size == 0:
|
| 231 |
+
# `mmap` raises ValueError on a zero-length file, and a governance check
|
| 232 |
+
# that crashes on an empty artefact is a governance check that takes the
|
| 233 |
+
# service down instead of reporting one bad model.
|
| 234 |
+
raise NotReadable(f"{path.name} is empty.")
|
| 235 |
+
with path.open("rb") as handle:
|
| 236 |
+
if handle.read(2) == b"PK":
|
| 237 |
+
# Named specifically rather than left to a protobuf stack trace.
|
| 238 |
+
# "Wire format was corrupt" is what every parser says here, and it
|
| 239 |
+
# sends a reader looking for a damaged download instead of at the
|
| 240 |
+
# `.pt` sitting where an `.onnx` was expected.
|
| 241 |
+
raise NotReadable(
|
| 242 |
+
f"{path.name} begins with the ZIP magic 'PK', so it is a torch "
|
| 243 |
+
f"checkpoint or another archive, not an ONNX protobuf."
|
| 244 |
+
)
|
| 245 |
+
handle.seek(0)
|
| 246 |
+
with mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) as buffer:
|
| 247 |
+
return _scan_model(buffer)
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def _scan_model(buffer) -> OnnxStructure:
|
| 251 |
+
structure = OnnxStructure()
|
| 252 |
+
saw_ir_version = False
|
| 253 |
+
for number, wire, start, end, value in _fields(buffer, 0, len(buffer)):
|
| 254 |
+
if number == 1 and wire == _WIRE_VARINT: # ir_version
|
| 255 |
+
structure.ir_version = _signed(value)
|
| 256 |
+
saw_ir_version = True
|
| 257 |
+
elif number == 2 and wire == _WIRE_LENGTH: # producer_name
|
| 258 |
+
structure.producer_name = _text(buffer, start, end)
|
| 259 |
+
elif number == 3 and wire == _WIRE_LENGTH: # producer_version
|
| 260 |
+
structure.producer_version = _text(buffer, start, end)
|
| 261 |
+
elif number == 8 and wire == _WIRE_LENGTH: # opset_import
|
| 262 |
+
for f2, w2, s2, e2, v2 in _fields(buffer, start, end):
|
| 263 |
+
if f2 == 2 and w2 == _WIRE_VARINT: # OperatorSetId.version
|
| 264 |
+
structure.opset = max(structure.opset, _signed(v2))
|
| 265 |
+
elif number == 14 and wire == _WIRE_LENGTH: # metadata_props
|
| 266 |
+
key = value_text = ""
|
| 267 |
+
for f2, w2, s2, e2, _v2 in _fields(buffer, start, end):
|
| 268 |
+
if f2 == 1 and w2 == _WIRE_LENGTH: # StringStringEntry.key
|
| 269 |
+
key = _text(buffer, s2, e2)
|
| 270 |
+
elif f2 == 2 and w2 == _WIRE_LENGTH: # .value
|
| 271 |
+
value_text = _text(buffer, s2, e2)
|
| 272 |
+
if key:
|
| 273 |
+
# Truncated because `names` on a COCO export is a 3 KB dict and
|
| 274 |
+
# this is going into a log line and a ledger entry.
|
| 275 |
+
structure.metadata[key] = value_text[:200]
|
| 276 |
+
elif number == 7 and wire == _WIRE_LENGTH: # graph
|
| 277 |
+
_scan_graph(buffer, start, end, structure)
|
| 278 |
+
if structure._first_conv_weight_name:
|
| 279 |
+
structure.first_conv_weight_dims = (
|
| 280 |
+
structure.initializer(structure._first_conv_weight_name) or []
|
| 281 |
+
)
|
| 282 |
+
if not saw_ir_version:
|
| 283 |
+
raise NotReadable(
|
| 284 |
+
"No ModelProto.ir_version field was found, so this is not an ONNX "
|
| 285 |
+
"model however well it parsed as protobuf."
|
| 286 |
+
)
|
| 287 |
+
return structure
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
def _scan_graph(buffer, start: int, end: int, structure: OnnxStructure) -> None:
|
| 291 |
+
for number, wire, from_, to, _ in _fields(buffer, start, end):
|
| 292 |
+
if wire != _WIRE_LENGTH:
|
| 293 |
+
continue
|
| 294 |
+
if number == 1: # GraphProto.node
|
| 295 |
+
structure.node_count += 1
|
| 296 |
+
_scan_node(buffer, from_, to, structure)
|
| 297 |
+
elif number == 2: # GraphProto.name
|
| 298 |
+
structure.graph_name = _text(buffer, from_, to)
|
| 299 |
+
elif number == 5: # GraphProto.initializer
|
| 300 |
+
_scan_initializer(buffer, from_, to, structure)
|
| 301 |
+
elif number == 11: # GraphProto.input
|
| 302 |
+
structure.inputs.append(_scan_value_info(buffer, from_, to))
|
| 303 |
+
elif number == 12: # GraphProto.output
|
| 304 |
+
structure.outputs.append(_scan_value_info(buffer, from_, to))
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def _scan_node(buffer, start: int, end: int, structure: OnnxStructure) -> None:
|
| 308 |
+
op_type = ""
|
| 309 |
+
inputs: list[str] = []
|
| 310 |
+
kernel: list[int] = []
|
| 311 |
+
strides: list[int] = []
|
| 312 |
+
for number, wire, from_, to, _ in _fields(buffer, start, end):
|
| 313 |
+
if wire != _WIRE_LENGTH:
|
| 314 |
+
continue
|
| 315 |
+
if number == 1: # NodeProto.input
|
| 316 |
+
inputs.append(_text(buffer, from_, to))
|
| 317 |
+
elif number == 4: # NodeProto.op_type
|
| 318 |
+
op_type = _text(buffer, from_, to)
|
| 319 |
+
elif number == 5: # NodeProto.attribute
|
| 320 |
+
name, values = _scan_attribute(buffer, from_, to)
|
| 321 |
+
if name == "kernel_shape":
|
| 322 |
+
kernel = values
|
| 323 |
+
elif name == "strides":
|
| 324 |
+
strides = values
|
| 325 |
+
if not op_type:
|
| 326 |
+
return
|
| 327 |
+
structure.op_counts[op_type] = structure.op_counts.get(op_type, 0) + 1
|
| 328 |
+
if op_type == "Conv" and not structure.first_conv_kernel:
|
| 329 |
+
structure.first_conv_kernel = kernel
|
| 330 |
+
structure.first_conv_strides = strides
|
| 331 |
+
if len(inputs) > 1:
|
| 332 |
+
structure._first_conv_weight_name = inputs[1]
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def _scan_attribute(buffer, start: int, end: int) -> tuple[str, list[int]]:
|
| 336 |
+
name = ""
|
| 337 |
+
values: list[int] = []
|
| 338 |
+
for number, wire, from_, to, value in _fields(buffer, start, end):
|
| 339 |
+
if number == 1 and wire == _WIRE_LENGTH: # AttributeProto.name
|
| 340 |
+
name = _text(buffer, from_, to)
|
| 341 |
+
# Field 8 is `ints`. Field 7 is `floats`, and reading it here returns an
|
| 342 |
+
# empty kernel shape with no error at all.
|
| 343 |
+
elif number == 8 and wire == _WIRE_LENGTH:
|
| 344 |
+
values.extend(_packed(buffer, from_, to))
|
| 345 |
+
elif number == 8 and wire == _WIRE_VARINT:
|
| 346 |
+
values.append(_signed(value))
|
| 347 |
+
return name, values
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
def _scan_initializer(buffer, start: int, end: int, structure: OnnxStructure) -> None:
|
| 351 |
+
dims: list[int] = []
|
| 352 |
+
name = ""
|
| 353 |
+
for number, wire, from_, to, value in _fields(buffer, start, end):
|
| 354 |
+
if number == 1 and wire == _WIRE_VARINT: # TensorProto.dims
|
| 355 |
+
dims.append(_signed(value))
|
| 356 |
+
elif number == 1 and wire == _WIRE_LENGTH:
|
| 357 |
+
dims.extend(_packed(buffer, from_, to))
|
| 358 |
+
elif number == 8 and wire == _WIRE_LENGTH: # TensorProto.name
|
| 359 |
+
name = _text(buffer, from_, to)
|
| 360 |
+
# Field 9 is `raw_data`. `_fields` has already skipped past it without
|
| 361 |
+
# reading a byte, which is the whole reason this is fast.
|
| 362 |
+
structure.initializers.append((name, dims))
|
| 363 |
+
count = 1
|
| 364 |
+
for dimension in dims:
|
| 365 |
+
count *= dimension
|
| 366 |
+
structure.total_params += count
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
def _scan_value_info(buffer, start: int, end: int) -> tuple[str, list[Any]]:
|
| 370 |
+
name = ""
|
| 371 |
+
dims: list[Any] = []
|
| 372 |
+
for number, wire, from_, to, _ in _fields(buffer, start, end):
|
| 373 |
+
if number == 1 and wire == _WIRE_LENGTH: # ValueInfoProto.name
|
| 374 |
+
name = _text(buffer, from_, to)
|
| 375 |
+
elif number == 2 and wire == _WIRE_LENGTH: # ValueInfoProto.type
|
| 376 |
+
for f2, w2, s2, e2, _v in _fields(buffer, from_, to):
|
| 377 |
+
if f2 != 1 or w2 != _WIRE_LENGTH: # TypeProto.tensor_type
|
| 378 |
+
continue
|
| 379 |
+
for f3, w3, s3, e3, _v3 in _fields(buffer, s2, e2):
|
| 380 |
+
if f3 != 2 or w3 != _WIRE_LENGTH: # Tensor.shape
|
| 381 |
+
continue
|
| 382 |
+
for f4, w4, s4, e4, _v4 in _fields(buffer, s3, e3):
|
| 383 |
+
if f4 != 1 or w4 != _WIRE_LENGTH: # Shape.dim
|
| 384 |
+
continue
|
| 385 |
+
dims.append(_scan_dimension(buffer, s4, e4))
|
| 386 |
+
return name, dims
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
def _scan_dimension(buffer, start: int, end: int) -> Any:
|
| 390 |
+
for number, wire, from_, to, value in _fields(buffer, start, end):
|
| 391 |
+
if number == 1 and wire == _WIRE_VARINT: # dim_value
|
| 392 |
+
return _signed(value)
|
| 393 |
+
if number == 2 and wire == _WIRE_LENGTH: # dim_param
|
| 394 |
+
return _text(buffer, from_, to)
|
| 395 |
+
return "?"
|
| 396 |
+
|
| 397 |
+
|
| 398 |
+
# --- The torch checkpoint reader ----------------------------------------------
|
| 399 |
+
|
| 400 |
+
#: How many pickle opcodes to decode before giving up on finding the header.
|
| 401 |
+
#: Ultralytics writes its metadata dict first, so the interesting keys arrive
|
| 402 |
+
#: inside the first couple of dozen opcodes; the cap exists so a hostile file
|
| 403 |
+
#: cannot turn a governance check into a long walk.
|
| 404 |
+
_PICKLE_OPCODE_LIMIT = 4000
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
@dataclass
|
| 408 |
+
class TorchStructure:
|
| 409 |
+
"""What a `.pt` says about itself, read without executing its pickle."""
|
| 410 |
+
|
| 411 |
+
#: Module paths named by `GLOBAL` and `STACK_GLOBAL` opcodes. This is the
|
| 412 |
+
#: identity: a checkpoint that reconstructs `ultralytics.nn.tasks.
|
| 413 |
+
#: DetectionModel` needs Ultralytics installed to load, whatever it is called.
|
| 414 |
+
globals: list[str] = field(default_factory=list)
|
| 415 |
+
#: Top-level string keys paired with the string that follows them. Enough
|
| 416 |
+
#: for `license`, `version`, `date` and `docs`, and no more.
|
| 417 |
+
header: dict[str, str] = field(default_factory=dict)
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
def scan_torch(path: Path | str) -> TorchStructure:
|
| 421 |
+
"""Read a torch checkpoint's pickle header without unpickling it.
|
| 422 |
+
|
| 423 |
+
`pickletools.genops` decodes the opcode stream and yields it. It builds no
|
| 424 |
+
objects and imports no modules, so reading an artefact to find out whether
|
| 425 |
+
it is Ultralytics does not run Ultralytics β which matters, because the
|
| 426 |
+
reason for asking is that this file might not be what its card says.
|
| 427 |
+
"""
|
| 428 |
+
path = Path(path)
|
| 429 |
+
try:
|
| 430 |
+
archive = zipfile.ZipFile(path)
|
| 431 |
+
except zipfile.BadZipFile as exc:
|
| 432 |
+
raise NotReadable(f"{path.name} is not a zip archive: {exc}") from exc
|
| 433 |
+
|
| 434 |
+
with archive:
|
| 435 |
+
members = [n for n in archive.namelist() if n.endswith("data.pkl")]
|
| 436 |
+
if not members:
|
| 437 |
+
raise NotReadable(
|
| 438 |
+
f"{path.name} is a zip archive with no `data.pkl` member, so it "
|
| 439 |
+
f"is not a torch checkpoint."
|
| 440 |
+
)
|
| 441 |
+
payload = archive.read(sorted(members, key=len)[0])
|
| 442 |
+
|
| 443 |
+
structure = TorchStructure()
|
| 444 |
+
strings: list[str] = []
|
| 445 |
+
try:
|
| 446 |
+
for index, (opcode, argument, _position) in enumerate(
|
| 447 |
+
pickletools.genops(payload)
|
| 448 |
+
):
|
| 449 |
+
if index > _PICKLE_OPCODE_LIMIT:
|
| 450 |
+
break
|
| 451 |
+
if opcode.name == "GLOBAL" and isinstance(argument, str):
|
| 452 |
+
structure.globals.append(argument.replace(" ", "."))
|
| 453 |
+
elif isinstance(argument, str):
|
| 454 |
+
strings.append(argument)
|
| 455 |
+
except Exception as exc: # pickletools raises bare ValueError on junk
|
| 456 |
+
raise NotReadable(f"{path.name} holds an unreadable pickle: {exc}") from exc
|
| 457 |
+
|
| 458 |
+
# `STACK_GLOBAL` takes its module and name off the stack rather than as an
|
| 459 |
+
# argument, so protocol-4 checkpoints leave the pair among the strings.
|
| 460 |
+
for index in range(len(strings) - 1):
|
| 461 |
+
if strings[index] in _TORCH_HEADER_KEYS:
|
| 462 |
+
structure.header.setdefault(strings[index], strings[index + 1])
|
| 463 |
+
return structure
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
#: The keys worth reading out of a checkpoint header. Not a general reader β
|
| 467 |
+
#: everything else in a `.pt` is weights.
|
| 468 |
+
_TORCH_HEADER_KEYS = frozenset({"license", "version", "date", "docs", "author"})
|
| 469 |
+
|
| 470 |
+
|
| 471 |
+
# --- The fingerprints themselves ----------------------------------------------
|
| 472 |
+
|
| 473 |
+
|
| 474 |
+
@dataclass(frozen=True)
|
| 475 |
+
class Fingerprint:
|
| 476 |
+
"""The structure one runtime's artefacts always have.
|
| 477 |
+
|
| 478 |
+
`must` holds predicates over the parsed structure. Every one has to hold,
|
| 479 |
+
and each carries the sentence that goes in a report when it does not β so a
|
| 480 |
+
mismatch says *which* property failed rather than "fingerprint failed",
|
| 481 |
+
which is the difference between a diagnosis and an alarm.
|
| 482 |
+
"""
|
| 483 |
+
|
| 484 |
+
runtime: str
|
| 485 |
+
kind: str # "onnx" or "torch"
|
| 486 |
+
must: tuple[tuple[str, Any], ...]
|
| 487 |
+
|
| 488 |
+
def match(self, structure: Any) -> tuple[bool, list[str]]:
|
| 489 |
+
failures = [
|
| 490 |
+
description
|
| 491 |
+
for description, predicate in self.must
|
| 492 |
+
if not _safe(predicate, structure)
|
| 493 |
+
]
|
| 494 |
+
return (not failures), failures
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
def _safe(predicate, structure) -> bool:
|
| 498 |
+
"""A predicate that raises is a predicate that did not hold.
|
| 499 |
+
|
| 500 |
+
A fingerprint runs against files that are not what they claim to be, so
|
| 501 |
+
indexing off the end of a two-element list is an expected outcome rather
|
| 502 |
+
than a bug.
|
| 503 |
+
"""
|
| 504 |
+
try:
|
| 505 |
+
return bool(predicate(structure))
|
| 506 |
+
except Exception:
|
| 507 |
+
return False
|
| 508 |
+
|
| 509 |
+
|
| 510 |
+
def _naming(structure: OnnxStructure, pattern: str, least: int) -> bool:
|
| 511 |
+
return structure.initializers_matching(pattern) >= least
|
| 512 |
+
|
| 513 |
+
|
| 514 |
+
#: DINOv3 ViT-S/16, as exported by `scripts/export_embedding.py`.
|
| 515 |
+
#:
|
| 516 |
+
#: `reg_token` is the strongest single property here β register tokens are the
|
| 517 |
+
#: architectural change DINOv3 introduced over DINOv2, and no DINOv2 export has
|
| 518 |
+
#: one. RoPE is the second: DINOv3 replaces the learned position embedding with
|
| 519 |
+
#: rotary embeddings, so `Sin` and `Cos` appear in the graph and `pos_embed`
|
| 520 |
+
#: does not exist as an initializer at all.
|
| 521 |
+
DINOV3_ONNX = Fingerprint(
|
| 522 |
+
runtime="dinov3-onnx",
|
| 523 |
+
kind="onnx",
|
| 524 |
+
must=(
|
| 525 |
+
("the patch convolution is not 16 pixels",
|
| 526 |
+
lambda s: s.first_conv_kernel == [16, 16] and s.first_conv_strides == [16, 16]),
|
| 527 |
+
("there is no `reg_token` of shape [1, 4, 384]",
|
| 528 |
+
lambda s: s.initializer("reg_token") == [1, 4, 384]),
|
| 529 |
+
("there is no `cls_token` of shape [1, 1, 384]",
|
| 530 |
+
lambda s: s.initializer("cls_token") == [1, 1, 384]),
|
| 531 |
+
("there are not 24 LayerScale `gamma_1`/`gamma_2` vectors",
|
| 532 |
+
lambda s: _naming(s, r"\.gamma_[12]$", 24)),
|
| 533 |
+
("the graph has no rotary position embedding (`Sin` and `Cos`)",
|
| 534 |
+
lambda s: s.op_counts.get("Sin", 0) >= 1 and s.op_counts.get("Cos", 0) >= 1),
|
| 535 |
+
("the initializers are not named in timm's `blocks.N.` convention",
|
| 536 |
+
lambda s: _naming(s, r"blocks\.\d+\.", 24)),
|
| 537 |
+
("it does not take one `pixel_values` tensor",
|
| 538 |
+
lambda s: [n for n, _ in s.inputs] == ["pixel_values"]),
|
| 539 |
+
),
|
| 540 |
+
)
|
| 541 |
+
|
| 542 |
+
#: DINOv2-small, as exported through `transformers`.
|
| 543 |
+
#:
|
| 544 |
+
#: The previous audit reported this one as identifiable by "Hugging Face
|
| 545 |
+
#: `Dinov2Model` naming". The literal string `Dinov2Model` does not occur
|
| 546 |
+
#: anywhere in the file β a byte search finds no `Dinov2`, no `transformers` and
|
| 547 |
+
#: no `facebook`. What is genuinely there is the *module-path convention* of
|
| 548 |
+
#: `transformers.models.dinov2`: `embeddings.patch_embeddings.projection.weight`
|
| 549 |
+
#: and `encoder.layer.N.attention.attention.query.bias`. That is a reliable
|
| 550 |
+
#: discriminator and it is a different fact, so it is written down as one.
|
| 551 |
+
DINOV2_ONNX = Fingerprint(
|
| 552 |
+
runtime="dinov2-onnx",
|
| 553 |
+
kind="onnx",
|
| 554 |
+
must=(
|
| 555 |
+
("the patch convolution is not 14 pixels",
|
| 556 |
+
lambda s: s.first_conv_kernel == [14, 14] and s.first_conv_strides == [14, 14]),
|
| 557 |
+
("there is no `cls_token` of shape [1, 1, 384]",
|
| 558 |
+
lambda s: s.initializer("cls_token") == [1, 1, 384]),
|
| 559 |
+
("it carries a `reg_token`, which DINOv2 does not have",
|
| 560 |
+
lambda s: s.initializer("reg_token") is None),
|
| 561 |
+
("the initializers are not named in the `encoder.layer.N.` convention "
|
| 562 |
+
"that transformers' Dinov2Model produces",
|
| 563 |
+
lambda s: _naming(s, r"encoder\.layer\.\d+\.", 24)),
|
| 564 |
+
("it has rotary position embedding nodes, which DINOv2 does not use",
|
| 565 |
+
lambda s: s.op_counts.get("Sin", 0) == 0 and s.op_counts.get("Cos", 0) == 0),
|
| 566 |
+
("it does not take one `pixel_values` tensor",
|
| 567 |
+
lambda s: [n for n, _ in s.inputs] == ["pixel_values"]),
|
| 568 |
+
),
|
| 569 |
+
)
|
| 570 |
+
|
| 571 |
+
#: YOLOX, at any of the four published sizes.
|
| 572 |
+
#:
|
| 573 |
+
#: Deliberately size-agnostic. All four share one licence, which is the question
|
| 574 |
+
#: this module exists to answer, and `sha256` on the card already pins which one
|
| 575 |
+
#: is installed. A fingerprint that also distinguished `-s` from `-m` would fail
|
| 576 |
+
#: for a reason the checksum had already caught, with a worse message.
|
| 577 |
+
YOLOX_ONNX = Fingerprint(
|
| 578 |
+
runtime="yolox-onnx",
|
| 579 |
+
kind="onnx",
|
| 580 |
+
must=(
|
| 581 |
+
("it does not take one 640-pixel `images` tensor",
|
| 582 |
+
lambda s: [n for n, _ in s.inputs] == ["images"]
|
| 583 |
+
and list(s.inputs[0][1])[1:] == [3, 640, 640]),
|
| 584 |
+
("it does not emit YOLOX's [1, 8400, 85] decode grid",
|
| 585 |
+
lambda s: list(s.outputs[0][1]) == [1, 8400, 85]),
|
| 586 |
+
("the stem convolution does not read a 12-channel focus slice",
|
| 587 |
+
lambda s: len(s.first_conv_weight_dims) == 4
|
| 588 |
+
and s.first_conv_weight_dims[1] == 12),
|
| 589 |
+
("it has fewer than 80 convolutions, so it is not a YOLOX backbone",
|
| 590 |
+
lambda s: s.op_counts.get("Conv", 0) >= 80),
|
| 591 |
+
("the initializers are not YOLOX's bare numeric names",
|
| 592 |
+
lambda s: _naming(s, r"^\d+$", 100)),
|
| 593 |
+
),
|
| 594 |
+
)
|
| 595 |
+
|
| 596 |
+
#: Ultralytics YOLO as a torch checkpoint.
|
| 597 |
+
#:
|
| 598 |
+
#: This is ADR 0017's exploit read from the bytes. A card claiming
|
| 599 |
+
#: `runtime: dinov3-onnx` over `yolo11m.pt` passes every other check in this
|
| 600 |
+
#: service; it fails here at the first predicate, because an Ultralytics
|
| 601 |
+
#: checkpoint reconstructs `ultralytics.nn.tasks.DetectionModel` and cannot be
|
| 602 |
+
#: loaded by anything else.
|
| 603 |
+
ULTRALYTICS_TORCH = Fingerprint(
|
| 604 |
+
runtime="ultralytics",
|
| 605 |
+
kind="torch",
|
| 606 |
+
must=(
|
| 607 |
+
("its pickle names no `ultralytics.` module, so nothing in it needs the "
|
| 608 |
+
"Ultralytics loader",
|
| 609 |
+
lambda s: any(g.startswith("ultralytics.") for g in s.globals)),
|
| 610 |
+
),
|
| 611 |
+
)
|
| 612 |
+
|
| 613 |
+
#: **Ultralytics after `yolo export format=onnx`, which is how the first version
|
| 614 |
+
#: of this module was defeated.**
|
| 615 |
+
#:
|
| 616 |
+
#: A watchdog exported `yolo11m.pt` to ONNX, wrote a card calling it
|
| 617 |
+
#: `yolox-onnx`, and served AGPL weights as `cattle_detection` under
|
| 618 |
+
#: `ANIMAP_LICENCE_POLICY=enforce` *and* `ANIMAP_ARTEFACT_IDENTITY=refuse`, with
|
| 619 |
+
#: an empty ledger. Nothing here recognised it: the torch fingerprint only reads
|
| 620 |
+
#: pickles, and an unidentified artefact loads on its card's word by design. A
|
| 621 |
+
#: ledger that reports success while the thing it audits walks past is worse
|
| 622 |
+
#: than no ledger.
|
| 623 |
+
#:
|
| 624 |
+
#: The predicates are structural rather than metadata-based on purpose. An
|
| 625 |
+
#: Ultralytics export does declare itself in `metadata_props` β `author:
|
| 626 |
+
#: Ultralytics`, `license: AGPL-3.0 License (β¦)` β and `_self_declared_refusal`
|
| 627 |
+
#: reads that too, but a stripped metadata block must not be a way through.
|
| 628 |
+
#: These survive stripping:
|
| 629 |
+
#:
|
| 630 |
+
#: | | YOLOX-m | YOLO11m exported |
|
| 631 |
+
#: |---|---|---|
|
| 632 |
+
#: | output | `output` `[1, 8400, 85]` | `output0` `[1, 84, 8400]` |
|
| 633 |
+
#: | initializer names | 224 bare numerals | 225 `model.N.β¦` |
|
| 634 |
+
#: | `Split` / `Softmax` | 0 / 0 | 10 / 2 (DFL head) |
|
| 635 |
+
#: | stem convolution | `[48, 12, 3, 3]`, focus slice | `[64, 3, 3, 3]` |
|
| 636 |
+
#:
|
| 637 |
+
#: Both take one `images` `[1, 3, 640, 640]` tensor, which is why input shape
|
| 638 |
+
#: alone was never going to be enough.
|
| 639 |
+
ULTRALYTICS_ONNX = Fingerprint(
|
| 640 |
+
runtime="ultralytics",
|
| 641 |
+
kind="onnx",
|
| 642 |
+
must=(
|
| 643 |
+
("it does not take one 640-pixel `images` tensor",
|
| 644 |
+
lambda s: [n for n, _ in s.inputs] == ["images"]
|
| 645 |
+
and list(s.inputs[0][1])[1:] == [3, 640, 640]),
|
| 646 |
+
("its output is not Ultralytics' channels-first [1, 84, N] decode head",
|
| 647 |
+
lambda s: len(list(s.outputs[0][1])) == 3
|
| 648 |
+
and list(s.outputs[0][1])[0] == 1
|
| 649 |
+
and 5 <= list(s.outputs[0][1])[1] <= 200
|
| 650 |
+
and list(s.outputs[0][1])[2] > list(s.outputs[0][1])[1]),
|
| 651 |
+
("the initializers are not named in Ultralytics' `model.N.` convention",
|
| 652 |
+
lambda s: _naming(s, r"^model\.\d+\.", 50)),
|
| 653 |
+
("it has no distribution-focal-loss head, which every YOLOv8-and-later "
|
| 654 |
+
"export carries",
|
| 655 |
+
lambda s: s.op_counts.get("Split", 0) >= 1
|
| 656 |
+
and s.op_counts.get("Softmax", 0) >= 1),
|
| 657 |
+
),
|
| 658 |
+
)
|
| 659 |
+
|
| 660 |
+
FINGERPRINTS: tuple[Fingerprint, ...] = (
|
| 661 |
+
DINOV3_ONNX, DINOV2_ONNX, ULTRALYTICS_ONNX, YOLOX_ONNX, ULTRALYTICS_TORCH,
|
| 662 |
+
)
|
| 663 |
+
|
| 664 |
+
|
| 665 |
+
def fingerprint_for(runtime: str, kind: str) -> Fingerprint | None:
|
| 666 |
+
"""The fingerprint that decides whether a file *is* this runtime.
|
| 667 |
+
|
| 668 |
+
Keyed on both, because `ultralytics` has two β a torch one and an ONNX one β
|
| 669 |
+
and asking whether a `.pt` matches the ONNX predicates would answer no for a
|
| 670 |
+
reason that says nothing about the file.
|
| 671 |
+
"""
|
| 672 |
+
for fingerprint in FINGERPRINTS:
|
| 673 |
+
if fingerprint.runtime == runtime and fingerprint.kind == kind:
|
| 674 |
+
return fingerprint
|
| 675 |
+
return None
|
| 676 |
+
|
| 677 |
+
#: Strings in an artefact's own metadata that name a runtime this service will
|
| 678 |
+
#: not serve from. Matched case-insensitively against every metadata value.
|
| 679 |
+
#:
|
| 680 |
+
#: **This may only make a verdict stricter, never laxer**, and that asymmetry is
|
| 681 |
+
#: the whole design. `licences.py` records the trap that proves it:
|
| 682 |
+
#: `BVRA/MegaDescriptor-L-384/config.json` declares `"license": "mit"` as
|
| 683 |
+
#: inherited timm boilerplate over CC-BY-NC-4.0 weights, so a loader that
|
| 684 |
+
#: believed an artefact's own permissive claim would ship a non-commercial
|
| 685 |
+
#: model. Believing a *restrictive* self-declaration has no such failure mode:
|
| 686 |
+
#: the worst case is refusing something that was fine, which is a conversation
|
| 687 |
+
#: rather than a breach.
|
| 688 |
+
_SELF_DECLARED_RUNTIMES: tuple[tuple[str, str], ...] = (
|
| 689 |
+
("ultralytics", "ultralytics"),
|
| 690 |
+
("agpl", "ultralytics"),
|
| 691 |
+
)
|
| 692 |
+
|
| 693 |
+
|
| 694 |
+
# --- The finding --------------------------------------------------------------
|
| 695 |
+
|
| 696 |
+
|
| 697 |
+
@dataclass(frozen=True)
|
| 698 |
+
class Identification:
|
| 699 |
+
"""What the bytes turned out to be, and how confidently.
|
| 700 |
+
|
| 701 |
+
`runtime` is `None` for both *"nobody has written a fingerprint for this"*
|
| 702 |
+
and *"this file could not be read at all"*, and `detail` is what tells them
|
| 703 |
+
apart. They are the same outcome for a caller β identity unverified, load
|
| 704 |
+
on the card's word β and different facts for a person reading the log.
|
| 705 |
+
"""
|
| 706 |
+
|
| 707 |
+
#: The runtime whose fingerprint matched, or `None` if none did.
|
| 708 |
+
runtime: str | None
|
| 709 |
+
#: True when the file was read and no fingerprint matched. False when the
|
| 710 |
+
#: file could not be read as either ONNX or a torch archive.
|
| 711 |
+
readable: bool
|
| 712 |
+
detail: str
|
| 713 |
+
#: What the file was read as β `"onnx"`, `"torch"`, or `""` when neither
|
| 714 |
+
#: reader could open it. A caller needs this to ask the inverse question:
|
| 715 |
+
#: *the card claims runtime X; was there a fingerprint for X this file could
|
| 716 |
+
#: have been tried against?* Only fingerprints of the same kind are ever
|
| 717 |
+
#: tried, so without it a caller cannot tell *"the card's claim was
|
| 718 |
+
#: disproved"* from *"the card's claim was never testable"*.
|
| 719 |
+
kind: str = ""
|
| 720 |
+
#: Facts worth keeping in the ledger whether or not anything matched.
|
| 721 |
+
evidence: dict[str, Any] = field(default_factory=dict)
|
| 722 |
+
#: Why each fingerprint that was tried did not match, keyed by runtime.
|
| 723 |
+
#: Present so a mismatch report can say what the file looked like instead.
|
| 724 |
+
#:
|
| 725 |
+
#: **These sentences are the predicates an attacker would have to break**,
|
| 726 |
+
#: and returning them is only defensible because nothing serialises them:
|
| 727 |
+
#: `app/main.py` puts no part of an `Identification` on `/health` or
|
| 728 |
+
#: `/capabilities`, and `tests/test_fingerprints.py` asserts that so a
|
| 729 |
+
#: future response model cannot quietly start carrying one. Anyone who can
|
| 730 |
+
#: read this field can already read this file β writing a model card means
|
| 731 |
+
#: writing into the models tree β so the leak that matters is the one across
|
| 732 |
+
#: an HTTP boundary, and that is the one under test.
|
| 733 |
+
near_misses: dict[str, list[str]] = field(default_factory=dict)
|
| 734 |
+
|
| 735 |
+
@property
|
| 736 |
+
def identified(self) -> bool:
|
| 737 |
+
return self.runtime is not None
|
| 738 |
+
|
| 739 |
+
def refutes(self, declared: str | None) -> list[str]:
|
| 740 |
+
"""Why these bytes are not `declared`, or an empty list.
|
| 741 |
+
|
| 742 |
+
**The inverse question, and nothing was asking it.** `identify` walks
|
| 743 |
+
the fingerprints looking for one that matches and answers `None` when
|
| 744 |
+
none does. That is the right answer for a toolchain nobody has
|
| 745 |
+
fingerprinted, and the wrong one for a card naming a runtime that *has*
|
| 746 |
+
a fingerprint.
|
| 747 |
+
|
| 748 |
+
A watchdog renamed 225 initializers in an Ultralytics ONNX export from
|
| 749 |
+
`model.N.` to `m.N.`, stripped `metadata_props`, and served it as
|
| 750 |
+
`cattle_detection` on a card reading `runtime: yolox-onnx`, under
|
| 751 |
+
`ANIMAP_LICENCE_POLICY=enforce` and `ANIMAP_ARTEFACT_IDENTITY=refuse`,
|
| 752 |
+
with an empty ledger. The rename left `onnxruntime`'s output
|
| 753 |
+
bit-identical, so nothing about the model changed β it only stopped
|
| 754 |
+
matching `ULTRALYTICS_ONNX`. Everything downstream then read the card,
|
| 755 |
+
because an unidentified artefact loads on its card's word by design.
|
| 756 |
+
|
| 757 |
+
But `YOLOX_ONNX` exists, the file was read as ONNX, and it fails three
|
| 758 |
+
of that fingerprint's five properties. That is not an unidentified
|
| 759 |
+
artefact; it is a claim this module can disprove and did. The two must
|
| 760 |
+
not produce the same outcome.
|
| 761 |
+
|
| 762 |
+
Empty when the claim was never testable β no `declared`, an unreadable
|
| 763 |
+
file, or no fingerprint for `declared` of the kind this file was read
|
| 764 |
+
as β because *"not checked"* has to stay distinguishable from *"checked
|
| 765 |
+
and false"*. That distinction is the whole reason an unfingerprinted
|
| 766 |
+
artefact still loads.
|
| 767 |
+
"""
|
| 768 |
+
if not declared or not self.readable:
|
| 769 |
+
return []
|
| 770 |
+
if fingerprint_for(declared, self.kind) is None:
|
| 771 |
+
return []
|
| 772 |
+
return list(self.near_misses.get(declared, []))
|
| 773 |
+
|
| 774 |
+
|
| 775 |
+
def identify(path: Path | str) -> Identification:
|
| 776 |
+
"""Read a model artefact and say which runtime's weights it holds."""
|
| 777 |
+
path = Path(path)
|
| 778 |
+
structure: Any
|
| 779 |
+
kind: str
|
| 780 |
+
try:
|
| 781 |
+
structure = scan_onnx(path)
|
| 782 |
+
kind = "onnx"
|
| 783 |
+
except NotReadable as onnx_failure:
|
| 784 |
+
try:
|
| 785 |
+
structure = scan_torch(path)
|
| 786 |
+
kind = "torch"
|
| 787 |
+
except NotReadable as torch_failure:
|
| 788 |
+
return Identification(
|
| 789 |
+
runtime=None, readable=False,
|
| 790 |
+
detail=(
|
| 791 |
+
f"{path.name} is neither an ONNX graph ({onnx_failure}) nor "
|
| 792 |
+
f"a torch checkpoint ({torch_failure}), so nothing here can "
|
| 793 |
+
f"say what it is."
|
| 794 |
+
),
|
| 795 |
+
)
|
| 796 |
+
|
| 797 |
+
evidence = _evidence(structure, kind)
|
| 798 |
+
near_misses: dict[str, list[str]] = {}
|
| 799 |
+
for fingerprint in FINGERPRINTS:
|
| 800 |
+
if fingerprint.kind != kind:
|
| 801 |
+
continue
|
| 802 |
+
matched, failures = fingerprint.match(structure)
|
| 803 |
+
if matched:
|
| 804 |
+
return Identification(
|
| 805 |
+
runtime=fingerprint.runtime, readable=True, kind=kind,
|
| 806 |
+
detail=(
|
| 807 |
+
f"{path.name} matches every structural property of "
|
| 808 |
+
f"{fingerprint.runtime}."
|
| 809 |
+
),
|
| 810 |
+
evidence=evidence,
|
| 811 |
+
)
|
| 812 |
+
near_misses[fingerprint.runtime] = failures
|
| 813 |
+
|
| 814 |
+
declared = _self_declared_refusal(structure, kind)
|
| 815 |
+
if declared is not None:
|
| 816 |
+
runtime, where = declared
|
| 817 |
+
return Identification(
|
| 818 |
+
runtime=runtime, readable=True, kind=kind,
|
| 819 |
+
detail=(
|
| 820 |
+
f"{path.name} matches no structural fingerprint, but it names "
|
| 821 |
+
f"itself: {where}. A self-declaration is not authority about "
|
| 822 |
+
f"what an artefact may be used for β a permissive one is "
|
| 823 |
+
f"exactly the MegaDescriptor trap β but a restrictive one can "
|
| 824 |
+
f"only ever make this stricter, so it is believed."
|
| 825 |
+
),
|
| 826 |
+
evidence=evidence, near_misses=near_misses,
|
| 827 |
+
)
|
| 828 |
+
|
| 829 |
+
return Identification(
|
| 830 |
+
runtime=None, readable=True, kind=kind,
|
| 831 |
+
detail=(
|
| 832 |
+
f"{path.name} was read as {kind} and matches no fingerprint in "
|
| 833 |
+
f"app/adapters/fingerprints.py. Its identity rests on its card, "
|
| 834 |
+
f"which is where it rested before this check existed."
|
| 835 |
+
),
|
| 836 |
+
evidence=evidence, near_misses=near_misses,
|
| 837 |
+
)
|
| 838 |
+
|
| 839 |
+
|
| 840 |
+
def _self_declared_refusal(
|
| 841 |
+
structure: Any, kind: str
|
| 842 |
+
) -> tuple[str, str] | None:
|
| 843 |
+
"""A runtime the artefact names in its own metadata, if it is a refused one.
|
| 844 |
+
|
| 845 |
+
The backstop behind the structural fingerprints, for the case they were
|
| 846 |
+
written to handle badly: a toolchain nobody has fingerprinted yet whose
|
| 847 |
+
exporter is honest about where the weights came from. Most exporters are β
|
| 848 |
+
Ultralytics writes `author` and `license` into `metadata_props` β and a
|
| 849 |
+
check that reads them costs nothing.
|
| 850 |
+
|
| 851 |
+
Returns `None` for a permissive self-declaration, always. That direction is
|
| 852 |
+
where the trap lives.
|
| 853 |
+
"""
|
| 854 |
+
declared = (
|
| 855 |
+
structure.metadata if kind == "onnx" else structure.header
|
| 856 |
+
)
|
| 857 |
+
for key, value in declared.items():
|
| 858 |
+
haystack = f"{key} {value}".lower()
|
| 859 |
+
for needle, runtime in _SELF_DECLARED_RUNTIMES:
|
| 860 |
+
if needle in haystack:
|
| 861 |
+
return runtime, f"{key}={value!r}"
|
| 862 |
+
return None
|
| 863 |
+
|
| 864 |
+
|
| 865 |
+
def _evidence(structure: Any, kind: str) -> dict[str, Any]:
|
| 866 |
+
"""The handful of facts worth keeping about any artefact, matched or not."""
|
| 867 |
+
if kind == "torch":
|
| 868 |
+
return {
|
| 869 |
+
"format": "torch",
|
| 870 |
+
"modules": sorted({g.rsplit(".", 1)[0] for g in structure.globals})[:8],
|
| 871 |
+
# Recorded, and deliberately not acted on. `licences.py` documents
|
| 872 |
+
# the MegaDescriptor trap: `config.json` there declares `"license":
|
| 873 |
+
# "mit"` as inherited timm boilerplate over CC-BY-NC-4.0 weights. An
|
| 874 |
+
# artefact's own claim about its terms is evidence about what
|
| 875 |
+
# upstream wrote, never authority about what Animap may serve.
|
| 876 |
+
"self_declared": dict(structure.header),
|
| 877 |
+
}
|
| 878 |
+
return {
|
| 879 |
+
"format": "onnx",
|
| 880 |
+
"producer": f"{structure.producer_name} {structure.producer_version}".strip(),
|
| 881 |
+
# Recorded, and read only in the strict direction. See
|
| 882 |
+
# `_SELF_DECLARED_RUNTIMES`.
|
| 883 |
+
"self_declared": dict(structure.metadata),
|
| 884 |
+
"opset": structure.opset,
|
| 885 |
+
"inputs": [f"{n}{d}" for n, d in structure.inputs],
|
| 886 |
+
"outputs": [f"{n}{d}" for n, d in structure.outputs],
|
| 887 |
+
"nodes": structure.node_count,
|
| 888 |
+
"initializers": len(structure.initializers),
|
| 889 |
+
"parameters": structure.total_params,
|
| 890 |
+
"first_conv_kernel": structure.first_conv_kernel,
|
| 891 |
+
}
|
app/adapters/geometry/__init__.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Geometry to weight: Β§22's pipeline from the mask onward, with no weights.
|
| 2 |
+
|
| 3 |
+
The models Β§22 names β SAM for the mask, ARCore and VGGT for metric scale β are
|
| 4 |
+
not here. What is here is everything downstream of them: a mask becomes body
|
| 5 |
+
dimensions (`body`), a scale turns those into centimetres, and a published
|
| 6 |
+
equation turns centimetres into a band (`equations`, `weight`). That split is
|
| 7 |
+
deliberate. The neural half needs a GPU host and a licence review; this half runs
|
| 8 |
+
on a phone, is exercised by unit tests, and is where most of the error lives.
|
| 9 |
+
|
| 10 |
+
**Nothing in this package invents a scale.** `BodyMeasurements` requires a
|
| 11 |
+
`scale_relative_error` with no default, and `app.adapters.signal.geometry`
|
| 12 |
+
already refuses to return a scale from a frame with no reference in it. Β§22 is
|
| 13 |
+
explicit that "a photograph has no scale" is true of an arbitrary photograph; the
|
| 14 |
+
answer is to make the photograph non-arbitrary, not to guess.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
from app.adapters.base import (
|
| 20 |
+
Adapter,
|
| 21 |
+
AdapterSpec,
|
| 22 |
+
Availability,
|
| 23 |
+
Modality,
|
| 24 |
+
Placement,
|
| 25 |
+
Task,
|
| 26 |
+
)
|
| 27 |
+
from app.adapters.geometry.body import (
|
| 28 |
+
MaskUnusable,
|
| 29 |
+
PixelProfile,
|
| 30 |
+
profile,
|
| 31 |
+
rear_width_px,
|
| 32 |
+
)
|
| 33 |
+
from app.adapters.geometry.equations import (
|
| 34 |
+
BY_ID,
|
| 35 |
+
EQUATIONS,
|
| 36 |
+
LESOSKY_SHZ,
|
| 37 |
+
ODADI_BORAN,
|
| 38 |
+
SCHAEFFER,
|
| 39 |
+
OutOfRange,
|
| 40 |
+
WeightEquation,
|
| 41 |
+
ellipse_perimeter_cm,
|
| 42 |
+
)
|
| 43 |
+
from app.adapters.geometry.weight import (
|
| 44 |
+
BodyMeasurements,
|
| 45 |
+
EquationResult,
|
| 46 |
+
NotEnoughGeometry,
|
| 47 |
+
WeightBand,
|
| 48 |
+
estimate,
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
BODY_GEOMETRY_SPEC = AdapterSpec(
|
| 52 |
+
adapter_id="body-geometry-weight",
|
| 53 |
+
runtime="opencv-numpy",
|
| 54 |
+
tasks=(Task.MEASURE,),
|
| 55 |
+
modalities=(Modality.IMAGE, Modality.VIDEO),
|
| 56 |
+
directive_role=(
|
| 57 |
+
"Β§22 cattle weight β the second half of the pipeline: body length, "
|
| 58 |
+
"height, girth proxy, classical livestock weight equation, broad "
|
| 59 |
+
"estimate. Β§4 prefers deterministic geometry to a network wherever it "
|
| 60 |
+
"wins, and this step is arithmetic."
|
| 61 |
+
),
|
| 62 |
+
requires_artefact=False,
|
| 63 |
+
placement=Placement.ON_DEVICE,
|
| 64 |
+
placement_reason=(
|
| 65 |
+
"Column sums over a mask and three closed-form equations. The expensive "
|
| 66 |
+
"parts of Β§22 are the mask and the metric depth, both of which are "
|
| 67 |
+
"somewhere else; this runs in microseconds wherever the mask is."
|
| 68 |
+
),
|
| 69 |
+
notes=(
|
| 70 |
+
"**The equations are published and unfitted, which is the point and "
|
| 71 |
+
"also the limit.** None of the three was derived on Nigerian cattle, "
|
| 72 |
+
"and the closest β Lesosky et al. on east African shorthorn zebu β "
|
| 73 |
+
"publishes a Β±20% interval on its own animals before any vision error "
|
| 74 |
+
"is added. `experiments/cattle_weight` measures all three on 623 real "
|
| 75 |
+
"cattle: 10.6β11.9% MAPE with a TAPE measurement, and Β±25% needed to "
|
| 76 |
+
"cover nine animals in ten.\n\n"
|
| 77 |
+
"**`body` has never run on a real animal mask.** Everything that turns "
|
| 78 |
+
"a mask into dimensions β the column profile, the trunk span, the chest "
|
| 79 |
+
"window, the rear width β is exercised only by unit tests over "
|
| 80 |
+
"rectangles in `tests/test_geometry_weight.py`. No segmented cow has "
|
| 81 |
+
"passed through it, because the one dataset carrying cattle "
|
| 82 |
+
"photographed from the side and the rear with the measurements beside "
|
| 83 |
+
"them serves zero files. Do not quote a dimension from this until "
|
| 84 |
+
"somebody has run it on a photograph."
|
| 85 |
+
),
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
class BodyGeometryAdapter(Adapter):
|
| 90 |
+
"""Β§22 from the mask onward. Always available; never guesses a scale."""
|
| 91 |
+
|
| 92 |
+
spec = BODY_GEOMETRY_SPEC
|
| 93 |
+
|
| 94 |
+
def availability(self) -> Availability:
|
| 95 |
+
# NumPy is a production dependency and the equations are arithmetic, so
|
| 96 |
+
# there is nothing that can be absent. The honesty property lives in
|
| 97 |
+
# `weight.estimate`, which refuses rather than widening when no equation
|
| 98 |
+
# applies, and in `BodyMeasurements`, which has no default scale error.
|
| 99 |
+
return Availability(True)
|
| 100 |
+
|
| 101 |
+
def load(self) -> "BodyGeometryAdapter":
|
| 102 |
+
return self
|
| 103 |
+
|
| 104 |
+
def measure(self, measurements: BodyMeasurements, **kwargs) -> WeightBand:
|
| 105 |
+
return estimate(measurements, **kwargs)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
__all__ = [
|
| 109 |
+
"BODY_GEOMETRY_SPEC",
|
| 110 |
+
"BY_ID",
|
| 111 |
+
"EQUATIONS",
|
| 112 |
+
"LESOSKY_SHZ",
|
| 113 |
+
"ODADI_BORAN",
|
| 114 |
+
"SCHAEFFER",
|
| 115 |
+
"BodyGeometryAdapter",
|
| 116 |
+
"BodyMeasurements",
|
| 117 |
+
"EquationResult",
|
| 118 |
+
"MaskUnusable",
|
| 119 |
+
"NotEnoughGeometry",
|
| 120 |
+
"OutOfRange",
|
| 121 |
+
"PixelProfile",
|
| 122 |
+
"WeightBand",
|
| 123 |
+
"WeightEquation",
|
| 124 |
+
"ellipse_perimeter_cm",
|
| 125 |
+
"estimate",
|
| 126 |
+
"profile",
|
| 127 |
+
"rear_width_px",
|
| 128 |
+
]
|
app/adapters/geometry/body.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""From an animal mask to body dimensions, and the places that step is weak.
|
| 2 |
+
|
| 3 |
+
Directive Β§22's pipeline goes `cattle masks β ARCore + VGGT geometry β body
|
| 4 |
+
length β height β girth/volume proxy`. This module is the arithmetic between a
|
| 5 |
+
mask and those three numbers. It is deterministic β contours and column sums, no
|
| 6 |
+
weights β which Β§4 says to prefer wherever it wins.
|
| 7 |
+
|
| 8 |
+
## Three known weaknesses, stated here rather than discovered later
|
| 9 |
+
|
| 10 |
+
**The silhouette's length is not the equation's length.** Schaeffer's rule wants
|
| 11 |
+
the distance from the point of the shoulder to the pin bone. A side-view mask
|
| 12 |
+
gives nose to tail, which is longer by however far the head sticks out, and the
|
| 13 |
+
head is the most mobile part of the animal. `trunk_span_px` exists to cut the
|
| 14 |
+
head and tail off by profile shape, and it is a heuristic; `experiments/
|
| 15 |
+
cattle_weight` measures what it costs against tape-measured oblique body length.
|
| 16 |
+
|
| 17 |
+
**The girth station is not visible.** Heart girth is measured immediately behind
|
| 18 |
+
the foreleg. A silhouette shows no foreleg boundary, so this module takes the
|
| 19 |
+
deepest part of the trunk inside a stated window and calls it the chest. On
|
| 20 |
+
cattle the deepest point is near the girth station, which is why the proxy is
|
| 21 |
+
defensible; it is not the same thing, which is why it is called a proxy.
|
| 22 |
+
|
| 23 |
+
**A circumference is not in the picture at all.** Depth comes from a side view
|
| 24 |
+
and width from a rear view, and everything between them is an assumption about
|
| 25 |
+
cross-sectional shape. `equations.ellipse_perimeter_cm` is that assumption.
|
| 26 |
+
|
| 27 |
+
None of these are reasons the capability is impossible. They are the reasons Β§22
|
| 28 |
+
asks for a guided sweep and a rear view rather than a snapshot, and they are the
|
| 29 |
+
error terms an experiment has to size.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
from __future__ import annotations
|
| 33 |
+
|
| 34 |
+
from dataclasses import dataclass
|
| 35 |
+
|
| 36 |
+
import numpy as np
|
| 37 |
+
|
| 38 |
+
#: Fraction of the body length, measured from the front, inside which the chest
|
| 39 |
+
#: depth is sought. Cattle are deepest through the barrel just behind the
|
| 40 |
+
#: foreleg; the window keeps the search off the hindquarters, which on a
|
| 41 |
+
#: well-conditioned animal can be as deep and are not the girth station.
|
| 42 |
+
CHEST_WINDOW = (0.20, 0.55)
|
| 43 |
+
|
| 44 |
+
#: A column holding fewer than this fraction of the mask's peak height is
|
| 45 |
+
#: treated as head, neck, tail or leg rather than trunk. Chosen so that a neck β
|
| 46 |
+
#: roughly a third of the trunk's depth on cattle β falls outside and the trunk
|
| 47 |
+
#: does not. It is a threshold on a silhouette, so it is a heuristic, and
|
| 48 |
+
#: `trunk_span_px` reports what it cut.
|
| 49 |
+
TRUNK_THRESHOLD = 0.55
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class MaskUnusable(ValueError):
|
| 53 |
+
"""The mask does not describe a whole animal seen side-on.
|
| 54 |
+
|
| 55 |
+
Raised rather than measured around. Β§22's `reject_if` already lists
|
| 56 |
+
`animal_heavily_occluded` and `wrong_pose`; a body length taken off a mask
|
| 57 |
+
with the hindquarters cut out of frame is a number that looks fine and is
|
| 58 |
+
wrong by 30%.
|
| 59 |
+
"""
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
@dataclass(frozen=True)
|
| 63 |
+
class PixelProfile:
|
| 64 |
+
"""The silhouette reduced to what the measurements need.
|
| 65 |
+
|
| 66 |
+
`column_height` is the mask's vertical extent in every column, which is the
|
| 67 |
+
only thing any of the derived numbers reads. Keeping it on the object means
|
| 68 |
+
an experiment can plot the profile that produced a bad measurement instead of
|
| 69 |
+
guessing at it.
|
| 70 |
+
"""
|
| 71 |
+
|
| 72 |
+
column_height: np.ndarray
|
| 73 |
+
#: Inclusive column indices of the whole animal, head and tail included.
|
| 74 |
+
full_span: tuple[int, int]
|
| 75 |
+
#: Inclusive column indices of the trunk, after the profile threshold.
|
| 76 |
+
trunk_span: tuple[int, int]
|
| 77 |
+
#: Column index of the deepest trunk column inside `CHEST_WINDOW`.
|
| 78 |
+
chest_column: int
|
| 79 |
+
#: Fraction of the mask's pixels that fell outside `trunk_span`. Large means
|
| 80 |
+
#: a long neck, an outstretched head β or a segmentation that included a
|
| 81 |
+
#: second animal, which is the failure worth catching.
|
| 82 |
+
trimmed_fraction: float
|
| 83 |
+
|
| 84 |
+
@property
|
| 85 |
+
def full_length_px(self) -> float:
|
| 86 |
+
return float(self.full_span[1] - self.full_span[0] + 1)
|
| 87 |
+
|
| 88 |
+
@property
|
| 89 |
+
def trunk_length_px(self) -> float:
|
| 90 |
+
return float(self.trunk_span[1] - self.trunk_span[0] + 1)
|
| 91 |
+
|
| 92 |
+
@property
|
| 93 |
+
def chest_depth_px(self) -> float:
|
| 94 |
+
return float(self.column_height[self.chest_column])
|
| 95 |
+
|
| 96 |
+
@property
|
| 97 |
+
def withers_height_px(self) -> float:
|
| 98 |
+
"""Peak trunk depth. **Not withers height above ground.**
|
| 99 |
+
|
| 100 |
+
A standing animal's withers height is measured from the floor and needs
|
| 101 |
+
the ground plane, which a mask alone does not give. This is the top of
|
| 102 |
+
the trunk to the bottom of the trunk in the same column β a body depth.
|
| 103 |
+
Named on the object as `withers_height_px` would be a lie, so it is not.
|
| 104 |
+
"""
|
| 105 |
+
low, high = self.trunk_span
|
| 106 |
+
return float(self.column_height[low:high + 1].max())
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def profile(mask: np.ndarray) -> PixelProfile:
|
| 110 |
+
"""Reduce a side-on animal mask to a column profile and its landmarks.
|
| 111 |
+
|
| 112 |
+
The mask is expected in image orientation β rows are image rows β and the
|
| 113 |
+
animal is expected to be roughly horizontal in frame. A cow photographed at
|
| 114 |
+
45Β° produces a profile whose peak is not the chest and whose span is not the
|
| 115 |
+
body length, and nothing here detects that. Β§22's capture prompt exists
|
| 116 |
+
partly for this reason.
|
| 117 |
+
"""
|
| 118 |
+
binary = np.asarray(mask) > 0
|
| 119 |
+
if binary.ndim != 2:
|
| 120 |
+
raise MaskUnusable(f"Expected a 2-D mask, got {binary.ndim} dimensions.")
|
| 121 |
+
if not binary.any():
|
| 122 |
+
raise MaskUnusable("The mask is empty; nothing was segmented.")
|
| 123 |
+
|
| 124 |
+
occupied_columns = np.flatnonzero(binary.any(axis=0))
|
| 125 |
+
first, last = int(occupied_columns[0]), int(occupied_columns[-1])
|
| 126 |
+
|
| 127 |
+
# Vertical extent rather than pixel count per column. A count would be
|
| 128 |
+
# thinned by a hole in the mask β a fence rail across the animal, a patch
|
| 129 |
+
# the segmenter lost β and the extent is what a depth means.
|
| 130 |
+
rows = np.arange(binary.shape[0])[:, None]
|
| 131 |
+
masked_rows = np.where(binary, rows, -1)
|
| 132 |
+
top = np.where(binary.any(axis=0), np.where(binary, rows, binary.shape[0]).min(axis=0), 0)
|
| 133 |
+
bottom = masked_rows.max(axis=0)
|
| 134 |
+
column_height = np.where(binary.any(axis=0), bottom - top + 1, 0).astype(float)
|
| 135 |
+
|
| 136 |
+
peak = float(column_height.max())
|
| 137 |
+
if peak <= 0:
|
| 138 |
+
raise MaskUnusable("The mask has no vertical extent.")
|
| 139 |
+
|
| 140 |
+
trunk_columns = np.flatnonzero(column_height >= TRUNK_THRESHOLD * peak)
|
| 141 |
+
if trunk_columns.size == 0:
|
| 142 |
+
raise MaskUnusable("No column reaches the trunk threshold.")
|
| 143 |
+
|
| 144 |
+
# The largest *contiguous* run, not simply the first and last column over
|
| 145 |
+
# threshold. A raised head can clear the threshold on its own and would
|
| 146 |
+
# otherwise stretch the trunk span across the neck's gap.
|
| 147 |
+
breaks = np.flatnonzero(np.diff(trunk_columns) > 1)
|
| 148 |
+
starts = np.concatenate(([0], breaks + 1))
|
| 149 |
+
ends = np.concatenate((breaks, [trunk_columns.size - 1]))
|
| 150 |
+
widest = int(np.argmax(ends - starts))
|
| 151 |
+
trunk_low = int(trunk_columns[starts[widest]])
|
| 152 |
+
trunk_high = int(trunk_columns[ends[widest]])
|
| 153 |
+
|
| 154 |
+
span = trunk_high - trunk_low + 1
|
| 155 |
+
window_low = trunk_low + int(CHEST_WINDOW[0] * span)
|
| 156 |
+
window_high = trunk_low + max(int(CHEST_WINDOW[1] * span), 1)
|
| 157 |
+
window = column_height[window_low:window_high + 1]
|
| 158 |
+
if window.size == 0:
|
| 159 |
+
raise MaskUnusable("The trunk is too short to contain a chest window.")
|
| 160 |
+
chest_column = window_low + int(np.argmax(window))
|
| 161 |
+
|
| 162 |
+
inside = binary[:, trunk_low:trunk_high + 1].sum()
|
| 163 |
+
total = binary.sum()
|
| 164 |
+
|
| 165 |
+
return PixelProfile(
|
| 166 |
+
column_height=column_height,
|
| 167 |
+
full_span=(first, last),
|
| 168 |
+
trunk_span=(trunk_low, trunk_high),
|
| 169 |
+
chest_column=chest_column,
|
| 170 |
+
trimmed_fraction=float(1.0 - inside / total) if total else 0.0,
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def rear_width_px(mask: np.ndarray) -> float:
|
| 175 |
+
"""Widest horizontal extent of a rear-view mask.
|
| 176 |
+
|
| 177 |
+
Β§22's fallback is "side + rear images", and this is the rear half of it: the
|
| 178 |
+
width that, with the side view's depth, gives an ellipse to take a girth
|
| 179 |
+
from. It returns the widest row rather than the mean because the chest is
|
| 180 |
+
the widest part of a rear silhouette below the hips β but a rear view also
|
| 181 |
+
contains the hips, which on many animals are wider, and nothing here
|
| 182 |
+
separates them. That confusion is a known bias of this proxy and it runs in
|
| 183 |
+
the direction of over-estimating girth.
|
| 184 |
+
"""
|
| 185 |
+
binary = np.asarray(mask) > 0
|
| 186 |
+
if binary.ndim != 2:
|
| 187 |
+
raise MaskUnusable(f"Expected a 2-D mask, got {binary.ndim} dimensions.")
|
| 188 |
+
if not binary.any():
|
| 189 |
+
raise MaskUnusable("The mask is empty; nothing was segmented.")
|
| 190 |
+
columns = np.flatnonzero(binary.any(axis=0))
|
| 191 |
+
left = np.where(binary, np.arange(binary.shape[1])[None, :], binary.shape[1]).min(axis=1)
|
| 192 |
+
right = np.where(binary, np.arange(binary.shape[1])[None, :], -1).max(axis=1)
|
| 193 |
+
widths = np.where(binary.any(axis=1), right - left + 1, 0)
|
| 194 |
+
if not columns.size:
|
| 195 |
+
raise MaskUnusable("The mask has no horizontal extent.")
|
| 196 |
+
return float(widths.max())
|
app/adapters/geometry/equations.py
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Published livestock weight equations, used exactly as published.
|
| 2 |
+
|
| 3 |
+
Directive Β§22 ends its pipeline at "classical livestock weight equation". This
|
| 4 |
+
module is that step, and the constraint on it is the whole directive's premise:
|
| 5 |
+
**nothing here is fitted.** Every coefficient below was published by somebody
|
| 6 |
+
else, against their own cattle, and is reproduced unchanged. Fitting a curve to
|
| 7 |
+
the animals this repository can measure would turn a zero-training capability
|
| 8 |
+
into a one-dataset model and would make every accuracy figure a training score.
|
| 9 |
+
|
| 10 |
+
That constraint costs something, and the cost is the point. An equation derived
|
| 11 |
+
from Kenyan shorthorn zebu applied to Chinese yellow cattle is out of domain, and
|
| 12 |
+
the error it makes is real information about how far a published relationship
|
| 13 |
+
travels. A fitted curve would hide exactly that.
|
| 14 |
+
|
| 15 |
+
## What each equation needs
|
| 16 |
+
|
| 17 |
+
Two shapes. `SCHAEFFER` is volumetric β it needs a girth **and** a length, and it
|
| 18 |
+
is the one Β§22's pipeline is written for. The other two are girth-only linear or
|
| 19 |
+
power laws, which need one measurement and are therefore the ones that survive a
|
| 20 |
+
capture where the animal's length could not be resolved.
|
| 21 |
+
|
| 22 |
+
## Why the error bands are here and not computed
|
| 23 |
+
|
| 24 |
+
`published_relative_error` is what the **authors** reported on **their** animals.
|
| 25 |
+
It is a `PRIOR` in the sense `experiments.harness.metrics.UncertaintyBasis` means
|
| 26 |
+
it, and it must never be presented as an Animap measurement. Directive Β§37 keeps
|
| 27 |
+
those apart, and `estimate()` labels which one it is holding.
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
from __future__ import annotations
|
| 31 |
+
|
| 32 |
+
import math
|
| 33 |
+
from dataclasses import dataclass
|
| 34 |
+
from typing import Callable
|
| 35 |
+
|
| 36 |
+
#: Conversion constants, spelled out so the derived Schaeffer coefficient below
|
| 37 |
+
#: cannot drift from them and cannot be mistaken for a fitted number.
|
| 38 |
+
KG_PER_POUND = 0.45359237
|
| 39 |
+
METRES_PER_INCH = 0.0254
|
| 40 |
+
|
| 41 |
+
#: Schaeffer's rule in pounds and inches: W = girthΒ² Γ length / 300.
|
| 42 |
+
SCHAEFFER_DIVISOR_LB_IN = 300.0
|
| 43 |
+
|
| 44 |
+
#: The same rule in kilograms and metres, derived rather than quoted:
|
| 45 |
+
#:
|
| 46 |
+
#: W_kg = KG_PER_POUND Γ (HG_m / m_per_in)Β² Γ (BL_m / m_per_in) / 300
|
| 47 |
+
#: = KG_PER_POUND / (m_per_inΒ³ Γ 300) Γ HG_mΒ² Γ BL_m
|
| 48 |
+
#:
|
| 49 |
+
#: which comes to 92.27. Worked check, matching the figure the rule is usually
|
| 50 |
+
#: quoted with: a 70 in girth and 78 in length give 1,274 lb; 177.8 cm and
|
| 51 |
+
#: 198.12 cm give 577.9 kg, and 1,274 lb is 577.9 kg.
|
| 52 |
+
SCHAEFFER_COEFFICIENT_KG_M = KG_PER_POUND / (METRES_PER_INCH ** 3 * SCHAEFFER_DIVISOR_LB_IN)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class OutOfRange(ValueError):
|
| 56 |
+
"""A measurement outside the range the equation was published over.
|
| 57 |
+
|
| 58 |
+
Raised rather than extrapolated. A power law fitted between 60 cm and 200 cm
|
| 59 |
+
of girth returns a number for 10 cm as readily as for 150 cm, and the number
|
| 60 |
+
it returns for 10 cm is not a weight.
|
| 61 |
+
"""
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
@dataclass(frozen=True)
|
| 65 |
+
class WeightEquation:
|
| 66 |
+
"""One published relationship, with the evidence for it attached.
|
| 67 |
+
|
| 68 |
+
Every field except `apply` is bibliography. It is here rather than in a
|
| 69 |
+
README because the README is not what runs, and because a coefficient whose
|
| 70 |
+
source is three documents away is a coefficient somebody will eventually
|
| 71 |
+
"improve".
|
| 72 |
+
"""
|
| 73 |
+
|
| 74 |
+
id: str
|
| 75 |
+
#: What it needs. `("heart_girth_cm",)` or `("heart_girth_cm", "body_length_cm")`.
|
| 76 |
+
inputs: tuple[str, ...]
|
| 77 |
+
citation: str
|
| 78 |
+
#: Verbatim, in the authors' own notation, so a reader can check the code
|
| 79 |
+
#: against the paper without opening the paper.
|
| 80 |
+
published_form: str
|
| 81 |
+
#: The animals it was fitted on. The single most important field here: it is
|
| 82 |
+
#: what makes an out-of-domain application visible as one.
|
| 83 |
+
population: str
|
| 84 |
+
sample_size: int
|
| 85 |
+
licence: str
|
| 86 |
+
source_url: str
|
| 87 |
+
#: Valid girth range in cm, from the paper where the paper states one and
|
| 88 |
+
#: from the paper's reported animals where it does not.
|
| 89 |
+
girth_range_cm: tuple[float, float]
|
| 90 |
+
#: The authors' own accuracy, as a fraction. A PRIOR, never an Animap
|
| 91 |
+
#: measurement. `None` where the paper reports no usable figure.
|
| 92 |
+
published_relative_error: float | None
|
| 93 |
+
published_error_basis: str
|
| 94 |
+
apply: Callable[..., float]
|
| 95 |
+
|
| 96 |
+
def __call__(self, **measurements: float) -> float:
|
| 97 |
+
"""Evaluate, or refuse.
|
| 98 |
+
|
| 99 |
+
Refuses on a missing input rather than defaulting one, because a
|
| 100 |
+
volumetric equation silently handed a default length returns a weight
|
| 101 |
+
that varies only with girth and looks nothing like a failure.
|
| 102 |
+
"""
|
| 103 |
+
missing = [name for name in self.inputs if measurements.get(name) is None]
|
| 104 |
+
if missing:
|
| 105 |
+
raise OutOfRange(
|
| 106 |
+
f"{self.id} needs {', '.join(self.inputs)}; "
|
| 107 |
+
f"{', '.join(missing)} was not measured."
|
| 108 |
+
)
|
| 109 |
+
girth = measurements["heart_girth_cm"]
|
| 110 |
+
low, high = self.girth_range_cm
|
| 111 |
+
if not low <= girth <= high:
|
| 112 |
+
raise OutOfRange(
|
| 113 |
+
f"{self.id} was published over girths of {low:g}β{high:g} cm and "
|
| 114 |
+
f"this animal measures {girth:g} cm. Extrapolating a fitted curve "
|
| 115 |
+
f"past its own data is how a calf becomes a bull."
|
| 116 |
+
)
|
| 117 |
+
return float(self.apply(**{k: measurements[k] for k in self.inputs}))
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def _schaeffer(heart_girth_cm: float, body_length_cm: float) -> float:
|
| 121 |
+
return SCHAEFFER_COEFFICIENT_KG_M * (heart_girth_cm / 100.0) ** 2 * (body_length_cm / 100.0)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _lesosky(heart_girth_cm: float) -> float:
|
| 125 |
+
# weight**0.262 = 0.95 + 0.022 Γ girth, inverted. The exponent is small, so
|
| 126 |
+
# the inverse power is large (1/0.262 β 3.82) and the equation is extremely
|
| 127 |
+
# sensitive to girth: a 1% girth error becomes roughly 3% of weight at a
|
| 128 |
+
# typical adult girth. That sensitivity is a property of the published
|
| 129 |
+
# relationship, not of this implementation, and `experiments/cattle_weight`
|
| 130 |
+
# measures what it does to a vision-derived girth.
|
| 131 |
+
return (0.95 + 0.022 * heart_girth_cm) ** (1.0 / 0.262)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def _odadi(heart_girth_cm: float) -> float:
|
| 135 |
+
return -265.0 + 3.37 * heart_girth_cm
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
#: Schaeffer's rule. The volumetric one, and the only one here that uses a body
|
| 139 |
+
#: length β which is why Β§22's pipeline derives a length at all.
|
| 140 |
+
SCHAEFFER = WeightEquation(
|
| 141 |
+
id="schaeffer",
|
| 142 |
+
inputs=("heart_girth_cm", "body_length_cm"),
|
| 143 |
+
citation=(
|
| 144 |
+
"Schaeffer's formula, in general agricultural use since the early 20th "
|
| 145 |
+
"century and with no single primary citation. Evaluated for Bos indicus "
|
| 146 |
+
"in Sarwar et al. (2018), 'Accuracy of estimates for live body weight "
|
| 147 |
+
"using Schaeffer's formula in non-descript cattle (Bos indicus), Nili "
|
| 148 |
+
"Ravi buffaloes and their calves using linear body measurements'."
|
| 149 |
+
),
|
| 150 |
+
published_form="W (lb) = girth (in)Β² Γ length (in) Γ· 300",
|
| 151 |
+
population=(
|
| 152 |
+
"No stated origin population. Applied across cattle generally, which is "
|
| 153 |
+
"both why it is the default here and why it should be expected to be "
|
| 154 |
+
"biased on any particular breed."
|
| 155 |
+
),
|
| 156 |
+
sample_size=0,
|
| 157 |
+
licence="Formula, not a copyrightable work. No licence attaches.",
|
| 158 |
+
source_url="https://www.tandfonline.com/doi/full/10.1080/09712119.2017.1302876",
|
| 159 |
+
# Deliberately wide, because this rule has no published population and so no
|
| 160 |
+
# published range. The bounds are the physical span of cattle from a large
|
| 161 |
+
# calf to a mature bull, and their purpose is to catch a segmentation failure
|
| 162 |
+
# that produced a girth of 8 cm, not to model a breed.
|
| 163 |
+
girth_range_cm=(80.0, 260.0),
|
| 164 |
+
published_relative_error=None,
|
| 165 |
+
published_error_basis=(
|
| 166 |
+
"No single published figure. Sarwar et al. report estimates 'not "
|
| 167 |
+
"significantly different from the weighbridge'; popular sources quote "
|
| 168 |
+
"around 5% near 500 kg. Neither is an interval this code can use, so "
|
| 169 |
+
"nothing is claimed."
|
| 170 |
+
),
|
| 171 |
+
apply=_schaeffer,
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
#: The strongest evidence of the three, and the closest to Animap's animals.
|
| 175 |
+
LESOSKY_SHZ = WeightEquation(
|
| 176 |
+
id="lesosky-shz-2012",
|
| 177 |
+
inputs=("heart_girth_cm",),
|
| 178 |
+
citation=(
|
| 179 |
+
"Lesosky M, Dumas S, Conradie I, et al. (2012). 'A live weightβheart "
|
| 180 |
+
"girth relationship for accurate dosing of east African shorthorn zebu "
|
| 181 |
+
"cattle.' Trop Anim Health Prod 45(1):311β316. doi:10.1007/s11250-012-0220-3"
|
| 182 |
+
),
|
| 183 |
+
published_form="weight^0.262 = 0.95 + 0.022 Γ girth (weight in kg, girth in cm)",
|
| 184 |
+
population=(
|
| 185 |
+
"East African shorthorn zebu, western Kenya, one week old to fully "
|
| 186 |
+
"mature. Indigenous tropical Bos indicus, which is the closest published "
|
| 187 |
+
"population to the cattle Animap serves."
|
| 188 |
+
),
|
| 189 |
+
sample_size=703,
|
| 190 |
+
licence="CC BY (the article states the Creative Commons Attribution License)",
|
| 191 |
+
source_url="https://pmc.ncbi.nlm.nih.gov/articles/PMC3552367/",
|
| 192 |
+
# The paper covers one-week-old calves upward; 40 cm is a newborn's girth
|
| 193 |
+
# and 220 cm covers a mature zebu bull with margin.
|
| 194 |
+
girth_range_cm=(40.0, 220.0),
|
| 195 |
+
published_relative_error=0.20,
|
| 196 |
+
published_error_basis=(
|
| 197 |
+
"The authors' own claim, quoted: '95% prediction intervals fall within "
|
| 198 |
+
"the Β±20% body weight error band regarded as acceptable when dosing "
|
| 199 |
+
"livestock'. RΒ²(adj) 0.98 over 703 animals, 300 modelling and 403 "
|
| 200 |
+
"validation. This is THEIR interval on THEIR cattle β a prior, not an "
|
| 201 |
+
"Animap measurement."
|
| 202 |
+
),
|
| 203 |
+
apply=_lesosky,
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
#: A linear girth-only rule, kept because it disagrees with the one above. Two
|
| 207 |
+
#: published equations that diverge on the same animal are the cheapest available
|
| 208 |
+
#: evidence about how much of the error is the equation rather than the geometry.
|
| 209 |
+
ODADI_BORAN = WeightEquation(
|
| 210 |
+
id="odadi-boran-2018",
|
| 211 |
+
inputs=("heart_girth_cm",),
|
| 212 |
+
citation=(
|
| 213 |
+
"Odadi WO (2018). 'Using heart girth to estimate live weight of heifers "
|
| 214 |
+
"(Bos indicus) in pastoral rangelands of northern Kenya.' Livestock "
|
| 215 |
+
"Research for Rural Development 30(1), article 16."
|
| 216 |
+
),
|
| 217 |
+
published_form="LW (kg) = β265 + 3.37 Γ HG (cm)",
|
| 218 |
+
population=(
|
| 219 |
+
"160 Boran zebu and Boran Γ Small East African Zebu heifers, aged 1β3 "
|
| 220 |
+
"years, on eight group ranches in Laikipia and Isiolo, northern Kenya. "
|
| 221 |
+
"**Heifers only** β a linear rule fitted to a narrow age band, so it is "
|
| 222 |
+
"expected to fail on calves and on mature bulls, and the intercept of "
|
| 223 |
+
"β265 kg makes that failure loud rather than subtle."
|
| 224 |
+
),
|
| 225 |
+
sample_size=160,
|
| 226 |
+
licence=(
|
| 227 |
+
"Livestock Research for Rural Development is open access; the article "
|
| 228 |
+
"states no explicit licence, so the equation is used as a fact and the "
|
| 229 |
+
"text is not reproduced beyond citation."
|
| 230 |
+
),
|
| 231 |
+
source_url="https://www.lrrd.org/lrrd30/1/wood30016.html",
|
| 232 |
+
# Below about 79 cm this line returns a negative weight, which is the
|
| 233 |
+
# clearest possible demonstration of why a fitted line needs a range on it.
|
| 234 |
+
girth_range_cm=(120.0, 200.0),
|
| 235 |
+
published_relative_error=0.073,
|
| 236 |
+
published_error_basis=(
|
| 237 |
+
"Residual standard error 12.8 kg, which the author gives as 7.3% of mean "
|
| 238 |
+
"live weight. RΒ² 0.90 over 160 heifers. A residual standard error is one "
|
| 239 |
+
"sigma on the fitting set, NOT a 95% interval and NOT an out-of-sample "
|
| 240 |
+
"figure; it is the most flattering of the three numbers here and should "
|
| 241 |
+
"be read as such."
|
| 242 |
+
),
|
| 243 |
+
apply=_odadi,
|
| 244 |
+
)
|
| 245 |
+
|
| 246 |
+
EQUATIONS: tuple[WeightEquation, ...] = (SCHAEFFER, LESOSKY_SHZ, ODADI_BORAN)
|
| 247 |
+
BY_ID = {equation.id: equation for equation in EQUATIONS}
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
def ellipse_perimeter_cm(depth_cm: float, width_cm: float) -> float:
|
| 251 |
+
"""Ramanujan's second approximation to an ellipse's perimeter.
|
| 252 |
+
|
| 253 |
+
This is the **girth proxy** Β§22's pipeline needs, and the place the pipeline
|
| 254 |
+
is most likely to be wrong. A side view gives chest depth, a rear view gives
|
| 255 |
+
chest width, and a circumference has to come from somewhere; treating the
|
| 256 |
+
cross-section as an ellipse is the standard move and the chest is not an
|
| 257 |
+
ellipse. It is flatter across the back and rounder underneath, so this is
|
| 258 |
+
expected to be biased, and `experiments/cattle_weight` measures the bias
|
| 259 |
+
against tape-measured heart girth rather than assuming it away.
|
| 260 |
+
|
| 261 |
+
Ramanujan's approximation itself is accurate to better than one part in 10β΅
|
| 262 |
+
for any eccentricity a chest could have, so none of the error this proxy
|
| 263 |
+
makes is the approximation's.
|
| 264 |
+
"""
|
| 265 |
+
a, b = depth_cm / 2.0, width_cm / 2.0
|
| 266 |
+
if a <= 0 or b <= 0:
|
| 267 |
+
raise OutOfRange("An ellipse needs two positive semi-axes.")
|
| 268 |
+
return float(math.pi * (3 * (a + b) - math.sqrt((3 * a + b) * (a + 3 * b))))
|
app/adapters/geometry/weight.py
ADDED
|
@@ -0,0 +1,334 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""A weight band, never a weight.
|
| 2 |
+
|
| 3 |
+
Directive Β§22's initial UI is *"Experimental weight estimate / 350β430 kg"* with
|
| 4 |
+
*"Add scale weight"* beside it, and Β§37 gives the reason: a single number is a
|
| 5 |
+
claim about an animal, and nothing in this pipeline can make that claim. So there
|
| 6 |
+
is no function here that returns a float. `estimate` returns a `WeightBand`, and
|
| 7 |
+
a `WeightBand` has no `.value`.
|
| 8 |
+
|
| 9 |
+
## Where the width comes from
|
| 10 |
+
|
| 11 |
+
Two sources, kept apart because they behave differently and a farm should not be
|
| 12 |
+
shown their sum as though it were one thing.
|
| 13 |
+
|
| 14 |
+
**Scale error** is the metric uncertainty on the measurements β how well ARCore,
|
| 15 |
+
VGGT or a reference marker pinned what a pixel is worth. It propagates through
|
| 16 |
+
the equation, and for a volumetric rule it amplifies: weight goes as girthΒ², so a
|
| 17 |
+
2% scale error is roughly 6% of weight once the length term is included. This
|
| 18 |
+
module propagates it numerically by re-evaluating the equation at the perturbed
|
| 19 |
+
inputs, which is exact for any equation shape and cannot get an analytic
|
| 20 |
+
derivative wrong.
|
| 21 |
+
|
| 22 |
+
**Equation error** is how wrong the published relationship is on an animal it was
|
| 23 |
+
not fitted to. It is a `PRIOR` until an Animap benchmark measures it, and
|
| 24 |
+
`WeightBand` records which of the two it is holding so Β§37's separation survives
|
| 25 |
+
into the object the API returns.
|
| 26 |
+
|
| 27 |
+
## Sensitivity is reported, not buried
|
| 28 |
+
|
| 29 |
+
`sensitivity` is d(ln W)/d(ln girth) computed numerically at the animal's own
|
| 30 |
+
measurements. It is the number that decides whether this capability is worth
|
| 31 |
+
building: if a 1% girth error costs 4% of weight, then the whole question is what
|
| 32 |
+
metric accuracy the capture can deliver, and no amount of model work substitutes
|
| 33 |
+
for it.
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
from __future__ import annotations
|
| 37 |
+
|
| 38 |
+
import math
|
| 39 |
+
from dataclasses import dataclass, field
|
| 40 |
+
|
| 41 |
+
from app.adapters.geometry.equations import (
|
| 42 |
+
EQUATIONS,
|
| 43 |
+
OutOfRange,
|
| 44 |
+
WeightEquation,
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
#: Fractional perturbation used for the numerical derivative. Small enough that
|
| 48 |
+
#: the second-order term is negligible for these equation shapes, large enough
|
| 49 |
+
#: to stay well clear of float64 cancellation.
|
| 50 |
+
DERIVATIVE_STEP = 1e-4
|
| 51 |
+
|
| 52 |
+
#: The narrowest band this module will publish, as a fraction of the estimate.
|
| 53 |
+
#: Β§22 says "the initial range can be wide" and says nothing about a floor; this
|
| 54 |
+
#: one exists because a band narrower than the best published equation's own
|
| 55 |
+
#: interval would be Animap claiming to have improved on the paper it is quoting.
|
| 56 |
+
MINIMUM_RELATIVE_HALF_WIDTH = 0.10
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
@dataclass(frozen=True)
|
| 60 |
+
class BodyMeasurements:
|
| 61 |
+
"""What the geometry stage produced, and how well scaled it is.
|
| 62 |
+
|
| 63 |
+
`scale_relative_error` is not optional and has no default. A measurement in
|
| 64 |
+
centimetres whose scale nobody characterised is a measurement in arbitrary
|
| 65 |
+
units with `cm` written after it, and this whole capability turns on the
|
| 66 |
+
difference.
|
| 67 |
+
"""
|
| 68 |
+
|
| 69 |
+
heart_girth_cm: float | None
|
| 70 |
+
body_length_cm: float | None
|
| 71 |
+
#: Fractional 1-sigma error on the metric scale. From
|
| 72 |
+
#: `signal.geometry.Scale.relative_error` for a marker, from the depth
|
| 73 |
+
#: source's own characterisation for ARCore or VGGT.
|
| 74 |
+
scale_relative_error: float
|
| 75 |
+
#: Free text naming what established the scale β a marker, an AR session, a
|
| 76 |
+
#: known-size object, or the fact that nothing did.
|
| 77 |
+
scale_source: str
|
| 78 |
+
withers_height_cm: float | None = None
|
| 79 |
+
chest_depth_cm: float | None = None
|
| 80 |
+
chest_width_cm: float | None = None
|
| 81 |
+
#: True when the girth came from an ellipse through a depth and a width
|
| 82 |
+
#: rather than from a tape. Carried because the proxy has a bias of its own
|
| 83 |
+
#: that is separate from the scale error.
|
| 84 |
+
girth_is_proxy: bool = False
|
| 85 |
+
|
| 86 |
+
def as_inputs(self) -> dict[str, float | None]:
|
| 87 |
+
return {
|
| 88 |
+
"heart_girth_cm": self.heart_girth_cm,
|
| 89 |
+
"body_length_cm": self.body_length_cm,
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
@dataclass(frozen=True)
|
| 94 |
+
class EquationResult:
|
| 95 |
+
"""One equation's answer on one animal, with what moves it."""
|
| 96 |
+
|
| 97 |
+
equation_id: str
|
| 98 |
+
kg: float
|
| 99 |
+
#: Half-width in kg from scale error alone.
|
| 100 |
+
scale_half_width_kg: float
|
| 101 |
+
#: d(ln W)/d(ln girth). Dimensionless amplification of girth error.
|
| 102 |
+
sensitivity: float
|
| 103 |
+
#: The authors' own error, as a fraction. `None` where they published none.
|
| 104 |
+
published_relative_error: float | None
|
| 105 |
+
population: str
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@dataclass(frozen=True)
|
| 109 |
+
class WeightBand:
|
| 110 |
+
"""What the app may show. Deliberately has no single value.
|
| 111 |
+
|
| 112 |
+
`statement` is words rather than a number because Β§37's worked example is
|
| 113 |
+
words. `basis` says whether the width came from an Animap measurement or from
|
| 114 |
+
the papers, and it is the field that must reach the screen alongside the
|
| 115 |
+
kilograms.
|
| 116 |
+
"""
|
| 117 |
+
|
| 118 |
+
low_kg: float
|
| 119 |
+
high_kg: float
|
| 120 |
+
equations: tuple[EquationResult, ...]
|
| 121 |
+
basis: str
|
| 122 |
+
caveats: tuple[str, ...] = field(default_factory=tuple)
|
| 123 |
+
|
| 124 |
+
@property
|
| 125 |
+
def statement(self) -> str:
|
| 126 |
+
return (
|
| 127 |
+
f"Experimental weight estimate, "
|
| 128 |
+
f"{round(self.low_kg / 5) * 5:g}β{round(self.high_kg / 5) * 5:g} kg"
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
@property
|
| 132 |
+
def relative_half_width(self) -> float:
|
| 133 |
+
midpoint = (self.low_kg + self.high_kg) / 2.0
|
| 134 |
+
return (self.high_kg - self.low_kg) / 2.0 / midpoint if midpoint else float("inf")
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
class NotEnoughGeometry(RuntimeError):
|
| 138 |
+
"""No equation could run on what the capture produced.
|
| 139 |
+
|
| 140 |
+
Raised rather than returning a very wide band. Β§22's `reject_if` list already
|
| 141 |
+
names `insufficient_geometry` as a re-capture prompt, and a band from no
|
| 142 |
+
usable measurement is not wide, it is meaningless.
|
| 143 |
+
"""
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def _sensitivity(equation: WeightEquation, inputs: dict[str, float | None]) -> float:
|
| 147 |
+
"""d(ln W)/d(ln girth), numerically, at this animal's own measurements.
|
| 148 |
+
|
| 149 |
+
Numerical rather than analytic on purpose. The three equations have three
|
| 150 |
+
different shapes β a power law, a cube-ish volume rule and a straight line β
|
| 151 |
+
and a hand-differentiated version of each is three more places to be wrong.
|
| 152 |
+
"""
|
| 153 |
+
girth = inputs["heart_girth_cm"]
|
| 154 |
+
if girth is None:
|
| 155 |
+
return float("nan")
|
| 156 |
+
up = dict(inputs, heart_girth_cm=girth * (1 + DERIVATIVE_STEP))
|
| 157 |
+
down = dict(inputs, heart_girth_cm=girth * (1 - DERIVATIVE_STEP))
|
| 158 |
+
try:
|
| 159 |
+
high, low = equation(**up), equation(**down)
|
| 160 |
+
except OutOfRange:
|
| 161 |
+
# An animal sitting exactly on the boundary of the equation's published
|
| 162 |
+
# range: the perturbation steps outside it and the range guard fires,
|
| 163 |
+
# correctly. A derivative that cannot be taken is `nan`, not a crash β
|
| 164 |
+
# `estimate` has already accepted this animal, and a band is still owed.
|
| 165 |
+
return float("nan")
|
| 166 |
+
if low <= 0 or high <= 0:
|
| 167 |
+
# The linear rule goes non-positive below its intercept. A log derivative
|
| 168 |
+
# does not exist there, and reporting one would be inventing a number.
|
| 169 |
+
return float("nan")
|
| 170 |
+
return (math.log(high) - math.log(low)) / (2 * DERIVATIVE_STEP)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _scale_half_width(equation: WeightEquation, inputs: dict[str, float | None],
|
| 174 |
+
relative_error: float) -> float:
|
| 175 |
+
"""Half-width in kg from perturbing every linear measurement together.
|
| 176 |
+
|
| 177 |
+
Together, not independently. A scale error is a single multiplicative factor
|
| 178 |
+
on every length the capture produced β it is one mistake about what a pixel
|
| 179 |
+
is worth, not several independent ones β so girth and length move in step and
|
| 180 |
+
the errors add rather than partially cancelling. Treating them as independent
|
| 181 |
+
would understate the band, which is the direction that matters.
|
| 182 |
+
"""
|
| 183 |
+
if relative_error <= 0:
|
| 184 |
+
return 0.0
|
| 185 |
+
centre = equation(**inputs)
|
| 186 |
+
sides: list[float] = []
|
| 187 |
+
for sign in (+1, -1):
|
| 188 |
+
perturbed = {k: (v * (1 + sign * relative_error) if v is not None else None)
|
| 189 |
+
for k, v in inputs.items()}
|
| 190 |
+
try:
|
| 191 |
+
sides.append(equation(**perturbed))
|
| 192 |
+
except OutOfRange:
|
| 193 |
+
# The perturbed animal falls outside the range this equation was
|
| 194 |
+
# published over. That is not nothing to report: it means a
|
| 195 |
+
# plausible capture error puts this animal off the end of the curve.
|
| 196 |
+
# Fall back to whichever side is still computable rather than
|
| 197 |
+
# silently returning a half-width of zero, and report `nan` when
|
| 198 |
+
# neither is β `estimate` turns that into a caveat.
|
| 199 |
+
continue
|
| 200 |
+
if len(sides) == 2:
|
| 201 |
+
return abs(sides[0] - sides[1]) / 2.0
|
| 202 |
+
if len(sides) == 1:
|
| 203 |
+
return abs(sides[0] - centre)
|
| 204 |
+
return float("nan")
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def estimate(measurements: BodyMeasurements, *,
|
| 208 |
+
equations: tuple[WeightEquation, ...] = EQUATIONS,
|
| 209 |
+
measured_relative_error: float | None = None,
|
| 210 |
+
measured_basis: str = "") -> WeightBand:
|
| 211 |
+
"""Every equation that can run, and the band that covers them.
|
| 212 |
+
|
| 213 |
+
`measured_relative_error` is an Animap benchmark's figure. When it is absent
|
| 214 |
+
the band falls back to the published intervals and says so, because Β§37 will
|
| 215 |
+
not let a prior be presented as a measurement. When it is present it
|
| 216 |
+
**replaces** the published error rather than being combined with it: the
|
| 217 |
+
benchmark measured the equation on Animap's own geometry, so the published
|
| 218 |
+
interval is already inside it and adding both would double-count.
|
| 219 |
+
"""
|
| 220 |
+
inputs = measurements.as_inputs()
|
| 221 |
+
results: list[EquationResult] = []
|
| 222 |
+
refusals: list[str] = []
|
| 223 |
+
#: Equations the scale error itself pushed off their published range.
|
| 224 |
+
scale_dropped: list[str] = []
|
| 225 |
+
|
| 226 |
+
for equation in equations:
|
| 227 |
+
try:
|
| 228 |
+
kg = equation(**inputs)
|
| 229 |
+
except OutOfRange as refused:
|
| 230 |
+
refusals.append(str(refused))
|
| 231 |
+
continue
|
| 232 |
+
results.append(EquationResult(
|
| 233 |
+
equation_id=equation.id,
|
| 234 |
+
kg=kg,
|
| 235 |
+
scale_half_width_kg=_scale_half_width(
|
| 236 |
+
equation, inputs, measurements.scale_relative_error),
|
| 237 |
+
sensitivity=_sensitivity(equation, inputs),
|
| 238 |
+
published_relative_error=equation.published_relative_error,
|
| 239 |
+
population=equation.population,
|
| 240 |
+
))
|
| 241 |
+
|
| 242 |
+
if not results:
|
| 243 |
+
raise NotEnoughGeometry(
|
| 244 |
+
"No published equation could be applied to this capture. "
|
| 245 |
+
+ " ".join(refusals)
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
caveats: list[str] = []
|
| 249 |
+
if measurements.girth_is_proxy:
|
| 250 |
+
caveats.append(
|
| 251 |
+
"The girth is an ellipse through a measured chest depth and width, "
|
| 252 |
+
"not a tape around the animal. A chest is not an ellipse."
|
| 253 |
+
)
|
| 254 |
+
if measurements.scale_relative_error <= 0:
|
| 255 |
+
caveats.append(
|
| 256 |
+
"No scale error was supplied, so the band below carries none. That "
|
| 257 |
+
"is almost certainly wrong: every metric scale has an error."
|
| 258 |
+
)
|
| 259 |
+
caveats.extend(refusals)
|
| 260 |
+
|
| 261 |
+
# The band spans every equation's answer, widened by each one's own
|
| 262 |
+
# uncertainty. Spanning rather than averaging is deliberate: two published
|
| 263 |
+
# equations disagreeing by 15% on the same animal is evidence about the
|
| 264 |
+
# equations, and averaging it away would produce a narrow band built on a
|
| 265 |
+
# disagreement nobody was told about.
|
| 266 |
+
lows, highs = [], []
|
| 267 |
+
for result in results:
|
| 268 |
+
if measured_relative_error is not None:
|
| 269 |
+
half = result.kg * measured_relative_error
|
| 270 |
+
elif result.published_relative_error is not None:
|
| 271 |
+
half = result.kg * result.published_relative_error
|
| 272 |
+
else:
|
| 273 |
+
# No published interval and no measurement. The scale term is all
|
| 274 |
+
# that is left, and it is not the dominant term, so the band would be
|
| 275 |
+
# falsely narrow. Widen to the floor and say why.
|
| 276 |
+
half = result.kg * MINIMUM_RELATIVE_HALF_WIDTH
|
| 277 |
+
caveats.append(
|
| 278 |
+
f"{result.equation_id} publishes no error interval, so its "
|
| 279 |
+
f"contribution to the band is a floor, not a measurement."
|
| 280 |
+
)
|
| 281 |
+
if math.isnan(result.scale_half_width_kg):
|
| 282 |
+
# **The equation is dropped, not banded on its published interval
|
| 283 |
+
# alone.** When a scale error this large moves the animal off the
|
| 284 |
+
# end of the published range, what this equation does under that
|
| 285 |
+
# error is *unknown*, and unknown is not zero. Falling back to the
|
| 286 |
+
# published interval made the band NARROWER as the scale error grew
|
| 287 |
+
# β at a 40% error the band spanned 0β1015 kg and at 80% it
|
| 288 |
+
# collapsed back to 317β521 kg, because every scale term had
|
| 289 |
+
# silently dropped out. A band that tightens as the capture gets
|
| 290 |
+
# worse is the most dangerous shape this function could have.
|
| 291 |
+
scale_dropped.append(result.equation_id)
|
| 292 |
+
continue
|
| 293 |
+
half = max(half, result.scale_half_width_kg)
|
| 294 |
+
lows.append(result.kg - half)
|
| 295 |
+
highs.append(result.kg + half)
|
| 296 |
+
|
| 297 |
+
# **Losing any equation to the scale error is a refusal, not a narrower
|
| 298 |
+
# band.** Dropping one still shrinks the span whenever the dropped equation
|
| 299 |
+
# was the one setting an extreme β at a 60% scale error two of the three
|
| 300 |
+
# fall off their published ranges and the band tightened from 1,015 kg wide
|
| 301 |
+
# to 799. A band that tightens as the capture gets worse is the most
|
| 302 |
+
# dangerous shape this function could have, so the capture is refused
|
| 303 |
+
# instead. Β§22's `reject_if` already lists `insufficient_geometry`, and a
|
| 304 |
+
# scale error this large is exactly that.
|
| 305 |
+
if scale_dropped or not lows:
|
| 306 |
+
raise NotEnoughGeometry(
|
| 307 |
+
f"A scale error of {measurements.scale_relative_error:.0%} moves "
|
| 308 |
+
f"this animal outside the published range of "
|
| 309 |
+
f"{', '.join(scale_dropped) or 'every equation'}, so what those "
|
| 310 |
+
f"equations would do under it is unknown rather than small. No band "
|
| 311 |
+
f"is given. Re-capture with a better scale reference."
|
| 312 |
+
)
|
| 313 |
+
|
| 314 |
+
low, high = min(lows), max(highs)
|
| 315 |
+
midpoint = (low + high) / 2.0
|
| 316 |
+
floor = midpoint * MINIMUM_RELATIVE_HALF_WIDTH
|
| 317 |
+
if (high - low) / 2.0 < floor:
|
| 318 |
+
low, high = midpoint - floor, midpoint + floor
|
| 319 |
+
|
| 320 |
+
if measured_relative_error is not None:
|
| 321 |
+
basis = measured_basis or "measured by an Animap benchmark"
|
| 322 |
+
else:
|
| 323 |
+
basis = (
|
| 324 |
+
"PRIOR β the equations' own published intervals, on the authors' own "
|
| 325 |
+
"cattle. No Animap benchmark stands behind this width."
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
return WeightBand(
|
| 329 |
+
low_kg=max(0.0, low),
|
| 330 |
+
high_kg=high,
|
| 331 |
+
equations=tuple(results),
|
| 332 |
+
basis=basis,
|
| 333 |
+
caveats=tuple(dict.fromkeys(caveats)),
|
| 334 |
+
)
|
app/adapters/licence_policy.py
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Whether a governance problem stops a load, or only records itself.
|
| 2 |
+
|
| 3 |
+
**Two settings, and they live here together so a deployment has one file to
|
| 4 |
+
read.** Both are environment variables, both are read at call time, and nothing
|
| 5 |
+
else in the codebase decides either.
|
| 6 |
+
|
| 7 |
+
ANIMAP_LICENCE_POLICY=enforce # default. A refused licence does not load.
|
| 8 |
+
ANIMAP_LICENCE_POLICY=record # it loads, loudly, and the fact is kept.
|
| 9 |
+
|
| 10 |
+
ANIMAP_ARTEFACT_IDENTITY=record # default. Bytes that contradict their card
|
| 11 |
+
# load, loudly, and the fact is kept.
|
| 12 |
+
ANIMAP_ARTEFACT_IDENTITY=refuse # they do not load.
|
| 13 |
+
|
| 14 |
+
They are separate variables because they answer different questions.
|
| 15 |
+
`ANIMAP_LICENCE_POLICY` is a permission question β *may Animap serve from these
|
| 16 |
+
terms?* β and the founder's standing instruction is that the answer is currently
|
| 17 |
+
yes for everything, so the work is not blocked while the ledger is kept.
|
| 18 |
+
`ANIMAP_ARTEFACT_IDENTITY` is an integrity question β *is this file what its
|
| 19 |
+
card says it is?* β and its default is `record` for the same reason and no
|
| 20 |
+
other: a refusal here would stop model work today, and the value of the check
|
| 21 |
+
right now is that the ledger says what actually shipped.
|
| 22 |
+
|
| 23 |
+
**`record` on either is only defensible while the record is true**, which is why
|
| 24 |
+
`app/adapters/fingerprints.py` exists at all: before it, an exception recorded
|
| 25 |
+
under `record` named the licence the *card* claimed, so a mislabelled artefact
|
| 26 |
+
produced a ledger entry that was wrong in exactly the case the ledger was for.
|
| 27 |
+
|
| 28 |
+
**Detection always runs, under both settings.** That is deliberate and it is the
|
| 29 |
+
part worth protecting: ADR 0017 records a watchdog defeating the label-based
|
| 30 |
+
version of this check by writing `"license": "Apache-2.0"` over a path to AGPL
|
| 31 |
+
weights, and the fix was to key the check to the runtime instead. `record` mode
|
| 32 |
+
does not weaken the detection, it changes what happens after it fires. A model
|
| 33 |
+
that is a licence problem stays a licence problem that anybody can find, which
|
| 34 |
+
is the whole point of keeping the machinery rather than deleting it.
|
| 35 |
+
|
| 36 |
+
**The default is `enforce`, and that is a decision rather than an oversight.**
|
| 37 |
+
Animap is a closed-source commercial product serving predictions from a private
|
| 38 |
+
API. AGPL-3.0's network clause and CC BY-NC's NonCommercial grant are live
|
| 39 |
+
exposures for exactly that shape of business, and ADR 0018 measured the cost of
|
| 40 |
+
avoiding the AGPL one at 8.8 points of coverage. Relaxing it is a decision with
|
| 41 |
+
a signature on it, so it is made by setting a variable on a deployment β an act
|
| 42 |
+
that appears in configuration and in `/health` β and not by a default that
|
| 43 |
+
nobody remembers choosing.
|
| 44 |
+
|
| 45 |
+
If the posture is relaxed, `record` mode is built so the debt is payable later:
|
| 46 |
+
every load that would have been refused is logged at ERROR, kept in
|
| 47 |
+
`RECORDED_EXCEPTIONS`, and surfaced by `scripts/licence_ledger.py`. The question
|
| 48 |
+
"what did we ship that we should not have?" has a one-command answer instead of
|
| 49 |
+
being archaeology.
|
| 50 |
+
"""
|
| 51 |
+
|
| 52 |
+
from __future__ import annotations
|
| 53 |
+
|
| 54 |
+
import logging
|
| 55 |
+
import os
|
| 56 |
+
from dataclasses import dataclass, field
|
| 57 |
+
from datetime import datetime, timezone
|
| 58 |
+
from enum import Enum
|
| 59 |
+
|
| 60 |
+
logger = logging.getLogger(__name__)
|
| 61 |
+
|
| 62 |
+
#: The one setting. Named here so a grep for it finds the definition, the
|
| 63 |
+
#: reader, and the documentation in the same file.
|
| 64 |
+
POLICY_ENV = "ANIMAP_LICENCE_POLICY"
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class LicencePolicy(str, Enum):
|
| 68 |
+
#: A licence Animap may not serve under stops the load. The capability goes
|
| 69 |
+
#: unavailable, which is a state the service already models honestly.
|
| 70 |
+
ENFORCE = "enforce"
|
| 71 |
+
#: It loads. The fact is logged at ERROR, recorded, and reported by
|
| 72 |
+
#: `/health` and the ledger. Nothing is silent and nothing is lost.
|
| 73 |
+
RECORD = "record"
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
DEFAULT_POLICY = LicencePolicy.ENFORCE
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def current() -> LicencePolicy:
|
| 80 |
+
"""Read at call time, not import time.
|
| 81 |
+
|
| 82 |
+
Same reasoning as `main._configured_token`: a change takes effect on restart
|
| 83 |
+
rather than needing a rebuild, and a test can set it without reloading the
|
| 84 |
+
module.
|
| 85 |
+
"""
|
| 86 |
+
raw = os.environ.get(POLICY_ENV, "").strip().lower()
|
| 87 |
+
if not raw:
|
| 88 |
+
return DEFAULT_POLICY
|
| 89 |
+
try:
|
| 90 |
+
return LicencePolicy(raw)
|
| 91 |
+
except ValueError:
|
| 92 |
+
# An unrecognised value falls back to the strict setting rather than to
|
| 93 |
+
# the permissive one. A typo in a deployment variable must not be the
|
| 94 |
+
# thing that puts AGPL weights in front of a farmer.
|
| 95 |
+
logger.error(
|
| 96 |
+
"%s=%r is not a recognised policy; falling back to %s. Valid: %s.",
|
| 97 |
+
POLICY_ENV, raw, DEFAULT_POLICY.value,
|
| 98 |
+
", ".join(p.value for p in LicencePolicy),
|
| 99 |
+
)
|
| 100 |
+
return DEFAULT_POLICY
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
#: The other setting. Same shape, same reading-at-call-time, same fallback rule.
|
| 104 |
+
IDENTITY_ENV = "ANIMAP_ARTEFACT_IDENTITY"
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
class IdentityPolicy(str, Enum):
|
| 108 |
+
#: An artefact whose bytes contradict its card loads, and the contradiction
|
| 109 |
+
#: is logged at ERROR and recorded. **The licence decision is still made on
|
| 110 |
+
#: what the bytes are**, never on what the card claimed β that is the part
|
| 111 |
+
#: that does not soften, because it is what makes the record true.
|
| 112 |
+
RECORD = "record"
|
| 113 |
+
#: It does not load. The capability goes unavailable, which the service
|
| 114 |
+
#: already models honestly.
|
| 115 |
+
REFUSE = "refuse"
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
#: `record`, and this is the founder's standing instruction rather than an
|
| 119 |
+
#: oversight: no model is dropped on a governance question right now, and the
|
| 120 |
+
#: value of this check today is an accurate ledger rather than a closed door.
|
| 121 |
+
DEFAULT_IDENTITY_POLICY = IdentityPolicy.RECORD
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def identity_policy() -> IdentityPolicy:
|
| 125 |
+
raw = os.environ.get(IDENTITY_ENV, "").strip().lower()
|
| 126 |
+
if not raw:
|
| 127 |
+
return DEFAULT_IDENTITY_POLICY
|
| 128 |
+
try:
|
| 129 |
+
return IdentityPolicy(raw)
|
| 130 |
+
except ValueError:
|
| 131 |
+
# Falls back to the *default* rather than to the strict setting, which
|
| 132 |
+
# is the opposite of `current()` above and is deliberate: here the
|
| 133 |
+
# default is the permissive one, and a typo must not silently make a
|
| 134 |
+
# deployment stricter than it was configured to be either.
|
| 135 |
+
logger.error(
|
| 136 |
+
"%s=%r is not a recognised policy; falling back to %s. Valid: %s.",
|
| 137 |
+
IDENTITY_ENV, raw, DEFAULT_IDENTITY_POLICY.value,
|
| 138 |
+
", ".join(p.value for p in IdentityPolicy),
|
| 139 |
+
)
|
| 140 |
+
return DEFAULT_IDENTITY_POLICY
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
@dataclass(frozen=True)
|
| 144 |
+
class RecordedException:
|
| 145 |
+
"""One load that the strict policy would have refused.
|
| 146 |
+
|
| 147 |
+
Kept in memory for `/health`, and re-derivable from the logs, which is what
|
| 148 |
+
makes this survivable if the process restarts. The ledger reads the model
|
| 149 |
+
cards directly and does not depend on this list.
|
| 150 |
+
"""
|
| 151 |
+
|
| 152 |
+
what: str
|
| 153 |
+
runtime: str
|
| 154 |
+
licence: str
|
| 155 |
+
reason: str
|
| 156 |
+
source_url: str = ""
|
| 157 |
+
at: str = field(
|
| 158 |
+
default_factory=lambda: datetime.now(timezone.utc).isoformat()
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
#: Everything loaded under `record` that `enforce` would have stopped. Empty
|
| 163 |
+
#: under the default policy, and its non-emptiness is itself the finding.
|
| 164 |
+
RECORDED_EXCEPTIONS: list[RecordedException] = []
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def record_exception(
|
| 168 |
+
*, what: str, runtime: str, licence: str, reason: str, source_url: str = ""
|
| 169 |
+
) -> RecordedException:
|
| 170 |
+
"""Note a licence problem that was allowed through, loudly."""
|
| 171 |
+
entry = RecordedException(
|
| 172 |
+
what=what, runtime=runtime, licence=licence,
|
| 173 |
+
reason=reason, source_url=source_url,
|
| 174 |
+
)
|
| 175 |
+
RECORDED_EXCEPTIONS.append(entry)
|
| 176 |
+
logger.error(
|
| 177 |
+
"LICENCE EXCEPTION: %s loaded on runtime %r under %s, which Animap may "
|
| 178 |
+
"not normally serve. Allowed because %s=%s. %s Reason: %s",
|
| 179 |
+
what, runtime, licence, POLICY_ENV, LicencePolicy.RECORD.value,
|
| 180 |
+
source_url, reason,
|
| 181 |
+
)
|
| 182 |
+
return entry
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def reset_recorded() -> None:
|
| 186 |
+
"""For tests. Production never clears this β a debt that can be cleared by
|
| 187 |
+
calling a function is not a debt."""
|
| 188 |
+
RECORDED_EXCEPTIONS.clear()
|
app/adapters/licences.py
ADDED
|
@@ -0,0 +1,639 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""What each runtime's weights are actually licensed under.
|
| 2 |
+
|
| 3 |
+
**This table is the control, and the model card is not.** ADR 0017 records why:
|
| 4 |
+
a card declares its own licence, so the threat the licence gate was written for β
|
| 5 |
+
somebody editing a card β was precisely the case it could not catch. A watchdog
|
| 6 |
+
wrote `"license": "Apache-2.0"` over a path to `yolo11m.pt` and `discover()`
|
| 7 |
+
waved it through. The fix named one runtime in a frozenset. This generalises it.
|
| 8 |
+
|
| 9 |
+
A runtime name is a fact about which loader executes, not a claim about terms.
|
| 10 |
+
`ultralytics` loads Ultralytics weights and nothing else; `dinov3-onnx` loads a
|
| 11 |
+
DINOv3 export and nothing else. So the licence is attached to the runtime, in
|
| 12 |
+
committed code, next to the URL it was read from and the date it was read.
|
| 13 |
+
|
| 14 |
+
Three things are refused, and the third is new:
|
| 15 |
+
|
| 16 |
+
1. A runtime this file does not know. No default, for the reason
|
| 17 |
+
`detectors.build` gives: a default would let a typo change which licensed
|
| 18 |
+
model produced a farmer's result.
|
| 19 |
+
2. A runtime whose real licence Animap may not serve under.
|
| 20 |
+
3. **A card whose declared licence disagrees with the runtime's real one.** This
|
| 21 |
+
is the check that catches the ADR 0017 exploit generically rather than by
|
| 22 |
+
name. A card claiming Apache-2.0 over a copyleft runtime is now a refusal
|
| 23 |
+
that says which of the two is lying.
|
| 24 |
+
|
| 25 |
+
Nothing here is legal advice, and `app/dispositions.py` carries the same caveat.
|
| 26 |
+
Every entry records the primary source so the next reader checks the licence
|
| 27 |
+
rather than this paragraph.
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
from __future__ import annotations
|
| 31 |
+
|
| 32 |
+
from dataclasses import dataclass
|
| 33 |
+
|
| 34 |
+
from app.adapters import licence_policy
|
| 35 |
+
|
| 36 |
+
#: Licence identifiers Animap may not serve from. Kept identical in spirit to
|
| 37 |
+
#: `providers.DISALLOWED_LICENSES`, which remains the gate on the card's own
|
| 38 |
+
#: declaration β the two checks are deliberately separate, because one reads a
|
| 39 |
+
#: claim and one reads a fact, and the whole point of ADR 0017's fifth mechanism
|
| 40 |
+
#: is that the claim can be false.
|
| 41 |
+
DISALLOWED = frozenset({
|
| 42 |
+
"AGPL-3.0", "AGPL-3.0-only", "AGPL-3.0-or-later",
|
| 43 |
+
"GPL-3.0", "GPL-3.0-only", "GPL-3.0-or-later", "GPL-2.0",
|
| 44 |
+
"SSPL-1.0",
|
| 45 |
+
"CC-BY-NC-4.0", "CC-BY-NC-SA-4.0", "CC BY-NC 4.0", "CC-BY-NC-SA-3.0",
|
| 46 |
+
"Non-Commercial Government Licence",
|
| 47 |
+
"CC-BY-NC-ND-4.0",
|
| 48 |
+
})
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
class LicenceRefused(RuntimeError):
|
| 52 |
+
"""This runtime may not be served, whatever its card says."""
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
@dataclass(frozen=True)
|
| 56 |
+
class RuntimeLicence:
|
| 57 |
+
"""The terms one loader's weights actually arrive under.
|
| 58 |
+
|
| 59 |
+
`permits_commercial_use` and `permits_network_service` are the two questions
|
| 60 |
+
that decide whether Animap may use a model at all β it is a closed-source
|
| 61 |
+
commercial product serving predictions from a private API β and they are
|
| 62 |
+
recorded separately from the SPDX-ish identifier because bespoke research
|
| 63 |
+
licences answer them differently from anything SPDX has a name for.
|
| 64 |
+
"""
|
| 65 |
+
|
| 66 |
+
runtime: str
|
| 67 |
+
#: The identifier as the publisher states it. Bespoke licences keep their
|
| 68 |
+
#: own name rather than being mapped onto the nearest SPDX id, because the
|
| 69 |
+
#: mapping is where the meaning gets lost.
|
| 70 |
+
licence: str
|
| 71 |
+
source_url: str
|
| 72 |
+
verified_on: str
|
| 73 |
+
permits_commercial_use: bool
|
| 74 |
+
permits_network_service: bool
|
| 75 |
+
#: Whether the weights sit behind an accepted-terms gate. A gated repo is not
|
| 76 |
+
#: automatically disqualifying, but it means `scripts/install_models.py`
|
| 77 |
+
#: cannot fetch it unattended, which changes the deployment story.
|
| 78 |
+
gated: bool = False
|
| 79 |
+
#: Text the product must display, verbatim, if the licence demands it.
|
| 80 |
+
#:
|
| 81 |
+
#: Machine-readable rather than a sentence in a note, because an attribution
|
| 82 |
+
#: obligation is the one licence term that is discharged by a line in a UI
|
| 83 |
+
#: rather than by a lawyer β so it needs to be something a test can assert
|
| 84 |
+
#: and a build can carry into an about page. Empty means none is required.
|
| 85 |
+
attribution_required: str = ""
|
| 86 |
+
#: Repository-relative files that carry `attribution_required` today.
|
| 87 |
+
#: `tests/test_attribution.py` opens each one and fails if the string is not
|
| 88 |
+
#: in it, so this list is a claim that can be wrong for exactly one commit.
|
| 89 |
+
attribution_displayed_in: tuple[str, ...] = ()
|
| 90 |
+
#: Surfaces that ought to carry it and do not yet, each with the reason.
|
| 91 |
+
#:
|
| 92 |
+
#: **This field exists because the alternative was a lie.** The DINOv3 model
|
| 93 |
+
#: card read "MITIGATION TAKEN: the product displays 'Built with DINOv3'"
|
| 94 |
+
#: while no Android screen, API response or web surface mentioned DINO at
|
| 95 |
+
#: all β a licence obligation discharged by writing that it had been
|
| 96 |
+
#: discharged. The same test asserts the *negative*: every path here must
|
| 97 |
+
#: exist and must NOT contain the string, so the day somebody adds it the
|
| 98 |
+
#: suite says to move the entry up rather than letting the record rot.
|
| 99 |
+
attribution_outstanding: tuple[tuple[str, str], ...] = ()
|
| 100 |
+
note: str = ""
|
| 101 |
+
|
| 102 |
+
@property
|
| 103 |
+
def attribution_satisfied(self) -> bool:
|
| 104 |
+
"""Whether every surface this obligation needs actually carries it."""
|
| 105 |
+
return bool(self.attribution_required) and not self.attribution_outstanding
|
| 106 |
+
|
| 107 |
+
@property
|
| 108 |
+
def servable(self) -> bool:
|
| 109 |
+
"""Whether Animap may answer a farmer's request with this."""
|
| 110 |
+
return (
|
| 111 |
+
self.licence not in DISALLOWED
|
| 112 |
+
and self.permits_commercial_use
|
| 113 |
+
and self.permits_network_service
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def register(licence: RuntimeLicence) -> RuntimeLicence:
|
| 118 |
+
RUNTIME_LICENCES[licence.runtime] = licence
|
| 119 |
+
return licence
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
#: Runtime β the terms its weights arrive under. Populated below.
|
| 123 |
+
RUNTIME_LICENCES: dict[str, RuntimeLicence] = {}
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def gate(runtime: str | None, declared_licence: str | None = None,
|
| 127 |
+
*, what: str = "") -> RuntimeLicence:
|
| 128 |
+
"""Raise unless this runtime may be served and its card agrees with it.
|
| 129 |
+
|
| 130 |
+
`declared_licence` is optional because the deterministic methods have no
|
| 131 |
+
card. When it is supplied it is checked *against* the table rather than
|
| 132 |
+
trusted, which is the whole difference between this and the card gate.
|
| 133 |
+
|
| 134 |
+
**Two of the three refusals below soften under
|
| 135 |
+
`ANIMAP_LICENCE_POLICY=record`, and one does not.** The two that soften are
|
| 136 |
+
licence *permission* questions β is this runtime unrecorded, and are its
|
| 137 |
+
terms ones Animap may not serve under. Those are the founder's call to
|
| 138 |
+
take, and under `record` they load with the fact kept.
|
| 139 |
+
|
| 140 |
+
The one that does not soften is the card-versus-runtime *disagreement*.
|
| 141 |
+
That is not a permission question, it is an integrity question: a card whose
|
| 142 |
+
declared licence is not what its loader actually loads is a card that
|
| 143 |
+
poisons the ledger. And a ledger is exactly what `record` mode depends on β
|
| 144 |
+
the whole argument for allowing a licence exception is that it stays
|
| 145 |
+
findable afterwards. Letting a lying card through would make the record of
|
| 146 |
+
the exception wrong, which defeats the mode rather than serving it. So it
|
| 147 |
+
raises under both policies, and the fix is to correct the card.
|
| 148 |
+
"""
|
| 149 |
+
policy = licence_policy.current()
|
| 150 |
+
|
| 151 |
+
if not runtime:
|
| 152 |
+
raise LicenceRefused(
|
| 153 |
+
"No runtime named, so there is no way to know what licence the "
|
| 154 |
+
"weights arrive under. A card without a runtime can be inspected "
|
| 155 |
+
"and not run."
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
licence = RUNTIME_LICENCES.get(runtime)
|
| 159 |
+
if licence is None:
|
| 160 |
+
message = (
|
| 161 |
+
f"Runtime {runtime!r} is not in the licence table, so nothing has "
|
| 162 |
+
f"checked what its weights are licensed under. Add an entry with "
|
| 163 |
+
f"the primary source you read, or the runtime cannot be served. "
|
| 164 |
+
f"Known: {', '.join(sorted(RUNTIME_LICENCES))}."
|
| 165 |
+
)
|
| 166 |
+
if policy is licence_policy.LicencePolicy.ENFORCE:
|
| 167 |
+
raise LicenceRefused(message)
|
| 168 |
+
licence_policy.record_exception(
|
| 169 |
+
what=what or runtime, runtime=runtime, licence="UNRECORDED",
|
| 170 |
+
reason="No entry in the runtime licence table; terms are unknown.",
|
| 171 |
+
)
|
| 172 |
+
# Returned as an explicit unknown rather than as something permissive,
|
| 173 |
+
# so a caller that reports `servable` reports False.
|
| 174 |
+
return RuntimeLicence(
|
| 175 |
+
runtime=runtime, licence="UNRECORDED", source_url="",
|
| 176 |
+
verified_on="", permits_commercial_use=False,
|
| 177 |
+
permits_network_service=False,
|
| 178 |
+
note="Loaded under a relaxed policy. Nobody has read its licence.",
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
if not licence.servable:
|
| 182 |
+
message = (
|
| 183 |
+
f"Runtime {runtime!r} loads weights under {licence.licence}, which "
|
| 184 |
+
f"Animap may not serve from"
|
| 185 |
+
+ ("" if licence.permits_commercial_use
|
| 186 |
+
else " β it does not permit commercial use")
|
| 187 |
+
+ ("" if licence.permits_network_service
|
| 188 |
+
else " β it does not permit serving over a network")
|
| 189 |
+
+ f". Read {licence.source_url}. {licence.note}"
|
| 190 |
+
)
|
| 191 |
+
if policy is licence_policy.LicencePolicy.ENFORCE:
|
| 192 |
+
raise LicenceRefused(message)
|
| 193 |
+
licence_policy.record_exception(
|
| 194 |
+
what=what or runtime, runtime=runtime, licence=licence.licence,
|
| 195 |
+
reason=message, source_url=licence.source_url,
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
if declared_licence and declared_licence != licence.licence:
|
| 199 |
+
# The generic form of the ADR 0017 exploit, and it raises under every
|
| 200 |
+
# policy β see the docstring. One of these two is wrong, and the card is
|
| 201 |
+
# the one that can be edited without review noticing.
|
| 202 |
+
raise LicenceRefused(
|
| 203 |
+
f"The card declares {declared_licence!r} but runtime {runtime!r} "
|
| 204 |
+
f"loads weights licensed {licence.licence!r} ({licence.source_url}). "
|
| 205 |
+
f"A card's licence field is a claim; the runtime is a fact about "
|
| 206 |
+
f"which loader runs. Fix whichever is wrong β do not widen this "
|
| 207 |
+
f"check. This is an integrity check, not a permission check, so "
|
| 208 |
+
f"{licence_policy.POLICY_ENV} does not relax it: a relaxed policy "
|
| 209 |
+
f"is only defensible while the record of what was relaxed is true."
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
return licence
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
# --- The table. Every entry names where it was read and when. -----------------
|
| 216 |
+
|
| 217 |
+
# The two runtimes that predate this file, from ADR 0017, which quotes both
|
| 218 |
+
# licences at length.
|
| 219 |
+
register(RuntimeLicence(
|
| 220 |
+
runtime="yolox-onnx",
|
| 221 |
+
licence="Apache-2.0",
|
| 222 |
+
source_url="https://github.com/Megvii-BaseDetection/YOLOX/blob/main/LICENSE",
|
| 223 |
+
verified_on="2026-08-20",
|
| 224 |
+
permits_commercial_use=True,
|
| 225 |
+
permits_network_service=True,
|
| 226 |
+
note=(
|
| 227 |
+
"Megvii publishes no separate licence for the released ONNX weights. "
|
| 228 |
+
"The repository's Apache-2.0 is read as covering the artefacts the "
|
| 229 |
+
"repository distributes, which is an inference and is recorded as one "
|
| 230 |
+
"in ADR 0017."
|
| 231 |
+
),
|
| 232 |
+
))
|
| 233 |
+
|
| 234 |
+
register(RuntimeLicence(
|
| 235 |
+
runtime="ultralytics",
|
| 236 |
+
licence="AGPL-3.0",
|
| 237 |
+
source_url="https://www.ultralytics.com/license",
|
| 238 |
+
verified_on="2026-08-20",
|
| 239 |
+
permits_commercial_use=False,
|
| 240 |
+
permits_network_service=False,
|
| 241 |
+
note=(
|
| 242 |
+
"The checkpoint carries `license: AGPL-3.0 License` in its own pickle "
|
| 243 |
+
"metadata, and the vendor's published position is that a closed-source "
|
| 244 |
+
"SaaS needs an Enterprise licence. Kept in the tree for evaluation/ "
|
| 245 |
+
"only (ADR 0017)."
|
| 246 |
+
),
|
| 247 |
+
))
|
| 248 |
+
|
| 249 |
+
#: Deterministic signal processing. No weights, so no artefact and no terms to
|
| 250 |
+
#: check β which is a substantial part of why Β§4 says to prefer it: "Do not use
|
| 251 |
+
#: a neural model when deterministic signal processing is better." The entry
|
| 252 |
+
#: exists so the gate has something to return rather than being special-cased,
|
| 253 |
+
#: and the licence named is the library's, since that is the only thing shipped.
|
| 254 |
+
register(RuntimeLicence(
|
| 255 |
+
runtime="opencv-numpy",
|
| 256 |
+
licence="Apache-2.0",
|
| 257 |
+
source_url="https://pypi.org/project/opencv-python/",
|
| 258 |
+
verified_on="2026-08-21",
|
| 259 |
+
permits_commercial_use=True,
|
| 260 |
+
permits_network_service=True,
|
| 261 |
+
note=(
|
| 262 |
+
"Read from the installed wheel's own metadata: opencv-python 5.0.0.93 "
|
| 263 |
+
"declares `License: Apache 2.0`, numpy 2.5.2 declares `BSD-3-Clause AND "
|
| 264 |
+
"0BSD AND MIT AND Zlib AND CC0-1.0`. **The wheel is not purely "
|
| 265 |
+
"permissive.** Its LICENSE-3RD-PARTY.txt says `FFmpeg is redistributed "
|
| 266 |
+
"within all opencv-python packages` and reproduces LGPL-2.1 and LGPL-3 "
|
| 267 |
+
"in full; on macOS wheels libbluray, libgnutls, libmp3lame, librtmp and "
|
| 268 |
+
"others are LGPL too. LGPL obligations attach on conveying, and Animap "
|
| 269 |
+
"conveys nothing to farmers, so this is materially weaker than the AGPL "
|
| 270 |
+
"question ADR 0017 declined to answer. It is recorded rather than "
|
| 271 |
+
"resolved: only the video-decode path touches FFmpeg, and the flow, FFT "
|
| 272 |
+
"and contour code does not."
|
| 273 |
+
),
|
| 274 |
+
))
|
| 275 |
+
|
| 276 |
+
#: Deterministic audio (Β§26, Β§4). NumPy does the arithmetic; an `ffmpeg` binary
|
| 277 |
+
#: does the decode, because this service has no audio library at all β no
|
| 278 |
+
#: `soundfile`, no `librosa`, no `av`, and `cv2` exposes no audio path.
|
| 279 |
+
#:
|
| 280 |
+
#: **The two halves of this entry answer different questions and both matter.**
|
| 281 |
+
#:
|
| 282 |
+
#: *Does FFmpeg's licence reach Animap's code?* No. `adapters/audio/decode.py`
|
| 283 |
+
#: runs `ffmpeg` as a separate program over a temporary file. Neither the LGPL
|
| 284 |
+
#: nor the GPL treats invoking a separate executable as creating a derivative
|
| 285 |
+
#: work, so nothing here is a combined work with FFmpeg, and this is materially
|
| 286 |
+
#: weaker than the linking question `opencv-numpy` above declines to answer.
|
| 287 |
+
#:
|
| 288 |
+
#: *Which build may be shipped?* Not the one this was developed against. Read
|
| 289 |
+
#: from `ffmpeg -version` on the development machine on 2026-08-22: the
|
| 290 |
+
#: configuration includes `--enable-gpl --enable-nonfree --enable-libfdk-aac`.
|
| 291 |
+
#: FFmpeg's own `LICENSE.md` says of `--enable-nonfree`: *"This will cause the
|
| 292 |
+
#: resulting binary to be unredistributable."* So the binary on this laptop
|
| 293 |
+
#: could not be put in a container image at all, whatever Animap's own licence
|
| 294 |
+
#: is. `permits_commercial_use` and `permits_network_service` below describe the
|
| 295 |
+
#: **default LGPL v2.1+ build** β *"Most files in FFmpeg are under the GNU
|
| 296 |
+
#: Lesser General Public License version 2.1 or later"* β which is what a
|
| 297 |
+
#: deployment must build or source.
|
| 298 |
+
register(RuntimeLicence(
|
| 299 |
+
runtime="ffmpeg-numpy",
|
| 300 |
+
licence="LGPL-2.1-or-later (default build); GPL-2.0-or-later with --enable-gpl; unredistributable with --enable-nonfree",
|
| 301 |
+
source_url="https://raw.githubusercontent.com/FFmpeg/FFmpeg/master/LICENSE.md",
|
| 302 |
+
verified_on="2026-08-22",
|
| 303 |
+
permits_commercial_use=True,
|
| 304 |
+
permits_network_service=True,
|
| 305 |
+
note=(
|
| 306 |
+
"Applies to a default LGPL v2.1+ FFmpeg invoked as a separate binary. "
|
| 307 |
+
"**The build present during development is not that build**: "
|
| 308 |
+
"`--enable-gpl --enable-nonfree --enable-libfdk-aac`, which FFmpeg's "
|
| 309 |
+
"LICENSE.md says makes the resulting binary unredistributable. Before "
|
| 310 |
+
"anything ships, build or source a default-configuration FFmpeg and "
|
| 311 |
+
"record its `-version` line beside this entry. LGPL obligations attach "
|
| 312 |
+
"on conveying; Animap conveys nothing to farmers from the server, but a "
|
| 313 |
+
"phone build that bundles a decoder does convey, and that is a different "
|
| 314 |
+
"question nobody here has answered. NumPy 2.5.2 declares `BSD-3-Clause "
|
| 315 |
+
"AND 0BSD AND MIT AND Zlib AND CC0-1.0`."
|
| 316 |
+
),
|
| 317 |
+
))
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
# --- Frozen embeddings (Β§3 DINOv3, Β§4 MegaDescriptor). -----------------------
|
| 321 |
+
|
| 322 |
+
register(RuntimeLicence(
|
| 323 |
+
runtime="dinov2-onnx",
|
| 324 |
+
licence="Apache-2.0",
|
| 325 |
+
source_url="https://github.com/facebookresearch/dinov2/blob/main/LICENSE",
|
| 326 |
+
verified_on="2026-08-21",
|
| 327 |
+
permits_commercial_use=True,
|
| 328 |
+
permits_network_service=True,
|
| 329 |
+
note=(
|
| 330 |
+
"Code and weights both Apache-2.0; `facebook/dinov2-small` carries "
|
| 331 |
+
"`license: apache-2.0` and is ungated. Its embedding is the same 384 "
|
| 332 |
+
"dimensions as DINOv3 ViT-S/16 at 22.06M against 21.60M parameters, so "
|
| 333 |
+
"it is the drop-in that costs nothing to be sure about."
|
| 334 |
+
),
|
| 335 |
+
))
|
| 336 |
+
|
| 337 |
+
register(RuntimeLicence(
|
| 338 |
+
runtime="dinov3-onnx",
|
| 339 |
+
licence="DINOv3 License",
|
| 340 |
+
source_url="https://github.com/facebookresearch/dinov3/blob/main/LICENSE.md",
|
| 341 |
+
verified_on="2026-08-21",
|
| 342 |
+
permits_commercial_use=True,
|
| 343 |
+
permits_network_service=True,
|
| 344 |
+
gated=True,
|
| 345 |
+
# Cheapest possible discharge of the ambiguity below: display it and both
|
| 346 |
+
# readings of the licence are satisfied. One line in an about page against
|
| 347 |
+
# an hour of counsel's time.
|
| 348 |
+
#
|
| 349 |
+
# **It is not discharged yet, and the second list is the honest half.** A
|
| 350 |
+
# NOTICES file in a source repository is where a lawyer looks, not where a
|
| 351 |
+
# farmer does, and the Meta-hosted text says "prominently display". Until
|
| 352 |
+
# the string is on a screen somebody using Animap can see, the obligation is
|
| 353 |
+
# met on paper and not in the product.
|
| 354 |
+
attribution_required="Built with DINOv3",
|
| 355 |
+
attribution_displayed_in=("THIRD_PARTY_NOTICES.md",),
|
| 356 |
+
attribution_outstanding=(
|
| 357 |
+
(
|
| 358 |
+
"apps/android/app/src/main/java/com/ccc2c/animap/ui/screens/more/"
|
| 359 |
+
"MoreScreen.kt",
|
| 360 |
+
"The app has no about screen and no third-party notices screen β "
|
| 361 |
+
"`ui/screens/Notices.kt` is a form-message component despite the "
|
| 362 |
+
"name, and nothing under `ui/` mentions a licence. `MoreScreen` is "
|
| 363 |
+
"where an about entry would hang, and this is the surface that "
|
| 364 |
+
"decides whether 'prominently display' is satisfied, because it is "
|
| 365 |
+
"the only one a farmer can reach.",
|
| 366 |
+
),
|
| 367 |
+
(
|
| 368 |
+
"services/inference/app/main.py",
|
| 369 |
+
"`/health` and `/capabilities` name every adapter and its licence "
|
| 370 |
+
"and do not carry the attribution string, so an integrator "
|
| 371 |
+
"embedding Animap has no way to learn it owes one.",
|
| 372 |
+
),
|
| 373 |
+
),
|
| 374 |
+
note=(
|
| 375 |
+
"Bespoke Meta licence, not Apache-2.0 β `facebookresearch/dinov3` "
|
| 376 |
+
"reports SPDX NOASSERTION and covers code and weights alike. It carries "
|
| 377 |
+
"no non-commercial clause, no acceptable-use policy and no monthly-user "
|
| 378 |
+
"trigger. **Two published versions of this licence disagree.** The "
|
| 379 |
+
"LICENSE.md shipped with the weights is dated 19 August 2025 and ends "
|
| 380 |
+
"clause 1.b.i at providing a copy of the agreement; the text at "
|
| 381 |
+
"ai.meta.com/resources/models-and-libraries/dinov3-license, which is "
|
| 382 |
+
"what the model card links to, is dated 14 August 2025 and adds a duty "
|
| 383 |
+
"to `prominently display \"Built with DINOv3\"`. Section 8 lets Meta "
|
| 384 |
+
"amend unilaterally with immediate effect. The `facebook/*` repos are "
|
| 385 |
+
"gated with manual approval; `timm/vit_small_patch16_dinov3.lvd1689m` "
|
| 386 |
+
"is ungated and ships the same LICENSE.md, which is where the shipped "
|
| 387 |
+
"export came from. Counsel's question, not an engineer's."
|
| 388 |
+
),
|
| 389 |
+
))
|
| 390 |
+
|
| 391 |
+
#: Refused, and registered *in order to be* refused. An unknown runtime produces
|
| 392 |
+
#: "nobody has checked this"; a known-and-disallowed one produces the reason,
|
| 393 |
+
#: which is what a reader needs when they ask why the wildlife re-ID model the
|
| 394 |
+
#: directive names is not here.
|
| 395 |
+
#:
|
| 396 |
+
#: **Installed since 2026-08-21, and no more servable for it.** The founder
|
| 397 |
+
#: lifted the rule that a refused licence stops a model being exported and
|
| 398 |
+
#: measured, so `models/alternates/megadescriptor/` now holds a real artefact
|
| 399 |
+
#: and Β§40.2's benchmark has a number in it. Nothing below changed: the terms
|
| 400 |
+
#: did not, `servable` is still False, and `gate` still raises under `enforce`.
|
| 401 |
+
#: The only difference is that the refusal now costs a measurement rather than
|
| 402 |
+
#: preventing one.
|
| 403 |
+
register(RuntimeLicence(
|
| 404 |
+
runtime="megadescriptor-timm",
|
| 405 |
+
licence="CC-BY-NC-4.0",
|
| 406 |
+
source_url="https://huggingface.co/BVRA/MegaDescriptor-L-384/raw/main/README.md",
|
| 407 |
+
verified_on="2026-08-21",
|
| 408 |
+
permits_commercial_use=False,
|
| 409 |
+
permits_network_service=True,
|
| 410 |
+
note=(
|
| 411 |
+
"All eight BVRA MegaDescriptor repos carry `license: cc-by-nc-4.0` in "
|
| 412 |
+
"their card front-matter. CC BY-NC 4.0 Β§2(a)(1) grants rights for "
|
| 413 |
+
"NonCommercial purposes only and the family has no commercial "
|
| 414 |
+
"exception, so Animap cannot use these weights at any size. "
|
| 415 |
+
"**There is a trap in the artefact itself**: "
|
| 416 |
+
"`BVRA/MegaDescriptor-L-384/config.json` contains "
|
| 417 |
+
"`\"pretrained_cfg\": {... \"license\": \"mit\" ...}`, inherited "
|
| 418 |
+
"boilerplate from timm's original Swin config that describes "
|
| 419 |
+
"Microsoft's weights and not these. A loader that read the licence out "
|
| 420 |
+
"of the checkpoint would conclude MIT and be wrong β which is the same "
|
| 421 |
+
"shape of failure as ADR 0017's edited card, arriving from upstream "
|
| 422 |
+
"instead of from a colleague."
|
| 423 |
+
),
|
| 424 |
+
))
|
| 425 |
+
|
| 426 |
+
#: Also refused, for a different and more easily missed reason: nothing was
|
| 427 |
+
#: granted at all. Installed and benchmarked on the same footing as
|
| 428 |
+
#: MegaDescriptor since 2026-08-21, and unservable on the same footing too.
|
| 429 |
+
#:
|
| 430 |
+
#: One extra fact worth carrying, because it is a build-time risk this table's
|
| 431 |
+
#: usual questions do not ask about: this repository ships its architecture as
|
| 432 |
+
#: Python rather than as a `transformers` class, so exporting it means running
|
| 433 |
+
#: a third party's code. See `scripts/export_embedding.py._load_remote`.
|
| 434 |
+
register(RuntimeLicence(
|
| 435 |
+
runtime="miewid",
|
| 436 |
+
licence="none stated",
|
| 437 |
+
source_url="https://huggingface.co/conservationxlabs/miewid-msv3",
|
| 438 |
+
verified_on="2026-08-21",
|
| 439 |
+
permits_commercial_use=False,
|
| 440 |
+
permits_network_service=False,
|
| 441 |
+
note=(
|
| 442 |
+
"The wildlife re-ID alternative to MegaDescriptor, and it carries no "
|
| 443 |
+
"licence tag, no LICENSE file, and `cardData` of "
|
| 444 |
+
"`{\"library_name\": \"transformers\", \"tags\": []}`. The upstream "
|
| 445 |
+
"repo returns `\"license\": null`. Absent an express grant the default "
|
| 446 |
+
"is all rights reserved, so silence is a refusal and not a permission. "
|
| 447 |
+
"Third-party re-uploads tagged MIT cannot grant what they were never "
|
| 448 |
+
"given."
|
| 449 |
+
),
|
| 450 |
+
))
|
| 451 |
+
|
| 452 |
+
|
| 453 |
+
# --- Open-vocabulary detection and counting (Β§4). -----------------------------
|
| 454 |
+
|
| 455 |
+
register(RuntimeLicence(
|
| 456 |
+
runtime="grounding-dino-hf",
|
| 457 |
+
licence="Apache-2.0",
|
| 458 |
+
source_url="https://github.com/IDEA-Research/GroundingDINO/blob/main/LICENSE",
|
| 459 |
+
verified_on="2026-08-21",
|
| 460 |
+
permits_commercial_use=True,
|
| 461 |
+
permits_network_service=True,
|
| 462 |
+
note=(
|
| 463 |
+
"Stock Apache-2.0 with no appended restrictions; the HF re-releases "
|
| 464 |
+
"`IDEA-Research/grounding-dino-tiny` and `-base` carry "
|
| 465 |
+
"`license: apache-2.0` explicitly, which the GitHub weights do not β "
|
| 466 |
+
"there the repo licence covering the release artefacts is an inference "
|
| 467 |
+
"from silence, the same shape as YOLOX above. The BERT text encoder it "
|
| 468 |
+
"loads is Apache-2.0. Objects365 and the other training sets carry "
|
| 469 |
+
"their own terms and whether those reach through to weights is "
|
| 470 |
+
"unsettled; not investigated."
|
| 471 |
+
),
|
| 472 |
+
))
|
| 473 |
+
|
| 474 |
+
register(RuntimeLicence(
|
| 475 |
+
runtime="countgd",
|
| 476 |
+
licence="MIT",
|
| 477 |
+
source_url="https://raw.githubusercontent.com/niki-amini-naieni/CountGD/main/LICENSE",
|
| 478 |
+
verified_on="2026-08-21",
|
| 479 |
+
permits_commercial_use=True,
|
| 480 |
+
permits_network_service=True,
|
| 481 |
+
note=(
|
| 482 |
+
"MIT for the code (`Copyright (c) 2024 Niki Amini-Naieni`) and "
|
| 483 |
+
"`license: mit` on the `nikigoli/CountGD` weights. It vendors a "
|
| 484 |
+
"GroundingDINO fork whose deformable-attention module falls back to a "
|
| 485 |
+
"pure-PyTorch `grid_sample` path when the CUDA extension is absent, so "
|
| 486 |
+
"CPU-only execution is possible in principle. No CPU throughput figure "
|
| 487 |
+
"is published and none was measured here."
|
| 488 |
+
),
|
| 489 |
+
))
|
| 490 |
+
|
| 491 |
+
|
| 492 |
+
#: Β§26's audio-embedding leg, and **not the model the directive names.**
|
| 493 |
+
#:
|
| 494 |
+
#: Β§26 asks for *"Perception Encoder audio / AV embeddings"*. Meta publishes the
|
| 495 |
+
#: vision ports of Perception Encoder (`facebook/PE-Core-*`, Apache-2.0) and
|
| 496 |
+
#: `facebook/perception_encoder` returns 401 to an unauthenticated request, so
|
| 497 |
+
#: no audio tower was reachable. `facebook/sam-audio` returns 401 as well.
|
| 498 |
+
#: CLAP is the substitute that was reachable, and
|
| 499 |
+
#: `experiments/poultry_respiratory/` records it as a substitute in its run
|
| 500 |
+
#: notes rather than letting it stand in for what was asked for.
|
| 501 |
+
register(RuntimeLicence(
|
| 502 |
+
runtime="clap-hf",
|
| 503 |
+
licence="Apache-2.0",
|
| 504 |
+
source_url="https://huggingface.co/laion/clap-htsat-unfused",
|
| 505 |
+
verified_on="2026-08-22",
|
| 506 |
+
permits_commercial_use=True,
|
| 507 |
+
permits_network_service=True,
|
| 508 |
+
note=(
|
| 509 |
+
"`laion/clap-htsat-unfused` carries `license: apache-2.0` in its card "
|
| 510 |
+
"front-matter and is ungated. The LAION-Audio-630K training data it was "
|
| 511 |
+
"fitted on carries its own terms, and whether those reach through to "
|
| 512 |
+
"weights is the same unsettled question the Grounding DINO entry above "
|
| 513 |
+
"records for Objects365; not investigated here either."
|
| 514 |
+
),
|
| 515 |
+
))
|
| 516 |
+
|
| 517 |
+
|
| 518 |
+
# --- Segmentation (Β§3 SAM 3.1). ----------------------------------------------
|
| 519 |
+
|
| 520 |
+
register(RuntimeLicence(
|
| 521 |
+
runtime="sam2-onnx",
|
| 522 |
+
licence="Apache-2.0",
|
| 523 |
+
source_url="https://github.com/facebookresearch/sam2/blob/main/LICENSE",
|
| 524 |
+
verified_on="2026-08-21",
|
| 525 |
+
permits_commercial_use=True,
|
| 526 |
+
permits_network_service=True,
|
| 527 |
+
note=(
|
| 528 |
+
"SAM 2.1 is the last Apache-2.0 generation, and "
|
| 529 |
+
"`facebook/sam2.1-hiera-tiny` is 39.0M parameters and ungated. It does "
|
| 530 |
+
"promptable segmentation only β no concept prompting β which is the "
|
| 531 |
+
"capability Β§3 wants SAM 3 for. Named here as the variant that fits the "
|
| 532 |
+
"container, not as an equivalent."
|
| 533 |
+
),
|
| 534 |
+
))
|
| 535 |
+
|
| 536 |
+
register(RuntimeLicence(
|
| 537 |
+
runtime="sam3",
|
| 538 |
+
licence="SAM License",
|
| 539 |
+
source_url="https://huggingface.co/facebook/sam3/resolve/main/LICENSE",
|
| 540 |
+
verified_on="2026-08-21",
|
| 541 |
+
permits_commercial_use=True,
|
| 542 |
+
permits_network_service=True,
|
| 543 |
+
gated=True,
|
| 544 |
+
note=(
|
| 545 |
+
"Bespoke Meta licence dated 19 November 2025, covering code and weights "
|
| 546 |
+
"alike β `facebookresearch/sam3` is SPDX NOASSERTION, not Apache-2.0 as "
|
| 547 |
+
"SAM 2 was. No non-commercial clause, no acceptable-use policy, no "
|
| 548 |
+
"monthly-user trigger. Clause 1.b.i binds distribution or making the "
|
| 549 |
+
"materials `available to a third party`, which serving inference "
|
| 550 |
+
"outputs is normally not, but the wording is broader than "
|
| 551 |
+
"`distribute` and unadjudicated. Section 8 permits unilateral "
|
| 552 |
+
"amendment with immediate effect, so archive the text you accepted. "
|
| 553 |
+
"Gated with manual approval, and a build needs an HF token. **The "
|
| 554 |
+
"licence is not what stops this being deployed β the size is.** See "
|
| 555 |
+
"`adapters/segmentation.py`."
|
| 556 |
+
),
|
| 557 |
+
))
|
| 558 |
+
|
| 559 |
+
|
| 560 |
+
# --- Hosted multimodal reasoning (Β§4). ---------------------------------------
|
| 561 |
+
|
| 562 |
+
register(RuntimeLicence(
|
| 563 |
+
runtime="hosted-multimodal",
|
| 564 |
+
licence="vendor terms of service",
|
| 565 |
+
source_url="",
|
| 566 |
+
verified_on="2026-08-21",
|
| 567 |
+
permits_commercial_use=True,
|
| 568 |
+
permits_network_service=True,
|
| 569 |
+
note=(
|
| 570 |
+
"No weights arrive, so there is no artefact licence to check and this "
|
| 571 |
+
"table cannot do its usual job. What governs is the vendor's API terms "
|
| 572 |
+
"and its data-retention posture, and both are procurement questions "
|
| 573 |
+
"rather than loader questions. Recorded so the runtime is known rather "
|
| 574 |
+
"than refused as unrecognised, and deliberately *not* recorded as "
|
| 575 |
+
"cleared: nobody has read a specific vendor's terms here."
|
| 576 |
+
),
|
| 577 |
+
))
|
| 578 |
+
|
| 579 |
+
|
| 580 |
+
# --- Quadruped pose (Β§24). ----------------------------------------------------
|
| 581 |
+
#
|
| 582 |
+
# Neither has run. Both are here because the licence question is decided long
|
| 583 |
+
# before the model is, and because Β§24 names the first one first β so whoever
|
| 584 |
+
# picks up cattle gait meets the commercial obstacle in the ledger rather than
|
| 585 |
+
# discovering it after building on it.
|
| 586 |
+
|
| 587 |
+
register(RuntimeLicence(
|
| 588 |
+
runtime="superanimal-quadruped-dlc",
|
| 589 |
+
licence="Modified MIT (academic and non-commercial use only)",
|
| 590 |
+
source_url=(
|
| 591 |
+
"https://huggingface.co/mwmathis/DeepLabCutModelZoo-SuperAnimal-Quadruped"
|
| 592 |
+
),
|
| 593 |
+
verified_on="2026-08-22",
|
| 594 |
+
# **The two fields that matter, and the reason this entry exists.** Animap
|
| 595 |
+
# is a closed-source commercial product; this model card restricts use to
|
| 596 |
+
# academic and non-commercial purposes and offers commercial licensing on
|
| 597 |
+
# application to Prof. Mackenzie W. Mathis or EPFL's technology transfer
|
| 598 |
+
# office. That is not a reason to skip evaluating it β the founder's
|
| 599 |
+
# standing instruction is that no model is excluded on licence today β it is
|
| 600 |
+
# a reason for the position to be written down where a sale would find it.
|
| 601 |
+
permits_commercial_use=False,
|
| 602 |
+
permits_network_service=False,
|
| 603 |
+
gated=False,
|
| 604 |
+
note=(
|
| 605 |
+
"Β§24 names this model first for cattle gait: zero-shot quadruped pose "
|
| 606 |
+
"over 39 bodyparts, trained on 40k+ images. Its 39-bodypart vocabulary "
|
| 607 |
+
"is mapped in `app/adapters/pose/vocabulary.py` and "
|
| 608 |
+
"`app/adapters/pose/gait.py` consumes its output unchanged, so the only "
|
| 609 |
+
"thing between here and running it is video β not code and not a "
|
| 610 |
+
"licence decision, which has not been taken. The card also forbids using "
|
| 611 |
+
"the model to harm any animal deliberately. **`servable` is False, so "
|
| 612 |
+
"`gate` refuses it under the default policy**; that is the correct "
|
| 613 |
+
"state for a model whose terms exclude the product's own business "
|
| 614 |
+
"model, and it should be revisited by asking EPFL rather than by "
|
| 615 |
+
"editing this line. Cite Ye et al., arXiv:2203.07436."
|
| 616 |
+
),
|
| 617 |
+
))
|
| 618 |
+
|
| 619 |
+
register(RuntimeLicence(
|
| 620 |
+
runtime="vitpose-hf",
|
| 621 |
+
licence="Apache-2.0",
|
| 622 |
+
source_url="https://huggingface.co/usyd-community/vitpose-base-simple",
|
| 623 |
+
verified_on="2026-08-22",
|
| 624 |
+
permits_commercial_use=True,
|
| 625 |
+
permits_network_service=True,
|
| 626 |
+
note=(
|
| 627 |
+
"Β§36's required alternative to SuperAnimal, and **the shippable one**: "
|
| 628 |
+
"permissive, already installed in this service's virtualenv via "
|
| 629 |
+
"`transformers`, and runnable with no CUDA extension. Its AP-10K "
|
| 630 |
+
"vocabulary covers 54 species including cattle and is mapped in "
|
| 631 |
+
"`app/adapters/pose/vocabulary.py`. It supplies 11 of the 12 landmarks "
|
| 632 |
+
"gait needs β AP-10K has no mid-back keypoint, so Β§24's back-line "
|
| 633 |
+
"movement is reported *unavailable* on it rather than zero. Not "
|
| 634 |
+
"measured: `experiments/cattle_gait` never ran a pose model, because no "
|
| 635 |
+
"openly licensed side-on cattle walking video with a locomotion score "
|
| 636 |
+
"could be obtained. AP-10K itself is CC BY 4.0, which is a fact about "
|
| 637 |
+
"the training data rather than about these weights."
|
| 638 |
+
),
|
| 639 |
+
))
|
app/adapters/multimodal.py
ADDED
|
@@ -0,0 +1,322 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The hosted multimodal reasoner, and the rails it runs inside.
|
| 2 |
+
|
| 3 |
+
Directive Β§4 asks for a Gemini-class model for BCS rubrics, dentition, wound
|
| 4 |
+
description, skin and hoof and footpad triage, breed suggestion, litter
|
| 5 |
+
condition, heat-stress signs and egg quality. It says two things about it that
|
| 6 |
+
are not decoration, and both are enforced here in code rather than in a prompt:
|
| 7 |
+
|
| 8 |
+
> All calls must return structured JSON.
|
| 9 |
+
|
| 10 |
+
> The multimodal model is an **experimental visual reasoner**, not an authority.
|
| 11 |
+
|
| 12 |
+
**The second one is why this file is longer than an HTTP call.** A hosted model
|
| 13 |
+
will cheerfully write "lumpy skin disease confirmed" into a field called
|
| 14 |
+
`diagnosis`, and Β§10 forbids exactly that sentence. A prompt asking it not to is
|
| 15 |
+
a request; `app/adapters/claims.py` is a control, and everything about how it
|
| 16 |
+
works and why it is shaped that way lives in that module's own docstring.
|
| 17 |
+
|
| 18 |
+
**This file used to claim a control it did not have.** The paragraph here said
|
| 19 |
+
every response was checked against `app/capabilities.FORBIDDEN_CLAIMS` and the
|
| 20 |
+
observation/interpretation split from `app/schemas.py`. The first was true and
|
| 21 |
+
useless β `FORBIDDEN_CLAIMS` is 29 snake_case registry keys, and no hosted model
|
| 22 |
+
writes `lsd_diagnosis` in prose. The second was not true at all: this module
|
| 23 |
+
never imported `app/schemas.py`. What actually stood between a model and a
|
| 24 |
+
published diagnosis was a tuple of seven hard-coded sentences, and a watchdog
|
| 25 |
+
published 24 evasions out of 24 attempts through it β a doubled space, a
|
| 26 |
+
Cyrillic `Ρ`, a zero-width character, Β§10's own sentence with the words
|
| 27 |
+
reordered, `best_estimate: 2.6347`, and `NaN`.
|
| 28 |
+
|
| 29 |
+
`claims.enforce` replaces it and `claims.schema_for` builds the schema from the
|
| 30 |
+
capability, so the closed vocabulary a model is handed and the closed vocabulary
|
| 31 |
+
its answer is judged against are the same object.
|
| 32 |
+
|
| 33 |
+
**No key is committed and none may be.** The adapter reads
|
| 34 |
+
`ANIMAP_MULTIMODAL_API_KEY` from the environment at call time, and its absence is
|
| 35 |
+
an honest `unavailable`, not a fallback to something plausible. On Azure
|
| 36 |
+
Container Apps that is a secret reference, the same posture `services/api`
|
| 37 |
+
already uses.
|
| 38 |
+
|
| 39 |
+
**This adapter chooses no vendor, and that is deliberate rather than
|
| 40 |
+
unfinished.** `_transport` is injected; `app/adapters/transports/` holds the
|
| 41 |
+
implementations and `registry.py` passes whichever one `ANIMAP_MULTIMODAL_PROVIDER`
|
| 42 |
+
names. With none named the adapter reports itself unavailable and says so.
|
| 43 |
+
|
| 44 |
+
Choosing a vendor commits Animap to a data-retention posture for photographs of
|
| 45 |
+
somebody's animals leaving the country, which is a procurement decision and not
|
| 46 |
+
one an adapter should make by importing an SDK. Keeping the transport injectable
|
| 47 |
+
is also what makes this file testable without a network and without a key: every
|
| 48 |
+
test below passes a callable.
|
| 49 |
+
"""
|
| 50 |
+
|
| 51 |
+
from __future__ import annotations
|
| 52 |
+
|
| 53 |
+
import os
|
| 54 |
+
from dataclasses import dataclass
|
| 55 |
+
from typing import Any, Callable
|
| 56 |
+
|
| 57 |
+
from PIL import Image
|
| 58 |
+
|
| 59 |
+
from app.adapters import claims
|
| 60 |
+
from app.adapters.base import (
|
| 61 |
+
Adapter,
|
| 62 |
+
AdapterError,
|
| 63 |
+
AdapterSpec,
|
| 64 |
+
AdapterUnavailable,
|
| 65 |
+
Availability,
|
| 66 |
+
Modality,
|
| 67 |
+
Placement,
|
| 68 |
+
Task,
|
| 69 |
+
)
|
| 70 |
+
from app.capabilities import Capability
|
| 71 |
+
|
| 72 |
+
# Re-exported rather than merely imported. `ContractViolation` was defined here
|
| 73 |
+
# before the contract moved to `claims.py`, and callers β including
|
| 74 |
+
# `tests/test_adapters.py` β import it from this module. Moving the definition
|
| 75 |
+
# without keeping the name would be a rename disguised as a refactor.
|
| 76 |
+
from app.adapters.claims import ContractViolation # noqa: F401
|
| 77 |
+
|
| 78 |
+
#: The environment variable this expects. Named here so a deployment can be
|
| 79 |
+
#: configured without reading the code, and so nobody invents a second name.
|
| 80 |
+
API_KEY_ENV = "ANIMAP_MULTIMODAL_API_KEY"
|
| 81 |
+
|
| 82 |
+
#: Which hosted model to call. Kept in the environment rather than in code
|
| 83 |
+
#: because a result must name the version that produced it β `schemas`
|
| 84 |
+
#: requires `model_version` β and a hosted model's version changes without
|
| 85 |
+
#: anyone here editing a file.
|
| 86 |
+
MODEL_ENV = "ANIMAP_MULTIMODAL_MODEL"
|
| 87 |
+
|
| 88 |
+
#: Prefixed to every rubric. It is not a safety incantation β the checks below
|
| 89 |
+
#: are what actually hold β but a model told the shape of the answer produces
|
| 90 |
+
#: fewer responses that have to be thrown away.
|
| 91 |
+
SYSTEM_RULES = """\
|
| 92 |
+
You are a visual observer for a livestock record system. You are not a
|
| 93 |
+
veterinarian and your output is not a diagnosis.
|
| 94 |
+
|
| 95 |
+
Return one JSON object matching the supplied schema and nothing else. No prose,
|
| 96 |
+
no markdown fence, no commentary.
|
| 97 |
+
|
| 98 |
+
Report only what is VISIBLE in the images. Separate observation from
|
| 99 |
+
interpretation: an observation is a fact about the picture ("multiple raised
|
| 100 |
+
nodular lesions on the flank"), an interpretation is what it might mean
|
| 101 |
+
("abnormal skin pattern, veterinary review recommended").
|
| 102 |
+
|
| 103 |
+
Never name a disease unless you qualify it in the same sentence β "a pattern
|
| 104 |
+
consistent with X", "could be X", "inconclusive for X". An unqualified name is a
|
| 105 |
+
diagnosis whatever the surrounding words, and it is discarded with the whole
|
| 106 |
+
response. Never state a body temperature, a pregnancy status, an exact weight,
|
| 107 |
+
an exact whole-house population, a cause for a wound, that a bird is dead rather
|
| 108 |
+
than resting, or a sex, breed or identity as settled. If the image does not
|
| 109 |
+
support a field, set it to null and say why in `limits`. A null is a correct
|
| 110 |
+
answer and a guess is not.
|
| 111 |
+
|
| 112 |
+
Make claims only by choosing identifiers from the schema's own `claims` list.
|
| 113 |
+
Do not invent a field. Any key the schema does not declare is discarded with the
|
| 114 |
+
whole response.
|
| 115 |
+
|
| 116 |
+
`evidence` and `limits` are closed lists, not free text. Choose the strings the
|
| 117 |
+
schema offers and change none of them. Do not paraphrase, do not combine two
|
| 118 |
+
into one, and do not add a word. If nothing in the list describes what you can
|
| 119 |
+
see, choose the limit that says so and let a person look; a response containing
|
| 120 |
+
a string the schema does not offer is discarded whole.
|
| 121 |
+
|
| 122 |
+
Put every number in a structured field β `range`, `best_estimate`, or an
|
| 123 |
+
observation's `value`. Never write a number into `evidence` or `limits`: the
|
| 124 |
+
strings there are fixed, and any figure a farmer reads has to come from a field
|
| 125 |
+
the capability declared it can measure.
|
| 126 |
+
|
| 127 |
+
Where the schema asks for a range, give a range. Do not narrow it to look
|
| 128 |
+
precise.\
|
| 129 |
+
"""
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
@dataclass(frozen=True)
|
| 133 |
+
class ReasonerResponse:
|
| 134 |
+
"""What came back, and what it cost.
|
| 135 |
+
|
| 136 |
+
`raw` is kept verbatim beside the parsed object because Β§33 requires the
|
| 137 |
+
model result to be preserved for later training, and a reparsed
|
| 138 |
+
reconstruction is not what the model said.
|
| 139 |
+
"""
|
| 140 |
+
|
| 141 |
+
parsed: dict[str, Any]
|
| 142 |
+
raw: str
|
| 143 |
+
model: str
|
| 144 |
+
warnings: tuple[str, ...] = ()
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
class HostedMultimodalAdapter(Adapter):
|
| 148 |
+
"""A hosted reasoner behind a structured-output contract.
|
| 149 |
+
|
| 150 |
+
Construct it with a `transport` β a callable taking `(prompt, images,
|
| 151 |
+
schema, model, api_key)` and returning the response text. There is no
|
| 152 |
+
default, so this adapter cannot silently start talking to a vendor nobody
|
| 153 |
+
chose.
|
| 154 |
+
"""
|
| 155 |
+
|
| 156 |
+
spec = AdapterSpec(
|
| 157 |
+
adapter_id="hosted-multimodal",
|
| 158 |
+
runtime="hosted-multimodal",
|
| 159 |
+
tasks=(Task.REASON,),
|
| 160 |
+
modalities=(Modality.IMAGE, Modality.VIDEO, Modality.AUDIO),
|
| 161 |
+
directive_role=(
|
| 162 |
+
"Β§4 hosted multimodal β BCS rubric scoring, dentition "
|
| 163 |
+
"interpretation, wound description, skin/hoof/footpad triage, breed "
|
| 164 |
+
"suggestion, litter condition, heat-stress signs, egg external "
|
| 165 |
+
"quality, structured evidence extraction. Β§4: 'an experimental "
|
| 166 |
+
"visual reasoner, not an authority'."
|
| 167 |
+
),
|
| 168 |
+
requires_artefact=False,
|
| 169 |
+
placement=Placement.CPU_SERVICE,
|
| 170 |
+
placement_reason=(
|
| 171 |
+
"No weights run here, so it costs the container a socket. It is "
|
| 172 |
+
"also the one leg that can never move to the phone: ADR 0002 makes "
|
| 173 |
+
"Animap offline-first, and every capability built on this one is a "
|
| 174 |
+
"capability a farm cannot use in a shed with no signal. That is a "
|
| 175 |
+
"connectivity tier, not a reason to drop it."
|
| 176 |
+
),
|
| 177 |
+
notes=(
|
| 178 |
+
"Unmeasured, because no vendor is wired and no key is configured. "
|
| 179 |
+
"Latency and cost per call are the vendor's and must be measured "
|
| 180 |
+
"against a real account before any capability depends on it."
|
| 181 |
+
),
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
def __init__(
|
| 185 |
+
self,
|
| 186 |
+
transport: Callable[..., str] | None = None,
|
| 187 |
+
*,
|
| 188 |
+
key_env: str = API_KEY_ENV,
|
| 189 |
+
model_env: str = MODEL_ENV,
|
| 190 |
+
) -> None:
|
| 191 |
+
self._transport = transport
|
| 192 |
+
self._key_env = key_env
|
| 193 |
+
self._model_env = model_env
|
| 194 |
+
|
| 195 |
+
# Read at call time rather than at import, so a secret rotation takes effect
|
| 196 |
+
# on restart rather than needing a rebuild β the same reasoning as
|
| 197 |
+
# `main._configured_token`.
|
| 198 |
+
def _api_key(self) -> str:
|
| 199 |
+
return os.environ.get(self._key_env, "")
|
| 200 |
+
|
| 201 |
+
def _model(self) -> str:
|
| 202 |
+
return os.environ.get(self._model_env, "")
|
| 203 |
+
|
| 204 |
+
def availability(self) -> Availability:
|
| 205 |
+
if self._transport is None:
|
| 206 |
+
from app.adapters.transports import PROVIDER_ENV, PROVIDERS
|
| 207 |
+
|
| 208 |
+
return Availability(
|
| 209 |
+
False,
|
| 210 |
+
"No hosted multimodal vendor is chosen for this deployment.",
|
| 211 |
+
f"Choosing one commits Animap to a retention posture for farm "
|
| 212 |
+
f"photographs, so it is a procurement decision rather than a "
|
| 213 |
+
f"deployment step. Set {PROVIDER_ENV} once it is made; this "
|
| 214 |
+
f"build can serve: {', '.join(sorted(PROVIDERS))}.",
|
| 215 |
+
)
|
| 216 |
+
if not self._api_key():
|
| 217 |
+
return Availability(
|
| 218 |
+
False,
|
| 219 |
+
f"{self._key_env} is not set, so the reasoner cannot be called.",
|
| 220 |
+
f"Set {self._key_env} as a Container App secret. Never commit "
|
| 221 |
+
f"it, and never add it to a provisioning script in this repo.",
|
| 222 |
+
)
|
| 223 |
+
if not self._model():
|
| 224 |
+
return Availability(
|
| 225 |
+
False,
|
| 226 |
+
f"{self._model_env} is not set, so a result could not name the "
|
| 227 |
+
f"model that produced it.",
|
| 228 |
+
f"Set {self._model_env} to the exact hosted model id. "
|
| 229 |
+
f"InferenceResult requires model_version, and the API refuses "
|
| 230 |
+
f"to store a run that cannot name its artefact.",
|
| 231 |
+
)
|
| 232 |
+
return Availability(True)
|
| 233 |
+
|
| 234 |
+
def load(self) -> "HostedMultimodalAdapter":
|
| 235 |
+
availability = self.availability()
|
| 236 |
+
if not availability.ready:
|
| 237 |
+
raise AdapterUnavailable(availability)
|
| 238 |
+
return self
|
| 239 |
+
|
| 240 |
+
def reason(
|
| 241 |
+
self,
|
| 242 |
+
images: list[Image.Image],
|
| 243 |
+
*,
|
| 244 |
+
rubric: str,
|
| 245 |
+
capability: Capability | None = None,
|
| 246 |
+
schema: dict | None = None,
|
| 247 |
+
) -> ReasonerResponse:
|
| 248 |
+
"""One call, one JSON object, checked before it is returned.
|
| 249 |
+
|
| 250 |
+
Pass `capability` and the schema is built from the registry, so the
|
| 251 |
+
vocabulary the model is handed is the vocabulary its answer is judged
|
| 252 |
+
against and the two cannot drift. Pass `schema` to override it β the
|
| 253 |
+
override is still hardened, still validated, and still scanned, because
|
| 254 |
+
a caller supplying a permissive schema must not be a way around the
|
| 255 |
+
checks.
|
| 256 |
+
|
| 257 |
+
**Passing neither a capability nor a schema is a refusal.** There is no
|
| 258 |
+
default open schema: an answer nobody declared a shape for is an answer
|
| 259 |
+
nobody can review.
|
| 260 |
+
"""
|
| 261 |
+
availability = self.availability()
|
| 262 |
+
if not availability.ready:
|
| 263 |
+
raise AdapterUnavailable(availability)
|
| 264 |
+
assert self._transport is not None
|
| 265 |
+
|
| 266 |
+
if not images:
|
| 267 |
+
raise AdapterError("A visual reasoner needs at least one image.")
|
| 268 |
+
if capability is None and schema is None:
|
| 269 |
+
raise AdapterError(
|
| 270 |
+
"reason() needs a capability or a schema. A hosted model asked "
|
| 271 |
+
"for an answer with no declared shape can put a claim in any "
|
| 272 |
+
"field it invents, which is the failure this adapter exists to "
|
| 273 |
+
"prevent."
|
| 274 |
+
)
|
| 275 |
+
|
| 276 |
+
contract = schema if schema is not None else claims.schema_for(capability)
|
| 277 |
+
|
| 278 |
+
prompt = f"{SYSTEM_RULES}\n\n{rubric}"
|
| 279 |
+
raw = self._transport(
|
| 280 |
+
prompt=prompt,
|
| 281 |
+
images=images,
|
| 282 |
+
schema=contract,
|
| 283 |
+
model=self._model(),
|
| 284 |
+
api_key=self._api_key(),
|
| 285 |
+
)
|
| 286 |
+
|
| 287 |
+
# Not repaired, not retried with a nudge, and not partially accepted.
|
| 288 |
+
# Β§4 says the calls must return structured JSON; one that did not is a
|
| 289 |
+
# failed call, and scraping an object out of prose is how a malformed
|
| 290 |
+
# answer becomes a stored result.
|
| 291 |
+
parsed = claims.parse_strict(raw)
|
| 292 |
+
claims.enforce(parsed, contract, capability=capability)
|
| 293 |
+
|
| 294 |
+
warnings = [
|
| 295 |
+
"This reading came from a general-purpose hosted model that has "
|
| 296 |
+
"never been trained on livestock, and it is experimental. A vet "
|
| 297 |
+
"confirms it."
|
| 298 |
+
]
|
| 299 |
+
return ReasonerResponse(
|
| 300 |
+
parsed=parsed, raw=raw, model=self._model(), warnings=tuple(warnings)
|
| 301 |
+
)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
# **`BCS_SCHEMA` used to be here, and it is gone rather than repaired.**
|
| 305 |
+
#
|
| 306 |
+
# It was Β§7's body-condition contract written by hand: `minimum: 1.0`,
|
| 307 |
+
# `maximum: 5.0`, `multipleOf: 0.5` on `best_estimate` and on `range.items` β
|
| 308 |
+
# the only bounded numbers anywhere in this service, and the reason the audit
|
| 309 |
+
# that found `best_estimate: 2.6347` could point at something that did work.
|
| 310 |
+
#
|
| 311 |
+
# It had no production caller. `reason(capability=cattle_bcs)` builds its
|
| 312 |
+
# contract from `claims.schema_for` on the line above, and the only imports of
|
| 313 |
+
# `BCS_SCHEMA` were in `tests/test_adapters.py` and `tests/test_claims.py`. **A
|
| 314 |
+
# control referenced only by its own tests is worse than none**, because a green
|
| 315 |
+
# suite reads as coverage of the path a farmer's result actually takes, and this
|
| 316 |
+
# one covered a path nothing takes.
|
| 317 |
+
#
|
| 318 |
+
# Deleting it costs nothing now that `schema_for` reads `OutputSpec`: for
|
| 319 |
+
# `cattle_bcs` it emits the same three keywords on the same two fields, from the
|
| 320 |
+
# registry rather than from a copy, and adds the closed `claims` vocabulary that
|
| 321 |
+
# the hand-written version never had. The tests that imported it now build the
|
| 322 |
+
# generated schema, so they exercise what production exercises.
|
app/adapters/pose/__init__.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Quadruped pose to gait features: Β§24 from the keypoints onward.
|
| 2 |
+
|
| 3 |
+
The pose model is not here. What is here is everything a pose model feeds:
|
| 4 |
+
a keypoint vocabulary that more than one model can be mapped into
|
| 5 |
+
(`vocabulary`), a body frame that removes the animal's translation and its
|
| 6 |
+
changing apparent size (`tracks`), and the Β§24 features plus a screen that
|
| 7 |
+
cannot express a diagnosis (`gait`).
|
| 8 |
+
|
| 9 |
+
That boundary is drawn where it is because of the licences, which are the part of
|
| 10 |
+
Β§24 most likely to be discovered late.
|
| 11 |
+
|
| 12 |
+
## Licence positions, recorded because they decide what can be sold
|
| 13 |
+
|
| 14 |
+
**DeepLabCut SuperAnimal-Quadruped β Modified MIT, academic and non-commercial
|
| 15 |
+
use only.** The model card on Hugging Face states it plainly: use is restricted
|
| 16 |
+
to academic and non-commercial purposes, the model "may not be used to harm any
|
| 17 |
+
animal deliberately", and commercial licensing is available on application to
|
| 18 |
+
Prof. Mackenzie W. Mathis or EPFL's technology transfer office. Β§24 names this
|
| 19 |
+
model first, and nothing about that is a reason not to evaluate it β but Animap
|
| 20 |
+
is a commercial product, so shipping it would need that licence. Read on
|
| 21 |
+
2026-08-22 from
|
| 22 |
+
`https://huggingface.co/mwmathis/DeepLabCutModelZoo-SuperAnimal-Quadruped`.
|
| 23 |
+
|
| 24 |
+
**ViTPose β Apache-2.0 for the code and the Hugging Face ports.** The AP-10K
|
| 25 |
+
vocabulary it can be run against covers 54 species including cattle. It is the
|
| 26 |
+
alternative Β§36 requires to be tried, and it is the one with no commercial
|
| 27 |
+
obstacle.
|
| 28 |
+
|
| 29 |
+
`experiments/cattle_gait/README.md` records which of these actually ran and what
|
| 30 |
+
it produced. This docstring records only what their terms say.
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
from __future__ import annotations
|
| 34 |
+
|
| 35 |
+
from app.adapters.base import (
|
| 36 |
+
Adapter,
|
| 37 |
+
AdapterSpec,
|
| 38 |
+
Availability,
|
| 39 |
+
Modality,
|
| 40 |
+
Placement,
|
| 41 |
+
Task,
|
| 42 |
+
)
|
| 43 |
+
from app.adapters.pose.gait import (
|
| 44 |
+
GaitFeatures,
|
| 45 |
+
GaitScreen,
|
| 46 |
+
LimbCycle,
|
| 47 |
+
PairAsymmetry,
|
| 48 |
+
STATEMENTS,
|
| 49 |
+
Verdict,
|
| 50 |
+
features,
|
| 51 |
+
screen,
|
| 52 |
+
)
|
| 53 |
+
from app.adapters.pose.tracks import (
|
| 54 |
+
BodyFrame,
|
| 55 |
+
PoseSequence,
|
| 56 |
+
TrackUnusable,
|
| 57 |
+
body_frame,
|
| 58 |
+
in_body_frame,
|
| 59 |
+
)
|
| 60 |
+
from app.adapters.pose.vocabulary import (
|
| 61 |
+
AP10K,
|
| 62 |
+
AP10K_ORDER,
|
| 63 |
+
CONTRALATERAL_PAIRS,
|
| 64 |
+
SUPERANIMAL_QUADRUPED,
|
| 65 |
+
VOCABULARIES,
|
| 66 |
+
Landmark,
|
| 67 |
+
missing,
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
GAIT_FEATURES_SPEC = AdapterSpec(
|
| 71 |
+
adapter_id="gait-features",
|
| 72 |
+
runtime="opencv-numpy",
|
| 73 |
+
tasks=(Task.MEASURE,),
|
| 74 |
+
modalities=(Modality.VIDEO,),
|
| 75 |
+
directive_role=(
|
| 76 |
+
"Β§24 cattle gait β stride timing, left/right symmetry, hoof "
|
| 77 |
+
"trajectories, back-line movement, head movement and stance duration, "
|
| 78 |
+
"derived from quadruped keypoint tracks. The pose model itself is a "
|
| 79 |
+
"separate adapter; this is the deterministic half Β§4 prefers."
|
| 80 |
+
),
|
| 81 |
+
requires_artefact=False,
|
| 82 |
+
placement=Placement.CPU_SERVICE,
|
| 83 |
+
placement_reason=(
|
| 84 |
+
"Gradients and one FFT over a few hundred frames of keypoints β "
|
| 85 |
+
"microseconds. It sits wherever the pose model does, and the pose model "
|
| 86 |
+
"is the thing that needs a GPU."
|
| 87 |
+
),
|
| 88 |
+
notes=(
|
| 89 |
+
"**No clinical threshold exists and none is offered.** `screen()` "
|
| 90 |
+
"requires the caller to supply the asymmetry index it judges against, "
|
| 91 |
+
"because deriving one needs cattle a vet locomotion-scored and this "
|
| 92 |
+
"code has never seen any. What `experiments/cattle_gait` measures "
|
| 93 |
+
"instead is the method's own resolution β the smallest asymmetry it can "
|
| 94 |
+
"separate from its noise β which is a different claim and is labelled "
|
| 95 |
+
"as one."
|
| 96 |
+
),
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
class GaitFeatureAdapter(Adapter):
|
| 101 |
+
"""Β§24 from the keypoints onward. Always available; never diagnoses."""
|
| 102 |
+
|
| 103 |
+
spec = GAIT_FEATURES_SPEC
|
| 104 |
+
|
| 105 |
+
def availability(self) -> Availability:
|
| 106 |
+
return Availability(True)
|
| 107 |
+
|
| 108 |
+
def load(self) -> "GaitFeatureAdapter":
|
| 109 |
+
return self
|
| 110 |
+
|
| 111 |
+
def measure(self, sequence: PoseSequence) -> GaitFeatures:
|
| 112 |
+
return features(sequence)
|
| 113 |
+
|
| 114 |
+
def screen(self, sequence: PoseSequence, *, threshold: float,
|
| 115 |
+
threshold_basis: str = "unstated") -> GaitScreen:
|
| 116 |
+
return screen(sequence, threshold=threshold, threshold_basis=threshold_basis)
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
__all__ = [
|
| 120 |
+
"AP10K",
|
| 121 |
+
"AP10K_ORDER",
|
| 122 |
+
"CONTRALATERAL_PAIRS",
|
| 123 |
+
"GAIT_FEATURES_SPEC",
|
| 124 |
+
"STATEMENTS",
|
| 125 |
+
"SUPERANIMAL_QUADRUPED",
|
| 126 |
+
"VOCABULARIES",
|
| 127 |
+
"BodyFrame",
|
| 128 |
+
"GaitFeatureAdapter",
|
| 129 |
+
"GaitFeatures",
|
| 130 |
+
"GaitScreen",
|
| 131 |
+
"Landmark",
|
| 132 |
+
"LimbCycle",
|
| 133 |
+
"PairAsymmetry",
|
| 134 |
+
"PoseSequence",
|
| 135 |
+
"TrackUnusable",
|
| 136 |
+
"Verdict",
|
| 137 |
+
"body_frame",
|
| 138 |
+
"features",
|
| 139 |
+
"in_body_frame",
|
| 140 |
+
"missing",
|
| 141 |
+
"screen",
|
| 142 |
+
]
|
app/adapters/pose/gait.py
ADDED
|
@@ -0,0 +1,566 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Gait features from keypoint tracks, and a screen that refuses to diagnose.
|
| 2 |
+
|
| 3 |
+
Directive Β§24 asks for stride timing, left/right symmetry, hoof trajectories,
|
| 4 |
+
back-line movement, head movement and stance duration, and it gives the exact
|
| 5 |
+
sentence the output may not be:
|
| 6 |
+
|
| 7 |
+
> Lameness score 3 caused by left rear hoof disease.
|
| 8 |
+
|
| 9 |
+
That sentence is forbidden three times over β it scores, it localises and it
|
| 10 |
+
attributes a cause β and none of the three is a thing this pipeline could know.
|
| 11 |
+
So `GaitScreen` has no score field, no limb field and no cause field. There is
|
| 12 |
+
nothing to fill in, which is a stronger guarantee than a rule about what to fill
|
| 13 |
+
in with. The permitted wording is a fixed map from a three-valued verdict; a
|
| 14 |
+
caller cannot compose a sentence out of the features because the features are
|
| 15 |
+
numbers with no words attached.
|
| 16 |
+
|
| 17 |
+
## What is measured, and in what units
|
| 18 |
+
|
| 19 |
+
Everything is in the body frame `tracks` establishes β trunk lengths and
|
| 20 |
+
seconds β so nothing here needs a metric scale. That is the structural reason
|
| 21 |
+
gait is a more tractable capability than Β§22's weight: an asymmetry is a ratio,
|
| 22 |
+
and a ratio survives not knowing how big the animal is.
|
| 23 |
+
|
| 24 |
+
**Asymmetry indices are unsigned.** `|left β right| / mean`, never `left β
|
| 25 |
+
right`. A signed index invites the reader to name a limb, and naming the limb is
|
| 26 |
+
half of the forbidden sentence.
|
| 27 |
+
|
| 28 |
+
## The threshold problem, stated rather than solved
|
| 29 |
+
|
| 30 |
+
Whether an asymmetry index of 0.12 means anything about a cow is a clinical
|
| 31 |
+
question, and answering it needs cattle whose locomotion a vet scored. This
|
| 32 |
+
module has never seen one. What it can establish without them is its own
|
| 33 |
+
**resolution**: the smallest asymmetry it can tell apart from its own noise, on
|
| 34 |
+
sequences where the true asymmetry is known because it was constructed. That is
|
| 35 |
+
what `RESOLUTION_FLOOR` holds, it is measured in `experiments/cattle_gait`, and
|
| 36 |
+
it is emphatically not a clinical threshold.
|
| 37 |
+
|
| 38 |
+
`screen()` therefore returns `ASYMMETRY_OBSERVED` only for an index above a
|
| 39 |
+
threshold the caller supplies. There is no default, and there is no module-level
|
| 40 |
+
constant a caller can reach for and mistake for a validated one.
|
| 41 |
+
"""
|
| 42 |
+
|
| 43 |
+
from __future__ import annotations
|
| 44 |
+
|
| 45 |
+
from dataclasses import dataclass, field
|
| 46 |
+
from enum import Enum
|
| 47 |
+
|
| 48 |
+
import numpy as np
|
| 49 |
+
|
| 50 |
+
from app.adapters.pose.tracks import (
|
| 51 |
+
PoseSequence,
|
| 52 |
+
TrackUnusable,
|
| 53 |
+
body_frame,
|
| 54 |
+
in_body_frame,
|
| 55 |
+
)
|
| 56 |
+
from app.adapters.pose.vocabulary import CONTRALATERAL_PAIRS, Landmark
|
| 57 |
+
from app.adapters.signal.periodicity import dominant_rate
|
| 58 |
+
|
| 59 |
+
#: Search band for stride rate, in strides per minute for one limb. A cow at a
|
| 60 |
+
#: walk completes roughly 0.6β1.3 strides per second. The band is deliberately
|
| 61 |
+
#: wider than that on both sides so the method is not a prior pulling the
|
| 62 |
+
#: estimate towards what a textbook says a cow does.
|
| 63 |
+
STRIDE_BAND_PER_MINUTE = (25.0, 120.0)
|
| 64 |
+
|
| 65 |
+
#: **The respiratory module's thresholds do not transfer to this band and are
|
| 66 |
+
#: not reused.** `periodicity.MIN_PEAK_PROMINENCE` was derived at (8, 180)
|
| 67 |
+
#: cycles per minute against a metronome and camera noise; prominence is peak
|
| 68 |
+
#: power over the *in-band* median, so it is a property of the band. These are
|
| 69 |
+
#: separate constants, set from what a hoof trace looks like rather than
|
| 70 |
+
#: inherited, and they are weaker gates because a swinging limb is a far
|
| 71 |
+
#: stronger oscillation than a breathing flank.
|
| 72 |
+
STRIDE_MIN_PROMINENCE = 8.0
|
| 73 |
+
STRIDE_MAX_HALF_DRIFT = 0.25
|
| 74 |
+
STRIDE_MIN_CYCLES = 3.0
|
| 75 |
+
|
| 76 |
+
#: Fractional change in apparent trunk length above which the capture is refused.
|
| 77 |
+
#: A walk perpendicular to the camera holds the trunk's apparent length nearly
|
| 78 |
+
#: constant; one angled towards the lens does not, and the foreshortening lands
|
| 79 |
+
#: unevenly on the two sides of the animal. That produces an asymmetry in a sound
|
| 80 |
+
#: cow, which is the worst failure this capability has.
|
| 81 |
+
#:
|
| 82 |
+
#: **Not derived from cattle footage.** 0.25 is a geometric argument β a 25%
|
| 83 |
+
#: change in apparent length is roughly a 25% change in distance, at which point
|
| 84 |
+
#: near-side and far-side limbs are being measured at materially different
|
| 85 |
+
#: scales. It should be re-derived the first time there is real footage.
|
| 86 |
+
MAX_TRUNK_DRIFT = 0.25
|
| 87 |
+
|
| 88 |
+
#: Width of the local-polynomial window used to differentiate a hoof's
|
| 89 |
+
#: body-frame position, as a fraction of that limb's own measured stride period.
|
| 90 |
+
#:
|
| 91 |
+
#: **Scaled to the stride rather than fixed in frames**, so the same constant
|
| 92 |
+
#: works at 15 fps and at 30, and on a slow walk and a brisk one. A fixed frame
|
| 93 |
+
#: count silently means something different in each of those cases.
|
| 94 |
+
#:
|
| 95 |
+
#: ## Why differentiation needs a window at all
|
| 96 |
+
#:
|
| 97 |
+
#: Stance is separated from swing by the sign of the hoof's velocity relative to
|
| 98 |
+
#: the trunk, and that velocity is small: at 30 fps, a 0.9 Hz stride and a
|
| 99 |
+
#: stride length of 0.55 trunk lengths, a planted hoof moves 0.0165 trunk
|
| 100 |
+
#: lengths per frame. Five pixels of keypoint noise on a 400 px trunk is 0.0125
|
| 101 |
+
#: trunk lengths, and a three-point difference of that noise is **larger than
|
| 102 |
+
#: the signal being measured**. The first version of this module used
|
| 103 |
+
#: `np.gradient` with a three-frame moving average and reported a 24% left-right
|
| 104 |
+
#: asymmetry on a perfectly symmetric walk at that noise level.
|
| 105 |
+
#:
|
| 106 |
+
#: ## How 0.35 was chosen
|
| 107 |
+
#:
|
| 108 |
+
#: **Regenerate this table rather than trusting it:**
|
| 109 |
+
#:
|
| 110 |
+
#: python -m experiments.cattle_gait.run --arm window-derivation
|
| 111 |
+
#:
|
| 112 |
+
#: The rule, fixed before the sweep ran: the **smallest** fraction that
|
| 113 |
+
#: separates a 12% injected asymmetry from a sound walk at **every** keypoint
|
| 114 |
+
#: noise level tested, where "separates" means the lame 5th percentile clears
|
| 115 |
+
#: the sound 95th. Protocol: seeds **100β111**, which are disjoint from the
|
| 116 |
+
#: `SEEDS = (0, 1, 2, 3, 4)` the benchmark runs on; hind pair; stance index;
|
| 117 |
+
#: 30 fps; `stance_reduction = stride_reduction = 0.12`. Margins as
|
| 118 |
+
#: `lame p5 β sound p95`, positive separates:
|
| 119 |
+
#:
|
| 120 |
+
#: | fraction | 0 px | 2 px | 5 px | 10 px | separates everywhere |
|
| 121 |
+
#: |---|---|---|---|---|---|
|
| 122 |
+
#: | 0.10 | +0.1593 | β0.1731 | β0.4312 | β0.1697 | no |
|
| 123 |
+
#: | 0.15 | +0.1651 | +0.1416 | β0.1490 | β0.4063 | no |
|
| 124 |
+
#: | 0.20 | +0.1853 | +0.1398 | +0.0993 | β0.4075 | no |
|
| 125 |
+
#: | 0.25 | +0.1510 | +0.1148 | +0.1008 | β0.1693 | no |
|
| 126 |
+
#: | 0.30 | +0.1510 | +0.1003 | +0.0748 | β0.0214 | no |
|
| 127 |
+
#: | **0.35** | **+0.1154** | **+0.0737** | **+0.0674** | **+0.0326** | **yes** |
|
| 128 |
+
#: | 0.40 | +0.1154 | +0.0737 | +0.0674 | +0.0326 | yes |
|
| 129 |
+
#: | 0.50 | +0.0582 | +0.0395 | +0.0179 | +0.0149 | yes |
|
| 130 |
+
#:
|
| 131 |
+
#: 0.40 gives an identical row because the rounded window is the same number of
|
| 132 |
+
#: frames; 0.35 is the smaller of the two and is taken.
|
| 133 |
+
#:
|
| 134 |
+
#: **The `--arm window-derivation` command exists because a hostile audit could
|
| 135 |
+
#: not reproduce this table from the paragraph above it.** Its best-matching
|
| 136 |
+
#: reconstruction put the load-bearing 0.35/10 px cell at +0.012 against the
|
| 137 |
+
#: published +0.033, and sixteen other plausible readings of the protocol made
|
| 138 |
+
#: it negative. The numbers were right; what was missing was enough of the
|
| 139 |
+
#: protocol to re-derive them, which is the same failure as not having them.
|
| 140 |
+
#:
|
| 141 |
+
#: The derivation set is disjoint from the benchmark set, so the published
|
| 142 |
+
#: figures are not a training score. It is still **synthetic on both sides**,
|
| 143 |
+
#: and a window chosen against constructed kinematics has no claim on real
|
| 144 |
+
#: footage.
|
| 145 |
+
#:
|
| 146 |
+
#: ## What it costs
|
| 147 |
+
#:
|
| 148 |
+
#: 0.35 of a stride period is nearly as long as the swing phase itself, so the
|
| 149 |
+
#: stance-to-swing transition is blurred and a stance duration is resolved to
|
| 150 |
+
#: roughly a tenth of a cycle rather than to a frame. That is the trade being
|
| 151 |
+
#: made: the sign of the velocity survives noise, and the exact instant of
|
| 152 |
+
#: touchdown does not. Nothing here should be read at single-frame resolution.
|
| 153 |
+
STANCE_WINDOW_FRACTION = 0.35
|
| 154 |
+
|
| 155 |
+
#: Order of the local polynomial fitted inside that window. Quadratic rather
|
| 156 |
+
#: than linear because a hoof's body-frame trace curves through the swing, and a
|
| 157 |
+
#: straight-line fit over a third of a cycle would bias the derivative towards
|
| 158 |
+
#: the window's mean slope β which is zero over a whole cycle.
|
| 159 |
+
STANCE_POLYNOMIAL_ORDER = 2
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
class Verdict(str, Enum):
|
| 163 |
+
"""The only three things this capability may conclude.
|
| 164 |
+
|
| 165 |
+
Note what is not here: a grade, a limb, a cause, a severity. Β§24 names all
|
| 166 |
+
four in the sentence it forbids.
|
| 167 |
+
"""
|
| 168 |
+
|
| 169 |
+
ASYMMETRY_OBSERVED = "asymmetry_observed"
|
| 170 |
+
NO_ASYMMETRY_OBSERVED = "no_asymmetry_observed"
|
| 171 |
+
NOT_ASSESSABLE = "not_assessable"
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
#: The permitted wording, fixed. A caller renders `screen().statement`; there is
|
| 175 |
+
#: no path by which a feature value becomes part of a sentence.
|
| 176 |
+
STATEMENTS: dict[Verdict, str] = {
|
| 177 |
+
Verdict.ASYMMETRY_OBSERVED: "Possible gait asymmetry",
|
| 178 |
+
Verdict.NO_ASYMMETRY_OBSERVED: (
|
| 179 |
+
"No gait asymmetry observed in this recording"
|
| 180 |
+
),
|
| 181 |
+
Verdict.NOT_ASSESSABLE: "Gait could not be assessed from this recording",
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
@dataclass(frozen=True)
|
| 186 |
+
class LimbCycle:
|
| 187 |
+
"""One limb's stride, as measured. All times in seconds.
|
| 188 |
+
|
| 189 |
+
`usable` false means the periodicity gates refused this limb, and every
|
| 190 |
+
figure below it should be read as diagnostic rather than as a measurement.
|
| 191 |
+
"""
|
| 192 |
+
|
| 193 |
+
landmark: str
|
| 194 |
+
usable: bool
|
| 195 |
+
reason: str = ""
|
| 196 |
+
strides_per_minute: float | None = None
|
| 197 |
+
#: Fraction of the cycle the hoof spends moving backwards relative to the
|
| 198 |
+
#: trunk, which is the definition of stance used here. Cattle at a walk sit
|
| 199 |
+
#: around 0.6; a figure near 0.5 or above 0.8 suggests the separation failed
|
| 200 |
+
#: rather than that the animal is unusual.
|
| 201 |
+
stance_fraction: float | None = None
|
| 202 |
+
#: Mean duration of one stance phase.
|
| 203 |
+
stance_seconds: float | None = None
|
| 204 |
+
#: Peak-to-peak excursion along the body axis, in trunk lengths.
|
| 205 |
+
stride_length_trunks: float | None = None
|
| 206 |
+
#: Peak-to-peak excursion across the body axis, in trunk lengths. Hoof lift.
|
| 207 |
+
hoof_lift_trunks: float | None = None
|
| 208 |
+
peak_prominence: float = 0.0
|
| 209 |
+
cycles_observed: float = 0.0
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
@dataclass(frozen=True)
|
| 213 |
+
class PairAsymmetry:
|
| 214 |
+
"""Left against right, for one pair of limbs. Unsigned, always.
|
| 215 |
+
|
| 216 |
+
Every index is `|left β right| / mean`, so 0 is symmetric and 0.2 means the
|
| 217 |
+
two sides differ by a fifth of their average. `None` where the underlying
|
| 218 |
+
limb measurement was refused.
|
| 219 |
+
"""
|
| 220 |
+
|
| 221 |
+
pair: str
|
| 222 |
+
stance_index: float | None
|
| 223 |
+
stride_length_index: float | None
|
| 224 |
+
#: Contralateral phase offset as a fraction of a stride, folded so that 0
|
| 225 |
+
#: means perfectly anti-phase (the sound pattern at a walk) and 1 means the
|
| 226 |
+
#: two limbs move together.
|
| 227 |
+
phase_index: float | None
|
| 228 |
+
left: LimbCycle
|
| 229 |
+
right: LimbCycle
|
| 230 |
+
|
| 231 |
+
@property
|
| 232 |
+
def worst(self) -> float | None:
|
| 233 |
+
values = [v for v in (self.stance_index, self.stride_length_index,
|
| 234 |
+
self.phase_index) if v is not None]
|
| 235 |
+
return max(values) if values else None
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
@dataclass(frozen=True)
|
| 239 |
+
class GaitFeatures:
|
| 240 |
+
"""Everything Β§24 asks to be derived, plus what could not be.
|
| 241 |
+
|
| 242 |
+
`unavailable` names the features the model's vocabulary or the capture made
|
| 243 |
+
impossible. It is a list of reasons rather than a set of nulls, because a
|
| 244 |
+
null reads as zero to whoever writes the next summary.
|
| 245 |
+
"""
|
| 246 |
+
|
| 247 |
+
pairs: tuple[PairAsymmetry, ...]
|
| 248 |
+
#: Peak-to-peak vertical excursion of the nose in trunk lengths. Head nodding
|
| 249 |
+
#: is a long-established lameness sign; the amplitude is reported and nothing
|
| 250 |
+
#: is concluded from it, because no threshold for it has been measured here.
|
| 251 |
+
head_movement_trunks: float | None
|
| 252 |
+
#: Deviation of the mid-back from the neck-to-tail line, in trunk lengths:
|
| 253 |
+
#: mean, then how much it varies over the clip. An arched back is another
|
| 254 |
+
#: established sign, and the same absence of a threshold applies.
|
| 255 |
+
back_arch_mean_trunks: float | None
|
| 256 |
+
back_arch_variation_trunks: float | None
|
| 257 |
+
trunk_drift: float
|
| 258 |
+
duration_seconds: float
|
| 259 |
+
fps: float
|
| 260 |
+
unavailable: tuple[str, ...] = field(default_factory=tuple)
|
| 261 |
+
|
| 262 |
+
@property
|
| 263 |
+
def worst_asymmetry(self) -> float | None:
|
| 264 |
+
values = [p.worst for p in self.pairs if p.worst is not None]
|
| 265 |
+
return max(values) if values else None
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
@dataclass(frozen=True)
|
| 269 |
+
class GaitScreen:
|
| 270 |
+
"""What the product may show, and the evidence under it.
|
| 271 |
+
|
| 272 |
+
There is no `score`, no `limb` and no `cause`. `statement` reads from
|
| 273 |
+
`STATEMENTS` by verdict and interpolates nothing.
|
| 274 |
+
"""
|
| 275 |
+
|
| 276 |
+
verdict: Verdict
|
| 277 |
+
features: GaitFeatures | None
|
| 278 |
+
reason: str
|
| 279 |
+
#: The asymmetry index the caller judged against, carried so a stored result
|
| 280 |
+
#: says what it was compared to. `None` on `NOT_ASSESSABLE`.
|
| 281 |
+
threshold: float | None = None
|
| 282 |
+
#: Whether that threshold came from measured cattle or from somewhere else.
|
| 283 |
+
#: There is no default: a caller that does not say gets "unstated", which is
|
| 284 |
+
#: what a reader needs to see.
|
| 285 |
+
threshold_basis: str = "unstated"
|
| 286 |
+
|
| 287 |
+
@property
|
| 288 |
+
def statement(self) -> str:
|
| 289 |
+
return STATEMENTS[self.verdict]
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
def _derivative(signal: np.ndarray, window: int,
|
| 293 |
+
order: int = STANCE_POLYNOMIAL_ORDER) -> np.ndarray:
|
| 294 |
+
"""First derivative by a sliding local polynomial fit (SavitzkyβGolay).
|
| 295 |
+
|
| 296 |
+
Fitting a polynomial across a window and reading its slope is far more
|
| 297 |
+
robust to independent per-frame noise than differencing neighbours, because
|
| 298 |
+
every sample in the window constrains the fit. The coefficients are the
|
| 299 |
+
second row of the pseudo-inverse of the Vandermonde matrix β the row that
|
| 300 |
+
recovers the linear term β so this is a plain least-squares fit written as a
|
| 301 |
+
convolution, with no SciPy dependency.
|
| 302 |
+
|
| 303 |
+
Edges are handled by repeating the end samples. That biases the derivative
|
| 304 |
+
towards zero in the first and last half-window, which is why `_limb` drops
|
| 305 |
+
the first and last stance run before averaging.
|
| 306 |
+
"""
|
| 307 |
+
# Forced odd. Defensive rather than load-bearing: `half` floors, so a
|
| 308 |
+
# window of 8 builds the same 9-tap kernel as a window of 9. What it
|
| 309 |
+
# does buy is that the two comparisons below test the width actually
|
| 310 |
+
# used. `tests/test_gait.py` records the equivalence.
|
| 311 |
+
window = int(window) | 1
|
| 312 |
+
if window < order + 2 or signal.size < window:
|
| 313 |
+
return np.gradient(signal)
|
| 314 |
+
half = window // 2
|
| 315 |
+
offsets = np.arange(-half, half + 1, dtype=np.float64)
|
| 316 |
+
design = np.vander(offsets, order + 1, increasing=True)
|
| 317 |
+
coefficients = np.linalg.pinv(design)[1]
|
| 318 |
+
padded = np.pad(signal, half, mode="edge")
|
| 319 |
+
return np.convolve(padded, coefficients[::-1], mode="valid")
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def _limb(sequence: PoseSequence, landmark: Landmark) -> LimbCycle:
|
| 323 |
+
"""Stride rate, stance fraction and excursions for one hoof."""
|
| 324 |
+
path = in_body_frame(sequence, landmark)
|
| 325 |
+
if path is None:
|
| 326 |
+
return LimbCycle(
|
| 327 |
+
landmark=landmark.value, usable=False,
|
| 328 |
+
reason=(
|
| 329 |
+
f"{landmark.value} was not tracked in enough frames, or the "
|
| 330 |
+
f"model has no keypoint for it."
|
| 331 |
+
),
|
| 332 |
+
)
|
| 333 |
+
|
| 334 |
+
along, across = path[:, 0], path[:, 1]
|
| 335 |
+
rate = dominant_rate(
|
| 336 |
+
along, sequence.fps, STRIDE_BAND_PER_MINUTE,
|
| 337 |
+
min_prominence=STRIDE_MIN_PROMINENCE,
|
| 338 |
+
max_drift=STRIDE_MAX_HALF_DRIFT,
|
| 339 |
+
min_cycles=STRIDE_MIN_CYCLES,
|
| 340 |
+
# The sub-band shoulder gate is disabled here and that is a deliberate
|
| 341 |
+
# difference from respiration. A hoof's body-frame trace is a sawtooth,
|
| 342 |
+
# not a sinusoid, and a sawtooth's own harmonic structure plus the
|
| 343 |
+
# residual of an imperfectly removed trend put real power below the
|
| 344 |
+
# band. Leaving the gate on refused sound synthetic walks. What replaces
|
| 345 |
+
# it is the trunk-drift check, which catches the same underlying
|
| 346 |
+
# problem β a capture that is drifting rather than cycling β at its
|
| 347 |
+
# source rather than in the spectrum.
|
| 348 |
+
max_shoulder=float("inf"),
|
| 349 |
+
)
|
| 350 |
+
|
| 351 |
+
# Stance and swing, separated by the sign of the body-frame velocity. In
|
| 352 |
+
# stance the hoof is planted, so relative to a forward-moving trunk it
|
| 353 |
+
# travels backwards; in swing it overtakes the trunk. The zero crossing is
|
| 354 |
+
# the natural boundary and needs no threshold to be chosen.
|
| 355 |
+
#
|
| 356 |
+
# The differentiating window is scaled to this limb's own measured stride.
|
| 357 |
+
# When the rate was refused there is no period to scale by, so the window
|
| 358 |
+
# falls back to a fifth of the clip's own length β the result is refused
|
| 359 |
+
# either way, and the figures below are diagnostics rather than a
|
| 360 |
+
# measurement.
|
| 361 |
+
period_frames = (sequence.fps * 60.0 / rate.cycles_per_minute
|
| 362 |
+
if rate.cycles_per_minute else along.size / 5.0)
|
| 363 |
+
window = max(3, int(round(STANCE_WINDOW_FRACTION * period_frames)))
|
| 364 |
+
velocity = _derivative(along, window)
|
| 365 |
+
in_stance = velocity < 0
|
| 366 |
+
stance_fraction = float(in_stance.mean())
|
| 367 |
+
|
| 368 |
+
# Mean length of a stance run, in seconds. Computed from the runs rather
|
| 369 |
+
# than as `stance_fraction Γ period`, so a limb that takes one very long
|
| 370 |
+
# stance and three short ones is distinguishable from one taking four even
|
| 371 |
+
# ones. The first and last runs are dropped because the clip truncates them
|
| 372 |
+
# and a truncated stance reads as a short one.
|
| 373 |
+
edges = np.flatnonzero(np.diff(in_stance.astype(np.int8)))
|
| 374 |
+
runs: list[int] = []
|
| 375 |
+
if edges.size >= 2:
|
| 376 |
+
boundaries = np.concatenate(([0], edges + 1, [in_stance.size]))
|
| 377 |
+
for start, end in zip(boundaries[:-1], boundaries[1:]):
|
| 378 |
+
if in_stance[start]:
|
| 379 |
+
runs.append(int(end - start))
|
| 380 |
+
runs = runs[1:-1] if len(runs) > 2 else runs
|
| 381 |
+
stance_seconds = float(np.mean(runs) / sequence.fps) if runs else None
|
| 382 |
+
|
| 383 |
+
return LimbCycle(
|
| 384 |
+
landmark=landmark.value,
|
| 385 |
+
usable=rate.usable,
|
| 386 |
+
reason=rate.reason,
|
| 387 |
+
strides_per_minute=round(rate.cycles_per_minute, 2) if rate.cycles_per_minute else None,
|
| 388 |
+
stance_fraction=round(stance_fraction, 4),
|
| 389 |
+
stance_seconds=round(stance_seconds, 4) if stance_seconds else None,
|
| 390 |
+
stride_length_trunks=round(float(along.max() - along.min()), 4),
|
| 391 |
+
hoof_lift_trunks=round(float(across.max() - across.min()), 4),
|
| 392 |
+
peak_prominence=round(rate.peak_prominence, 2),
|
| 393 |
+
cycles_observed=round(rate.cycles_observed, 2),
|
| 394 |
+
)
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
def _index(left: float | None, right: float | None) -> float | None:
|
| 398 |
+
"""`|l β r| / mean`, or `None` when either side is missing.
|
| 399 |
+
|
| 400 |
+
Unsigned on purpose β see this module's docstring. Returns `None` rather
|
| 401 |
+
than 0 when the mean is zero, because two limbs that both measured zero have
|
| 402 |
+
not been shown to be symmetric, they have not been measured.
|
| 403 |
+
"""
|
| 404 |
+
if left is None or right is None:
|
| 405 |
+
return None
|
| 406 |
+
mean = (left + right) / 2.0
|
| 407 |
+
if mean == 0:
|
| 408 |
+
return None
|
| 409 |
+
return round(abs(left - right) / mean, 4)
|
| 410 |
+
|
| 411 |
+
|
| 412 |
+
def _phase_index(sequence: PoseSequence, left: Landmark, right: Landmark,
|
| 413 |
+
strides_per_minute: float | None) -> float | None:
|
| 414 |
+
"""How far the two limbs are from anti-phase, as a fraction of a stride.
|
| 415 |
+
|
| 416 |
+
Cross-correlation of the two body-frame along-axis traces gives the lag at
|
| 417 |
+
which they best agree. At a walk the contralateral pair is half a cycle
|
| 418 |
+
apart, so a lag of half a period is the sound pattern and is mapped to 0.
|
| 419 |
+
|
| 420 |
+
Returns `None` without a stride rate: a lag in frames means nothing until
|
| 421 |
+
there is a period to express it as a fraction of.
|
| 422 |
+
"""
|
| 423 |
+
if not strides_per_minute:
|
| 424 |
+
return None
|
| 425 |
+
left_path = in_body_frame(sequence, left)
|
| 426 |
+
right_path = in_body_frame(sequence, right)
|
| 427 |
+
if left_path is None or right_path is None:
|
| 428 |
+
return None
|
| 429 |
+
|
| 430 |
+
a = left_path[:, 0] - left_path[:, 0].mean()
|
| 431 |
+
b = right_path[:, 0] - right_path[:, 0].mean()
|
| 432 |
+
if not np.any(a) or not np.any(b):
|
| 433 |
+
return None
|
| 434 |
+
|
| 435 |
+
period_frames = sequence.fps * 60.0 / strides_per_minute
|
| 436 |
+
correlation = np.correlate(a, b, mode="full")
|
| 437 |
+
lags = np.arange(-len(a) + 1, len(b))
|
| 438 |
+
# Only lags inside one period are meaningful; beyond that the correlation
|
| 439 |
+
# peak repeats and picking the global maximum would report a lag of three
|
| 440 |
+
# cycles as easily as one.
|
| 441 |
+
inside = np.abs(lags) <= period_frames
|
| 442 |
+
if not inside.any():
|
| 443 |
+
return None
|
| 444 |
+
lag = float(lags[inside][int(np.argmax(correlation[inside]))])
|
| 445 |
+
|
| 446 |
+
offset = abs(lag) / period_frames # 0 = in phase, 0.5 = anti-phase
|
| 447 |
+
folded = offset % 1.0
|
| 448 |
+
# Distance from 0.5, doubled so the index runs 0 (sound) to 1 (limbs moving
|
| 449 |
+
# together), matching the direction of every other index here.
|
| 450 |
+
return round(abs(folded - 0.5) * 2.0, 4)
|
| 451 |
+
|
| 452 |
+
|
| 453 |
+
def features(sequence: PoseSequence) -> GaitFeatures:
|
| 454 |
+
"""Every Β§24 quantity this sequence supports, and a reason for each it does not."""
|
| 455 |
+
frame = body_frame(sequence)
|
| 456 |
+
unavailable: list[str] = []
|
| 457 |
+
|
| 458 |
+
pairs: list[PairAsymmetry] = []
|
| 459 |
+
for name, left_landmark, right_landmark in CONTRALATERAL_PAIRS:
|
| 460 |
+
left = _limb(sequence, left_landmark)
|
| 461 |
+
right = _limb(sequence, right_landmark)
|
| 462 |
+
|
| 463 |
+
# **Both limbs must be usable before any index is computed.** A refused
|
| 464 |
+
# limb still carries a stance fraction and an excursion β they are
|
| 465 |
+
# diagnostics, emitted so a threshold can be re-derived later β and an
|
| 466 |
+
# earlier version of this function fed them straight into `_index`. On a
|
| 467 |
+
# two-second clip, where every limb is refused for holding too few
|
| 468 |
+
# strides, that published `Possible gait asymmetry` at an index of
|
| 469 |
+
# 0.145. A refusal that reaches the farm as a finding is the exact
|
| 470 |
+
# failure this whole service is built against; `tests/test_gait.py`
|
| 471 |
+
# asserts the short-clip case so it cannot come back.
|
| 472 |
+
both = left.usable and right.usable
|
| 473 |
+
rate = left.strides_per_minute if both else None
|
| 474 |
+
pairs.append(PairAsymmetry(
|
| 475 |
+
pair=name,
|
| 476 |
+
stance_index=_index(left.stance_seconds, right.stance_seconds) if both else None,
|
| 477 |
+
stride_length_index=_index(left.stride_length_trunks,
|
| 478 |
+
right.stride_length_trunks) if both else None,
|
| 479 |
+
phase_index=_phase_index(sequence, left_landmark, right_landmark, rate),
|
| 480 |
+
left=left,
|
| 481 |
+
right=right,
|
| 482 |
+
))
|
| 483 |
+
if not both:
|
| 484 |
+
unavailable.append(
|
| 485 |
+
f"{name} pair: {left.reason or right.reason or 'limb not measurable'}"
|
| 486 |
+
)
|
| 487 |
+
|
| 488 |
+
nose = in_body_frame(sequence, Landmark.NOSE)
|
| 489 |
+
if nose is None:
|
| 490 |
+
head_movement = None
|
| 491 |
+
unavailable.append("head movement: the nose was not tracked well enough.")
|
| 492 |
+
else:
|
| 493 |
+
head_movement = round(float(nose[:, 1].max() - nose[:, 1].min()), 4)
|
| 494 |
+
|
| 495 |
+
back = in_body_frame(sequence, Landmark.BACK_MIDDLE)
|
| 496 |
+
if back is None:
|
| 497 |
+
arch_mean = arch_variation = None
|
| 498 |
+
unavailable.append(
|
| 499 |
+
f"back-line movement: {sequence.vocabulary or 'this model'} supplies "
|
| 500 |
+
f"no mid-back keypoint, or it was not tracked well enough. Β§24 lists "
|
| 501 |
+
f"back-line movement among the quantities to derive and this "
|
| 502 |
+
f"vocabulary cannot supply it."
|
| 503 |
+
)
|
| 504 |
+
else:
|
| 505 |
+
# Already the perpendicular distance from the neck-to-tail line, in
|
| 506 |
+
# trunk lengths: `in_body_frame`'s second column is the across-axis
|
| 507 |
+
# component and the axis runs between exactly those two anchors.
|
| 508 |
+
arch_mean = round(float(np.mean(back[:, 1])), 4)
|
| 509 |
+
arch_variation = round(float(np.std(back[:, 1])), 4)
|
| 510 |
+
|
| 511 |
+
return GaitFeatures(
|
| 512 |
+
pairs=tuple(pairs),
|
| 513 |
+
head_movement_trunks=head_movement,
|
| 514 |
+
back_arch_mean_trunks=arch_mean,
|
| 515 |
+
back_arch_variation_trunks=arch_variation,
|
| 516 |
+
trunk_drift=round(frame.drift, 4),
|
| 517 |
+
duration_seconds=round(sequence.duration_seconds, 3),
|
| 518 |
+
fps=sequence.fps,
|
| 519 |
+
unavailable=tuple(unavailable),
|
| 520 |
+
)
|
| 521 |
+
|
| 522 |
+
|
| 523 |
+
def screen(sequence: PoseSequence, *, threshold: float,
|
| 524 |
+
threshold_basis: str = "unstated") -> GaitScreen:
|
| 525 |
+
"""Β§24's output: a screen, with the evidence, and never a diagnosis.
|
| 526 |
+
|
| 527 |
+
`threshold` is required and has no default. The module has no validated
|
| 528 |
+
value to offer β deriving one needs cattle a vet locomotion-scored, and this
|
| 529 |
+
code has never seen any β so making the caller supply it forces the question
|
| 530 |
+
"where did this number come from?" to be answered at every call site rather
|
| 531 |
+
than inherited from a constant.
|
| 532 |
+
"""
|
| 533 |
+
try:
|
| 534 |
+
measured = features(sequence)
|
| 535 |
+
except TrackUnusable as refused:
|
| 536 |
+
return GaitScreen(Verdict.NOT_ASSESSABLE, None, str(refused))
|
| 537 |
+
|
| 538 |
+
if measured.trunk_drift > MAX_TRUNK_DRIFT:
|
| 539 |
+
return GaitScreen(
|
| 540 |
+
Verdict.NOT_ASSESSABLE, measured,
|
| 541 |
+
f"The animal's apparent size changed by {measured.trunk_drift:.0%} "
|
| 542 |
+
f"during the recording, against a {MAX_TRUNK_DRIFT:.0%} limit. It "
|
| 543 |
+
f"was walking towards or away from the camera rather than across "
|
| 544 |
+
f"it, and near-side and far-side limbs would be measured at "
|
| 545 |
+
f"different scales. Record again from the side, standing still.",
|
| 546 |
+
)
|
| 547 |
+
|
| 548 |
+
worst = measured.worst_asymmetry
|
| 549 |
+
if worst is None:
|
| 550 |
+
return GaitScreen(
|
| 551 |
+
Verdict.NOT_ASSESSABLE, measured,
|
| 552 |
+
"No limb pair produced a usable stride. "
|
| 553 |
+
+ " ".join(measured.unavailable),
|
| 554 |
+
)
|
| 555 |
+
|
| 556 |
+
verdict = (Verdict.ASYMMETRY_OBSERVED if worst >= threshold
|
| 557 |
+
else Verdict.NO_ASYMMETRY_OBSERVED)
|
| 558 |
+
return GaitScreen(
|
| 559 |
+
verdict, measured,
|
| 560 |
+
reason=(
|
| 561 |
+
f"Largest left-right index {worst:.3f} against a threshold of "
|
| 562 |
+
f"{threshold:.3f}."
|
| 563 |
+
),
|
| 564 |
+
threshold=threshold,
|
| 565 |
+
threshold_basis=threshold_basis,
|
| 566 |
+
)
|
app/adapters/pose/tracks.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Keypoint tracks over time, and the body frame that makes them comparable.
|
| 2 |
+
|
| 3 |
+
A pose model returns pixel coordinates. Pixel coordinates are useless for gait on
|
| 4 |
+
their own, for two reasons that both have to be removed before any feature means
|
| 5 |
+
anything:
|
| 6 |
+
|
| 7 |
+
**The animal moves across the frame.** Β§24's capture is a cow walking 5β10 m
|
| 8 |
+
side-on, so a hoof's image x-coordinate is dominated by the animal's own
|
| 9 |
+
translation. Every hoof's trace looks the same: a ramp.
|
| 10 |
+
|
| 11 |
+
**The animal's apparent size changes.** It walks closer to or further from the
|
| 12 |
+
camera, and a stride that measures 300 px at one end of the run measures 200 px
|
| 13 |
+
at the other. Comparing a left stride recorded at the near end to a right stride
|
| 14 |
+
recorded at the far end would find an asymmetry in a perfectly sound animal, and
|
| 15 |
+
that is the single most dangerous false positive this capability can produce.
|
| 16 |
+
|
| 17 |
+
So everything downstream reads the **body frame**: positions expressed relative
|
| 18 |
+
to the trunk and divided by the trunk's own apparent length. The result is
|
| 19 |
+
dimensionless, immune to both problems, and needs no metric scale at all β which
|
| 20 |
+
is why gait is tractable on a phone in a way that Β§22's weight is not.
|
| 21 |
+
|
| 22 |
+
**What the body frame does not fix** is the camera not being perpendicular to the
|
| 23 |
+
walk. A cow walking towards the camera at an angle has a foreshortened stride
|
| 24 |
+
that shortens further as it approaches, and the trunk-length normalisation
|
| 25 |
+
partly absorbs that and partly does not. `PoseSequence.trunk_drift` reports how
|
| 26 |
+
much the trunk length changed over the clip so a caller can refuse a capture
|
| 27 |
+
where it changed a lot, rather than analysing it anyway.
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
from __future__ import annotations
|
| 31 |
+
|
| 32 |
+
from dataclasses import dataclass
|
| 33 |
+
|
| 34 |
+
import numpy as np
|
| 35 |
+
|
| 36 |
+
from app.adapters.pose.vocabulary import Landmark
|
| 37 |
+
|
| 38 |
+
#: Below this the keypoint is treated as absent for that frame. Pose models
|
| 39 |
+
#: report a heatmap peak value, not a probability, so this is a threshold on an
|
| 40 |
+
#: uncalibrated score and Β§37 forbids presenting it as a confidence anywhere.
|
| 41 |
+
#: 0.3 is the value ViTPose's and DeepLabCut's own example code use; it has not
|
| 42 |
+
#: been tuned on cattle and should not be quoted as if it had.
|
| 43 |
+
MIN_KEYPOINT_SCORE = 0.3
|
| 44 |
+
|
| 45 |
+
#: Fraction of frames a landmark must be present in before it is used at all.
|
| 46 |
+
#: A trace that exists in a third of the frames is mostly interpolation, and
|
| 47 |
+
#: interpolation between two distant stance phases invents a swing that never
|
| 48 |
+
#: happened.
|
| 49 |
+
MIN_LANDMARK_COVERAGE = 0.6
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class TrackUnusable(ValueError):
|
| 53 |
+
"""The pose sequence cannot support a body frame.
|
| 54 |
+
|
| 55 |
+
Raised rather than worked around. Without both trunk anchors there is no
|
| 56 |
+
body frame, and without a body frame every feature below measures the
|
| 57 |
+
camera's motion as though it were the animal's.
|
| 58 |
+
"""
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@dataclass(frozen=True)
|
| 62 |
+
class PoseSequence:
|
| 63 |
+
"""Keypoints over frames, plus the model vocabulary they are named in.
|
| 64 |
+
|
| 65 |
+
`xy` is `(frames, landmarks, 2)` in image pixels and `score` is
|
| 66 |
+
`(frames, landmarks)`. `landmarks` is the tuple naming the second axis, so
|
| 67 |
+
an experiment holding a stored sequence never has to remember an ordering.
|
| 68 |
+
"""
|
| 69 |
+
|
| 70 |
+
xy: np.ndarray
|
| 71 |
+
score: np.ndarray
|
| 72 |
+
landmarks: tuple[Landmark, ...]
|
| 73 |
+
fps: float
|
| 74 |
+
#: Which vocabulary the underlying model used. Carried for the record: a
|
| 75 |
+
#: feature that is unavailable because the model has no mid-back keypoint
|
| 76 |
+
#: should say which model.
|
| 77 |
+
vocabulary: str = ""
|
| 78 |
+
|
| 79 |
+
def __post_init__(self) -> None:
|
| 80 |
+
frames, count, _ = np.shape(self.xy)
|
| 81 |
+
if (frames, count) != np.shape(self.score):
|
| 82 |
+
raise TrackUnusable("xy and score describe different numbers of keypoints.")
|
| 83 |
+
if count != len(self.landmarks):
|
| 84 |
+
raise TrackUnusable(
|
| 85 |
+
f"{count} keypoint columns against {len(self.landmarks)} names."
|
| 86 |
+
)
|
| 87 |
+
if self.fps <= 0:
|
| 88 |
+
raise TrackUnusable("The clip reports no frame rate.")
|
| 89 |
+
|
| 90 |
+
@property
|
| 91 |
+
def frames(self) -> int:
|
| 92 |
+
return int(np.shape(self.xy)[0])
|
| 93 |
+
|
| 94 |
+
@property
|
| 95 |
+
def duration_seconds(self) -> float:
|
| 96 |
+
return self.frames / self.fps
|
| 97 |
+
|
| 98 |
+
def index(self, landmark: Landmark) -> int | None:
|
| 99 |
+
try:
|
| 100 |
+
return self.landmarks.index(landmark)
|
| 101 |
+
except ValueError:
|
| 102 |
+
return None
|
| 103 |
+
|
| 104 |
+
def coverage(self, landmark: Landmark) -> float:
|
| 105 |
+
"""Fraction of frames where this landmark scored above threshold."""
|
| 106 |
+
column = self.index(landmark)
|
| 107 |
+
if column is None:
|
| 108 |
+
return 0.0
|
| 109 |
+
return float((self.score[:, column] >= MIN_KEYPOINT_SCORE).mean())
|
| 110 |
+
|
| 111 |
+
def trace(self, landmark: Landmark) -> np.ndarray | None:
|
| 112 |
+
"""This landmark's `(frames, 2)` path, low-score frames filled in.
|
| 113 |
+
|
| 114 |
+
Returns `None` when the landmark is absent from the vocabulary or falls
|
| 115 |
+
under `MIN_LANDMARK_COVERAGE`. Gaps inside a well-covered trace are
|
| 116 |
+
linearly interpolated and the ends are held, which is the standard
|
| 117 |
+
treatment and is also a lie about the frames it fills; nothing here
|
| 118 |
+
should be read at single-frame resolution.
|
| 119 |
+
"""
|
| 120 |
+
column = self.index(landmark)
|
| 121 |
+
if column is None or self.coverage(landmark) < MIN_LANDMARK_COVERAGE:
|
| 122 |
+
return None
|
| 123 |
+
present = self.score[:, column] >= MIN_KEYPOINT_SCORE
|
| 124 |
+
frames = np.arange(self.frames, dtype=np.float64)
|
| 125 |
+
filled = np.empty((self.frames, 2), dtype=np.float64)
|
| 126 |
+
for axis in (0, 1):
|
| 127 |
+
filled[:, axis] = np.interp(
|
| 128 |
+
frames, frames[present], self.xy[present, column, axis]
|
| 129 |
+
)
|
| 130 |
+
return filled
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
@dataclass(frozen=True)
|
| 134 |
+
class BodyFrame:
|
| 135 |
+
"""The trunk, per frame: where it is, how long it looks, which way it faces.
|
| 136 |
+
|
| 137 |
+
`length` is the neck-to-tail distance in pixels. It is the normaliser for
|
| 138 |
+
everything, so its stability over the clip is reported rather than assumed β
|
| 139 |
+
see `drift`.
|
| 140 |
+
"""
|
| 141 |
+
|
| 142 |
+
origin: np.ndarray
|
| 143 |
+
length: np.ndarray
|
| 144 |
+
#: Unit vector from tail to neck. The animal's forward direction in image
|
| 145 |
+
#: coordinates, so a cow walking right-to-left is handled without the caller
|
| 146 |
+
#: having to know which way it went.
|
| 147 |
+
forward: np.ndarray
|
| 148 |
+
|
| 149 |
+
@property
|
| 150 |
+
def drift(self) -> float:
|
| 151 |
+
"""Peak-to-peak change in apparent trunk length, as a fraction of the median.
|
| 152 |
+
|
| 153 |
+
The single best available warning that the walk was not perpendicular to
|
| 154 |
+
the camera. A cow crossing the frame at a right angle holds its apparent
|
| 155 |
+
length within a few per cent; one walking towards the lens does not.
|
| 156 |
+
"""
|
| 157 |
+
median = float(np.median(self.length))
|
| 158 |
+
if median <= 0:
|
| 159 |
+
return float("inf")
|
| 160 |
+
return float(self.length.max() - self.length.min()) / median
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def body_frame(sequence: PoseSequence) -> BodyFrame:
|
| 164 |
+
"""Trunk position, apparent length and heading, per frame."""
|
| 165 |
+
neck = sequence.trace(Landmark.NECK)
|
| 166 |
+
tail = sequence.trace(Landmark.TAIL_BASE)
|
| 167 |
+
if neck is None or tail is None:
|
| 168 |
+
missing_names = [
|
| 169 |
+
landmark.value for landmark, trace in
|
| 170 |
+
((Landmark.NECK, neck), (Landmark.TAIL_BASE, tail)) if trace is None
|
| 171 |
+
]
|
| 172 |
+
raise TrackUnusable(
|
| 173 |
+
f"No body frame: {', '.join(missing_names)} was not tracked in at "
|
| 174 |
+
f"least {MIN_LANDMARK_COVERAGE:.0%} of frames. Without both trunk "
|
| 175 |
+
f"anchors every gait feature would measure the camera's motion."
|
| 176 |
+
)
|
| 177 |
+
|
| 178 |
+
span = neck - tail
|
| 179 |
+
length = np.linalg.norm(span, axis=1)
|
| 180 |
+
if not np.all(length > 0):
|
| 181 |
+
raise TrackUnusable("The neck and tail keypoints coincide in some frames.")
|
| 182 |
+
return BodyFrame(
|
| 183 |
+
origin=tail,
|
| 184 |
+
length=length,
|
| 185 |
+
forward=span / length[:, None],
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def in_body_frame(sequence: PoseSequence, landmark: Landmark) -> np.ndarray | None:
|
| 190 |
+
"""A landmark's path in trunk lengths, along and across the body axis.
|
| 191 |
+
|
| 192 |
+
Returns `(frames, 2)` where column 0 is the along-body coordinate β positive
|
| 193 |
+
towards the head β and column 1 is across it, positive downwards in image
|
| 194 |
+
terms once the axis is fixed. Both are in units of trunk length, so a stride
|
| 195 |
+
amplitude of 0.4 means "four tenths of this animal's own body".
|
| 196 |
+
|
| 197 |
+
Projecting onto the trunk axis rather than onto the image axes is what makes
|
| 198 |
+
the result independent of which way the animal walked and of a camera held
|
| 199 |
+
slightly off level.
|
| 200 |
+
"""
|
| 201 |
+
trace = sequence.trace(landmark)
|
| 202 |
+
if trace is None:
|
| 203 |
+
return None
|
| 204 |
+
frame = body_frame(sequence)
|
| 205 |
+
relative = trace - frame.origin
|
| 206 |
+
forward = frame.forward
|
| 207 |
+
# The perpendicular of a 2-D unit vector, taken consistently so "across" has
|
| 208 |
+
# a fixed sign for a given walk direction.
|
| 209 |
+
across = np.stack([-forward[:, 1], forward[:, 0]], axis=1)
|
| 210 |
+
along_component = (relative * forward).sum(axis=1) / frame.length
|
| 211 |
+
across_component = (relative * across).sum(axis=1) / frame.length
|
| 212 |
+
return np.stack([along_component, across_component], axis=1)
|
app/adapters/pose/vocabulary.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One keypoint vocabulary, and the maps from the models that use another.
|
| 2 |
+
|
| 3 |
+
Directive Β§24 names DeepLabCut SuperAnimal-Quadruped, and Β§36 requires the
|
| 4 |
+
alternatives to be attempted before anything is called impossible. Those two
|
| 5 |
+
sentences together mean more than one pose model will be tried, and every pose
|
| 6 |
+
model has its own names for the same anatomy. Wiring gait analysis to any one of
|
| 7 |
+
them would make swapping models a rewrite of the analysis.
|
| 8 |
+
|
| 9 |
+
So the analysis reads `Landmark`, and each model gets a map. A map that has no
|
| 10 |
+
entry for a landmark leaves it **absent**, and absent propagates: a gait feature
|
| 11 |
+
that needs the mid-back is reported as unavailable on a model with no mid-back
|
| 12 |
+
keypoint, never as zero. That distinction is the whole reason this file exists
|
| 13 |
+
rather than an integer index.
|
| 14 |
+
|
| 15 |
+
## The maps are copied from the models' own definitions, not remembered
|
| 16 |
+
|
| 17 |
+
`SUPERANIMAL_QUADRUPED` is the `bodyparts` list in DeepLabCut's own
|
| 18 |
+
`superanimal_quadruped.yaml`, read on 2026-08-22. `AP10K` is the `keypoint_info`
|
| 19 |
+
ordering in mmpose's `configs/_base_/datasets/ap10k.py`, read the same day. Both
|
| 20 |
+
are recorded in source order with their indices, because a pose model returns an
|
| 21 |
+
array and an off-by-one in this table silently swaps a cow's left hind hoof for
|
| 22 |
+
its right β which would produce a confident, entirely fictional asymmetry.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
from __future__ import annotations
|
| 26 |
+
|
| 27 |
+
from enum import Enum
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class Landmark(str, Enum):
|
| 31 |
+
"""The anatomy gait analysis needs, named once.
|
| 32 |
+
|
| 33 |
+
Deliberately small. Every entry here is read by something in `gait`; a
|
| 34 |
+
landmark nothing uses is a landmark whose mapping nobody checks.
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
NOSE = "nose"
|
| 38 |
+
#: Where the neck meets the trunk. The front anchor of the body frame.
|
| 39 |
+
NECK = "neck"
|
| 40 |
+
#: Base of the tail. The rear anchor of the body frame.
|
| 41 |
+
TAIL_BASE = "tail_base"
|
| 42 |
+
#: The middle of the topline, between neck and tail. Only some models have
|
| 43 |
+
#: it, and back-arch analysis is unavailable without it.
|
| 44 |
+
BACK_MIDDLE = "back_middle"
|
| 45 |
+
LEFT_FRONT_HOOF = "left_front_hoof"
|
| 46 |
+
RIGHT_FRONT_HOOF = "right_front_hoof"
|
| 47 |
+
LEFT_HIND_HOOF = "left_hind_hoof"
|
| 48 |
+
RIGHT_HIND_HOOF = "right_hind_hoof"
|
| 49 |
+
LEFT_FRONT_KNEE = "left_front_knee"
|
| 50 |
+
RIGHT_FRONT_KNEE = "right_front_knee"
|
| 51 |
+
LEFT_HIND_KNEE = "left_hind_knee"
|
| 52 |
+
RIGHT_HIND_KNEE = "right_hind_knee"
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
#: Contralateral pairs, front and hind. Symmetry is only ever computed within a
|
| 56 |
+
#: pair β comparing a fore hoof to a hind hoof would find a phase difference on
|
| 57 |
+
#: every sound animal that walks.
|
| 58 |
+
CONTRALATERAL_PAIRS: tuple[tuple[str, Landmark, Landmark], ...] = (
|
| 59 |
+
("front", Landmark.LEFT_FRONT_HOOF, Landmark.RIGHT_FRONT_HOOF),
|
| 60 |
+
("hind", Landmark.LEFT_HIND_HOOF, Landmark.RIGHT_HIND_HOOF),
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
#: DeepLabCut SuperAnimal-Quadruped, 39 bodyparts. Source: the `bodyparts` list
|
| 64 |
+
#: in `deeplabcut/modelzoo/project_configs/superanimal_quadruped.yaml` on the
|
| 65 |
+
#: DeepLabCut `main` branch, read 2026-08-22.
|
| 66 |
+
#:
|
| 67 |
+
#: **Only 8 of the 39 are mapped**, and that is correct rather than lazy: the
|
| 68 |
+
#: other 31 are ears, antlers, jaw and eye points that no gait feature reads.
|
| 69 |
+
#: Mapping them would create thirty-one more entries nobody verifies.
|
| 70 |
+
#:
|
| 71 |
+
#: Two notes on the choices, because both are judgement calls:
|
| 72 |
+
#: `back_base` is taken as the neck-side trunk anchor rather than `neck_end`,
|
| 73 |
+
#: because `back_base` sits on the topline and `neck_end` does not, and a body
|
| 74 |
+
#: axis is more stable between two topline points. The model spells the upper
|
| 75 |
+
#: limb segment `thai` (its own spelling of "thigh"); the `knee` points are the
|
| 76 |
+
#: lower joint and the `paw` points are the ground contact, which is what stance
|
| 77 |
+
#: detection needs.
|
| 78 |
+
SUPERANIMAL_QUADRUPED: dict[Landmark, str] = {
|
| 79 |
+
Landmark.NOSE: "nose",
|
| 80 |
+
Landmark.NECK: "back_base",
|
| 81 |
+
Landmark.TAIL_BASE: "tail_base",
|
| 82 |
+
Landmark.BACK_MIDDLE: "back_middle",
|
| 83 |
+
Landmark.LEFT_FRONT_HOOF: "front_left_paw",
|
| 84 |
+
Landmark.RIGHT_FRONT_HOOF: "front_right_paw",
|
| 85 |
+
Landmark.LEFT_HIND_HOOF: "back_left_paw",
|
| 86 |
+
Landmark.RIGHT_HIND_HOOF: "back_right_paw",
|
| 87 |
+
Landmark.LEFT_FRONT_KNEE: "front_left_knee",
|
| 88 |
+
Landmark.RIGHT_FRONT_KNEE: "front_right_knee",
|
| 89 |
+
Landmark.LEFT_HIND_KNEE: "back_left_knee",
|
| 90 |
+
Landmark.RIGHT_HIND_KNEE: "back_right_knee",
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
#: AP-10K's 17 keypoints, in the order mmpose defines them. Source:
|
| 94 |
+
#: `configs/_base_/datasets/ap10k.py`, `keypoint_info` ids 0β16, read 2026-08-22.
|
| 95 |
+
#: AP-10K includes cattle among its 54 species, which is why it is here at all.
|
| 96 |
+
#:
|
| 97 |
+
#: **It has no mid-back point.** `Neck` and `Root of tail` are the only topline
|
| 98 |
+
#: keypoints, so `BACK_MIDDLE` is absent and back-arch analysis cannot run on a
|
| 99 |
+
#: model trained to this vocabulary. That is a real limitation of the model
|
| 100 |
+
#: choice, not a gap in this table, and `gait` reports it as unavailable.
|
| 101 |
+
AP10K_ORDER: tuple[str, ...] = (
|
| 102 |
+
"L_Eye", "R_Eye", "Nose", "Neck", "Root of tail",
|
| 103 |
+
"L_Shoulder", "L_Elbow", "L_F_Paw",
|
| 104 |
+
"R_Shoulder", "R_Elbow", "R_F_Paw",
|
| 105 |
+
"L_Hip", "L_Knee", "L_B_Paw",
|
| 106 |
+
"R_Hip", "R_Knee", "R_B_Paw",
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
+
AP10K: dict[Landmark, str] = {
|
| 110 |
+
Landmark.NOSE: "Nose",
|
| 111 |
+
Landmark.NECK: "Neck",
|
| 112 |
+
Landmark.TAIL_BASE: "Root of tail",
|
| 113 |
+
Landmark.LEFT_FRONT_HOOF: "L_F_Paw",
|
| 114 |
+
Landmark.RIGHT_FRONT_HOOF: "R_F_Paw",
|
| 115 |
+
Landmark.LEFT_HIND_HOOF: "L_B_Paw",
|
| 116 |
+
Landmark.RIGHT_HIND_HOOF: "R_B_Paw",
|
| 117 |
+
Landmark.LEFT_FRONT_KNEE: "L_Elbow",
|
| 118 |
+
Landmark.RIGHT_FRONT_KNEE: "R_Elbow",
|
| 119 |
+
Landmark.LEFT_HIND_KNEE: "L_Knee",
|
| 120 |
+
Landmark.RIGHT_HIND_KNEE: "R_Knee",
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
VOCABULARIES: dict[str, dict[Landmark, str]] = {
|
| 124 |
+
"superanimal_quadruped": SUPERANIMAL_QUADRUPED,
|
| 125 |
+
"ap10k": AP10K,
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def missing(vocabulary: str) -> tuple[Landmark, ...]:
|
| 130 |
+
"""Landmarks this model cannot supply.
|
| 131 |
+
|
| 132 |
+
Called before analysis so a caller learns what will be unavailable up front
|
| 133 |
+
rather than reading a result with silent holes in it.
|
| 134 |
+
"""
|
| 135 |
+
mapping = VOCABULARIES[vocabulary]
|
| 136 |
+
return tuple(landmark for landmark in Landmark if landmark not in mapping)
|
app/adapters/registry.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Every adapter the zero-training stack names, and the state each one is in.
|
| 2 |
+
|
| 3 |
+
One listing, built the same way whether an adapter runs, is refused on licence,
|
| 4 |
+
is merely not installed, or needs a credential nobody has set. That uniformity is
|
| 5 |
+
the point: `main._unavailable_reason` already distinguishes *"no validated model
|
| 6 |
+
exists"* from *"this is not planned"*, and the same distinction has to survive
|
| 7 |
+
one level down or a reader is back to guessing which of six models is the one
|
| 8 |
+
blocking a capability.
|
| 9 |
+
|
| 10 |
+
**Nothing here can produce a result.** The registry hands back adapters, and an
|
| 11 |
+
adapter that cannot run raises from `load()`. There is no path through this
|
| 12 |
+
module that returns an answer, which is the property `providers.discover()` has
|
| 13 |
+
and the reason this is a sibling of it rather than a replacement for it.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import json
|
| 19 |
+
|
| 20 |
+
from app.adapters.base import Adapter, Placement
|
| 21 |
+
from app.adapters.deterministic import deterministic_adapters
|
| 22 |
+
from app.adapters.embedding import (
|
| 23 |
+
DINOV2_SPEC,
|
| 24 |
+
DINOV3_SPEC,
|
| 25 |
+
MEGADESCRIPTOR_SPEC,
|
| 26 |
+
MIEWID_SPEC,
|
| 27 |
+
OnnxEmbeddingAdapter,
|
| 28 |
+
)
|
| 29 |
+
from app.adapters.multimodal import HostedMultimodalAdapter
|
| 30 |
+
from app.adapters.transports import transport_from_env
|
| 31 |
+
from app.adapters.unavailable import unavailable_adapters
|
| 32 |
+
from app.providers import MODELS_DIR, ArtefactError, load_card
|
| 33 |
+
|
| 34 |
+
#: Cards for adapters this package knows how to build, by adapter id. An
|
| 35 |
+
#: adapter whose card is absent still appears in the listing β as unavailable,
|
| 36 |
+
#: with the path it was looking for β because a silently missing entry is how a
|
| 37 |
+
#: capability disappears without anybody deciding it should.
|
| 38 |
+
#:
|
| 39 |
+
#: The last two are installed and permanently non-servable, which is a state
|
| 40 |
+
#: this listing has not had before. They are here rather than in
|
| 41 |
+
#: `unavailable.py` because they are no longer absences to explain: the
|
| 42 |
+
#: artefacts exist, `load()` works under `ANIMAP_LICENCE_POLICY=record`, and
|
| 43 |
+
#: what stops them reaching a farm is `licences.gate` refusing every request
|
| 44 |
+
#: under the default policy. `refused()` below still names exactly these two.
|
| 45 |
+
_EMBEDDING_CARDS = {
|
| 46 |
+
"dinov3-vits16": ("cattle_identity", "model_card.json"),
|
| 47 |
+
"dinov2-small": ("alternates/dinov2_embedding", "model_card.json"),
|
| 48 |
+
"megadescriptor": ("alternates/megadescriptor", "model_card.json"),
|
| 49 |
+
"miewid-msv3": ("alternates/miewid", "model_card.json"),
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _embedding(spec, folder: str, filename: str) -> Adapter:
|
| 54 |
+
card_path = MODELS_DIR / folder / filename
|
| 55 |
+
artefact = None
|
| 56 |
+
card: dict = {}
|
| 57 |
+
if card_path.is_file():
|
| 58 |
+
try:
|
| 59 |
+
artefact = load_card(card_path)
|
| 60 |
+
card = json.loads(card_path.read_text())
|
| 61 |
+
except (ArtefactError, json.JSONDecodeError):
|
| 62 |
+
# An unreadable card leaves the adapter unavailable rather than
|
| 63 |
+
# taking the listing down, matching `discover()`'s behaviour. The
|
| 64 |
+
# detail is logged there, not swallowed twice here.
|
| 65 |
+
artefact = None
|
| 66 |
+
return OnnxEmbeddingAdapter(
|
| 67 |
+
artefact, spec,
|
| 68 |
+
input_size=card.get("input_size", 224),
|
| 69 |
+
mean=tuple(card.get("image_mean", (0.485, 0.456, 0.406))),
|
| 70 |
+
std=tuple(card.get("image_std", (0.229, 0.224, 0.225))),
|
| 71 |
+
dimensions=card.get("embedding_dimensions", 768),
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def all_adapters() -> list[Adapter]:
|
| 76 |
+
"""Every adapter, runnable or not, in the order the directive names them."""
|
| 77 |
+
adapters: list[Adapter] = []
|
| 78 |
+
for spec in (DINOV3_SPEC, DINOV2_SPEC, MEGADESCRIPTOR_SPEC, MIEWID_SPEC):
|
| 79 |
+
folder, filename = _EMBEDDING_CARDS[spec.adapter_id]
|
| 80 |
+
adapters.append(_embedding(spec, folder, filename))
|
| 81 |
+
adapters.extend(unavailable_adapters())
|
| 82 |
+
# **The transport is chosen by the deployment, not by this import.**
|
| 83 |
+
# `transport_from_env` returns None where no provider is configured, which
|
| 84 |
+
# leaves the adapter reporting itself unavailable with the reason it always
|
| 85 |
+
# gave β and that is the whole posture: a vendor commits Animap to a
|
| 86 |
+
# retention stance for photographs of somebody's animals, so nothing here
|
| 87 |
+
# picks one by default.
|
| 88 |
+
#
|
| 89 |
+
# An unknown provider name raises rather than falling back. A typo that
|
| 90 |
+
# silently disabled fifteen capabilities would produce a listing identical
|
| 91 |
+
# to a deployment where nobody had chosen yet, which is the one failure
|
| 92 |
+
# nobody would go looking for.
|
| 93 |
+
adapters.append(HostedMultimodalAdapter(transport=transport_from_env()))
|
| 94 |
+
adapters.extend(deterministic_adapters())
|
| 95 |
+
return adapters
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def describe_all() -> list[dict]:
|
| 99 |
+
"""The listing, for `/health` and for a person reading a deployment."""
|
| 100 |
+
return [adapter.describe() for adapter in all_adapters()]
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def ready() -> list[Adapter]:
|
| 104 |
+
return [a for a in all_adapters() if a.availability().ready]
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def refused() -> list[Adapter]:
|
| 108 |
+
"""Adapters a licence forbids, as distinct from ones nobody installed.
|
| 109 |
+
|
| 110 |
+
Worth its own function because the two look identical in a listing and mean
|
| 111 |
+
completely different things to whoever is planning the next fortnight: one
|
| 112 |
+
is a download and one is never.
|
| 113 |
+
"""
|
| 114 |
+
from app.adapters.licences import RUNTIME_LICENCES
|
| 115 |
+
|
| 116 |
+
out = []
|
| 117 |
+
for adapter in all_adapters():
|
| 118 |
+
licence = RUNTIME_LICENCES.get(adapter.spec.runtime)
|
| 119 |
+
if licence is not None and not licence.servable:
|
| 120 |
+
out.append(adapter)
|
| 121 |
+
return out
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def by_placement() -> dict[str, list[str]]:
|
| 125 |
+
"""Which leg belongs where.
|
| 126 |
+
|
| 127 |
+
The container size is a hosting decision and not a verdict on a model, so
|
| 128 |
+
this is a recommendation with a measurement behind it rather than a filter.
|
| 129 |
+
"""
|
| 130 |
+
grouped: dict[str, list[str]] = {p.value: [] for p in Placement}
|
| 131 |
+
for adapter in all_adapters():
|
| 132 |
+
grouped[adapter.spec.placement.value].append(adapter.spec.adapter_id)
|
| 133 |
+
return grouped
|
app/adapters/signal/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deterministic signal processing. Directive Β§4: "Do not use a neural model
|
| 2 |
+
when deterministic signal processing is better."
|
| 3 |
+
"""
|
app/adapters/signal/flow.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Turning frames into a one-dimensional motion signal.
|
| 2 |
+
|
| 3 |
+
Directive Β§4 lists optical flow first among the things to use OpenCV for, and
|
| 4 |
+
Β§14's respiration pipeline is built on it. This module does the video half β
|
| 5 |
+
decode, downscale, dense flow, project to one number per frame β and hands the
|
| 6 |
+
result to `periodicity`, which does the arithmetic half.
|
| 7 |
+
|
| 8 |
+
**The projection is the only interesting decision here.** Dense flow gives a
|
| 9 |
+
vector field; a rate needs a scalar. Averaging the magnitude would work and is
|
| 10 |
+
wrong, because magnitude rectifies: a flank moving out and a flank moving back
|
| 11 |
+
both read positive, so the signal comes out at twice the breathing rate and the
|
| 12 |
+
error looks like a plausible answer. The mean *signed* flow keeps the direction,
|
| 13 |
+
and projecting onto the field's own principal axis means the caller does not
|
| 14 |
+
have to know whether the phone was held upright.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
from dataclasses import dataclass
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
|
| 22 |
+
import numpy as np
|
| 23 |
+
|
| 24 |
+
#: Width the frames are resized to before flow is computed. FarnebΓ€ck cost is
|
| 25 |
+
#: linear in pixels and the signal being measured is a whole region's mean, so
|
| 26 |
+
#: resolution buys nothing above a few hundred pixels. Measured on a 1920Γ1080
|
| 27 |
+
#: clip: 4.3 ms per frame pair at 320 px. At full resolution the same clip is
|
| 28 |
+
#: roughly thirty-six times the pixels for the same number.
|
| 29 |
+
FLOW_WIDTH = 320
|
| 30 |
+
|
| 31 |
+
#: FarnebΓ€ck's own parameters, at OpenCV's documented defaults for these
|
| 32 |
+
#: arguments. They are named rather than passed positionally because a reader
|
| 33 |
+
#: cannot otherwise tell `poly_n` from `levels`, and getting one wrong degrades
|
| 34 |
+
#: the field quietly.
|
| 35 |
+
_FARNEBACK = dict(
|
| 36 |
+
pyr_scale=0.5, levels=3, winsize=15,
|
| 37 |
+
iterations=3, poly_n=5, poly_sigma=1.2, flags=0,
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class VideoUnreadable(RuntimeError):
|
| 42 |
+
"""The clip could not be decoded, so there is nothing to measure."""
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass(frozen=True)
|
| 46 |
+
class MotionSignal:
|
| 47 |
+
"""One number per frame pair, plus what it took to get there."""
|
| 48 |
+
|
| 49 |
+
#: Signed displacement along the dominant motion axis, in resized pixels.
|
| 50 |
+
values: np.ndarray
|
| 51 |
+
sample_rate_hz: float
|
| 52 |
+
frames: int
|
| 53 |
+
#: Ratio of the first principal component to the second. Near 1 means the
|
| 54 |
+
#: motion has no preferred direction, which is what a field of noise looks
|
| 55 |
+
#: like β reported rather than acted on, because a real oscillation seen
|
| 56 |
+
#: head-on is also near 1.
|
| 57 |
+
anisotropy: float
|
| 58 |
+
frame_size: tuple[int, int]
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def read_frames(
|
| 62 |
+
path: Path | str,
|
| 63 |
+
*,
|
| 64 |
+
max_frames: int = 1800,
|
| 65 |
+
width: int = FLOW_WIDTH,
|
| 66 |
+
roi: tuple[float, float, float, float] | None = None,
|
| 67 |
+
) -> tuple[list[np.ndarray], float]:
|
| 68 |
+
"""Decode to greyscale, cropped and downscaled.
|
| 69 |
+
|
| 70 |
+
`roi` is fractional β `(x0, y0, x1, y1)` in 0β1 β so a caller that got a
|
| 71 |
+
flank box from a segmenter does not have to know what resolution the clip
|
| 72 |
+
is. Β§14's capture protocol is "hold the cow's flank in frame", and cropping
|
| 73 |
+
to it is what keeps a swishing tail out of the average.
|
| 74 |
+
"""
|
| 75 |
+
import cv2
|
| 76 |
+
|
| 77 |
+
capture = cv2.VideoCapture(str(path))
|
| 78 |
+
if not capture.isOpened():
|
| 79 |
+
raise VideoUnreadable(f"{Path(path).name} could not be opened.")
|
| 80 |
+
|
| 81 |
+
sample_rate = float(capture.get(cv2.CAP_PROP_FPS))
|
| 82 |
+
frames: list[np.ndarray] = []
|
| 83 |
+
try:
|
| 84 |
+
while len(frames) < max_frames:
|
| 85 |
+
ok, frame = capture.read()
|
| 86 |
+
if not ok:
|
| 87 |
+
break
|
| 88 |
+
if roi is not None:
|
| 89 |
+
x0, y0, x1, y1 = roi
|
| 90 |
+
height, frame_width = frame.shape[:2]
|
| 91 |
+
frame = frame[
|
| 92 |
+
int(y0 * height):int(y1 * height),
|
| 93 |
+
int(x0 * frame_width):int(x1 * frame_width),
|
| 94 |
+
]
|
| 95 |
+
if frame.size == 0:
|
| 96 |
+
raise VideoUnreadable("The requested region is outside the frame.")
|
| 97 |
+
height, frame_width = frame.shape[:2]
|
| 98 |
+
scale = width / frame_width
|
| 99 |
+
frame = cv2.resize(
|
| 100 |
+
frame, (width, max(1, int(height * scale))),
|
| 101 |
+
interpolation=cv2.INTER_AREA,
|
| 102 |
+
)
|
| 103 |
+
frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY))
|
| 104 |
+
finally:
|
| 105 |
+
capture.release()
|
| 106 |
+
|
| 107 |
+
if len(frames) < 2:
|
| 108 |
+
raise VideoUnreadable(
|
| 109 |
+
f"{Path(path).name} yielded {len(frames)} frames. There is no motion "
|
| 110 |
+
f"in a single frame."
|
| 111 |
+
)
|
| 112 |
+
return frames, sample_rate
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def motion_signal(frames: list[np.ndarray], sample_rate_hz: float) -> MotionSignal:
|
| 116 |
+
"""Mean signed dense flow per frame pair, projected onto its principal axis."""
|
| 117 |
+
import cv2
|
| 118 |
+
|
| 119 |
+
means = np.empty((len(frames) - 1, 2), dtype=np.float64)
|
| 120 |
+
previous = frames[0]
|
| 121 |
+
for index, current in enumerate(frames[1:]):
|
| 122 |
+
field = cv2.calcOpticalFlowFarneback(previous, current, None, **_FARNEBACK)
|
| 123 |
+
means[index] = (float(field[..., 0].mean()), float(field[..., 1].mean()))
|
| 124 |
+
previous = current
|
| 125 |
+
|
| 126 |
+
centred = means - means.mean(axis=0)
|
| 127 |
+
# SVD rather than an eigendecomposition of the covariance: same axis, and it
|
| 128 |
+
# cannot return a negative eigenvalue when the field is nearly degenerate.
|
| 129 |
+
_, singular, components = np.linalg.svd(centred, full_matrices=False)
|
| 130 |
+
projected = centred @ components[0]
|
| 131 |
+
|
| 132 |
+
return MotionSignal(
|
| 133 |
+
values=projected,
|
| 134 |
+
sample_rate_hz=sample_rate_hz,
|
| 135 |
+
frames=len(frames),
|
| 136 |
+
anisotropy=float(singular[0] / max(singular[1], 1e-12)),
|
| 137 |
+
frame_size=(frames[0].shape[1], frames[0].shape[0]),
|
| 138 |
+
)
|
app/adapters/signal/geometry.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Measuring physical size from a photograph, when something in it has a size.
|
| 2 |
+
|
| 3 |
+
Directive Β§4 lists geometry, contour measurement and reference-marker
|
| 4 |
+
calibration among the things to do with OpenCV rather than a network, and Β§9
|
| 5 |
+
names the payoff: a wound reported as *"approximate visible area: 12β16 cmΒ²"*
|
| 6 |
+
instead of "a wound", so a follow-up scan can say whether it is getting smaller.
|
| 7 |
+
|
| 8 |
+
**The whole capability rests on one honest premise.** Β§22 puts it plainly β "a
|
| 9 |
+
photograph has no scale" is true of an arbitrary photograph and not of one with
|
| 10 |
+
a known-size object in it. So there are exactly two entry points here: one that
|
| 11 |
+
takes a marker of stated physical size, and one that takes a scale somebody
|
| 12 |
+
else established. There is no third that guesses, because a wound area computed
|
| 13 |
+
from a guessed scale is a number with a unit attached to nothing.
|
| 14 |
+
|
| 15 |
+
The `Β±` on every result is not decoration either. A marker localised to within a
|
| 16 |
+
pixel or two at the edges puts a few per cent of error into the linear scale and
|
| 17 |
+
twice that into an area, and Β§37 requires that to reach the farmer as a range
|
| 18 |
+
rather than being rounded away.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import math
|
| 24 |
+
from dataclasses import dataclass
|
| 25 |
+
|
| 26 |
+
import numpy as np
|
| 27 |
+
|
| 28 |
+
#: Aruco dictionary used for the Animap reference marker. 4Γ4 with 50 ids has
|
| 29 |
+
#: the largest cell size for a given printed square, which is what survives
|
| 30 |
+
#: being photographed at arm's length on a phone in a barn.
|
| 31 |
+
ARUCO_DICTIONARY = "DICT_4X4_50"
|
| 32 |
+
|
| 33 |
+
#: Assumed localisation error on each marker corner, in pixels. Propagated into
|
| 34 |
+
#: the reported range rather than ignored.
|
| 35 |
+
#:
|
| 36 |
+
#: **This is an assumption and not a measurement.** OpenCV's corner refinement
|
| 37 |
+
#: is typically sub-pixel on a well-lit marker, and 1.5 px is a deliberately
|
| 38 |
+
#: pessimistic stand-in for the barn case β motion blur, a marker at an angle,
|
| 39 |
+
#: a printed square that has been in a pocket. It should be replaced by a
|
| 40 |
+
#: measurement the first time anybody photographs a marker at a known distance.
|
| 41 |
+
CORNER_UNCERTAINTY_PX = 1.5
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class NoReference(RuntimeError):
|
| 45 |
+
"""Nothing in the frame establishes a physical scale."""
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
@dataclass(frozen=True)
|
| 49 |
+
class Scale:
|
| 50 |
+
"""Pixels per millimetre, with the error that comes from measuring it."""
|
| 51 |
+
|
| 52 |
+
pixels_per_mm: float
|
| 53 |
+
#: Fractional 1-sigma uncertainty on the above.
|
| 54 |
+
relative_error: float
|
| 55 |
+
source: str
|
| 56 |
+
|
| 57 |
+
def length_mm(self, pixels: float) -> tuple[float, float]:
|
| 58 |
+
"""A length and its half-width, both in millimetres."""
|
| 59 |
+
value = pixels / self.pixels_per_mm
|
| 60 |
+
return value, value * self.relative_error
|
| 61 |
+
|
| 62 |
+
def area_mm2(self, pixels: float) -> tuple[float, float]:
|
| 63 |
+
"""An area and its half-width, in square millimetres.
|
| 64 |
+
|
| 65 |
+
The relative error doubles going from a length to an area, which is the
|
| 66 |
+
reason Β§9's example is a range β "12β16 cmΒ²" β and not a figure.
|
| 67 |
+
"""
|
| 68 |
+
value = pixels / (self.pixels_per_mm ** 2)
|
| 69 |
+
return value, value * 2.0 * self.relative_error
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def scale_from_marker(
|
| 73 |
+
image: np.ndarray, marker_side_mm: float, *, dictionary: str = ARUCO_DICTIONARY
|
| 74 |
+
) -> Scale:
|
| 75 |
+
"""Find a printed square marker of known size and derive pixels per mm.
|
| 76 |
+
|
| 77 |
+
Raises rather than returning a default. A frame with no marker has no
|
| 78 |
+
scale, and the caller's correct response is to ask for a re-capture with
|
| 79 |
+
the card in shot β not to receive a number that happens to be plausible.
|
| 80 |
+
"""
|
| 81 |
+
import cv2
|
| 82 |
+
|
| 83 |
+
grey = image if image.ndim == 2 else cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
|
| 84 |
+
aruco = cv2.aruco
|
| 85 |
+
detector = aruco.ArucoDetector(
|
| 86 |
+
aruco.getPredefinedDictionary(getattr(aruco, dictionary)),
|
| 87 |
+
aruco.DetectorParameters(),
|
| 88 |
+
)
|
| 89 |
+
corners, ids, _ = detector.detectMarkers(grey)
|
| 90 |
+
if ids is None or len(corners) == 0:
|
| 91 |
+
raise NoReference(
|
| 92 |
+
"No reference marker in this frame, so nothing establishes what a "
|
| 93 |
+
"pixel is worth. Photograph again with the Animap card beside the "
|
| 94 |
+
"subject and in the same plane."
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
# The mean of the four sides, per marker, then across markers. Averaging the
|
| 98 |
+
# sides absorbs a little perspective; it does not correct for it, and a
|
| 99 |
+
# marker photographed at a steep angle still reads short. That is why the
|
| 100 |
+
# capture instruction says "in the same plane" rather than "in frame".
|
| 101 |
+
sides: list[float] = []
|
| 102 |
+
for quad in corners:
|
| 103 |
+
points = quad.reshape(4, 2)
|
| 104 |
+
sides.extend(
|
| 105 |
+
float(np.linalg.norm(points[i] - points[(i + 1) % 4])) for i in range(4)
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
mean_side = float(np.mean(sides))
|
| 109 |
+
if mean_side <= 0:
|
| 110 |
+
raise NoReference("The marker was found but has no measurable size.")
|
| 111 |
+
|
| 112 |
+
pixels_per_mm = mean_side / marker_side_mm
|
| 113 |
+
# Two corners contribute to each side, independently, hence the root two.
|
| 114 |
+
relative_error = (CORNER_UNCERTAINTY_PX * math.sqrt(2.0)) / mean_side
|
| 115 |
+
# Spread between markers, when there is more than one, is real evidence
|
| 116 |
+
# about perspective and is folded in rather than averaged away.
|
| 117 |
+
if len(sides) > 4:
|
| 118 |
+
relative_error = math.hypot(relative_error, float(np.std(sides)) / mean_side)
|
| 119 |
+
|
| 120 |
+
return Scale(
|
| 121 |
+
pixels_per_mm=pixels_per_mm,
|
| 122 |
+
relative_error=relative_error,
|
| 123 |
+
source=f"{len(sides) // 4} Γ {marker_side_mm:g} mm {dictionary} marker",
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def contour_area_px(mask: np.ndarray) -> float:
|
| 128 |
+
"""Area of the largest connected region in a boolean mask, in pixels.
|
| 129 |
+
|
| 130 |
+
The largest region rather than the sum, because a segmenter that returns a
|
| 131 |
+
wound plus three specks of noise should report the wound. A caller that
|
| 132 |
+
genuinely wants the total already has `mask.sum()`.
|
| 133 |
+
|
| 134 |
+
**This is the polygon's area, not a pixel count, and it is smaller.** OpenCV
|
| 135 |
+
traces the contour through pixel centres, so a filled 40x40 square measures
|
| 136 |
+
39x39 = 1,521 rather than 1,600 β **4.9% low**, not 2.5%. The 2.5% is the
|
| 137 |
+
error on each *side*, and an area loses it twice; this comment used to quote
|
| 138 |
+
the linear figure for an area, and so did the 300x300 case, where the true
|
| 139 |
+
shortfall is **0.67%** rather than 0.3%. Both were measured on 2026-08-21.
|
| 140 |
+
|
| 141 |
+
The gap is a perimeter effect and shrinks as the region grows, so at any
|
| 142 |
+
size worth reporting it sits inside the uncertainty `Scale` already carries
|
| 143 |
+
β `CORNER_UNCERTAINTY_PX` alone puts a few per cent into the linear scale
|
| 144 |
+
and twice that into an area. Recorded because a reader checking the
|
| 145 |
+
arithmetic against `mask.sum()` will otherwise find a discrepancy and wonder
|
| 146 |
+
which is wrong.
|
| 147 |
+
"""
|
| 148 |
+
import cv2
|
| 149 |
+
|
| 150 |
+
binary = (np.asarray(mask) > 0).astype(np.uint8)
|
| 151 |
+
contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
| 152 |
+
if not contours:
|
| 153 |
+
return 0.0
|
| 154 |
+
return float(max(cv2.contourArea(c) for c in contours))
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def measure_region(mask: np.ndarray, scale: Scale) -> dict[str, float | str]:
|
| 158 |
+
"""A masked region's physical size, as a range.
|
| 159 |
+
|
| 160 |
+
Returns square centimetres because that is the unit Β§9 reports in, and
|
| 161 |
+
because a wound quoted in square millimetres invites the false precision
|
| 162 |
+
Β§37 warns about.
|
| 163 |
+
"""
|
| 164 |
+
pixels = contour_area_px(mask)
|
| 165 |
+
area_mm2, half_width_mm2 = scale.area_mm2(pixels)
|
| 166 |
+
return {
|
| 167 |
+
"area_cm2": round(area_mm2 / 100.0, 2),
|
| 168 |
+
"area_cm2_low": round(max(0.0, area_mm2 - half_width_mm2) / 100.0, 2),
|
| 169 |
+
"area_cm2_high": round((area_mm2 + half_width_mm2) / 100.0, 2),
|
| 170 |
+
"area_px": round(pixels, 1),
|
| 171 |
+
"scale_source": scale.source,
|
| 172 |
+
"scale_relative_error": round(scale.relative_error, 4),
|
| 173 |
+
}
|
app/adapters/signal/periodicity.py
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Finding a rate in a noisy 1-D signal, and refusing when there isn't one.
|
| 2 |
+
|
| 3 |
+
Directive Β§4: "Do not use a neural model when deterministic signal processing is
|
| 4 |
+
better." Respiration is the clearest case in the whole capability matrix β Β§14
|
| 5 |
+
spells the pipeline out as video β flank region β optical flow β periodicity β
|
| 6 |
+
FFT β breaths per minute, and none of those steps wants a network.
|
| 7 |
+
|
| 8 |
+
**The hard part is not finding a peak. Every spectrum has a peak.** The hard part
|
| 9 |
+
is knowing whether the peak means anything, and this module is mostly that. It
|
| 10 |
+
mirrors the shape `app/counting.py` arrived at: measure whether you are still in
|
| 11 |
+
the regime the method works in, and when you are not, publish nothing.
|
| 12 |
+
|
| 13 |
+
Four gates, and all must pass:
|
| 14 |
+
|
| 15 |
+
**Enough cycles** β the clip has to hold `MIN_CYCLES` full periods, or the
|
| 16 |
+
estimate moves with where the recording happened to start.
|
| 17 |
+
|
| 18 |
+
**Prominence** β peak power over the median in-band power. Noise is flat, so a
|
| 19 |
+
real oscillation stands far above its neighbours and sensor noise does not.
|
| 20 |
+
|
| 21 |
+
**Stability** β split the clip, measure each half independently, and require both
|
| 22 |
+
to agree with the whole-clip estimate. A signal that is genuinely periodic gives
|
| 23 |
+
the same answer on any window of it. One that is a slow drift, a pan, or an
|
| 24 |
+
animal shifting its weight does not.
|
| 25 |
+
|
| 26 |
+
**Sub-band shoulder** β the power sitting *below* the search band, against the
|
| 27 |
+
in-band peak. This is the newest gate and it exists because the other three
|
| 28 |
+
published a confident rate for signals that do not oscillate at all.
|
| 29 |
+
|
| 30 |
+
**Every threshold here belongs to one search band, and the band is now recorded
|
| 31 |
+
with them.** Prominence is peak power over median *in-band* power, so it is a
|
| 32 |
+
property of the band as much as of the signal, and it does not transfer. The
|
| 33 |
+
constants were derived at `(8, 180)` cycles per minute and the caller shipped
|
| 34 |
+
`(8, 90)`; at the narrower band the ordering inverts, a static patch of sensor
|
| 35 |
+
noise outranks a positive control, and no threshold separates the two classes at
|
| 36 |
+
all. `DERIVATION_BAND_CPM` pins the band the numbers came from, and
|
| 37 |
+
`tests/test_periodicity.py` asserts that the band the service actually searches
|
| 38 |
+
is that one.
|
| 39 |
+
|
| 40 |
+
**The stability rule is written the way it is because of a bug real footage
|
| 41 |
+
caught.** The first version compared the two halves *to each other*. On a clip of
|
| 42 |
+
resting cattle both halves said ~26 cycles/min while the whole clip said 45.7,
|
| 43 |
+
and the two halves agreeing with each other sailed through a check that never
|
| 44 |
+
looked at the number being published. Each half is now compared to the estimate
|
| 45 |
+
that would actually be reported.
|
| 46 |
+
|
| 47 |
+
**The shoulder rule is written because of a failure a synthetic sweep caught.**
|
| 48 |
+
`sqrt(t)`, `log(1+t)`, `sigmoid(t)` and `exp(t)` contain no oscillation of any
|
| 49 |
+
kind, and over 2,804 runs across durations from 20 to 90 seconds, 1,504 of them
|
| 50 |
+
β 53.6% β published a confident 8.34 to 10.39 cycles per minute through the
|
| 51 |
+
prominence and stability gates. They did not scrape past: at sixty seconds their
|
| 52 |
+
prominence runs to seven figures and their half-drift to 0.059, twice as good as
|
| 53 |
+
the tolerance. On a respiratory screen a fabricated rate is the dangerous
|
| 54 |
+
direction, and this was the exact case the gates were described as catching.
|
| 55 |
+
"""
|
| 56 |
+
|
| 57 |
+
from __future__ import annotations
|
| 58 |
+
|
| 59 |
+
from dataclasses import dataclass
|
| 60 |
+
|
| 61 |
+
import numpy as np
|
| 62 |
+
|
| 63 |
+
#: The search band every threshold below was derived at, in cycles per minute.
|
| 64 |
+
#:
|
| 65 |
+
#: **Recorded as a constant because a threshold without its band is not a
|
| 66 |
+
#: threshold.** Prominence is peak power over the *in-band* median, so narrowing
|
| 67 |
+
#: the band changes the denominator and every number in the tables below moves.
|
| 68 |
+
#: The service shipped `(8, 90)` against constants derived here, and at that
|
| 69 |
+
#: band:
|
| 70 |
+
#:
|
| 71 |
+
#: - the positive control is out of range entirely β the metronome's stated
|
| 72 |
+
#: 96 cycles per minute sits above a 90 ceiling, so its whole-frame peak is
|
| 73 |
+
#: forced onto the pendulum's 48 subharmonic and its prominence collapses from
|
| 74 |
+
#: 1944.3 to 16.9;
|
| 75 |
+
#: - a walking cow scores 114.2 and a static patch of sensor noise scores 30.9,
|
| 76 |
+
#: so **two negatives outrank two of the three positives**;
|
| 77 |
+
#: - no prominence threshold separates the classes. The sorted list interleaves
|
| 78 |
+
#: three times and the best achievable is two misclassifications at any value.
|
| 79 |
+
#:
|
| 80 |
+
#: So the band was widened back to the one the evidence belongs to rather than
|
| 81 |
+
#: the constants being re-derived at a band where they cannot exist.
|
| 82 |
+
#: `tests/test_periodicity.py` asserts that `respiration.SEARCH_BAND_BPM` is
|
| 83 |
+
#: this band, which is what stops the two drifting apart again.
|
| 84 |
+
#:
|
| 85 |
+
#: **What widening costs is untested rather than nil.** No negative in the
|
| 86 |
+
#: derivation set has any content between 90 and 180 cycles per minute, so the
|
| 87 |
+
#: claim that the band is "narrow enough to exclude a swishing tail at the top"
|
| 88 |
+
#: is now unexamined at the top. Narrowing it again is defensible and it is a
|
| 89 |
+
#: re-derivation, not an edit.
|
| 90 |
+
DERIVATION_BAND_CPM = (8.0, 180.0)
|
| 91 |
+
|
| 92 |
+
#: Peak power divided by median in-band power, below which no rate is reported.
|
| 93 |
+
#:
|
| 94 |
+
#: **Derived, on four real Wikimedia Commons clips, not chosen.** The clips are
|
| 95 |
+
#: pinned by sha256 in `experiments/cattle_respiratory/examples/sources.json`,
|
| 96 |
+
#: which is how this derivation is reproduced. The positives
|
| 97 |
+
#: are three regions of `Metronome.webm` (CC BY 4.0), whose Commons description
|
| 98 |
+
#: states the mechanism ticks at 96 beats per minute; the negatives are regions
|
| 99 |
+
#: of two cattle clips, a walking cow, and two static background patches that
|
| 100 |
+
#: contain nothing but sensor noise.
|
| 101 |
+
#:
|
| 102 |
+
#: All figures below are at `DERIVATION_BAND_CPM` and were reproduced from the
|
| 103 |
+
#: pinned clips on 2026-08-21, matching the recorded values to the last digit.
|
| 104 |
+
#:
|
| 105 |
+
#: | region | prominence | shoulder | reported |
|
| 106 |
+
#: |---|---|---|---|
|
| 107 |
+
#: | metronome, whole frame | 1944.3 | 0.0031 | 96.48 cyc/min |
|
| 108 |
+
#: | metronome, pendulum crop | 1480.7 | 0.0009 | 48.38 cyc/min |
|
| 109 |
+
#: | metronome, upper crop | 225.5 | 0.0033 | 96.49 cyc/min |
|
| 110 |
+
#: | cow walking across a grid, first 600 frames | 92.8 | 4.327 | β |
|
| 111 |
+
#: | static background, metronome corner | 37.4 | 0.323 | β |
|
| 112 |
+
#: | cattle at rest, whole frame | 12.1 | 0.064 | β |
|
| 113 |
+
#: | cattle defecating, flank crop | 5.9 | 0.148 | β |
|
| 114 |
+
#: | static background, road corner, first 600 frames | 5.4 | 4.312 | β |
|
| 115 |
+
#:
|
| 116 |
+
#: The gap runs from 92.8 to 225.5 and 150 sits in it. The walking-cow row is
|
| 117 |
+
#: the worst case and it is a truncation: the first 600 frames read 92.8, and
|
| 118 |
+
#: the whole 925-frame clip reads 16.2. The tighter of the two is quoted,
|
| 119 |
+
#: because a threshold should be set against the hardest window a capture can
|
| 120 |
+
#: present and not against the average of one. **Three positives, all
|
| 121 |
+
#: from one video of one metronome, is a very thin basis** and this threshold
|
| 122 |
+
#: should be re-derived the moment there is real cattle footage that anybody has
|
| 123 |
+
#: counted breaths on. It is recorded here as a starting point with its evidence
|
| 124 |
+
#: attached, not as a validated constant.
|
| 125 |
+
MIN_PEAK_PROMINENCE = 150.0
|
| 126 |
+
|
| 127 |
+
#: How far either half's estimate may sit from the whole-clip estimate, as a
|
| 128 |
+
#: fraction of it.
|
| 129 |
+
#:
|
| 130 |
+
#: Same eight regions, same band. The three metronome regions drift 0.003, 0.015
|
| 131 |
+
#: and 0.003; everything else drifts 0.372 or more, up to 2.885. 0.15 sits in
|
| 132 |
+
#: that gap with room on both sides.
|
| 133 |
+
#:
|
| 134 |
+
#: **This rule and the one above do not overlap on this set**, which is a
|
| 135 |
+
#: stronger separation than `app/counting.py` gets from its two rules β and with
|
| 136 |
+
#: eleven regions measured, it is also much weaker evidence. Both are required
|
| 137 |
+
#: rather than either, because the cost of publishing a wrong breathing rate is
|
| 138 |
+
#: a clinical decision and the cost of refusing is a re-capture.
|
| 139 |
+
MAX_HALF_DRIFT = 0.15
|
| 140 |
+
|
| 141 |
+
#: Full cycles that must fit in the clip before a rate is worth reporting.
|
| 142 |
+
#: Below this the FFT has too few periods to resolve one, and the estimate moves
|
| 143 |
+
#: with where the clip happened to start. The walking-cow clip fails here too
|
| 144 |
+
#: (2.5 cycles), so this rule is not load-bearing on the current set β it is a
|
| 145 |
+
#: guard for the short-capture case the set does not contain.
|
| 146 |
+
MIN_CYCLES = 4.0
|
| 147 |
+
|
| 148 |
+
#: The strongest spectral line *below* the search band, over the in-band peak.
|
| 149 |
+
#: Above this, no rate is published.
|
| 150 |
+
#:
|
| 151 |
+
#: **This is the gate that catches drift wearing an oscillation's clothes.**
|
| 152 |
+
#: `_detrend` removes a straight line exactly, and nothing removes a curve. A
|
| 153 |
+
#: signal that only ever increases has a `1/f`-shaped residual spectrum: power
|
| 154 |
+
#: keeps climbing as frequency falls, so whatever the in-band peak is, there is
|
| 155 |
+
#: far more power under the band than in it. A real oscillation puts its power
|
| 156 |
+
#: at its own frequency and leaves the sub-band bins empty.
|
| 157 |
+
#:
|
| 158 |
+
#: Measured, all at `DERIVATION_BAND_CPM`:
|
| 159 |
+
#:
|
| 160 |
+
#: | class | worst case | value |
|
| 161 |
+
#: |---|---|---|
|
| 162 |
+
#: | metronome positives | upper crop | **0.0033** |
|
| 163 |
+
#: | synthetic breathers, 8.5β45 cyc/min with drift and noise | 8.5 at noise 0.3 | **0.040** |
|
| 164 |
+
#: | real footage, nearest miss | static road corner, full 925 frames | **0.962** |
|
| 165 |
+
#: | monotonic curves, worst of a 2,804-run sweep | β | **253.8** |
|
| 166 |
+
#:
|
| 167 |
+
#: The sweep is `sqrt(t)`, `log(1+t)`, `sigmoid((t β mid)/6)` and `exp(t/12)` at
|
| 168 |
+
#: 30 fps, over every duration from 20.0 to 90.0 seconds in 0.1 s steps, giving
|
| 169 |
+
#: 2,804 runs. **With the shoulder gate off, 1,504 of them publish a confident
|
| 170 |
+
#: 8.34 to 10.39 cycles per minute β 53.6%. With it on, 0 do**, and all three
|
| 171 |
+
#: metronome positives and every synthetic breather still publish.
|
| 172 |
+
#:
|
| 173 |
+
#: The exact constants matter to the first number and not to the second. 1,504
|
| 174 |
+
#: is what these four shapes give and it moves if the shapes or their constants
|
| 175 |
+
#: do β an independent reviewer using a different sigmoid midpoint reported
|
| 176 |
+
#: 1,500. **The figure that carries the argument is the zero**, which is stable
|
| 177 |
+
#: across every variation tried, and `tests/test_periodicity.py` re-derives a
|
| 178 |
+
#: 284-run subset of the sweep so the claim is checked rather than remembered.
|
| 179 |
+
#:
|
| 180 |
+
#: **Two things it is not.** It is not a universal monotonic detector: after
|
| 181 |
+
#: `_detrend` removes a straight line *exactly*, a pure linear ramp leaves only
|
| 182 |
+
#: float rounding noise, whose spectrum is numerically unstable β measured
|
| 183 |
+
#: shoulders from 0.14 to 1,312 depending on duration and scale. The ramp is
|
| 184 |
+
#: always refused, but which gate refuses it is not predictable and no claim
|
| 185 |
+
#: should rest on it. And the headroom above is against synthetic curves; the
|
| 186 |
+
#: **nearest real region is 0.962**, four per cent under the threshold, so the
|
| 187 |
+
#: margin against real-world footage is far thinner than the sweep suggests.
|
| 188 |
+
#: That row is the one to watch if this is ever tightened.
|
| 189 |
+
MAX_SUBBAND_SHOULDER = 1.0
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
@dataclass(frozen=True)
|
| 193 |
+
class Periodicity:
|
| 194 |
+
"""A rate, or an account of why there isn't one.
|
| 195 |
+
|
| 196 |
+
`cycles_per_minute` is `None` whenever `usable` is False. There is
|
| 197 |
+
deliberately no way to read a number out of a failed measurement β the same
|
| 198 |
+
property `app/counting.py` has, where a withheld count is not a count of
|
| 199 |
+
zero.
|
| 200 |
+
"""
|
| 201 |
+
|
| 202 |
+
usable: bool
|
| 203 |
+
cycles_per_minute: float | None
|
| 204 |
+
reason: str = ""
|
| 205 |
+
#: The diagnostics, emitted whether or not a rate is published, so a
|
| 206 |
+
#: threshold can be re-derived from stored results rather than by going back
|
| 207 |
+
#: to footage nobody kept.
|
| 208 |
+
peak_prominence: float = 0.0
|
| 209 |
+
half_drift: float = 0.0
|
| 210 |
+
cycles_observed: float = 0.0
|
| 211 |
+
resolution_cycles_per_minute: float = 0.0
|
| 212 |
+
duration_seconds: float = 0.0
|
| 213 |
+
#: Strongest sub-band line over the in-band peak. `inf` when the clip is too
|
| 214 |
+
#: short to have a bin below the band, which is a refusal rather than a pass.
|
| 215 |
+
subband_shoulder: float = 0.0
|
| 216 |
+
#: The band this was measured in. **Recorded on every result**, because a
|
| 217 |
+
#: prominence without its band cannot be compared to anything and a stored
|
| 218 |
+
#: diagnostic that nobody can re-derive a threshold from is not a diagnostic.
|
| 219 |
+
band_cycles_per_minute: tuple[float, float] = DERIVATION_BAND_CPM
|
| 220 |
+
#: What each half said on its own. Kept because when a measurement is
|
| 221 |
+
#: refused, this pair is usually the reason a reader can see it.
|
| 222 |
+
half_estimates: tuple[float, float] | None = None
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def _detrend(signal: np.ndarray) -> np.ndarray:
|
| 226 |
+
"""Remove the linear component.
|
| 227 |
+
|
| 228 |
+
A handheld capture drifts. Left in, the drift dumps power into the lowest
|
| 229 |
+
bins and drags the peak down towards zero, which is how a still frame comes
|
| 230 |
+
out as a very slow, very confident oscillation.
|
| 231 |
+
"""
|
| 232 |
+
n = len(signal)
|
| 233 |
+
t = np.arange(n, dtype=np.float64)
|
| 234 |
+
design = np.vstack([t, np.ones(n)]).T
|
| 235 |
+
coefficients, *_ = np.linalg.lstsq(design, signal, rcond=None)
|
| 236 |
+
return signal - design @ coefficients
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
@dataclass(frozen=True)
|
| 240 |
+
class _Peak:
|
| 241 |
+
"""One spectrum's verdict, before any threshold is applied."""
|
| 242 |
+
|
| 243 |
+
frequency_hz: float
|
| 244 |
+
prominence: float
|
| 245 |
+
bin_width_hz: float
|
| 246 |
+
shoulder: float
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def _peak(
|
| 250 |
+
signal: np.ndarray, sample_rate_hz: float, band_hz: tuple[float, float]
|
| 251 |
+
) -> _Peak | None:
|
| 252 |
+
"""Dominant in-band frequency, its prominence, the bin width, the shoulder."""
|
| 253 |
+
n = len(signal)
|
| 254 |
+
if n < 8:
|
| 255 |
+
return None
|
| 256 |
+
|
| 257 |
+
windowed = _detrend(signal) * np.hanning(n)
|
| 258 |
+
power = np.abs(np.fft.rfft(windowed)) ** 2
|
| 259 |
+
frequencies = np.fft.rfftfreq(n, d=1.0 / sample_rate_hz)
|
| 260 |
+
|
| 261 |
+
low, high = band_hz
|
| 262 |
+
in_band = np.flatnonzero((frequencies >= low) & (frequencies <= high))
|
| 263 |
+
if in_band.size < 3:
|
| 264 |
+
return None
|
| 265 |
+
|
| 266 |
+
peak = in_band[int(np.argmax(power[in_band]))]
|
| 267 |
+
|
| 268 |
+
# Bin 0 is excluded because `_detrend` has already taken DC out, so
|
| 269 |
+
# whatever is left there is arithmetic rather than signal.
|
| 270 |
+
below_band = power[1:int(in_band[0])]
|
| 271 |
+
if below_band.size == 0:
|
| 272 |
+
# A clip too short to have a single bin under the band cannot be asked
|
| 273 |
+
# this question, and the answer to a question that cannot be asked is
|
| 274 |
+
# not "pass". At the 8 cycles/min floor this needs 7.5 seconds, and
|
| 275 |
+
# `respiration.MIN_CAPTURE_SECONDS` is 20, so it is unreachable through
|
| 276 |
+
# the respiratory path and reachable by calling this function directly.
|
| 277 |
+
shoulder = float("inf")
|
| 278 |
+
else:
|
| 279 |
+
shoulder = float(
|
| 280 |
+
np.max(below_band) / max(float(power[peak]), 1e-30)
|
| 281 |
+
)
|
| 282 |
+
|
| 283 |
+
# Parabolic interpolation in log power. The bin width at 30 fps over ten
|
| 284 |
+
# seconds is about 5 cycles/min, which is coarse enough to matter for a
|
| 285 |
+
# breathing rate; three points around the peak give sub-bin resolution for
|
| 286 |
+
# four lines of arithmetic and no extra dependency.
|
| 287 |
+
offset = 0.0
|
| 288 |
+
if 0 < peak < len(power) - 1:
|
| 289 |
+
left, centre, right = (
|
| 290 |
+
np.log(power[peak - 1] + 1e-30),
|
| 291 |
+
np.log(power[peak] + 1e-30),
|
| 292 |
+
np.log(power[peak + 1] + 1e-30),
|
| 293 |
+
)
|
| 294 |
+
curvature = left - 2.0 * centre + right
|
| 295 |
+
if abs(curvature) > 1e-30:
|
| 296 |
+
offset = float(np.clip(0.5 * (left - right) / curvature, -0.5, 0.5))
|
| 297 |
+
|
| 298 |
+
bin_width = float(frequencies[1] - frequencies[0])
|
| 299 |
+
frequency = float(frequencies[peak]) + offset * bin_width
|
| 300 |
+
prominence = float(power[peak] / max(float(np.median(power[in_band])), 1e-30))
|
| 301 |
+
return _Peak(frequency, prominence, bin_width, shoulder)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def dominant_rate(
|
| 305 |
+
signal: np.ndarray,
|
| 306 |
+
sample_rate_hz: float,
|
| 307 |
+
band_cycles_per_minute: tuple[float, float],
|
| 308 |
+
*,
|
| 309 |
+
min_prominence: float = MIN_PEAK_PROMINENCE,
|
| 310 |
+
max_drift: float = MAX_HALF_DRIFT,
|
| 311 |
+
min_cycles: float = MIN_CYCLES,
|
| 312 |
+
max_shoulder: float = MAX_SUBBAND_SHOULDER,
|
| 313 |
+
) -> Periodicity:
|
| 314 |
+
"""The rate this signal repeats at, or a refusal.
|
| 315 |
+
|
| 316 |
+
`band_cycles_per_minute` is a *search range*, not a claim about what is
|
| 317 |
+
normal for an animal. Narrowing it is how a caller stops the method locking
|
| 318 |
+
on to a gait or a fan; it is not a prior that pulls the estimate.
|
| 319 |
+
|
| 320 |
+
**It is also the band every default threshold here was derived at**, and
|
| 321 |
+
passing a different one silently invalidates all four. It is a parameter
|
| 322 |
+
rather than a constant because the derivation itself has to sweep it, and
|
| 323 |
+
the result records the band it used so a stored diagnostic can be compared
|
| 324 |
+
with the constants it was judged against.
|
| 325 |
+
"""
|
| 326 |
+
signal = np.asarray(signal, dtype=np.float64).ravel()
|
| 327 |
+
n = len(signal)
|
| 328 |
+
duration = n / sample_rate_hz if sample_rate_hz > 0 else 0.0
|
| 329 |
+
|
| 330 |
+
band = (float(band_cycles_per_minute[0]), float(band_cycles_per_minute[1]))
|
| 331 |
+
|
| 332 |
+
if sample_rate_hz <= 0:
|
| 333 |
+
return Periodicity(
|
| 334 |
+
False, None, "The clip reports no frame rate.",
|
| 335 |
+
band_cycles_per_minute=band,
|
| 336 |
+
)
|
| 337 |
+
|
| 338 |
+
low_hz, high_hz = (v / 60.0 for v in band_cycles_per_minute)
|
| 339 |
+
|
| 340 |
+
# Nyquist. Asked before anything is computed, because above it the method
|
| 341 |
+
# does not degrade β it aliases, and returns a confident wrong number.
|
| 342 |
+
if sample_rate_hz < 2.0 * high_hz:
|
| 343 |
+
return Periodicity(
|
| 344 |
+
False, None,
|
| 345 |
+
f"The clip samples at {sample_rate_hz:.1f} Hz, which cannot resolve "
|
| 346 |
+
f"{band_cycles_per_minute[1]:.0f} cycles per minute. Capture at a "
|
| 347 |
+
f"higher frame rate.",
|
| 348 |
+
duration_seconds=duration,
|
| 349 |
+
band_cycles_per_minute=band,
|
| 350 |
+
)
|
| 351 |
+
|
| 352 |
+
whole = _peak(signal, sample_rate_hz, (low_hz, high_hz))
|
| 353 |
+
if whole is None:
|
| 354 |
+
return Periodicity(
|
| 355 |
+
False, None,
|
| 356 |
+
"The clip is too short to hold a spectrum.",
|
| 357 |
+
duration_seconds=duration,
|
| 358 |
+
band_cycles_per_minute=band,
|
| 359 |
+
)
|
| 360 |
+
|
| 361 |
+
rate = whole.frequency_hz * 60.0
|
| 362 |
+
cycles = whole.frequency_hz * duration
|
| 363 |
+
resolution = whole.bin_width_hz * 60.0
|
| 364 |
+
|
| 365 |
+
half = n // 2
|
| 366 |
+
first = _peak(signal[:half], sample_rate_hz, (low_hz, high_hz))
|
| 367 |
+
second = _peak(signal[half:], sample_rate_hz, (low_hz, high_hz))
|
| 368 |
+
|
| 369 |
+
if first is None or second is None:
|
| 370 |
+
drift = float("inf")
|
| 371 |
+
halves = None
|
| 372 |
+
else:
|
| 373 |
+
halves = (first.frequency_hz * 60.0, second.frequency_hz * 60.0)
|
| 374 |
+
# Each half against the number that would be *published*, never against
|
| 375 |
+
# each other. Two halves can agree on something the whole clip does not
|
| 376 |
+
# say, and that is the case this rule exists to catch.
|
| 377 |
+
drift = max(abs(h - rate) for h in halves) / max(rate, 1e-9)
|
| 378 |
+
|
| 379 |
+
diagnostics = dict(
|
| 380 |
+
peak_prominence=whole.prominence,
|
| 381 |
+
half_drift=drift,
|
| 382 |
+
cycles_observed=cycles,
|
| 383 |
+
resolution_cycles_per_minute=resolution,
|
| 384 |
+
duration_seconds=duration,
|
| 385 |
+
subband_shoulder=whole.shoulder,
|
| 386 |
+
band_cycles_per_minute=band,
|
| 387 |
+
half_estimates=halves,
|
| 388 |
+
)
|
| 389 |
+
|
| 390 |
+
if cycles < min_cycles:
|
| 391 |
+
return Periodicity(
|
| 392 |
+
False, None,
|
| 393 |
+
f"Only {cycles:.1f} cycles fit in {duration:.1f} s. At least "
|
| 394 |
+
f"{min_cycles:.0f} are needed before a rate means anything β record "
|
| 395 |
+
f"for longer.",
|
| 396 |
+
**diagnostics,
|
| 397 |
+
)
|
| 398 |
+
|
| 399 |
+
if whole.prominence < min_prominence:
|
| 400 |
+
return Periodicity(
|
| 401 |
+
False, None,
|
| 402 |
+
f"No clear rhythm. The strongest rate in the search band stands only "
|
| 403 |
+
f"{whole.prominence:.0f}Γ above the background, and "
|
| 404 |
+
f"{min_prominence:.0f}Γ is the least that has separated a real "
|
| 405 |
+
f"oscillation from camera noise.",
|
| 406 |
+
**diagnostics,
|
| 407 |
+
)
|
| 408 |
+
|
| 409 |
+
if whole.shoulder > max_shoulder:
|
| 410 |
+
return Periodicity(
|
| 411 |
+
False, None,
|
| 412 |
+
f"This is drift, not a rhythm. Below the search band the spectrum "
|
| 413 |
+
f"holds {whole.shoulder:.0f}Γ more power than the peak inside it, "
|
| 414 |
+
f"against a {max_shoulder:.0f}Γ limit β the signature of something "
|
| 415 |
+
f"that only moves one way. A rate measured from a slow drift is a "
|
| 416 |
+
f"number with nothing behind it.",
|
| 417 |
+
**diagnostics,
|
| 418 |
+
)
|
| 419 |
+
|
| 420 |
+
if drift > max_drift:
|
| 421 |
+
detail = (
|
| 422 |
+
f" The two halves of the clip read {halves[0]:.0f} and "
|
| 423 |
+
f"{halves[1]:.0f}." if halves else ""
|
| 424 |
+
)
|
| 425 |
+
return Periodicity(
|
| 426 |
+
False, None,
|
| 427 |
+
f"The rhythm is not steady: measured over each half of the clip it "
|
| 428 |
+
f"moves by {drift * 100:.0f}%, against a {max_drift * 100:.0f}% "
|
| 429 |
+
f"tolerance.{detail} Something in the frame is moving that is not "
|
| 430 |
+
f"the thing being measured.",
|
| 431 |
+
**diagnostics,
|
| 432 |
+
)
|
| 433 |
+
|
| 434 |
+
return Periodicity(True, rate, "", **diagnostics)
|
app/adapters/signal/respiration.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Respiratory rate from a video, by optical flow and an FFT.
|
| 2 |
+
|
| 3 |
+
Directive Β§14, and it is the clearest case in the whole capability matrix for
|
| 4 |
+
Β§4's rule that a neural model should not be used where deterministic signal
|
| 5 |
+
processing is better. The pipeline the directive specifies is exactly this:
|
| 6 |
+
|
| 7 |
+
video β flank region segmentation β optical flow / pixel-motion signal
|
| 8 |
+
β periodicity β FFT / peak detection β breaths per minute
|
| 9 |
+
|
| 10 |
+
**What this module adds to that list is the refusal.** `flow` produces a signal
|
| 11 |
+
from any video and `periodicity` finds a peak in any signal, so the only thing
|
| 12 |
+
standing between a still frame of a fence post and a confident respiratory rate
|
| 13 |
+
is the pair of gates in `periodicity`. Those gates were derived on real footage
|
| 14 |
+
and they are the reason this file is worth having.
|
| 15 |
+
|
| 16 |
+
**Measured on every real cattle clip available, this publishes nothing**, and
|
| 17 |
+
that is the current honest state of the capability rather than a bug. None of
|
| 18 |
+
the three is the capture Β§14 asks for. Two are under twenty seconds and are
|
| 19 |
+
refused before any signal processing runs; the third is thirty-one seconds of a
|
| 20 |
+
cow walking across a cattle grid, gets as far as the spectrum, and is refused
|
| 21 |
+
there for having no clear rhythm β peak prominence 16 against a threshold of 150.
|
| 22 |
+
|
| 23 |
+
What the module demonstrates positively is the metronome: on footage whose
|
| 24 |
+
Commons description states 96 beats per minute, `dominant_rate` returns 96.48,
|
| 25 |
+
and on a crop of the pendulum alone 48.38, the pendulum's own cycle being half
|
| 26 |
+
the tick rate. That is a 0.5% error against a stated rate, on real video.
|
| 27 |
+
|
| 28 |
+
**That demonstration does not run through `respiratory_rate`, and saying it does
|
| 29 |
+
would be false.** `Metronome.webm` is 11.71 seconds and `MIN_CAPTURE_SECONDS` is
|
| 30 |
+
20, so this function refuses it before `motion_signal` is called. The only clip
|
| 31 |
+
anybody has a ground-truth rate for cannot reach the gates it was used to
|
| 32 |
+
derive. That is a gap in the evidence rather than a bug in the code β the
|
| 33 |
+
capture minimum is right, and what is missing is a long enough clip of something
|
| 34 |
+
with a known rate.
|
| 35 |
+
|
| 36 |
+
**No cattle respiratory rate has been validated and none can be from what is
|
| 37 |
+
here.** What is missing is not model work β it is a thirty-second clip of a
|
| 38 |
+
cow's flank with somebody's counted breath rate beside it.
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
from __future__ import annotations
|
| 42 |
+
|
| 43 |
+
from dataclasses import dataclass
|
| 44 |
+
from pathlib import Path
|
| 45 |
+
|
| 46 |
+
from app.adapters.base import Measurement
|
| 47 |
+
from app.adapters.signal.flow import motion_signal, read_frames
|
| 48 |
+
from app.adapters.signal.periodicity import DERIVATION_BAND_CPM, dominant_rate
|
| 49 |
+
|
| 50 |
+
#: The range the peak is searched in, in breaths per minute.
|
| 51 |
+
#:
|
| 52 |
+
#: **A search range, not a claim about what is normal for cattle.** Nothing in
|
| 53 |
+
#: this repository establishes a normal respiratory range for a White Fulani in
|
| 54 |
+
#: Kaduna, and a band that encoded one would be a physiological claim smuggled
|
| 55 |
+
#: in as a constant. It is set wide enough to contain any plausible rate and
|
| 56 |
+
#: narrow enough to exclude a slow pan at the bottom.
|
| 57 |
+
#:
|
| 58 |
+
#: **It is `periodicity.DERIVATION_BAND_CPM` because it has to be.** This used
|
| 59 |
+
#: to read `(8, 90)` while every threshold in `periodicity` was derived at
|
| 60 |
+
#: `(8, 180)`, and prominence is peak power over the *in-band* median, so it
|
| 61 |
+
#: does not survive a change of band. At `(8, 90)` the positive control is out
|
| 62 |
+
#: of range β the metronome's stated 96 breaths per minute is above the ceiling
|
| 63 |
+
#: β two negatives outrank two of the three positives, and no threshold
|
| 64 |
+
#: separates the classes at all. The band was widened back to where the evidence
|
| 65 |
+
#: is rather than the thresholds being re-derived where they cannot exist, and
|
| 66 |
+
#: `tests/test_periodicity.py` asserts the two stay equal.
|
| 67 |
+
#:
|
| 68 |
+
#: **What that costs is untested, not nil.** No clip in the derivation set has
|
| 69 |
+
#: content between 90 and 180 breaths per minute, so a ceiling of 180 leaves the
|
| 70 |
+
#: top of the band unexamined β a swishing tail is the case to worry about.
|
| 71 |
+
#: Narrowing it again is a re-derivation, not an edit.
|
| 72 |
+
SEARCH_BAND_BPM = DERIVATION_BAND_CPM
|
| 73 |
+
|
| 74 |
+
#: Β§14 asks the farmer to "hold the cow's flank in frame for 30β60 seconds".
|
| 75 |
+
#: Below this there are not enough breaths to resolve one β at the bottom of
|
| 76 |
+
#: the search band, four cycles take thirty seconds β and the periodicity gate
|
| 77 |
+
#: would refuse anyway. Checking it here lets the app say so before spending
|
| 78 |
+
#: the compute.
|
| 79 |
+
MIN_CAPTURE_SECONDS = 20.0
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
@dataclass(frozen=True)
|
| 83 |
+
class RespirationResult:
|
| 84 |
+
"""A rate, or a refusal, plus everything needed to re-derive a threshold."""
|
| 85 |
+
|
| 86 |
+
measurement: Measurement
|
| 87 |
+
frames: int
|
| 88 |
+
sample_rate_hz: float
|
| 89 |
+
duration_seconds: float
|
| 90 |
+
#: Where the signal came from, so a result recorded against the wrong part
|
| 91 |
+
#: of the animal is diagnosable later.
|
| 92 |
+
region: tuple[float, float, float, float] | None
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def respiratory_rate(
|
| 96 |
+
video_path: Path | str,
|
| 97 |
+
*,
|
| 98 |
+
region: tuple[float, float, float, float] | None = None,
|
| 99 |
+
max_frames: int = 1800,
|
| 100 |
+
) -> RespirationResult:
|
| 101 |
+
"""Breaths per minute from a clip, or an account of why not.
|
| 102 |
+
|
| 103 |
+
`region` is the fractional flank box β `(x0, y0, x1, y1)` in 0β1 β from a
|
| 104 |
+
segmenter or from the capture UI's own guide box. Passing one is strongly
|
| 105 |
+
preferable to not: the whole-frame signal averages the flank together with
|
| 106 |
+
everything else that moved, and on a clip of two animals it is the other
|
| 107 |
+
animal that wins.
|
| 108 |
+
"""
|
| 109 |
+
frames, sample_rate = read_frames(
|
| 110 |
+
video_path, max_frames=max_frames, roi=region
|
| 111 |
+
)
|
| 112 |
+
duration = len(frames) / sample_rate if sample_rate > 0 else 0.0
|
| 113 |
+
|
| 114 |
+
if duration < MIN_CAPTURE_SECONDS:
|
| 115 |
+
return RespirationResult(
|
| 116 |
+
measurement=Measurement(
|
| 117 |
+
kind="respiratory_rate", value=None, unit="breaths/min",
|
| 118 |
+
usable=False,
|
| 119 |
+
support={"duration_seconds": round(duration, 2)},
|
| 120 |
+
detail=(
|
| 121 |
+
f"The clip is {duration:.0f} seconds. Hold the flank in "
|
| 122 |
+
f"frame for at least {MIN_CAPTURE_SECONDS:.0f} β a slow "
|
| 123 |
+
f"breather needs half a minute before four breaths have "
|
| 124 |
+
f"happened."
|
| 125 |
+
),
|
| 126 |
+
),
|
| 127 |
+
frames=len(frames), sample_rate_hz=sample_rate,
|
| 128 |
+
duration_seconds=duration, region=region,
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
signal = motion_signal(frames, sample_rate)
|
| 132 |
+
rate = dominant_rate(signal.values, sample_rate, SEARCH_BAND_BPM)
|
| 133 |
+
|
| 134 |
+
support = {
|
| 135 |
+
"peak_prominence": round(rate.peak_prominence, 2),
|
| 136 |
+
"half_drift": round(rate.half_drift, 4),
|
| 137 |
+
"cycles_observed": round(rate.cycles_observed, 2),
|
| 138 |
+
"resolution_bpm": round(rate.resolution_cycles_per_minute, 2),
|
| 139 |
+
"subband_shoulder": round(rate.subband_shoulder, 4),
|
| 140 |
+
# Stored with every result, because a prominence is a property of the
|
| 141 |
+
# band it was measured in and a stored diagnostic that does not name its
|
| 142 |
+
# band cannot be used to re-derive anything.
|
| 143 |
+
"band_low_bpm": rate.band_cycles_per_minute[0],
|
| 144 |
+
"band_high_bpm": rate.band_cycles_per_minute[1],
|
| 145 |
+
"anisotropy": round(signal.anisotropy, 3),
|
| 146 |
+
"duration_seconds": round(duration, 2),
|
| 147 |
+
}
|
| 148 |
+
if rate.half_estimates:
|
| 149 |
+
support["half_1_bpm"] = round(rate.half_estimates[0], 2)
|
| 150 |
+
support["half_2_bpm"] = round(rate.half_estimates[1], 2)
|
| 151 |
+
|
| 152 |
+
return RespirationResult(
|
| 153 |
+
measurement=Measurement(
|
| 154 |
+
kind="respiratory_rate",
|
| 155 |
+
# A refused measurement carries no number. The diagnostics are in
|
| 156 |
+
# `support` where a threshold can be re-derived from them, and
|
| 157 |
+
# nowhere that a caller could mistake for a result.
|
| 158 |
+
value=round(rate.cycles_per_minute, 1) if rate.usable else None,
|
| 159 |
+
unit="breaths/min",
|
| 160 |
+
usable=rate.usable,
|
| 161 |
+
support=support,
|
| 162 |
+
detail=rate.reason,
|
| 163 |
+
),
|
| 164 |
+
frames=len(frames),
|
| 165 |
+
sample_rate_hz=sample_rate,
|
| 166 |
+
duration_seconds=duration,
|
| 167 |
+
region=region,
|
| 168 |
+
)
|
app/adapters/tiled/__init__.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""High-resolution tiled inference (Β§25), and the two ways to fill the tiles.
|
| 2 |
+
|
| 3 |
+
Directive Β§25 lists four things to build for cattle ticks:
|
| 4 |
+
|
| 5 |
+
1. SAM 3.1 visual exemplar prompting;
|
| 6 |
+
2. Grounding DINO;
|
| 7 |
+
3. **high-resolution tiled inference**;
|
| 8 |
+
4. a multimodal verifier.
|
| 9 |
+
|
| 10 |
+
This package is the third, plus what the first two turn into when SAM 3 is out
|
| 11 |
+
of reach. `experiments/cattle_ticks/` is where the four are attempted and where
|
| 12 |
+
the record of what could not be attempted lives, which is what Β§36 asks for
|
| 13 |
+
before anything is called unavailable.
|
| 14 |
+
|
| 15 |
+
## The pieces
|
| 16 |
+
|
| 17 |
+
`tiles.py` β cut a photograph into overlapping fixed-pixel tiles and merge what
|
| 18 |
+
comes back. Model-free arithmetic, and the part Β§25 actually names.
|
| 19 |
+
|
| 20 |
+
`blobs.py` β a difference-of-Gaussians proposer. Β§4's classical route: no
|
| 21 |
+
weights, milliseconds, generous by design.
|
| 22 |
+
|
| 23 |
+
`openvocab.py` β Grounding DINO prompted with a word. Β§4's open-vocabulary leg.
|
| 24 |
+
|
| 25 |
+
`exemplar.py` β a frozen DINOv3 embedding and a cosine, which is the reachable
|
| 26 |
+
form of Β§25's visual-exemplar prompting.
|
| 27 |
+
|
| 28 |
+
## Why the pipeline is two-stage
|
| 29 |
+
|
| 30 |
+
Because the counting result says so. `experiments/poultry_house_count/` measured
|
| 31 |
+
a COCO detector out by 157 birds a frame and the same frames counted to within
|
| 32 |
+
15 once the model was **shown three examples**. The exemplar was what closed the
|
| 33 |
+
gap, and it did not need to come from the frame being scored. Propose cheaply,
|
| 34 |
+
then verify against an exemplar, and report both stages separately so a failure
|
| 35 |
+
is attributable to one of them.
|
| 36 |
+
|
| 37 |
+
## What a tiled pass costs
|
| 38 |
+
|
| 39 |
+
Linear in tiles, and the tile count is quadratic in the photograph's side. A
|
| 40 |
+
4,000 Γ 3,000 photo at an 800 px side with 20% overlap is 6 Γ 5 = 30 tiles. With
|
| 41 |
+
the blob proposer that is milliseconds. With Grounding DINO at a measured 5.5 s
|
| 42 |
+
a frame on a laptop CPU it is nearly three minutes, which is why
|
| 43 |
+
`openvocab.GROUNDING_DINO_TILED_SPEC` is a queued GPU placement and says so.
|
| 44 |
+
"""
|
| 45 |
+
|
| 46 |
+
from __future__ import annotations
|
| 47 |
+
|
| 48 |
+
import time
|
| 49 |
+
from dataclasses import dataclass, field
|
| 50 |
+
from typing import Callable
|
| 51 |
+
|
| 52 |
+
from PIL import Image
|
| 53 |
+
|
| 54 |
+
from app.adapters.base import OpenVocabularyDetector, Region
|
| 55 |
+
from app.adapters.tiled.tiles import (
|
| 56 |
+
MERGE_CONTAINMENT,
|
| 57 |
+
MERGE_IOU,
|
| 58 |
+
TILE_OVERLAP,
|
| 59 |
+
TILE_PIXELS,
|
| 60 |
+
Tile,
|
| 61 |
+
merge,
|
| 62 |
+
tiles_for,
|
| 63 |
+
to_frame,
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
#: A function that turns one tile image into regions in **tile** coordinates.
|
| 67 |
+
#: `blobs.propose` is one; a bound `GroundingDinoAdapter.detect_text` is another.
|
| 68 |
+
OnTile = Callable[[Image.Image], list[Region]]
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
@dataclass(frozen=True)
|
| 72 |
+
class TiledResult:
|
| 73 |
+
"""What a tiled pass found, and enough to say what tiling did.
|
| 74 |
+
|
| 75 |
+
**`raw_count` and `count` are both here on purpose.** The difference between
|
| 76 |
+
them is how much of the answer was duplicate detections across seams, and a
|
| 77 |
+
result that reported only the merged number would make an over-aggressive
|
| 78 |
+
merge and a genuinely sparse image look identical. `experiments/cattle_ticks/`
|
| 79 |
+
reads both.
|
| 80 |
+
"""
|
| 81 |
+
|
| 82 |
+
regions: list[Region]
|
| 83 |
+
tiles: tuple[Tile, ...]
|
| 84 |
+
#: Regions before the cross-tile merge, summed over tiles.
|
| 85 |
+
raw_count: int
|
| 86 |
+
#: Per-tile counts before the merge, in tile order. Kept because a tiled
|
| 87 |
+
#: pass whose detections all come from one tile is a different finding from
|
| 88 |
+
#: one spread evenly, and only this shows it.
|
| 89 |
+
per_tile: tuple[int, ...]
|
| 90 |
+
seconds: float
|
| 91 |
+
tile_pixels: int
|
| 92 |
+
overlap: float
|
| 93 |
+
#: Empty when every tile ran. Otherwise `(tile index, reason)` β a tile that
|
| 94 |
+
#: raised is recorded, never silently skipped, because a pass that dropped a
|
| 95 |
+
#: third of its tiles must not report a count as though it had not.
|
| 96 |
+
failures: tuple[tuple[int, str], ...] = ()
|
| 97 |
+
notes: dict[str, object] = field(default_factory=dict)
|
| 98 |
+
|
| 99 |
+
@property
|
| 100 |
+
def count(self) -> int:
|
| 101 |
+
return len(self.regions)
|
| 102 |
+
|
| 103 |
+
@property
|
| 104 |
+
def complete(self) -> bool:
|
| 105 |
+
return not self.failures
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def run_tiled(
|
| 109 |
+
image: Image.Image,
|
| 110 |
+
on_tile: OnTile,
|
| 111 |
+
*,
|
| 112 |
+
tile_pixels: int = TILE_PIXELS,
|
| 113 |
+
overlap: float = TILE_OVERLAP,
|
| 114 |
+
iou: float = MERGE_IOU,
|
| 115 |
+
containment: float = MERGE_CONTAINMENT,
|
| 116 |
+
) -> TiledResult:
|
| 117 |
+
"""Run `on_tile` over every tile and merge the results into frame coordinates.
|
| 118 |
+
|
| 119 |
+
A tile that raises is recorded in `failures` and the pass continues. That is
|
| 120 |
+
the same choice `experiments/poultry_house_count/run.py` makes per frame, and
|
| 121 |
+
for the same reason: one unreadable tile should cost one tile, and the count
|
| 122 |
+
that comes back has to carry the fact that it was short.
|
| 123 |
+
"""
|
| 124 |
+
tiles = tuple(tiles_for(image.size, tile_pixels=tile_pixels, overlap=overlap))
|
| 125 |
+
began = time.perf_counter()
|
| 126 |
+
|
| 127 |
+
gathered: list[Region] = []
|
| 128 |
+
per_tile: list[int] = []
|
| 129 |
+
failures: list[tuple[int, str]] = []
|
| 130 |
+
for tile in tiles:
|
| 131 |
+
try:
|
| 132 |
+
found = on_tile(tile.crop(image))
|
| 133 |
+
except Exception as failure: # noqa: BLE001 β recorded, not swallowed
|
| 134 |
+
failures.append((tile.index, f"{type(failure).__name__}: {failure}"))
|
| 135 |
+
per_tile.append(0)
|
| 136 |
+
continue
|
| 137 |
+
per_tile.append(len(found))
|
| 138 |
+
gathered.extend(to_frame(region, tile, image.size) for region in found)
|
| 139 |
+
|
| 140 |
+
merged = merge(gathered, image.size, iou=iou, containment=containment)
|
| 141 |
+
return TiledResult(
|
| 142 |
+
regions=merged,
|
| 143 |
+
tiles=tiles,
|
| 144 |
+
raw_count=len(gathered),
|
| 145 |
+
per_tile=tuple(per_tile),
|
| 146 |
+
seconds=round(time.perf_counter() - began, 3),
|
| 147 |
+
tile_pixels=tile_pixels,
|
| 148 |
+
overlap=overlap,
|
| 149 |
+
failures=tuple(failures),
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
class TiledDetector:
|
| 154 |
+
"""An `OpenVocabularyDetector` run tile by tile instead of whole-frame.
|
| 155 |
+
|
| 156 |
+
Not an `Adapter`. It owns no weights, has no licence of its own and cannot
|
| 157 |
+
be unavailable β it is a strategy applied to an adapter, and giving it a
|
| 158 |
+
spec would put a second entry in the listing for one model, which is exactly
|
| 159 |
+
the drift `adapters/registry.py` exists to prevent.
|
| 160 |
+
"""
|
| 161 |
+
|
| 162 |
+
def __init__(
|
| 163 |
+
self,
|
| 164 |
+
detector: OpenVocabularyDetector,
|
| 165 |
+
*,
|
| 166 |
+
tile_pixels: int = TILE_PIXELS,
|
| 167 |
+
overlap: float = TILE_OVERLAP,
|
| 168 |
+
) -> None:
|
| 169 |
+
self.detector = detector
|
| 170 |
+
self.tile_pixels = tile_pixels
|
| 171 |
+
self.overlap = overlap
|
| 172 |
+
|
| 173 |
+
def detect_text(
|
| 174 |
+
self, image: Image.Image, prompts: tuple[str, ...]
|
| 175 |
+
) -> TiledResult:
|
| 176 |
+
return run_tiled(
|
| 177 |
+
image,
|
| 178 |
+
lambda tile: self.detector.detect_text(tile, prompts),
|
| 179 |
+
tile_pixels=self.tile_pixels,
|
| 180 |
+
overlap=self.overlap,
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
__all__ = [
|
| 185 |
+
"MERGE_CONTAINMENT",
|
| 186 |
+
"MERGE_IOU",
|
| 187 |
+
"TILE_OVERLAP",
|
| 188 |
+
"TILE_PIXELS",
|
| 189 |
+
"OnTile",
|
| 190 |
+
"Tile",
|
| 191 |
+
"TiledDetector",
|
| 192 |
+
"TiledResult",
|
| 193 |
+
"merge",
|
| 194 |
+
"run_tiled",
|
| 195 |
+
"tiles_for",
|
| 196 |
+
"to_frame",
|
| 197 |
+
]
|
app/adapters/tiled/blobs.py
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Small dark blobs on a textured surface, found with a filter bank and no model.
|
| 2 |
+
|
| 3 |
+
Directive Β§4: *"Do not use a neural model when deterministic signal processing
|
| 4 |
+
is better."* A tick on a cow is a small dark ellipse on hide, and finding small
|
| 5 |
+
dark ellipses is what a difference-of-Gaussians scale space has done since 1980.
|
| 6 |
+
It costs milliseconds where a tiled Grounding DINO pass costs minutes, it needs
|
| 7 |
+
no weights and no licence, and β the part that matters for Β§25 β it does not
|
| 8 |
+
have to be *right*. It has to be a **proposer**: cheap, generous, and biased
|
| 9 |
+
towards recall, so that an exemplar check or a multimodal verifier decides what
|
| 10 |
+
is actually a tick.
|
| 11 |
+
|
| 12 |
+
That two-stage shape is the one the counting result argued for. CountGD did not
|
| 13 |
+
need a better frame; it needed to be shown once what the target looks like. A
|
| 14 |
+
proposer that finds every dark speck and an exemplar that says which specks
|
| 15 |
+
match is the same idea with the parts separated, and the separation is what lets
|
| 16 |
+
`experiments/cattle_ticks/` measure which half is failing.
|
| 17 |
+
|
| 18 |
+
**What this cannot do, stated up front.** It has no idea what a tick is. On a
|
| 19 |
+
Friesian's flank it will propose every black patch edge; on a dusty hide it will
|
| 20 |
+
propose dirt; on an ear it will propose the shadow inside the ear. Every figure
|
| 21 |
+
in `experiments/cattle_ticks/` for this arm is a proposer's figure and the
|
| 22 |
+
false-positive rate is the interesting half of it, not an embarrassment.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
from __future__ import annotations
|
| 26 |
+
|
| 27 |
+
import numpy as np
|
| 28 |
+
from PIL import Image
|
| 29 |
+
|
| 30 |
+
from app.adapters.base import Region
|
| 31 |
+
|
| 32 |
+
#: Blob radii searched, in pixels, on the image as handed in.
|
| 33 |
+
#:
|
| 34 |
+
#: **These are pixel sizes, so they only mean something at a known scale**, and
|
| 35 |
+
#: that is the whole reason `tiles.py` tiles at a fixed pixel side rather than a
|
| 36 |
+
#: fraction. An engorged Rhipicephalus microplus female is around 8β12 mm long
|
| 37 |
+
#: and an unfed one around 2β3 mm; at a phone macro distance where a cow's ear
|
| 38 |
+
#: fills an 800 px tile β roughly 150 mm across β that is 10β65 px. The range
|
| 39 |
+
#: below is wider at both ends because nothing here has measured a real capture
|
| 40 |
+
#: distance, and a proposer should over-propose.
|
| 41 |
+
RADII_PIXELS = (3.0, 4.5, 6.5, 9.0, 13.0, 18.0)
|
| 42 |
+
|
| 43 |
+
#: Minimum response, as a multiple of the robust spread of the scale-space
|
| 44 |
+
#: response over the whole tile. Median plus kΓMAD, for the reason
|
| 45 |
+
#: `audio/features.adaptive_threshold` gives: the things being looked for are
|
| 46 |
+
#: the outliers, so a mean-and-standard-deviation threshold is raised by its own
|
| 47 |
+
#: targets.
|
| 48 |
+
RESPONSE_K = 4.0
|
| 49 |
+
|
| 50 |
+
#: Floor on the raw difference-of-Gaussians response, in units of image
|
| 51 |
+
#: intensity where full scale is 1.0. A blob must be an outlier for its tile
|
| 52 |
+
#: **and** actually visible.
|
| 53 |
+
#:
|
| 54 |
+
#: **Measured, and it does exactly one job.** On a synthetic field of Gaussian
|
| 55 |
+
#: noise at Ο = 0.05 holding four discs of 0.45 contrast, this floor takes the
|
| 56 |
+
#: proposal count from 120 β the cap, saturated by noise peaks β to **4, the
|
| 57 |
+
#: four discs and nothing else**. On three real cattle photographs from
|
| 58 |
+
#: `evaluation/images/` it changes the count by **nothing at all**: 120 with it
|
| 59 |
+
#: and 120 without, because a real photograph's response distribution reaches
|
| 60 |
+
#: p99 0.102 and a maximum of 0.230, which is where the synthetic discs
|
| 61 |
+
#: themselves sit at 0.143β0.150.
|
| 62 |
+
#:
|
| 63 |
+
#: So it removes noise-only proposals from a flat field and is inert on a real
|
| 64 |
+
#: image. It is not what separates ticks from hide, and nothing at this stage is.
|
| 65 |
+
MIN_ABSOLUTE_RESPONSE = 0.02
|
| 66 |
+
|
| 67 |
+
#: Most proposals one tile may return, strongest first.
|
| 68 |
+
#:
|
| 69 |
+
#: **On every real photograph tried this cap binds, and that is the honest
|
| 70 |
+
#: description of this stage rather than a defect.** Measured on
|
| 71 |
+
#: `cattle_ng_red_bororo.jpg` (1600 Γ 1067), `cattle_ng_kaduna_market_01.jpg`
|
| 72 |
+
#: and `cattle_ke_maasai.jpg`: 120 proposals whole-frame on each, and 720 raw
|
| 73 |
+
#: over six 800 px tiles merging to 445β525. The proposer's job is recall, and
|
| 74 |
+
#: the discrimination happens in `exemplar.verify`.
|
| 75 |
+
#:
|
| 76 |
+
#: The number is a cost decision. Exemplar verification is a DINOv3 forward pass
|
| 77 |
+
#: per candidate at a **measured 122.5 ms** on a small crop under load β not the
|
| 78 |
+
#: 59 ms `adapters/embedding.py` records for a whole frame β so 120 per tile
|
| 79 |
+
#: over a 35-tile photograph is about **eight and a half minutes** of embedding
|
| 80 |
+
#: on a laptop CPU. An earlier version of this comment said 59 ms, 30 tiles and
|
| 81 |
+
#: 3.5 minutes, and was out by roughly a factor of two on each.
|
| 82 |
+
#:
|
| 83 |
+
#: Rank-and-truncate rather than a stricter threshold because a cap is a cost
|
| 84 |
+
#: bound that says what it is, and a threshold tuned until the count looks right
|
| 85 |
+
#: is a threshold tuned on the answer.
|
| 86 |
+
MAX_PER_TILE = 120
|
| 87 |
+
|
| 88 |
+
#: Centres closer than this multiple of the larger blob's radius are the same
|
| 89 |
+
#: blob found at two scales.
|
| 90 |
+
MERGE_RADIUS_FACTOR = 1.0
|
| 91 |
+
|
| 92 |
+
#: Label every proposal carries. Not `"tick"` β that would be the proposer
|
| 93 |
+
#: claiming to have identified something, and it has not. `tiles.merge` compares
|
| 94 |
+
#: labels, so this also keeps proposals from merging with a detector's boxes.
|
| 95 |
+
LABEL = "blob"
|
| 96 |
+
|
| 97 |
+
#: Which way round a blob has to be against its surroundings.
|
| 98 |
+
#:
|
| 99 |
+
#: **`both` is the default because the field images say it has to be.** The
|
| 100 |
+
#: obvious assumption is that a tick is a dark speck on pale hide, and on
|
| 101 |
+
#: `tick_cattle_calf_groin` β dark Hyalomma on a calf's pale groin β it is. On
|
| 102 |
+
#: `tick_cattle_hereford_neck` it is exactly backwards: engorged
|
| 103 |
+
#: Rhipicephalus microplus on a dark Hereford neck photograph as **bright**
|
| 104 |
+
#: specks against the coat. Same species of problem, opposite sign, and a
|
| 105 |
+
#: dark-only filter finds nothing at all on the second image.
|
| 106 |
+
#:
|
| 107 |
+
#: This was found by looking at the photographs rather than by reasoning about
|
| 108 |
+
#: ticks, and it is the single most useful thing the field set contributed to
|
| 109 |
+
#: the method.
|
| 110 |
+
POLARITY = ("dark", "light", "both")
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def _scale_space(grey: np.ndarray, radii: tuple[float, ...]) -> np.ndarray:
|
| 114 |
+
"""Scale-normalised Laplacian response at each radius, `(scales, h, w)`.
|
| 115 |
+
|
| 116 |
+
Approximated by a difference of Gaussians, which is the standard and much
|
| 117 |
+
cheaper stand-in for the Laplacian of Gaussian. **Sign convention: positive
|
| 118 |
+
response means darker than surroundings**, which is what a tick is against
|
| 119 |
+
hide, and is why the difference is taken the way round it is.
|
| 120 |
+
|
| 121 |
+
**The response is not multiplied by ΟΒ², and an earlier version of this
|
| 122 |
+
function was.** Scale normalisation is genuinely required β without it the
|
| 123 |
+
smallest scale wins everywhere β but a difference of Gaussians already
|
| 124 |
+
carries it: DoG(Ο, kΟ) β (kβ1)Β·ΟΒ²βΒ²G, so the ΟΒ² is in the approximation.
|
| 125 |
+
Applying it a second time biases every response towards the largest radius,
|
| 126 |
+
and a smoke test on four synthetic discs of radius 5, 6, 9 and 13 px
|
| 127 |
+
reported **every one of them at the 18 px maximum**. Found because the test
|
| 128 |
+
checked the radii and not only the count.
|
| 129 |
+
"""
|
| 130 |
+
import cv2
|
| 131 |
+
|
| 132 |
+
responses = np.empty((len(radii), *grey.shape), dtype=np.float32)
|
| 133 |
+
for index, radius in enumerate(radii):
|
| 134 |
+
# Ο = r/β2 is the scale at which a DoG's response peaks for a circular
|
| 135 |
+
# blob of radius r.
|
| 136 |
+
sigma = float(radius) / np.sqrt(2.0)
|
| 137 |
+
inner = cv2.GaussianBlur(grey, (0, 0), sigmaX=sigma)
|
| 138 |
+
outer = cv2.GaussianBlur(grey, (0, 0), sigmaX=sigma * 1.6)
|
| 139 |
+
responses[index] = outer - inner
|
| 140 |
+
return responses
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def propose(
|
| 144 |
+
image: Image.Image,
|
| 145 |
+
*,
|
| 146 |
+
radii: tuple[float, ...] = RADII_PIXELS,
|
| 147 |
+
response_k: float = RESPONSE_K,
|
| 148 |
+
max_proposals: int = MAX_PER_TILE,
|
| 149 |
+
polarity: str = "both",
|
| 150 |
+
) -> list[Region]:
|
| 151 |
+
"""Candidate small dark blobs, strongest first, in image pixels.
|
| 152 |
+
|
| 153 |
+
`score` is the response over the tile's own robust spread. **It is not a
|
| 154 |
+
probability and nothing calibrated it**, which is the same Β§37 position every
|
| 155 |
+
model score in this repository is in; the run records carry
|
| 156 |
+
`confidence_is_calibrated: false` and the harness enforces it.
|
| 157 |
+
|
| 158 |
+
**The reported box is a size estimate and it runs about 30% small.** The
|
| 159 |
+
scale that peaks for a hard-edged disc is not the scale that peaks for the
|
| 160 |
+
Gaussian blob the filter is matched to. Measured on synthetic discs of
|
| 161 |
+
radius 5, 6, 9 and 13 px, the selected radii were 4.5, 4.5, 6.5 and 9.0 β
|
| 162 |
+
the right ordering, consistently under. That is harmless for a proposer and
|
| 163 |
+
would not be harmless if a box from here were ever used to *measure* a tick,
|
| 164 |
+
which nothing does.
|
| 165 |
+
|
| 166 |
+
Boxes are clipped to the frame, so a blob on an edge reports a box narrower
|
| 167 |
+
than its radius. Also harmless here, also worth knowing before reading a
|
| 168 |
+
width out of one.
|
| 169 |
+
"""
|
| 170 |
+
import cv2
|
| 171 |
+
|
| 172 |
+
grey = np.asarray(image.convert("L"), dtype=np.float32) / 255.0
|
| 173 |
+
height, width = grey.shape
|
| 174 |
+
if height < 8 or width < 8:
|
| 175 |
+
return []
|
| 176 |
+
|
| 177 |
+
usable = tuple(r for r in radii if 2.0 * r < min(height, width))
|
| 178 |
+
if not usable:
|
| 179 |
+
return []
|
| 180 |
+
|
| 181 |
+
if polarity not in POLARITY:
|
| 182 |
+
raise ValueError(f"polarity must be one of {POLARITY}, not {polarity!r}")
|
| 183 |
+
|
| 184 |
+
responses = _scale_space(grey, usable)
|
| 185 |
+
if polarity == "light":
|
| 186 |
+
responses = -responses
|
| 187 |
+
elif polarity == "both":
|
| 188 |
+
# Magnitude, so a bright tick on a dark Hereford scores the same as a
|
| 189 |
+
# dark one on a pale calf. It costs the *sign*, which nothing here uses
|
| 190 |
+
# and which would be worth keeping the day something wants to know
|
| 191 |
+
# whether a hide is dark or pale.
|
| 192 |
+
responses = np.abs(responses)
|
| 193 |
+
|
| 194 |
+
best_scale = np.argmax(responses, axis=0)
|
| 195 |
+
best = np.take_along_axis(responses, best_scale[None], axis=0)[0]
|
| 196 |
+
|
| 197 |
+
centre = float(np.median(best))
|
| 198 |
+
spread = float(np.median(np.abs(best - centre)))
|
| 199 |
+
if spread <= 0.0:
|
| 200 |
+
# A flat tile β a blown-out highlight, a uniform background, a synthetic
|
| 201 |
+
# patch. No spread means no threshold can be set, and proposing
|
| 202 |
+
# everything would be worse than proposing nothing.
|
| 203 |
+
return []
|
| 204 |
+
threshold = max(centre + response_k * spread, MIN_ABSOLUTE_RESPONSE)
|
| 205 |
+
|
| 206 |
+
# Local maxima only. A 3x3 dilation equals the original exactly where a
|
| 207 |
+
# pixel is the largest in its neighbourhood, which is a peak test in one
|
| 208 |
+
# OpenCV call rather than a Python loop over a megapixel.
|
| 209 |
+
peak = (best >= cv2.dilate(best, np.ones((3, 3), np.uint8))) & (best > threshold)
|
| 210 |
+
ys, xs = np.nonzero(peak)
|
| 211 |
+
if ys.size == 0:
|
| 212 |
+
return []
|
| 213 |
+
|
| 214 |
+
order = np.argsort(-best[ys, xs])
|
| 215 |
+
frame_area = float(width * height)
|
| 216 |
+
|
| 217 |
+
kept: list[Region] = []
|
| 218 |
+
centres: list[tuple[float, float, float]] = []
|
| 219 |
+
for index in order:
|
| 220 |
+
y, x = float(ys[index]), float(xs[index])
|
| 221 |
+
radius = float(usable[int(best_scale[int(ys[index]), int(xs[index])])])
|
| 222 |
+
|
| 223 |
+
# Same blob at two scales, or two peaks on one blob's shoulder.
|
| 224 |
+
duplicate = False
|
| 225 |
+
for cx, cy, cr in centres:
|
| 226 |
+
if (x - cx) ** 2 + (y - cy) ** 2 < (
|
| 227 |
+
MERGE_RADIUS_FACTOR * max(radius, cr)
|
| 228 |
+
) ** 2:
|
| 229 |
+
duplicate = True
|
| 230 |
+
break
|
| 231 |
+
if duplicate:
|
| 232 |
+
continue
|
| 233 |
+
|
| 234 |
+
x0, y0 = max(0.0, x - radius), max(0.0, y - radius)
|
| 235 |
+
x1, y1 = min(float(width), x + radius), min(float(height), y + radius)
|
| 236 |
+
kept.append(Region(
|
| 237 |
+
label=LABEL,
|
| 238 |
+
score=round(float((best[int(y), int(x)] - centre) / spread), 4),
|
| 239 |
+
box=(x0, y0, x1, y1),
|
| 240 |
+
area_fraction=((x1 - x0) * (y1 - y0)) / frame_area,
|
| 241 |
+
))
|
| 242 |
+
centres.append((x, y, radius))
|
| 243 |
+
if len(kept) >= max_proposals:
|
| 244 |
+
break
|
| 245 |
+
return kept
|
app/adapters/tiled/exemplar.py
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Showing the system one example, and asking which candidates look like it.
|
| 2 |
+
|
| 3 |
+
**This is the lesson from counting, applied to a different problem.**
|
| 4 |
+
`experiments/poultry_house_count/` is the strongest result in this tree: a COCO
|
| 5 |
+
detector was out by 157 birds a frame on commercial broiler-house imagery, and
|
| 6 |
+
CountGD given **three exemplar boxes** was out by 15. The exemplars were not
|
| 7 |
+
from the frame being counted β they came from one frame in a different split,
|
| 8 |
+
fixed once and reused for all 452 β and that arm was the *best* of the three. The
|
| 9 |
+
conclusion in that README is the one this module is built on: *"Nothing CountGD
|
| 10 |
+
needs is specific to the frame in front of it. It needs to be shown, once, what
|
| 11 |
+
a bird looks like from this camera."*
|
| 12 |
+
|
| 13 |
+
Directive Β§25 asks for the same mechanism by name β **SAM 3.1 visual exemplar
|
| 14 |
+
prompting** β for ticks. SAM 3 is gated behind manual Meta approval, needs an
|
| 15 |
+
HF token a build cannot obtain unattended, and is a 3.44 GB checkpoint that
|
| 16 |
+
wants a CUDA 12.6 GPU. It was not reachable from this machine and
|
| 17 |
+
`experiments/cattle_ticks/README.md` records that as an attempt that could not be
|
| 18 |
+
made rather than as one that failed.
|
| 19 |
+
|
| 20 |
+
So the exemplar mechanism here is built from what Β§3 *does* make reachable: a
|
| 21 |
+
frozen DINOv3 embedding and a cosine similarity. Embed one crop of a tick, embed
|
| 22 |
+
each candidate, keep the candidates whose vector points the same way. It is
|
| 23 |
+
weaker than SAM 3's β there is no segmentation and no joint text-and-exemplar
|
| 24 |
+
conditioning β and it is the same idea.
|
| 25 |
+
|
| 26 |
+
## The three ways this is honest about what it is
|
| 27 |
+
|
| 28 |
+
**It never proposes.** It only ranks and filters what a proposer found, so its
|
| 29 |
+
recall ceiling is the proposer's recall. A tick the blob filter missed cannot be
|
| 30 |
+
recovered here, and the experiment reports both numbers separately for exactly
|
| 31 |
+
that reason.
|
| 32 |
+
|
| 33 |
+
**A similarity is not a probability.** Cosine between two frozen embeddings is
|
| 34 |
+
an uncalibrated number in [-1, 1] whose useful range depends entirely on what
|
| 35 |
+
the exemplars were. Β§37 forbids it reaching a farm, and the run records say
|
| 36 |
+
`confidence_is_calibrated: false`.
|
| 37 |
+
|
| 38 |
+
**The exemplars are part of the method.** Change them and every figure changes.
|
| 39 |
+
So `ExemplarBank` records what each vector came from and hashes the images, and
|
| 40 |
+
the experiment pins that hash in its run record β the same discipline
|
| 41 |
+
`poultry_house_count` applies to its three boxes from `train[0]`.
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
from __future__ import annotations
|
| 45 |
+
|
| 46 |
+
import hashlib
|
| 47 |
+
from dataclasses import dataclass, replace
|
| 48 |
+
|
| 49 |
+
import numpy as np
|
| 50 |
+
from PIL import Image
|
| 51 |
+
|
| 52 |
+
from app.adapters.base import Embedder, Region
|
| 53 |
+
|
| 54 |
+
#: How much larger than the candidate box the embedded crop is.
|
| 55 |
+
#:
|
| 56 |
+
#: **Not 1.0, and the reason is what the backbone is.** DINOv3 was trained on
|
| 57 |
+
#: photographs of things in context, and its embedding of a 12 px brown speck
|
| 58 |
+
#: filling the frame is dominated by colour and blur. Giving it the speck plus
|
| 59 |
+
#: the hide around it produces a vector about *a tick on a cow*, which is the
|
| 60 |
+
#: thing being matched. 1.8 is a starting point, swept in
|
| 61 |
+
#: `experiments/cattle_ticks/`, and the sweep is in that directory's config.
|
| 62 |
+
CONTEXT_FACTOR = 1.8
|
| 63 |
+
|
| 64 |
+
#: Smallest crop, in pixels, handed to the backbone. Below roughly this size the
|
| 65 |
+
#: 224 px resize is pure upsampling and the vector is describing interpolation.
|
| 66 |
+
MIN_CROP_PIXELS = 24
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
@dataclass(frozen=True)
|
| 70 |
+
class ExemplarBank:
|
| 71 |
+
"""Unit vectors for a handful of example crops, and where they came from.
|
| 72 |
+
|
| 73 |
+
Frozen and content-addressed for the same reason `provenance.InputSet` is:
|
| 74 |
+
an accuracy figure measured against one set of exemplars is not an accuracy
|
| 75 |
+
figure for another, and a bank identified only by a variable name drifts
|
| 76 |
+
without anybody noticing.
|
| 77 |
+
"""
|
| 78 |
+
|
| 79 |
+
#: `(n, d)`, L2-normalised. The embedding adapters normalise inside the ONNX
|
| 80 |
+
#: graph, so this is asserted rather than re-done.
|
| 81 |
+
vectors: np.ndarray
|
| 82 |
+
#: One per row: a human-readable id for the crop it came from.
|
| 83 |
+
ids: tuple[str, ...]
|
| 84 |
+
#: sha256 over the raw crop bytes, in row order. What pins the bank.
|
| 85 |
+
digests: tuple[str, ...]
|
| 86 |
+
#: Which backbone produced the vectors. A bank built with DINOv3 and queried
|
| 87 |
+
#: with DINOv2 is a silent failure, and this is what makes it a loud one.
|
| 88 |
+
embedder_id: str
|
| 89 |
+
|
| 90 |
+
def __post_init__(self) -> None:
|
| 91 |
+
if self.vectors.ndim != 2 or self.vectors.shape[0] == 0:
|
| 92 |
+
raise ValueError(
|
| 93 |
+
"An exemplar bank with no exemplars in it is a bank that will "
|
| 94 |
+
"match nothing and report a threshold failure. Build it with at "
|
| 95 |
+
"least one crop."
|
| 96 |
+
)
|
| 97 |
+
norms = np.linalg.norm(self.vectors, axis=1)
|
| 98 |
+
if not np.allclose(norms, 1.0, atol=1e-3):
|
| 99 |
+
raise ValueError(
|
| 100 |
+
f"Exemplar vectors are not unit length (norms "
|
| 101 |
+
f"{norms.min():.4f}β{norms.max():.4f}). Cosine similarity here "
|
| 102 |
+
f"is a dot product and assumes they are."
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
@property
|
| 106 |
+
def digest(self) -> str:
|
| 107 |
+
"""One hash over the whole bank, in a stable order."""
|
| 108 |
+
joined = "\n".join(
|
| 109 |
+
f"{i}:{d}" for i, d in sorted(zip(self.ids, self.digests))
|
| 110 |
+
)
|
| 111 |
+
return hashlib.sha256(joined.encode("utf-8")).hexdigest()
|
| 112 |
+
|
| 113 |
+
def similarity(self, vector: np.ndarray) -> tuple[float, str]:
|
| 114 |
+
"""Best cosine against the bank, and which exemplar gave it.
|
| 115 |
+
|
| 116 |
+
Maximum rather than mean. Ticks differ enormously by species, sex and
|
| 117 |
+
engorgement β a flat unfed male and a grey engorged female do not look
|
| 118 |
+
alike β so a bank is a set of appearances rather than a cluster with a
|
| 119 |
+
centre, and averaging them produces a vector that matches none of them.
|
| 120 |
+
"""
|
| 121 |
+
scores = self.vectors @ np.asarray(vector, dtype=np.float32)
|
| 122 |
+
best = int(np.argmax(scores))
|
| 123 |
+
return float(scores[best]), self.ids[best]
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
@dataclass(frozen=True)
|
| 127 |
+
class Verified:
|
| 128 |
+
"""One candidate, and what the exemplar check said about it.
|
| 129 |
+
|
| 130 |
+
Both scores are kept. The proposer's response and the exemplar's similarity
|
| 131 |
+
fail in different directions β a strong blob that looks nothing like a tick,
|
| 132 |
+
a faint blob that looks exactly like one β and collapsing them into a single
|
| 133 |
+
number loses the ability to say which stage is wrong.
|
| 134 |
+
"""
|
| 135 |
+
|
| 136 |
+
region: Region
|
| 137 |
+
similarity: float
|
| 138 |
+
nearest_exemplar: str
|
| 139 |
+
proposer_score: float
|
| 140 |
+
passed: bool
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def _crop(image: Image.Image, box: tuple[float, float, float, float],
|
| 144 |
+
context: float) -> Image.Image:
|
| 145 |
+
"""The candidate plus context, square, clipped to the frame.
|
| 146 |
+
|
| 147 |
+
Square because the backbone centre-crops to a square anyway, and letting it
|
| 148 |
+
do that on a wide crop silently discards one axis of the context this
|
| 149 |
+
function exists to add.
|
| 150 |
+
"""
|
| 151 |
+
x0, y0, x1, y1 = box
|
| 152 |
+
cx, cy = (x0 + x1) / 2.0, (y0 + y1) / 2.0
|
| 153 |
+
half = max(x1 - x0, y1 - y0) * context / 2.0
|
| 154 |
+
half = max(half, MIN_CROP_PIXELS / 2.0)
|
| 155 |
+
|
| 156 |
+
width, height = image.size
|
| 157 |
+
left = int(max(0, round(cx - half)))
|
| 158 |
+
top = int(max(0, round(cy - half)))
|
| 159 |
+
right = int(min(width, round(cx + half)))
|
| 160 |
+
bottom = int(min(height, round(cy + half)))
|
| 161 |
+
if right - left < 2 or bottom - top < 2:
|
| 162 |
+
return image.crop((0, 0, min(2, width), min(2, height)))
|
| 163 |
+
return image.crop((left, top, right, bottom))
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def build_bank(
|
| 167 |
+
embedder: Embedder,
|
| 168 |
+
crops: list[Image.Image],
|
| 169 |
+
ids: list[str],
|
| 170 |
+
*,
|
| 171 |
+
embedder_id: str,
|
| 172 |
+
) -> ExemplarBank:
|
| 173 |
+
"""Embed example images once. The bank is then reused for every candidate.
|
| 174 |
+
|
| 175 |
+
`crops` are whole images of the target β an isolated tick, a close-up of one
|
| 176 |
+
on an ear β not boxes on a scene. Cropping is the caller's decision, because
|
| 177 |
+
what counts as "the exemplar" is exactly the thing an experiment has to pin
|
| 178 |
+
and vary.
|
| 179 |
+
"""
|
| 180 |
+
if len(crops) != len(ids):
|
| 181 |
+
raise ValueError(f"{len(crops)} crops against {len(ids)} ids.")
|
| 182 |
+
|
| 183 |
+
vectors = np.stack([embedder.embed(crop) for crop in crops]).astype(np.float32)
|
| 184 |
+
digests = tuple(
|
| 185 |
+
hashlib.sha256(crop.convert("RGB").tobytes()).hexdigest() for crop in crops
|
| 186 |
+
)
|
| 187 |
+
return ExemplarBank(
|
| 188 |
+
vectors=vectors, ids=tuple(ids), digests=digests, embedder_id=embedder_id
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def verify(
|
| 193 |
+
embedder: Embedder,
|
| 194 |
+
image: Image.Image,
|
| 195 |
+
candidates: list[Region],
|
| 196 |
+
bank: ExemplarBank,
|
| 197 |
+
*,
|
| 198 |
+
threshold: float,
|
| 199 |
+
context: float = CONTEXT_FACTOR,
|
| 200 |
+
) -> list[Verified]:
|
| 201 |
+
"""Score every candidate against the bank. Nothing is dropped.
|
| 202 |
+
|
| 203 |
+
**Rejected candidates come back too**, carrying `passed=False`. A run that
|
| 204 |
+
proposed 400 blobs and passed 3 is a completely different situation from one
|
| 205 |
+
that proposed 3, and a function that returned only the survivors would make
|
| 206 |
+
the two indistinguishable in the record. The experiment stores both counts;
|
| 207 |
+
`app/counting.py` and `audio/events.py` keep their rejects for the same
|
| 208 |
+
reason.
|
| 209 |
+
|
| 210 |
+
The label is rewritten to `probable_tick` on the ones that pass, so a merged
|
| 211 |
+
result cannot mix a verified candidate with a raw proposal β `tiles.merge`
|
| 212 |
+
compares labels, and two objects with different labels are two objects.
|
| 213 |
+
"""
|
| 214 |
+
out: list[Verified] = []
|
| 215 |
+
for candidate in candidates:
|
| 216 |
+
vector = embedder.embed(_crop(image, candidate.box, context))
|
| 217 |
+
similarity, nearest = bank.similarity(vector)
|
| 218 |
+
passed = similarity >= threshold
|
| 219 |
+
out.append(Verified(
|
| 220 |
+
region=replace(
|
| 221 |
+
candidate,
|
| 222 |
+
label="probable_tick" if passed else candidate.label,
|
| 223 |
+
score=round(similarity, 4),
|
| 224 |
+
),
|
| 225 |
+
similarity=round(similarity, 6),
|
| 226 |
+
nearest_exemplar=nearest,
|
| 227 |
+
proposer_score=candidate.score,
|
| 228 |
+
passed=passed,
|
| 229 |
+
))
|
| 230 |
+
return out
|
app/adapters/tiled/openvocab.py
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Grounding DINO, prompted with a word, as an `OpenVocabularyDetector`.
|
| 2 |
+
|
| 3 |
+
Directive Β§4 names it for *"open-vocabulary object detection, fallback detection
|
| 4 |
+
when SAM concept prompting is weak, text-prompted localization"*, and Β§25 names
|
| 5 |
+
it for ticks specifically. It is also the only model in the whole Β§3/Β§4 stack
|
| 6 |
+
that is Apache-2.0 for code, weights and text encoder alike, ungated, and small
|
| 7 |
+
enough to run on a laptop β `adapters/licences.py:grounding-dino-hf` has the
|
| 8 |
+
detail.
|
| 9 |
+
|
| 10 |
+
**This is a real adapter and `adapters/unavailable.py` still lists a placeholder
|
| 11 |
+
for the same model.** That is deliberate and it is not a contradiction:
|
| 12 |
+
`GROUNDING_DINO_SPEC` there describes an adapter registered in
|
| 13 |
+
`adapters/registry.py` and driven from a `models/` card, which is the shape
|
| 14 |
+
every served model in this service has, and which this one does not have yet.
|
| 15 |
+
Nothing here is wired into `registry.all_adapters()`, so `/health` and
|
| 16 |
+
`/capabilities` report exactly what they reported before. What this file adds is
|
| 17 |
+
a way for `experiments/cattle_ticks/` to drive the model that Β§25 asks for, from
|
| 18 |
+
code that is in the diff. Promoting it to a registered adapter means exporting
|
| 19 |
+
it and writing a card, and that belongs to whoever owns `registry.py`.
|
| 20 |
+
|
| 21 |
+
## Two things worth knowing before reading a number out of it
|
| 22 |
+
|
| 23 |
+
**It is not a counter.** `experiments/poultry_house_count/` measured this exact
|
| 24 |
+
model, at this exact port, on 452 annotated frames: MAE 158.37 against a COCO
|
| 25 |
+
detector's 156.80, finding nothing at all in 222 of them. Its own paper's
|
| 26 |
+
Table 1 puts text-only Grounding DINO at FSC-147 val MAE 54.45 where CountGD
|
| 27 |
+
scores 7.10. Prompting an open-vocabulary detector with a word is not counting,
|
| 28 |
+
and this repository has the receipt.
|
| 29 |
+
|
| 30 |
+
**Its box score is not a probability.** It is a sigmoid over a text-conditioned
|
| 31 |
+
logit that nothing has calibrated β not on ticks, not on cattle, not on
|
| 32 |
+
anything. Β§37 forbids it reaching a farm as a confidence, and the run records in
|
| 33 |
+
`experiments/cattle_ticks/results/` carry `confidence_is_calibrated: false` so
|
| 34 |
+
the harness enforces that rather than trusting a docstring.
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
from __future__ import annotations
|
| 38 |
+
|
| 39 |
+
from dataclasses import dataclass
|
| 40 |
+
|
| 41 |
+
from PIL import Image
|
| 42 |
+
|
| 43 |
+
from app.adapters.base import (
|
| 44 |
+
Adapter,
|
| 45 |
+
AdapterError,
|
| 46 |
+
AdapterSpec,
|
| 47 |
+
AdapterUnavailable,
|
| 48 |
+
Availability,
|
| 49 |
+
Modality,
|
| 50 |
+
Placement,
|
| 51 |
+
Region,
|
| 52 |
+
Task,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
#: The port used throughout. `-tiny` rather than `-base` because it is what is
|
| 56 |
+
#: in the Hugging Face cache on the development machine and because
|
| 57 |
+
#: `experiments/poultry_house_count/` already benchmarked this exact checkpoint,
|
| 58 |
+
#: so a figure here is comparable with one there. `-base` is the stronger model
|
| 59 |
+
#: and nobody has run it; both directories say so.
|
| 60 |
+
MODEL_ID = "IDEA-Research/grounding-dino-tiny"
|
| 61 |
+
|
| 62 |
+
#: Box and text thresholds. 0.3 is the value in the model card's own usage
|
| 63 |
+
#: example, used unchanged so it cannot have been tuned to any set here. It is
|
| 64 |
+
#: also what `experiments/poultry_house_count/` used, which is the other reason
|
| 65 |
+
#: not to move it.
|
| 66 |
+
THRESHOLD = 0.3
|
| 67 |
+
|
| 68 |
+
GROUNDING_DINO_TILED_SPEC = AdapterSpec(
|
| 69 |
+
adapter_id="grounding-dino-tiny-openvocab",
|
| 70 |
+
runtime="grounding-dino-hf",
|
| 71 |
+
tasks=(Task.DETECT,),
|
| 72 |
+
modalities=(Modality.IMAGE,),
|
| 73 |
+
directive_role=(
|
| 74 |
+
"Β§4 Grounding DINO β open-vocabulary detection and text-prompted "
|
| 75 |
+
"localisation. Β§25 names it for tick detection over tiled close-ups, "
|
| 76 |
+
"which is what `adapters/tiled/` drives it for."
|
| 77 |
+
),
|
| 78 |
+
placement=Placement.GPU_SERVICE,
|
| 79 |
+
requires_gpu=False,
|
| 80 |
+
placement_reason=(
|
| 81 |
+
"It runs on CPU β the transformers port's deformable attention is plain "
|
| 82 |
+
"`grid_sample` with no CUDA extension β and should not. ADR-adjacent "
|
| 83 |
+
"measurement in `adapters/unavailable.py` records 5.5 s a frame and a "
|
| 84 |
+
"2,121 MB peak on one whole frame, against a 4 GiB container also "
|
| 85 |
+
"holding the media. **Tiling multiplies that by the tile count**, which "
|
| 86 |
+
"is the cost that decides the placement: a 4,000 Γ 3,000 photograph is "
|
| 87 |
+
"30 tiles at an 800 px side and a 4,598 Γ 2,997 one is 35, so a tiled "
|
| 88 |
+
"pass is minutes on CPU and seconds on a GPU. This is a queued GPU "
|
| 89 |
+
"capability."
|
| 90 |
+
),
|
| 91 |
+
notes=(
|
| 92 |
+
"172,250,626 parameters, 657 MiB of safetensors, Apache-2.0 for code, "
|
| 93 |
+
"weights and the BERT text encoder alike β the cleanest licence in the "
|
| 94 |
+
"Β§3/Β§4 stack. Not registered in `adapters/registry.py`: there is no "
|
| 95 |
+
"`models/` card and no ONNX export, so the CPU worker's no-torch "
|
| 96 |
+
"property (ADR 0017) would be lost by wiring it in as it stands."
|
| 97 |
+
),
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
@dataclass(frozen=True)
|
| 102 |
+
class _Loaded:
|
| 103 |
+
processor: object
|
| 104 |
+
model: object
|
| 105 |
+
device: str
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
class GroundingDinoAdapter(Adapter):
|
| 109 |
+
"""Text-prompted detection on one image. Tiling is `tiled.TiledDetector`'s job.
|
| 110 |
+
|
| 111 |
+
Kept single-image on purpose. Mixing the prompt-and-threshold concerns with
|
| 112 |
+
the cut-up-and-merge concerns is how a tiling bug becomes indistinguishable
|
| 113 |
+
from a prompting bug, and `experiments/cattle_ticks/` has to be able to run
|
| 114 |
+
this model with tiling on and off to say what tiling did.
|
| 115 |
+
"""
|
| 116 |
+
|
| 117 |
+
spec = GROUNDING_DINO_TILED_SPEC
|
| 118 |
+
|
| 119 |
+
def __init__(
|
| 120 |
+
self,
|
| 121 |
+
*,
|
| 122 |
+
model_id: str = MODEL_ID,
|
| 123 |
+
device: str = "cpu",
|
| 124 |
+
threshold: float = THRESHOLD,
|
| 125 |
+
) -> None:
|
| 126 |
+
self.model_id = model_id
|
| 127 |
+
self.device = device
|
| 128 |
+
self.threshold = threshold
|
| 129 |
+
self._loaded: _Loaded | None = None
|
| 130 |
+
|
| 131 |
+
def availability(self) -> Availability:
|
| 132 |
+
"""Whether this could run without a download, checked without loading.
|
| 133 |
+
|
| 134 |
+
Deliberately does not import torch. `/health` calling this must not pay
|
| 135 |
+
for a two-second import, and β more to the point β an adapter that
|
| 136 |
+
reports availability by loading the thing has no way to report that
|
| 137 |
+
loading it failed.
|
| 138 |
+
"""
|
| 139 |
+
try:
|
| 140 |
+
from app.adapters.licences import LicenceRefused, gate
|
| 141 |
+
except ImportError as absent: # pragma: no cover β package always present
|
| 142 |
+
return Availability(False, f"licence table unreadable: {absent}", "")
|
| 143 |
+
try:
|
| 144 |
+
gate(self.spec.runtime)
|
| 145 |
+
except LicenceRefused as refusal:
|
| 146 |
+
return Availability(
|
| 147 |
+
False, str(refusal),
|
| 148 |
+
"This one does not become available by installing it.",
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
try:
|
| 152 |
+
import transformers # noqa: F401
|
| 153 |
+
except ImportError:
|
| 154 |
+
return Availability(
|
| 155 |
+
False,
|
| 156 |
+
"transformers is not installed, so the Grounding DINO port "
|
| 157 |
+
"cannot be built.",
|
| 158 |
+
"pip install transformers β nothing here is gated and nothing "
|
| 159 |
+
"needs a token.",
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
try:
|
| 163 |
+
from huggingface_hub import try_to_load_from_cache
|
| 164 |
+
except ImportError: # pragma: no cover β a transformers dependency
|
| 165 |
+
return Availability(True)
|
| 166 |
+
|
| 167 |
+
cached = try_to_load_from_cache(self.model_id, "config.json")
|
| 168 |
+
if not isinstance(cached, str):
|
| 169 |
+
return Availability(
|
| 170 |
+
False,
|
| 171 |
+
f"{self.model_id} is not in the Hugging Face cache, so running "
|
| 172 |
+
f"it needs a 657 MiB download.",
|
| 173 |
+
f"huggingface-cli download {self.model_id}, or run once with "
|
| 174 |
+
f"network access.",
|
| 175 |
+
)
|
| 176 |
+
return Availability(True)
|
| 177 |
+
|
| 178 |
+
def load(self) -> "GroundingDinoAdapter":
|
| 179 |
+
state = self.availability()
|
| 180 |
+
if not state.ready:
|
| 181 |
+
raise AdapterUnavailable(state)
|
| 182 |
+
if self._loaded is not None:
|
| 183 |
+
return self
|
| 184 |
+
|
| 185 |
+
from transformers import AutoModelForZeroShotObjectDetection, AutoProcessor
|
| 186 |
+
|
| 187 |
+
processor = AutoProcessor.from_pretrained(self.model_id)
|
| 188 |
+
model = AutoModelForZeroShotObjectDetection.from_pretrained(self.model_id)
|
| 189 |
+
model = model.to(self.device)
|
| 190 |
+
model.eval()
|
| 191 |
+
# Held rather than re-read per call: session construction is 16.3 s and
|
| 192 |
+
# 948 MB before a single frame, and a tiled pass makes dozens of calls.
|
| 193 |
+
self._loaded = _Loaded(processor=processor, model=model, device=self.device)
|
| 194 |
+
return self
|
| 195 |
+
|
| 196 |
+
def detect_text(
|
| 197 |
+
self, image: Image.Image, prompts: tuple[str, ...]
|
| 198 |
+
) -> list[Region]:
|
| 199 |
+
"""Boxes for each prompt, in image pixels.
|
| 200 |
+
|
| 201 |
+
Grounding DINO wants a caption, not a class list: the documented form is
|
| 202 |
+
lower-case phrases separated by " . " and terminated by one. Building it
|
| 203 |
+
here rather than asking the caller to means a caller cannot silently
|
| 204 |
+
prompt the model with something it was not trained to parse.
|
| 205 |
+
"""
|
| 206 |
+
if self._loaded is None:
|
| 207 |
+
raise AdapterError(
|
| 208 |
+
"Call load() before detect_text(). There is no path from an "
|
| 209 |
+
"unloaded adapter to a detection, which is the point."
|
| 210 |
+
)
|
| 211 |
+
if not prompts:
|
| 212 |
+
return []
|
| 213 |
+
|
| 214 |
+
import torch
|
| 215 |
+
|
| 216 |
+
loaded = self._loaded
|
| 217 |
+
# Lower-cased and dot-terminated per the model card. A capitalised
|
| 218 |
+
# prompt tokenises differently and the model's own examples are lower
|
| 219 |
+
# case throughout.
|
| 220 |
+
caption = " . ".join(p.strip().strip(".").lower() for p in prompts) + " ."
|
| 221 |
+
rgb = image.convert("RGB")
|
| 222 |
+
inputs = loaded.processor(
|
| 223 |
+
images=rgb, text=caption, return_tensors="pt"
|
| 224 |
+
).to(loaded.device)
|
| 225 |
+
|
| 226 |
+
with torch.no_grad():
|
| 227 |
+
outputs = loaded.model(**inputs)
|
| 228 |
+
|
| 229 |
+
results = loaded.processor.post_process_grounded_object_detection(
|
| 230 |
+
outputs,
|
| 231 |
+
inputs.input_ids,
|
| 232 |
+
threshold=self.threshold,
|
| 233 |
+
text_threshold=self.threshold,
|
| 234 |
+
target_sizes=[rgb.size[::-1]],
|
| 235 |
+
)[0]
|
| 236 |
+
|
| 237 |
+
frame_area = float(rgb.size[0] * rgb.size[1]) or 1.0
|
| 238 |
+
regions: list[Region] = []
|
| 239 |
+
for box, score, label in zip(
|
| 240 |
+
results["boxes"].cpu().tolist(),
|
| 241 |
+
results["scores"].cpu().tolist(),
|
| 242 |
+
# `text_labels` in transformers v5; `labels` was the v4 name and
|
| 243 |
+
# holds token ids there, which would silently become integer labels.
|
| 244 |
+
results.get("text_labels", results.get("labels", [])),
|
| 245 |
+
):
|
| 246 |
+
x0, y0, x1, y1 = (float(v) for v in box)
|
| 247 |
+
regions.append(Region(
|
| 248 |
+
label=str(label),
|
| 249 |
+
score=float(score),
|
| 250 |
+
box=(x0, y0, x1, y1),
|
| 251 |
+
area_fraction=((x1 - x0) * (y1 - y0)) / frame_area,
|
| 252 |
+
))
|
| 253 |
+
return regions
|
app/adapters/tiled/tiles.py
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Cutting a high-resolution photograph up so a small object survives the resize.
|
| 2 |
+
|
| 3 |
+
Directive Β§25 lists **high-resolution tiled inference** as one of four things to
|
| 4 |
+
build for cattle ticks, and it is there for a concrete reason. A detector reads
|
| 5 |
+
a fixed input β Grounding DINO's shortest side is 800 px, DINOv3's is 224 β
|
| 6 |
+
so a 4,000 px close-up of a cow's ear is downsampled by five before the network
|
| 7 |
+
sees it, and a 40 px tick arrives as 8 px. Cut the same photograph into 800 px
|
| 8 |
+
tiles and the tick arrives at 40 px. Nothing about the model changed; the
|
| 9 |
+
object got five times larger.
|
| 10 |
+
|
| 11 |
+
## Why this is not `app/tiling.py`
|
| 12 |
+
|
| 13 |
+
That module exists and does the same arithmetic, and it is the right module for
|
| 14 |
+
what it does. It is bound to `detectors.Detection` and to a **fractional grid** β
|
| 15 |
+
1Γ1, 2Γ2, 3Γ3 β which is the correct control when the question is *"has the
|
| 16 |
+
count converged?"* over an unknown frame. It is the wrong control here.
|
| 17 |
+
|
| 18 |
+
Ticks are a fixed-size object in an unknown-resolution frame, so what matters is
|
| 19 |
+
the ratio between the tick's pixels and the model's input, and that is set by a
|
| 20 |
+
**tile side in pixels**, not by a fraction of the picture. A 3Γ3 grid over a
|
| 21 |
+
1,200 px phone photo gives 400 px tiles that get *upsampled*; the same grid over
|
| 22 |
+
an 8,000 px macro gives 2,600 px tiles that are still downsampled by three. The
|
| 23 |
+
grid is the same and the thing that matters is not.
|
| 24 |
+
|
| 25 |
+
So this module tiles at a fixed pixel side, and works on
|
| 26 |
+
`adapters.base.Region`, which carries a mask and comes from the open-vocabulary
|
| 27 |
+
and exemplar routes Β§25 actually names.
|
| 28 |
+
|
| 29 |
+
## The merge, and the trap `app/tiling.py` already fell into
|
| 30 |
+
|
| 31 |
+
Overlapping tiles find the same object twice, so duplicates have to go. IoU
|
| 32 |
+
alone does not do it: an object cut into quarters by tile boundaries gives four
|
| 33 |
+
quarter-boxes that barely overlap *each other* and survive an IoU test, and the
|
| 34 |
+
whole-frame box that contains all four is what removes them. That is why the
|
| 35 |
+
merge is **largest box first** and why containment is a separate rule from IoU.
|
| 36 |
+
Both were learned in `app/tiling.py` β its comment records one cow becoming
|
| 37 |
+
three β and re-deriving them here rather than importing them would have been
|
| 38 |
+
the same bug twice.
|
| 39 |
+
|
| 40 |
+
The thresholds are deliberately the same values, for one reason worth stating:
|
| 41 |
+
nobody has re-derived them for objects of this size, and inventing new numbers
|
| 42 |
+
would have made it look as though somebody had.
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
from __future__ import annotations
|
| 46 |
+
|
| 47 |
+
from dataclasses import dataclass, replace
|
| 48 |
+
|
| 49 |
+
import numpy as np
|
| 50 |
+
from PIL import Image
|
| 51 |
+
|
| 52 |
+
from app.adapters.base import Region
|
| 53 |
+
|
| 54 |
+
#: Default tile side, in source pixels. Grounding DINO resizes its input to a
|
| 55 |
+
#: shortest side of 800, so an 800 px tile passes through at roughly 1:1 and a
|
| 56 |
+
#: tick keeps every pixel the camera gave it. Larger tiles throw resolution
|
| 57 |
+
#: away; smaller ones multiply the inference count without adding detail, and
|
| 58 |
+
#: cost is linear in tile count.
|
| 59 |
+
TILE_PIXELS = 800
|
| 60 |
+
|
| 61 |
+
#: How much neighbouring tiles overlap, as a share of the tile side. An object
|
| 62 |
+
#: sitting exactly on a cut would otherwise be two partial objects, neither
|
| 63 |
+
#: complete enough to detect. 0.20 matches `app/tiling.py`; at an 800 px tile it
|
| 64 |
+
#: is 160 px, which is four times the longest tick measured on the set in
|
| 65 |
+
#: `experiments/cattle_ticks/`, so no tick can straddle a seam without being
|
| 66 |
+
#: whole in at least one tile.
|
| 67 |
+
TILE_OVERLAP = 0.20
|
| 68 |
+
|
| 69 |
+
#: IoU above which two boxes are the same object. From `app/tiling.py`, which
|
| 70 |
+
#: derived it against cattle; **not re-derived for objects this small**, and the
|
| 71 |
+
#: cattle_ticks README lists that as a gap rather than hiding it here.
|
| 72 |
+
MERGE_IOU = 0.55
|
| 73 |
+
|
| 74 |
+
#: Intersection over the *smaller* box's area, above which the smaller box is
|
| 75 |
+
#: part of the larger rather than a second object. This is the rule that stops
|
| 76 |
+
#: an object cut across tile seams becoming several objects. See the module
|
| 77 |
+
#: docstring.
|
| 78 |
+
MERGE_CONTAINMENT = 0.85
|
| 79 |
+
|
| 80 |
+
#: Centre separation, as a fraction of the **larger** box's mean side, below
|
| 81 |
+
#: which two same-label regions are the same object.
|
| 82 |
+
#:
|
| 83 |
+
#: **The rule `app/tiling.py` does not have, added because small objects break
|
| 84 |
+
#: the two that it does.** IoU is a poor duplicate test at small sizes: two
|
| 85 |
+
#: 36 px boxes on the same tick, found in overlapping tiles and disagreeing by
|
| 86 |
+
#: 14 px, score IoU 0.44 and containment 0.61 β under both thresholds, so the
|
| 87 |
+
#: tick is counted twice. That is what a smoke test on synthetic discs produced:
|
| 88 |
+
#: 208 raw regions merging to 150, where the four real discs appeared two or
|
| 89 |
+
#: three times each. The same 14 px offset on a cow-sized box is nothing, which
|
| 90 |
+
#: is why `app/tiling.py` never needed this and why adding it there is not
|
| 91 |
+
#: implied.
|
| 92 |
+
#:
|
| 93 |
+
#: **Normalised by the larger box, not the smaller, and that is what makes 0.6
|
| 94 |
+
#: work.** Two detections of one 9 px-radius disc, found in overlapping tiles at
|
| 95 |
+
#: radii 4.5 and 6.5 with centres 7 px apart, score 0.54 against the larger box
|
| 96 |
+
#: and 0.78 against the smaller β so only the larger normalisation separates
|
| 97 |
+
#: them from the case that must survive: two ticks touching. Touching circles of
|
| 98 |
+
#: equal radius r have centres 2r apart over a box side of 2r, which is exactly
|
| 99 |
+
#: 1.0, well clear of 0.6. Objects that overlap by half score 0.5 and merge,
|
| 100 |
+
#: which for small round objects found twice across a seam is the right answer.
|
| 101 |
+
MERGE_CENTRE_DISTANCE = 0.6
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
@dataclass(frozen=True)
|
| 105 |
+
class Tile:
|
| 106 |
+
"""One crop, and where it sits in the source frame."""
|
| 107 |
+
|
| 108 |
+
index: int
|
| 109 |
+
#: `(x0, y0, x1, y1)` in source-image pixels, left-top inclusive.
|
| 110 |
+
box: tuple[int, int, int, int]
|
| 111 |
+
#: Grid position, for a caller that wants to lay results out.
|
| 112 |
+
row: int
|
| 113 |
+
column: int
|
| 114 |
+
|
| 115 |
+
@property
|
| 116 |
+
def origin(self) -> tuple[int, int]:
|
| 117 |
+
return self.box[0], self.box[1]
|
| 118 |
+
|
| 119 |
+
@property
|
| 120 |
+
def size(self) -> tuple[int, int]:
|
| 121 |
+
return self.box[2] - self.box[0], self.box[3] - self.box[1]
|
| 122 |
+
|
| 123 |
+
def crop(self, image: Image.Image) -> Image.Image:
|
| 124 |
+
return image.crop(self.box)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def tiles_for(
|
| 128 |
+
size: tuple[int, int],
|
| 129 |
+
*,
|
| 130 |
+
tile_pixels: int = TILE_PIXELS,
|
| 131 |
+
overlap: float = TILE_OVERLAP,
|
| 132 |
+
) -> list[Tile]:
|
| 133 |
+
"""Cover a frame with overlapping fixed-size tiles.
|
| 134 |
+
|
| 135 |
+
A frame smaller than one tile returns a single tile covering all of it,
|
| 136 |
+
which is whole-frame inference β the honest degenerate case, and the one a
|
| 137 |
+
phone photo that has already been downscaled by a messaging app will hit.
|
| 138 |
+
|
| 139 |
+
The last tile in each direction is pulled back to the frame edge rather than
|
| 140 |
+
being a narrow remainder. A 120 px strip is a strip in which nothing is
|
| 141 |
+
detectable and which still costs a full inference.
|
| 142 |
+
"""
|
| 143 |
+
width, height = int(size[0]), int(size[1])
|
| 144 |
+
if width <= 0 or height <= 0:
|
| 145 |
+
return []
|
| 146 |
+
tile_pixels = max(1, int(tile_pixels))
|
| 147 |
+
if width <= tile_pixels and height <= tile_pixels:
|
| 148 |
+
return [Tile(index=0, box=(0, 0, width, height), row=0, column=0)]
|
| 149 |
+
|
| 150 |
+
step = max(1, int(round(tile_pixels * (1.0 - overlap))))
|
| 151 |
+
|
| 152 |
+
def starts(extent: int) -> list[int]:
|
| 153 |
+
if extent <= tile_pixels:
|
| 154 |
+
return [0]
|
| 155 |
+
positions = list(range(0, extent - tile_pixels, step))
|
| 156 |
+
# The final position is always flush with the far edge, so the last
|
| 157 |
+
# tile is a full tile rather than a remainder.
|
| 158 |
+
positions.append(extent - tile_pixels)
|
| 159 |
+
return positions
|
| 160 |
+
|
| 161 |
+
found: list[Tile] = []
|
| 162 |
+
for row, y0 in enumerate(starts(height)):
|
| 163 |
+
for column, x0 in enumerate(starts(width)):
|
| 164 |
+
found.append(Tile(
|
| 165 |
+
index=len(found),
|
| 166 |
+
box=(x0, y0, min(width, x0 + tile_pixels),
|
| 167 |
+
min(height, y0 + tile_pixels)),
|
| 168 |
+
row=row, column=column,
|
| 169 |
+
))
|
| 170 |
+
return found
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def to_frame(region: Region, tile: Tile, frame_size: tuple[int, int]) -> Region:
|
| 174 |
+
"""Move a tile-relative region into source-frame coordinates.
|
| 175 |
+
|
| 176 |
+
`area_fraction` is recomputed against the **whole frame**, not the tile. A
|
| 177 |
+
tick covering a fiftieth of its tile covers a two-thousandth of an 8,000 px
|
| 178 |
+
photograph, and everything downstream β the quality gates, the burden band β
|
| 179 |
+
reasons about the photograph.
|
| 180 |
+
|
| 181 |
+
A mask, when one is present, is pasted into a frame-sized array rather than
|
| 182 |
+
being returned tile-relative. A boolean array whose coordinate system
|
| 183 |
+
depends on which tile produced it is the kind of thing that works until two
|
| 184 |
+
tiles disagree.
|
| 185 |
+
"""
|
| 186 |
+
x0, y0 = tile.origin
|
| 187 |
+
bx0, by0, bx1, by1 = region.box
|
| 188 |
+
box = (bx0 + x0, by0 + y0, bx1 + x0, by1 + y0)
|
| 189 |
+
frame_area = float(frame_size[0] * frame_size[1]) or 1.0
|
| 190 |
+
|
| 191 |
+
mask = region.mask
|
| 192 |
+
if mask is not None:
|
| 193 |
+
placed = np.zeros((frame_size[1], frame_size[0]), dtype=bool)
|
| 194 |
+
tile_width, tile_height = tile.size
|
| 195 |
+
placed[y0:y0 + tile_height, x0:x0 + tile_width] = mask[:tile_height, :tile_width]
|
| 196 |
+
mask = placed
|
| 197 |
+
|
| 198 |
+
return replace(
|
| 199 |
+
region,
|
| 200 |
+
box=box,
|
| 201 |
+
mask=mask,
|
| 202 |
+
area_fraction=((box[2] - box[0]) * (box[3] - box[1])) / frame_area,
|
| 203 |
+
)
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def _overlaps(
|
| 207 |
+
a: tuple[float, float, float, float], b: tuple[float, float, float, float]
|
| 208 |
+
) -> tuple[float, float, float]:
|
| 209 |
+
"""`(IoU, intersection over the smaller area, centre distance in objects)`.
|
| 210 |
+
|
| 211 |
+
The third is the centre separation divided by the **larger** box's mean
|
| 212 |
+
side, so it is dimensionless and comparable between a tick and a cow. See
|
| 213 |
+
`MERGE_CENTRE_DISTANCE` for why the larger and not the smaller. It is
|
| 214 |
+
computed even when the boxes do not intersect, because two boxes can be
|
| 215 |
+
nearly concentric and still score zero on the first two when one is a
|
| 216 |
+
sliver.
|
| 217 |
+
"""
|
| 218 |
+
ax0, ay0, ax1, ay1 = a
|
| 219 |
+
bx0, by0, bx1, by1 = b
|
| 220 |
+
area_a = (ax1 - ax0) * (ay1 - ay0)
|
| 221 |
+
area_b = (bx1 - bx0) * (by1 - by0)
|
| 222 |
+
|
| 223 |
+
centre_gap = float(np.hypot(
|
| 224 |
+
(ax0 + ax1) / 2.0 - (bx0 + bx1) / 2.0,
|
| 225 |
+
(ay0 + ay1) / 2.0 - (by0 + by1) / 2.0,
|
| 226 |
+
))
|
| 227 |
+
larger_side = max(
|
| 228 |
+
((ax1 - ax0) + (ay1 - ay0)) / 2.0,
|
| 229 |
+
((bx1 - bx0) + (by1 - by0)) / 2.0,
|
| 230 |
+
)
|
| 231 |
+
separation = centre_gap / larger_side if larger_side > 0 else float("inf")
|
| 232 |
+
|
| 233 |
+
x0, y0 = max(ax0, bx0), max(ay0, by0)
|
| 234 |
+
x1, y1 = min(ax1, bx1), min(ay1, by1)
|
| 235 |
+
overlap = max(0.0, x1 - x0) * max(0.0, y1 - y0)
|
| 236 |
+
if overlap <= 0.0:
|
| 237 |
+
return 0.0, 0.0, separation
|
| 238 |
+
union = area_a + area_b - overlap
|
| 239 |
+
smaller = min(area_a, area_b)
|
| 240 |
+
return (
|
| 241 |
+
overlap / union if union > 0 else 0.0,
|
| 242 |
+
overlap / smaller if smaller > 0 else 0.0,
|
| 243 |
+
separation,
|
| 244 |
+
)
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def merge(
|
| 248 |
+
regions: list[Region],
|
| 249 |
+
frame_size: tuple[int, int],
|
| 250 |
+
*,
|
| 251 |
+
iou: float = MERGE_IOU,
|
| 252 |
+
containment: float = MERGE_CONTAINMENT,
|
| 253 |
+
centre_distance: float = MERGE_CENTRE_DISTANCE,
|
| 254 |
+
) -> list[Region]:
|
| 255 |
+
"""One object, one region, whichever tile found it.
|
| 256 |
+
|
| 257 |
+
**Largest box first**, which is the ordering the containment rule needs: the
|
| 258 |
+
whole object has to be in the kept set before its fragments are tested
|
| 259 |
+
against it. Score order β the usual choice for non-maximum suppression β
|
| 260 |
+
would let a confident fragment claim the object and leave its siblings
|
| 261 |
+
unmatched, which is how one cow became three in `app/tiling.py`'s first
|
| 262 |
+
attempt.
|
| 263 |
+
|
| 264 |
+
Labels are compared, so a tick overlapping a tag is two objects. Comparison
|
| 265 |
+
is case-insensitive because open-vocabulary detectors return the caption's
|
| 266 |
+
own casing and a caller prompting `"tick"` and `"Tick"` did not mean two
|
| 267 |
+
classes.
|
| 268 |
+
"""
|
| 269 |
+
def area(region: Region) -> float:
|
| 270 |
+
x0, y0, x1, y1 = region.box
|
| 271 |
+
return (x1 - x0) * (y1 - y0)
|
| 272 |
+
|
| 273 |
+
frame_area = float(frame_size[0] * frame_size[1]) or 1.0
|
| 274 |
+
kept: list[Region] = []
|
| 275 |
+
for region in sorted(regions, key=area, reverse=True):
|
| 276 |
+
duplicate = False
|
| 277 |
+
for other in kept:
|
| 278 |
+
if other.label.casefold() != region.label.casefold():
|
| 279 |
+
continue
|
| 280 |
+
overlap_iou, overlap_containment, separation = _overlaps(
|
| 281 |
+
other.box, region.box
|
| 282 |
+
)
|
| 283 |
+
if (overlap_iou > iou or overlap_containment > containment
|
| 284 |
+
or separation < centre_distance):
|
| 285 |
+
duplicate = True
|
| 286 |
+
break
|
| 287 |
+
if duplicate:
|
| 288 |
+
continue
|
| 289 |
+
kept.append(replace(region, area_fraction=area(region) / frame_area))
|
| 290 |
+
kept.sort(key=lambda r: r.score, reverse=True)
|
| 291 |
+
return kept
|
app/adapters/transports/__init__.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""How a hosted reasoner is actually called, and which one.
|
| 2 |
+
|
| 3 |
+
`app/adapters/multimodal.py` has the whole contract β the rubric, the closed
|
| 4 |
+
schema, the structured-output validation, the forbidden-claims gate β and takes
|
| 5 |
+
an injectable `transport: Callable[..., str]`. Until this package existed,
|
| 6 |
+
`registry.py` constructed it with **none**. There were zero HTTP calls in the
|
| 7 |
+
service and nothing had ever talked to a hosted model; the adapter reported
|
| 8 |
+
itself `unavailable` and fifteen capabilities waited on it.
|
| 9 |
+
|
| 10 |
+
This package is the missing half, and nothing else. The transport's whole job:
|
| 11 |
+
|
| 12 |
+
(prompt, images, schema, model, api_key) -> the model's response text
|
| 13 |
+
|
| 14 |
+
Everything on either side of that already exists. A transport does not decide
|
| 15 |
+
what to ask, does not read the registry, does not interpret an answer and does
|
| 16 |
+
not get to relax a check β `claims.parse_strict` and `claims.enforce` run on
|
| 17 |
+
whatever it returns, from the caller, where a transport cannot reach them.
|
| 18 |
+
|
| 19 |
+
## Why the provider is a choice and not an import
|
| 20 |
+
|
| 21 |
+
Choosing a vendor commits Animap to that vendor's retention posture for
|
| 22 |
+
photographs of somebody's animals, leaving the country. That is a procurement
|
| 23 |
+
decision rather than an engineering one, and `multimodal.py` refuses to make it
|
| 24 |
+
by importing an SDK β which is why the transport is injected at all.
|
| 25 |
+
|
| 26 |
+
So the shape here is a **registry keyed on a name**, and the name comes from the
|
| 27 |
+
environment:
|
| 28 |
+
|
| 29 |
+
ANIMAP_MULTIMODAL_PROVIDER=anthropic
|
| 30 |
+
|
| 31 |
+
An unset provider is the honest `unavailable` that was there before, not a
|
| 32 |
+
default that quietly picks somebody. A provider this build does not implement is
|
| 33 |
+
an error naming the ones it does, because a typo that silently disabled fifteen
|
| 34 |
+
capabilities would look exactly like the state this package exists to leave.
|
| 35 |
+
|
| 36 |
+
## What a provider has to do to belong here
|
| 37 |
+
|
| 38 |
+
Three things, and the third is the one that decides whether a capability is
|
| 39 |
+
worth shipping at all:
|
| 40 |
+
|
| 41 |
+
* **Take images and a prompt.** All fifteen waiting capabilities are visual.
|
| 42 |
+
* **Return one JSON object matching a supplied schema**, natively rather than
|
| 43 |
+
by being asked nicely in prose. Directive Β§4 is explicit β *"All calls must
|
| 44 |
+
return structured JSON"* β and a provider without server-side schema
|
| 45 |
+
enforcement makes `claims.parse_strict` the only thing standing between a
|
| 46 |
+
farm and a model's prose, which is a retry loop rather than a contract.
|
| 47 |
+
* **Actually see what is in the photograph.** A model that cannot tell a
|
| 48 |
+
raised nodular lesion from mud on a flank produces confident, well-formed,
|
| 49 |
+
wrong JSON, and every gate downstream will pass it: the schema is closed,
|
| 50 |
+
the claim is in the vocabulary, the sentence is qualified. Nothing in this
|
| 51 |
+
service can catch a plausible wrong answer, and that is why the choice of
|
| 52 |
+
model is a capability decision rather than a cost one.
|
| 53 |
+
"""
|
| 54 |
+
|
| 55 |
+
from __future__ import annotations
|
| 56 |
+
|
| 57 |
+
import os
|
| 58 |
+
from typing import Callable
|
| 59 |
+
|
| 60 |
+
#: Which hosted provider to call. Read at call time rather than at import, for
|
| 61 |
+
#: the same reason `multimodal.API_KEY_ENV` is: a change takes effect on a
|
| 62 |
+
#: restart rather than needing a rebuild.
|
| 63 |
+
PROVIDER_ENV = "ANIMAP_MULTIMODAL_PROVIDER"
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class UnknownProvider(ValueError):
|
| 67 |
+
"""A provider name this build has no transport for."""
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _anthropic():
|
| 71 |
+
from app.adapters.transports.anthropic_transport import anthropic_transport
|
| 72 |
+
|
| 73 |
+
return anthropic_transport
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
#: Name -> a factory returning the transport callable.
|
| 77 |
+
#:
|
| 78 |
+
#: Factories rather than the callables themselves, so importing this module does
|
| 79 |
+
#: not import a vendor SDK. The inference container runs on CPU with no torch
|
| 80 |
+
#: and starts in under a second; that is a property worth keeping, and a
|
| 81 |
+
#: deployment that has chosen no provider should not be paying for an import it
|
| 82 |
+
#: will never call.
|
| 83 |
+
PROVIDERS: dict[str, Callable[[], Callable[..., str]]] = {
|
| 84 |
+
"anthropic": _anthropic,
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def configured_provider() -> str:
|
| 89 |
+
"""The provider name in the environment, lower-cased, or empty."""
|
| 90 |
+
return os.environ.get(PROVIDER_ENV, "").strip().lower()
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def transport_from_env() -> Callable[..., str] | None:
|
| 94 |
+
"""The transport this deployment has chosen, or None if it has chosen none.
|
| 95 |
+
|
| 96 |
+
None rather than a default, and it is the whole posture of this package:
|
| 97 |
+
an unset provider leaves `HostedMultimodalAdapter` reporting itself
|
| 98 |
+
unavailable with the reason it always gave, and no farm gets an answer from
|
| 99 |
+
a vendor nobody picked.
|
| 100 |
+
|
| 101 |
+
Raises `UnknownProvider` for a name this build cannot serve. Loud, because
|
| 102 |
+
the failure it replaces is silent: a typo would leave fifteen capabilities
|
| 103 |
+
off with a listing that says *"no vendor is wired"* β indistinguishable from
|
| 104 |
+
a deployment where nobody had chosen one yet.
|
| 105 |
+
"""
|
| 106 |
+
name = configured_provider()
|
| 107 |
+
if not name:
|
| 108 |
+
return None
|
| 109 |
+
factory = PROVIDERS.get(name)
|
| 110 |
+
if factory is None:
|
| 111 |
+
raise UnknownProvider(
|
| 112 |
+
f"{PROVIDER_ENV} is {name!r}, which this build has no transport "
|
| 113 |
+
f"for. It knows: {', '.join(sorted(PROVIDERS))}."
|
| 114 |
+
)
|
| 115 |
+
return factory()
|
app/adapters/transports/anthropic_transport.py
ADDED
|
@@ -0,0 +1,342 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The Claude transport: images and a rubric in, one JSON object out.
|
| 2 |
+
|
| 3 |
+
The first concrete provider behind `HostedMultimodalAdapter`, and the reason it
|
| 4 |
+
is first is the third requirement in this package's header β a model that cannot
|
| 5 |
+
actually see what is in a photograph produces confident, well-formed, wrong JSON
|
| 6 |
+
that every gate downstream will pass.
|
| 7 |
+
|
| 8 |
+
## What this file does and does not decide
|
| 9 |
+
|
| 10 |
+
It does not build the prompt, does not build the schema, does not read the
|
| 11 |
+
capability registry, and does not judge the answer. `multimodal.reason` composes
|
| 12 |
+
`SYSTEM_RULES` with the capability's rubric, `claims.schema_for` builds the
|
| 13 |
+
closed schema from the registry, and `claims.parse_strict` plus `claims.enforce`
|
| 14 |
+
run on whatever comes back β from the caller, where this module cannot reach
|
| 15 |
+
them. Everything here is the HTTP call and the shape of its arguments.
|
| 16 |
+
|
| 17 |
+
## The schema the API accepts is narrower than the one the contract enforces
|
| 18 |
+
|
| 19 |
+
`claims.schema_for` emits a full JSON Schema. Structured outputs accepts most of
|
| 20 |
+
it and refuses four things, each verified against the live API rather than read
|
| 21 |
+
off a page:
|
| 22 |
+
|
| 23 |
+
array maxItems not supported
|
| 24 |
+
array minItems, where the value is > 1 not supported
|
| 25 |
+
number minimum / maximum / multipleOf not supported
|
| 26 |
+
enum containing null beside a union type β "Enum value 'a' does not match
|
| 27 |
+
declared type '['string', 'null']'"
|
| 28 |
+
|
| 29 |
+
`_for_structured_output` removes exactly those and changes nothing else.
|
| 30 |
+
|
| 31 |
+
The `minItems` rule was the one a keyword probe missed: `minItems: 1` is
|
| 32 |
+
accepted, so only a real capability's schema β `range`, which is exactly two
|
| 33 |
+
entries β produced *"values other than 0 or 1 are not supported"*.
|
| 34 |
+
|
| 35 |
+
**This narrows what the model is guided by and not what the answer is judged
|
| 36 |
+
against.** `multimodal.reason` calls `claims.enforce(parsed, contract, β¦)` with
|
| 37 |
+
the **original** schema, and `claims.validate` implements `maxItems`, `minimum`,
|
| 38 |
+
`maximum` and `multipleOf` itself β the `2.6347` a watchdog once published still
|
| 39 |
+
fails on `multipleOf`, from the same line it always did. What is lost is the
|
| 40 |
+
model being *told* the bound up front, which costs a retry rather than a
|
| 41 |
+
control.
|
| 42 |
+
|
| 43 |
+
The `unit` enum is the one worth naming. It is dropped rather than reshaped, and
|
| 44 |
+
that is safe for a stated reason: the field is decorative for enforcement β
|
| 45 |
+
`_declared_quantities` reads `ClaimQuantity.unit` from the registry and never
|
| 46 |
+
from the response, precisely so a model cannot declare its own corroboration.
|
| 47 |
+
|
| 48 |
+
## Structured output is server-side, not a request in prose
|
| 49 |
+
|
| 50 |
+
`output_config.format` with a `json_schema` constrains the response at the API
|
| 51 |
+
rather than asking for JSON and hoping. That matters more than it looks:
|
| 52 |
+
`claims.parse_strict` does not repair, retry with a nudge, or partially accept β
|
| 53 |
+
directive Β§4 says the calls must return structured JSON, so one that did not is
|
| 54 |
+
a failed call. Without server-side enforcement every malformed answer is a
|
| 55 |
+
capability that intermittently returns nothing, and the schema this service
|
| 56 |
+
already builds would be doing nothing but rejecting.
|
| 57 |
+
|
| 58 |
+
The schema `claims.schema_for` emits is already closed β `additionalProperties:
|
| 59 |
+
false`, explicit `required`, enums on every claim, evidence and limit string β
|
| 60 |
+
and every one of those crosses intact. What is dropped is the numeric and length
|
| 61 |
+
*bounds* named in the section above, and `claims.validate` implements all of
|
| 62 |
+
them itself.
|
| 63 |
+
|
| 64 |
+
## Why the images go in as base64 rather than by URL
|
| 65 |
+
|
| 66 |
+
The capture lives in `animapmedia`, which the inference service reads through
|
| 67 |
+
its own managed identity (ADR 0012). A URL the vendor fetches would need that
|
| 68 |
+
blob to be publicly reachable, which is a photograph of somebody's animals on
|
| 69 |
+
the open internet in exchange for saving a base64 encode.
|
| 70 |
+
|
| 71 |
+
## No retry, and no fallback model
|
| 72 |
+
|
| 73 |
+
A refusal, a timeout or a malformed answer is reported rather than worked
|
| 74 |
+
around. The SDK's own transport-level retries stay on β they cover connection
|
| 75 |
+
errors and 429s, which are not answers β but nothing here re-asks a question
|
| 76 |
+
that was answered badly, and nothing silently substitutes a different model.
|
| 77 |
+
`InferenceResult` requires `model_version` and the release row records it, so a
|
| 78 |
+
result that cannot name the model that produced it is not evidence.
|
| 79 |
+
"""
|
| 80 |
+
|
| 81 |
+
from __future__ import annotations
|
| 82 |
+
|
| 83 |
+
import base64
|
| 84 |
+
import io
|
| 85 |
+
import json
|
| 86 |
+
import logging
|
| 87 |
+
from typing import Any
|
| 88 |
+
|
| 89 |
+
from PIL import Image
|
| 90 |
+
|
| 91 |
+
from app.adapters.base import AdapterError
|
| 92 |
+
|
| 93 |
+
logger = logging.getLogger(__name__)
|
| 94 |
+
|
| 95 |
+
#: What a capability may spend on one answer.
|
| 96 |
+
#:
|
| 97 |
+
#: These are structured extractions β a claim list, a handful of observations,
|
| 98 |
+
#: an evidence phrase β and not prose. The ceiling is generous against that
|
| 99 |
+
#: shape rather than tuned, because the cost of a truncated answer is a whole
|
| 100 |
+
#: capture refused by `parse_strict` and re-queued.
|
| 101 |
+
MAX_TOKENS = 8_000
|
| 102 |
+
|
| 103 |
+
#: How long one call may take before it is a failure rather than a wait.
|
| 104 |
+
#:
|
| 105 |
+
#: Under `INFERENCE_TIMEOUT_SECONDS`' fifteen-second default on purpose: a run
|
| 106 |
+
#: that outlives the request path is kept queued and retried by
|
| 107 |
+
#: `drain_inference_queue`, and a transport that sat past that would turn a
|
| 108 |
+
#: recoverable wait into a request the API has already given up on.
|
| 109 |
+
TIMEOUT_SECONDS = 120.0
|
| 110 |
+
|
| 111 |
+
#: Sent as `image/jpeg` unless Pillow says otherwise. Captures are JPEG.
|
| 112 |
+
_MEDIA_TYPES = {
|
| 113 |
+
"JPEG": "image/jpeg",
|
| 114 |
+
"PNG": "image/png",
|
| 115 |
+
"GIF": "image/gif",
|
| 116 |
+
"WEBP": "image/webp",
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def _encode(image: Image.Image) -> tuple[str, str]:
|
| 121 |
+
"""One image as `(media_type, base64)`.
|
| 122 |
+
|
| 123 |
+
Re-encoded to JPEG unless it is already one of the four types the API
|
| 124 |
+
accepts. A capture that reached this service as something else β a frame
|
| 125 |
+
grabbed from a clip, a PNG from a screenshot harness β would otherwise be
|
| 126 |
+
refused for its container rather than for anything about the animal.
|
| 127 |
+
|
| 128 |
+
RGBA is flattened onto white before a JPEG encode, because JPEG has no alpha
|
| 129 |
+
and Pillow raises rather than guessing. White rather than black: a
|
| 130 |
+
transparent margin around a photograph is padding, and padding a farmer sees
|
| 131 |
+
is paper.
|
| 132 |
+
"""
|
| 133 |
+
fmt = (image.format or "").upper()
|
| 134 |
+
if fmt in _MEDIA_TYPES:
|
| 135 |
+
buffer = io.BytesIO()
|
| 136 |
+
image.save(buffer, format=fmt)
|
| 137 |
+
return _MEDIA_TYPES[fmt], base64.standard_b64encode(buffer.getvalue()).decode()
|
| 138 |
+
|
| 139 |
+
prepared = image
|
| 140 |
+
if image.mode in ("RGBA", "LA", "P"):
|
| 141 |
+
prepared = Image.new("RGB", image.size, (255, 255, 255))
|
| 142 |
+
converted = image.convert("RGBA")
|
| 143 |
+
prepared.paste(converted, mask=converted.split()[-1])
|
| 144 |
+
elif image.mode != "RGB":
|
| 145 |
+
prepared = image.convert("RGB")
|
| 146 |
+
|
| 147 |
+
buffer = io.BytesIO()
|
| 148 |
+
# Quality 90 rather than Pillow's default 75. The capabilities waiting on
|
| 149 |
+
# this read skin texture, tooth wear and footpad lesions, and a compression
|
| 150 |
+
# artefact at that scale is indistinguishable from the thing being looked
|
| 151 |
+
# for. The bytes are not the constraint here; the reading is.
|
| 152 |
+
prepared.save(buffer, format="JPEG", quality=90)
|
| 153 |
+
return "image/jpeg", base64.standard_b64encode(buffer.getvalue()).decode()
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
#: Keywords structured outputs refuses, by the type they sit on.
|
| 157 |
+
#:
|
| 158 |
+
#: Discovered by probing the live API one keyword at a time, not by reading a
|
| 159 |
+
#: list: `minItems`, `maxLength`, a bare union type and a plain `enum` on a
|
| 160 |
+
#: string are all accepted, so a broader strip would remove controls the API was
|
| 161 |
+
#: willing to enforce.
|
| 162 |
+
_UNSUPPORTED = {
|
| 163 |
+
"array": ("maxItems",),
|
| 164 |
+
"number": ("minimum", "maximum", "multipleOf"),
|
| 165 |
+
"integer": ("minimum", "maximum", "multipleOf"),
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
#: `minItems` survives only as 0 or 1.
|
| 169 |
+
#:
|
| 170 |
+
#: *"'minItems' values other than 0 or 1 are not supported (got: [2, 5])"* β the
|
| 171 |
+
#: `range` array is exactly two entries and this is what the API says about it.
|
| 172 |
+
#: A probe with `minItems: 1` passed, which is why the first pass missed it and
|
| 173 |
+
#: why this is a value rule rather than another entry above.
|
| 174 |
+
_MIN_ITEMS_CEILING = 1
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def _for_structured_output(node):
|
| 178 |
+
"""`node` with the keywords structured outputs refuses taken out.
|
| 179 |
+
|
| 180 |
+
Recursive and non-destructive β the caller keeps the original, because the
|
| 181 |
+
original is what judges the answer.
|
| 182 |
+
"""
|
| 183 |
+
if isinstance(node, list):
|
| 184 |
+
return [_for_structured_output(item) for item in node]
|
| 185 |
+
if not isinstance(node, dict):
|
| 186 |
+
return node
|
| 187 |
+
|
| 188 |
+
out = {key: _for_structured_output(value) for key, value in node.items()}
|
| 189 |
+
|
| 190 |
+
# **Only when `type` is a type.** `claims.schema_for` has a property
|
| 191 |
+
# literally named `type` β an observation's own kind β so inside a
|
| 192 |
+
# `properties` node `out["type"]` is a schema rather than a declaration.
|
| 193 |
+
# Reading it as one raised `TypeError: unhashable type: 'dict'` the first
|
| 194 |
+
# time this ran against a real capability.
|
| 195 |
+
declared = out.get("type")
|
| 196 |
+
if isinstance(declared, str):
|
| 197 |
+
types = [declared]
|
| 198 |
+
elif isinstance(declared, list) and all(isinstance(t, str) for t in declared):
|
| 199 |
+
types = declared
|
| 200 |
+
else:
|
| 201 |
+
types = []
|
| 202 |
+
|
| 203 |
+
for kind in types:
|
| 204 |
+
for keyword in _UNSUPPORTED.get(kind, ()):
|
| 205 |
+
out.pop(keyword, None)
|
| 206 |
+
if kind == "array":
|
| 207 |
+
floor = out.get("minItems")
|
| 208 |
+
if isinstance(floor, int) and floor > _MIN_ITEMS_CEILING:
|
| 209 |
+
out.pop("minItems", None)
|
| 210 |
+
|
| 211 |
+
# An enum whose values cannot all be the declared type. The API reads a
|
| 212 |
+
# union type plus an enum as a contradiction rather than as a widening, so
|
| 213 |
+
# the enum goes and the type β which is what makes the field nullable β
|
| 214 |
+
# stays. See the header for why this one costs nothing.
|
| 215 |
+
enum = out.get("enum")
|
| 216 |
+
if isinstance(enum, list) and len(types) > 1 and None in enum:
|
| 217 |
+
out.pop("enum", None)
|
| 218 |
+
|
| 219 |
+
return out
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def anthropic_transport(
|
| 223 |
+
*,
|
| 224 |
+
prompt: str,
|
| 225 |
+
images: list[Image.Image],
|
| 226 |
+
schema: dict[str, Any],
|
| 227 |
+
model: str,
|
| 228 |
+
api_key: str,
|
| 229 |
+
) -> str:
|
| 230 |
+
"""One call to Claude, returning the response text verbatim.
|
| 231 |
+
|
| 232 |
+
The signature is `HostedMultimodalAdapter`'s, keyword-only, and it returns
|
| 233 |
+
the raw string rather than a parsed object β `ReasonerResponse` keeps `raw`
|
| 234 |
+
beside `parsed` because Β§33 requires what the model said to be preserved for
|
| 235 |
+
later training, and a reparsed reconstruction is not that.
|
| 236 |
+
|
| 237 |
+
:param model: the exact hosted model id, from `ANIMAP_MULTIMODAL_MODEL`.
|
| 238 |
+
Never defaulted here. A result has to name the model that produced it,
|
| 239 |
+
and a transport quietly substituting one it preferred would make every
|
| 240 |
+
stored `model_version` a guess.
|
| 241 |
+
"""
|
| 242 |
+
# Imported inside the call, not at module scope. `transports/__init__` keeps
|
| 243 |
+
# its provider table as factories for the same reason: a deployment that has
|
| 244 |
+
# chosen no provider should not pay for a vendor SDK import, and the
|
| 245 |
+
# inference container's sub-second start is a property worth keeping.
|
| 246 |
+
import anthropic
|
| 247 |
+
|
| 248 |
+
if not images:
|
| 249 |
+
raise ValueError("A visual reasoner needs at least one image.")
|
| 250 |
+
|
| 251 |
+
client = anthropic.Anthropic(
|
| 252 |
+
api_key=api_key,
|
| 253 |
+
timeout=TIMEOUT_SECONDS,
|
| 254 |
+
# The SDK's default of 2. Connection errors and 429s are not answers,
|
| 255 |
+
# so retrying them is not re-asking a question that was answered badly β
|
| 256 |
+
# which is the thing this transport does not do.
|
| 257 |
+
max_retries=2,
|
| 258 |
+
)
|
| 259 |
+
|
| 260 |
+
content: list[dict[str, Any]] = [
|
| 261 |
+
{
|
| 262 |
+
"type": "image",
|
| 263 |
+
"source": {"type": "base64", "media_type": media_type, "data": data},
|
| 264 |
+
}
|
| 265 |
+
for media_type, data in (_encode(image) for image in images)
|
| 266 |
+
]
|
| 267 |
+
# The rubric last, after the images. It refers to them.
|
| 268 |
+
content.append({"type": "text", "text": prompt})
|
| 269 |
+
|
| 270 |
+
response = client.messages.create(
|
| 271 |
+
model=model,
|
| 272 |
+
max_tokens=MAX_TOKENS,
|
| 273 |
+
messages=[{"role": "user", "content": content}],
|
| 274 |
+
# **Server-side, from the registry's own schema**, minus the four
|
| 275 |
+
# keywords structured outputs refuses. `claims.enforce` still judges the
|
| 276 |
+
# answer against the original, and implements all four itself β see the
|
| 277 |
+
# module header.
|
| 278 |
+
output_config={
|
| 279 |
+
"format": {
|
| 280 |
+
"type": "json_schema",
|
| 281 |
+
"schema": _for_structured_output(schema),
|
| 282 |
+
}
|
| 283 |
+
},
|
| 284 |
+
# Adaptive rather than a fixed budget: these are visual judgements whose
|
| 285 |
+
# difficulty varies by photograph β a clear muzzle and an occluded hock
|
| 286 |
+
# are not the same question β and a fixed budget is either wasted on the
|
| 287 |
+
# first or short on the second.
|
| 288 |
+
thinking={"type": "adaptive"},
|
| 289 |
+
)
|
| 290 |
+
|
| 291 |
+
# **A refusal is reported, not retried and not swallowed.** It arrives as a
|
| 292 |
+
# 200 with `stop_reason: "refusal"` and no usable content, so reading
|
| 293 |
+
# `content` first would raise something that looks like a parse failure and
|
| 294 |
+
# hide what actually happened.
|
| 295 |
+
if response.stop_reason == "refusal":
|
| 296 |
+
details = getattr(response, "stop_details", None)
|
| 297 |
+
category = getattr(details, "category", None)
|
| 298 |
+
raise AdapterError(
|
| 299 |
+
f"The hosted model declined this request"
|
| 300 |
+
f"{f' ({category})' if category else ''}. Nothing is stored for it."
|
| 301 |
+
)
|
| 302 |
+
|
| 303 |
+
if response.stop_reason == "max_tokens":
|
| 304 |
+
# Truncated JSON is not partial evidence, it is malformed. Said here
|
| 305 |
+
# rather than left to `parse_strict`, because the fix is a ceiling and
|
| 306 |
+
# not a recapture, and a farm should not be asked to re-photograph an
|
| 307 |
+
# animal over a token limit.
|
| 308 |
+
raise AdapterError(
|
| 309 |
+
f"The hosted model's answer was cut off at {MAX_TOKENS} tokens, so "
|
| 310 |
+
f"it is not a complete JSON object. Nothing is stored for it."
|
| 311 |
+
)
|
| 312 |
+
|
| 313 |
+
text = next((b.text for b in response.content if b.type == "text"), None)
|
| 314 |
+
if text is None:
|
| 315 |
+
# Reachable when a response carries only thinking blocks. Structured
|
| 316 |
+
# output makes it very unlikely and never impossible, and an empty
|
| 317 |
+
# string here would reach `parse_strict` as a malformed answer with no
|
| 318 |
+
# explanation attached.
|
| 319 |
+
raise AdapterError(
|
| 320 |
+
"The hosted model returned no text block, so there is no JSON "
|
| 321 |
+
"object to read. Nothing is stored for it."
|
| 322 |
+
)
|
| 323 |
+
|
| 324 |
+
logger.info(
|
| 325 |
+
"Hosted reasoner %s answered in %s input and %s output tokens.",
|
| 326 |
+
model, response.usage.input_tokens, response.usage.output_tokens,
|
| 327 |
+
)
|
| 328 |
+
return text
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def _describe() -> str:
|
| 332 |
+
"""What this transport is, for a listing. Imports nothing."""
|
| 333 |
+
return json.dumps(
|
| 334 |
+
{
|
| 335 |
+
"provider": "anthropic",
|
| 336 |
+
"structured_output": "server-side json_schema",
|
| 337 |
+
"images": "base64, inline",
|
| 338 |
+
"max_tokens": MAX_TOKENS,
|
| 339 |
+
"timeout_seconds": TIMEOUT_SECONDS,
|
| 340 |
+
},
|
| 341 |
+
sort_keys=True,
|
| 342 |
+
)
|
app/adapters/unavailable.py
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Adapters for models that are not installed, and what each one is waiting on.
|
| 2 |
+
|
| 3 |
+
**Every class here is a real adapter with a real reason, not a stub.** The
|
| 4 |
+
difference matters and it is the whole point of the file: a stub returns
|
| 5 |
+
something plausible, and these cannot return anything at all. `load()` raises,
|
| 6 |
+
there is no `detect` or `embed` or `count` to call, and the only thing they can
|
| 7 |
+
produce is an account of what is missing.
|
| 8 |
+
|
| 9 |
+
Directive Β§36 says a capability may only be called unavailable after the
|
| 10 |
+
alternatives have been attempted and the attempt documented. This is where that
|
| 11 |
+
documentation lives in code rather than in a document nobody re-reads β each
|
| 12 |
+
adapter carries the licence position, the measured or unmeasured cost, and the
|
| 13 |
+
specific thing that would make it runnable.
|
| 14 |
+
|
| 15 |
+
Two kinds of absence are represented, and collapsing them would lose the
|
| 16 |
+
information a reader needs:
|
| 17 |
+
|
| 18 |
+
- **Not installed.** It is usable and nobody has stood it up yet. SAM 3,
|
| 19 |
+
Grounding DINO, CountGD.
|
| 20 |
+
- **Not configured.** It needs a credential the deployment does not have. The
|
| 21 |
+
hosted reasoner.
|
| 22 |
+
|
| 23 |
+
There used to be a third β **refused**, for MegaDescriptor and MiewID β and it
|
| 24 |
+
is gone because a refused licence is no longer a reason to leave a model
|
| 25 |
+
uninstalled. Both now run, both are still unservable, and both live in
|
| 26 |
+
`adapters/embedding.py`. `NotInstalled.availability()` still checks the licence
|
| 27 |
+
first, because the distinction it draws is the one that matters here: a model
|
| 28 |
+
Animap may not serve does not become available by installing it, and saying so
|
| 29 |
+
keeps somebody from spending a week finding out.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
from __future__ import annotations
|
| 33 |
+
|
| 34 |
+
from app.adapters.base import (
|
| 35 |
+
Adapter,
|
| 36 |
+
AdapterSpec,
|
| 37 |
+
AdapterUnavailable,
|
| 38 |
+
Availability,
|
| 39 |
+
MeasuredCost,
|
| 40 |
+
Modality,
|
| 41 |
+
Placement,
|
| 42 |
+
Task,
|
| 43 |
+
)
|
| 44 |
+
from app.adapters.licences import LicenceRefused, gate
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class NotInstalled(Adapter):
|
| 48 |
+
"""An adapter whose weights are not on this machine.
|
| 49 |
+
|
| 50 |
+
Holds a spec and a reason and nothing else. It exists so `/capabilities`
|
| 51 |
+
can say *which* model is missing and what it would take, rather than
|
| 52 |
+
returning the same "coming soon" for a model nobody has tried and a model
|
| 53 |
+
that is one download away.
|
| 54 |
+
"""
|
| 55 |
+
|
| 56 |
+
def __init__(self, spec: AdapterSpec, *, reason: str, remedy: str) -> None:
|
| 57 |
+
self.spec = spec
|
| 58 |
+
self._reason = reason
|
| 59 |
+
self._remedy = remedy
|
| 60 |
+
|
| 61 |
+
def availability(self) -> Availability:
|
| 62 |
+
# The licence is checked first and separately. A model Animap may not
|
| 63 |
+
# serve is not "not installed yet" β installing it would not help, and
|
| 64 |
+
# saying so keeps somebody from spending a week on it.
|
| 65 |
+
try:
|
| 66 |
+
gate(self.spec.runtime)
|
| 67 |
+
except LicenceRefused as refusal:
|
| 68 |
+
return Availability(
|
| 69 |
+
False, str(refusal),
|
| 70 |
+
"This one does not become available by installing it.",
|
| 71 |
+
)
|
| 72 |
+
return Availability(False, self._reason, self._remedy)
|
| 73 |
+
|
| 74 |
+
def load(self) -> "NotInstalled":
|
| 75 |
+
raise AdapterUnavailable(self.availability())
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
# --- Β§3 SAM 3.1 ---------------------------------------------------------------
|
| 79 |
+
|
| 80 |
+
SAM3_SPEC = AdapterSpec(
|
| 81 |
+
adapter_id="sam3",
|
| 82 |
+
runtime="sam3",
|
| 83 |
+
tasks=(Task.SEGMENT, Task.DETECT, Task.TRACK),
|
| 84 |
+
modalities=(Modality.IMAGE, Modality.VIDEO),
|
| 85 |
+
directive_role=(
|
| 86 |
+
"Β§3 SAM 3.1 β detection, segmentation, tracking, concept prompting, "
|
| 87 |
+
"visual exemplar prompting, auto-labelling, body-region extraction, "
|
| 88 |
+
"wound and lesion masks. The teacher model for most of Β§6βΒ§20."
|
| 89 |
+
),
|
| 90 |
+
placement=Placement.GPU_SERVICE,
|
| 91 |
+
requires_gpu=True,
|
| 92 |
+
placement_reason=(
|
| 93 |
+
"848M parameters and a 3.44 GB fp32 checkpoint, and the official "
|
| 94 |
+
"install requirements name a CUDA 12.6 GPU. It will not sit beside the "
|
| 95 |
+
"API on the 2 vCPU / 4 GiB worker, and it does not need to β this is "
|
| 96 |
+
"the leg to host on a GPU. A quantised CPU build is conceivable and "
|
| 97 |
+
"nobody has measured one."
|
| 98 |
+
),
|
| 99 |
+
notes=(
|
| 100 |
+
"SAM 3.1 (27 March 2026) is a video-tracking speed-up over SAM 3, not a "
|
| 101 |
+
"new image model, and ships no transformers integration β its own model "
|
| 102 |
+
"card says so. For still images, SAM 3 is the one to stand up; "
|
| 103 |
+
"transformers has supported it since v5.0.0. Both are gated with manual "
|
| 104 |
+
"approval, so a build needs an HF token and Meta can revoke access."
|
| 105 |
+
),
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
SAM2_SPEC = AdapterSpec(
|
| 109 |
+
adapter_id="sam2.1-hiera-tiny",
|
| 110 |
+
runtime="sam2-onnx",
|
| 111 |
+
tasks=(Task.SEGMENT,),
|
| 112 |
+
modalities=(Modality.IMAGE, Modality.VIDEO),
|
| 113 |
+
directive_role=(
|
| 114 |
+
"Β§3 SAM's segmentation role at a size the CPU worker can hold. 39.0M "
|
| 115 |
+
"parameters, Apache-2.0, ungated."
|
| 116 |
+
),
|
| 117 |
+
placement=Placement.CPU_SERVICE,
|
| 118 |
+
notes=(
|
| 119 |
+
"**Not a substitute for SAM 3 and must not be recorded as one.** It "
|
| 120 |
+
"does promptable segmentation from a point or a box; it has no concept "
|
| 121 |
+
"prompting and no text, which is most of what Β§3 wants SAM 3 for. It is "
|
| 122 |
+
"here as the mask source for the pipelines that already know where to "
|
| 123 |
+
"look β Β§14's flank region, Β§9's wound outline β where a box from the "
|
| 124 |
+
"existing YOLOX detector is prompt enough."
|
| 125 |
+
),
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
# --- Β§4 Grounding DINO --------------------------------------------------------
|
| 130 |
+
|
| 131 |
+
GROUNDING_DINO_SPEC = AdapterSpec(
|
| 132 |
+
adapter_id="grounding-dino-tiny",
|
| 133 |
+
runtime="grounding-dino-hf",
|
| 134 |
+
tasks=(Task.DETECT,),
|
| 135 |
+
modalities=(Modality.IMAGE,),
|
| 136 |
+
directive_role=(
|
| 137 |
+
"Β§4 Grounding DINO β open-vocabulary detection, text-prompted "
|
| 138 |
+
"localisation, and the fallback when SAM's concept prompting is weak. "
|
| 139 |
+
"Β§25 also names it for tick detection over tiled close-ups."
|
| 140 |
+
),
|
| 141 |
+
placement=Placement.GPU_SERVICE,
|
| 142 |
+
placement_reason=(
|
| 143 |
+
"It does run on CPU β the HF implementation's deformable attention is "
|
| 144 |
+
"plain PyTorch `grid_sample` with no CUDA extension β and the measured "
|
| 145 |
+
"cost below is why it should not run there anyway: 5.5 s a frame and a "
|
| 146 |
+
"2,121 MB peak, against a 4 GiB container that is also holding the "
|
| 147 |
+
"media. The second obstacle is torch, which ADR 0017 deliberately "
|
| 148 |
+
"removed; an ONNX export would fix that and nobody has attempted one."
|
| 149 |
+
),
|
| 150 |
+
measured=MeasuredCost(
|
| 151 |
+
hardware="Apple M-series laptop (NOT the target container)",
|
| 152 |
+
threads=1,
|
| 153 |
+
sample=(
|
| 154 |
+
"3 Commons frames β cattle_ng_kaduna_market_01, cattle_ng_red_bororo, "
|
| 155 |
+
"poultry_free_range_flock β at 1600β1920 px, prompts 'a cow.' and "
|
| 156 |
+
"'a chicken.', box and text thresholds 0.3"
|
| 157 |
+
),
|
| 158 |
+
runs=3,
|
| 159 |
+
median_seconds=5.53,
|
| 160 |
+
peak_rss_mb=2121.0,
|
| 161 |
+
measured_on="2026-08-21",
|
| 162 |
+
),
|
| 163 |
+
notes=(
|
| 164 |
+
"172,250,626 parameters, 657 MiB of safetensors, Apache-2.0 for code, "
|
| 165 |
+
"weights and the BERT text encoder alike β the cleanest licence in the "
|
| 166 |
+
"whole Β§3/Β§4 stack. Session load was 16.3 s and 948 MB before a single "
|
| 167 |
+
"frame. "
|
| 168 |
+
"**The detection quality above is a spike and not a benchmark.** At "
|
| 169 |
+
"threshold 0.3 it returned 1 box on a Kaduna market frame, 2 on "
|
| 170 |
+
"cattle_ng_red_bororo β which ADR 0018 records as holding 3 cattle, all "
|
| 171 |
+
"3 found by the shipped YOLOX β and 5 on a free-range flock frame whose "
|
| 172 |
+
"human count is 12. One prompt, one threshold, no tuning, three frames. "
|
| 173 |
+
"It says the pipeline runs, and nothing about whether it is better than "
|
| 174 |
+
"what ships."
|
| 175 |
+
),
|
| 176 |
+
)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
# --- Β§4 CountGD ---------------------------------------------------------------
|
| 180 |
+
|
| 181 |
+
COUNTGD_SPEC = AdapterSpec(
|
| 182 |
+
adapter_id="countgd",
|
| 183 |
+
runtime="countgd",
|
| 184 |
+
tasks=(Task.COUNT,),
|
| 185 |
+
modalities=(Modality.IMAGE,),
|
| 186 |
+
directive_role=(
|
| 187 |
+
"Β§4 and Β§6.3 CountGD β zero-shot open-world counting, exemplar-guided, "
|
| 188 |
+
"for the poultry visible-count that a COCO detector cannot do. Β§40.3 "
|
| 189 |
+
"names the specific job: benchmark it against the 452 annotated "
|
| 190 |
+
"commercial broiler frames."
|
| 191 |
+
),
|
| 192 |
+
placement=Placement.GPU_SERVICE,
|
| 193 |
+
requires_gpu=False,
|
| 194 |
+
placement_reason=(
|
| 195 |
+
"MIT, and its vendored GroundingDINO falls back to a pure-PyTorch path "
|
| 196 |
+
"when the CUDA extension is absent, so CPU execution is possible in "
|
| 197 |
+
"principle. 894 MiB of weights on top of torch, and no published CPU "
|
| 198 |
+
"throughput figure exists. GPU is the sane host; CPU is a measurement "
|
| 199 |
+
"nobody has taken."
|
| 200 |
+
),
|
| 201 |
+
notes=(
|
| 202 |
+
"**This is the highest-value unbuilt thing in the stack.** ADR 0018 "
|
| 203 |
+
"measured the shipped detector finding nothing at all in 254 of 452 PIO "
|
| 204 |
+
"frames and 8.5% of the birds that were there β a 92% undercount β and "
|
| 205 |
+
"concluded a density method is needed. CountGD is that method, the "
|
| 206 |
+
"benchmark set is already on disk under evaluation/pio, and the licence "
|
| 207 |
+
"is MIT. Skip the optional --sam_tt_norm flag and the 2.4 GB SAM ViT-H "
|
| 208 |
+
"checkpoint is not needed."
|
| 209 |
+
),
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
# --- Β§4 MegaDescriptor and MiewID have left this file ------------------------
|
| 214 |
+
#
|
| 215 |
+
# They were here as placeholders while their licences were treated as a reason
|
| 216 |
+
# not to install them. The founder lifted that: a licence is no longer grounds
|
| 217 |
+
# for dropping a model, only for refusing to serve it. Both are now exported,
|
| 218 |
+
# benchmarked and registered as real adapters in `adapters/embedding.py`, with
|
| 219 |
+
# their cards under `models/alternates/megadescriptor/` and `.../miewid/`.
|
| 220 |
+
#
|
| 221 |
+
# Nothing was weakened by the move. `licences.gate` still refuses both under the
|
| 222 |
+
# default `enforce` policy, `registry.refused()` still names exactly these two,
|
| 223 |
+
# and `describe()` still reports `servable: False` whatever the policy β the
|
| 224 |
+
# terms have not changed, and neither has what Animap may put in front of a
|
| 225 |
+
# farmer. What changed is that "refused" is now a fact with a measurement behind
|
| 226 |
+
# it rather than a reason the measurement never happened.
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
# --- Β§4 hosted multimodal -----------------------------------------------------
|
| 230 |
+
|
| 231 |
+
HOSTED_MULTIMODAL_SPEC = AdapterSpec(
|
| 232 |
+
adapter_id="hosted-multimodal",
|
| 233 |
+
runtime="hosted-multimodal",
|
| 234 |
+
tasks=(Task.REASON,),
|
| 235 |
+
modalities=(Modality.IMAGE, Modality.VIDEO, Modality.AUDIO),
|
| 236 |
+
directive_role=(
|
| 237 |
+
"Β§4 hosted multimodal β BCS rubric scoring, dentition, wound "
|
| 238 |
+
"description, skin and hoof and footpad triage, breed suggestion, "
|
| 239 |
+
"litter condition, heat-stress signs, egg quality, structured evidence "
|
| 240 |
+
"extraction. Β§4 is explicit that it is 'an experimental visual "
|
| 241 |
+
"reasoner, not an authority'."
|
| 242 |
+
),
|
| 243 |
+
requires_artefact=False,
|
| 244 |
+
placement=Placement.CPU_SERVICE,
|
| 245 |
+
placement_reason=(
|
| 246 |
+
"No weights run here, so it costs the container nothing but a socket. "
|
| 247 |
+
"It is the one leg that cannot ever move to the phone, which matters: "
|
| 248 |
+
"ADR 0002 makes Animap offline-first, and every capability built on "
|
| 249 |
+
"this one is a capability a farm cannot use in a shed with no signal."
|
| 250 |
+
),
|
| 251 |
+
notes=(
|
| 252 |
+
"Unavailable because no API key is configured. See adapters/multimodal.py "
|
| 253 |
+
"for the env var and the structured-output contract."
|
| 254 |
+
),
|
| 255 |
+
)
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
#: Every adapter the directive names that is not runnable here, with its reason.
|
| 259 |
+
#: Built as a function rather than a module-level dict so a caller gets fresh
|
| 260 |
+
#: objects and cannot mutate a shared registry.
|
| 261 |
+
def unavailable_adapters() -> list[Adapter]:
|
| 262 |
+
return [
|
| 263 |
+
NotInstalled(
|
| 264 |
+
SAM3_SPEC,
|
| 265 |
+
reason="SAM 3 is not installed.",
|
| 266 |
+
remedy=(
|
| 267 |
+
"Request access on huggingface.co/facebook/sam3, set HF_TOKEN, "
|
| 268 |
+
"and host it on a GPU worker β it will not fit the CPU one."
|
| 269 |
+
),
|
| 270 |
+
),
|
| 271 |
+
NotInstalled(
|
| 272 |
+
SAM2_SPEC,
|
| 273 |
+
reason="SAM 2.1 is not installed.",
|
| 274 |
+
remedy=(
|
| 275 |
+
"Export facebook/sam2.1-hiera-tiny to ONNX the way "
|
| 276 |
+
"scripts/export_embedding.py does, so the CPU worker keeps its "
|
| 277 |
+
"no-torch property."
|
| 278 |
+
),
|
| 279 |
+
),
|
| 280 |
+
NotInstalled(
|
| 281 |
+
GROUNDING_DINO_SPEC,
|
| 282 |
+
reason="Grounding DINO is not installed.",
|
| 283 |
+
remedy=(
|
| 284 |
+
"pip install transformers and fetch "
|
| 285 |
+
"IDEA-Research/grounding-dino-tiny, or export it to ONNX for the "
|
| 286 |
+
"CPU worker. Nothing is gated and nothing needs a token."
|
| 287 |
+
),
|
| 288 |
+
),
|
| 289 |
+
NotInstalled(
|
| 290 |
+
COUNTGD_SPEC,
|
| 291 |
+
reason="CountGD is not installed.",
|
| 292 |
+
remedy=(
|
| 293 |
+
"Clone niki-amini-naieni/CountGD, fetch the MIT weights from "
|
| 294 |
+
"nikigoli/CountGD, and benchmark against evaluation/pio β the "
|
| 295 |
+
"452-frame set ADR 0018 says the shipped detector fails on."
|
| 296 |
+
),
|
| 297 |
+
),
|
| 298 |
+
]
|
app/capabilities.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
app/counting.py
ADDED
|
@@ -0,0 +1,526 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Counting, and knowing when not to.
|
| 2 |
+
|
| 3 |
+
This is the layer that turns boxes into a claim. It exists separately from
|
| 4 |
+
`detection.py` because the hard part of counting livestock is not detection β it
|
| 5 |
+
is being honest about the frames where detection stops working.
|
| 6 |
+
|
| 7 |
+
Two capabilities share every line of it:
|
| 8 |
+
|
| 9 |
+
**`cattle_detection`** counts cattle in a paddock. Cattle are large, separated,
|
| 10 |
+
and there are tens of them. A COCO detector already has a `cow` class and this
|
| 11 |
+
is genuinely the right tool.
|
| 12 |
+
|
| 13 |
+
**`poultry_count`** counts birds in a frame. For a backyard flock or a yard of
|
| 14 |
+
fifty layers, the same detector works. For 12,000 broilers in a shed it does
|
| 15 |
+
not, and no threshold tuning will make it: the birds overlap, each one covers a
|
| 16 |
+
few hundred pixels, and non-maximum suppression merges the ones that remain. The
|
| 17 |
+
answer is a density head, not a better detector (ADR 0014).
|
| 18 |
+
|
| 19 |
+
So this module measures whether it is still in the regime it was validated for,
|
| 20 |
+
and when it is not, **it reports no count at all**. A sample presented as a count
|
| 21 |
+
is the failure mode that costs the product its credibility: a farmer shown "18"
|
| 22 |
+
for a shed of several hundred does not conclude that the number means something
|
| 23 |
+
narrower than they thought.
|
| 24 |
+
|
| 25 |
+
**How it knows.** It counts the frame three ways β whole, 2x2, 3x3 β and watches
|
| 26 |
+
what the count does (`app/tiling.py`). In a frame the detector can read, the
|
| 27 |
+
count stops moving, because there was nothing left to find. In a shed it never
|
| 28 |
+
stops, because there are always more birds behind the ones in front. That is a
|
| 29 |
+
measurement of what the detector is *missing*, which is the thing a saturation
|
| 30 |
+
guard has to know and the thing box sizes cannot tell it.
|
| 31 |
+
|
| 32 |
+
The first version of this file guarded on box size instead, and the evaluation
|
| 33 |
+
set caught it: on a broiler house of a thousand birds it found a handful of large
|
| 34 |
+
foreground birds, concluded the frame was sparse, and published the handful as a
|
| 35 |
+
count. It withheld a number on three of twenty uncountable frames; the tiled test
|
| 36 |
+
withholds on twenty of twenty (ADR 0018).
|
| 37 |
+
|
| 38 |
+
**The thresholds belong to the detector, not to the problem.** They were first
|
| 39 |
+
derived on YOLO11m and then inherited unchanged when the shipped model became
|
| 40 |
+
YOLOX-m, which cost 22 points of coverage for no gain in safety β the grid a
|
| 41 |
+
frame settles at depends on how much the detector's whole-frame pass resolves,
|
| 42 |
+
and that is precisely what differs between detectors. Re-deriving them removed a
|
| 43 |
+
rule entirely and restored the coverage. If the artefact changes again, re-run
|
| 44 |
+
`evaluation/run.py` before trusting a number in this file.
|
| 45 |
+
"""
|
| 46 |
+
|
| 47 |
+
from __future__ import annotations
|
| 48 |
+
|
| 49 |
+
from dataclasses import dataclass
|
| 50 |
+
from statistics import median
|
| 51 |
+
from uuid import UUID, uuid4
|
| 52 |
+
|
| 53 |
+
from app.capabilities import Capability, FORBIDDEN_CLAIMS
|
| 54 |
+
from app.detectors import Detection, build
|
| 55 |
+
from app.media import MediaRef, MediaStore
|
| 56 |
+
from app.providers import ModelArtefact
|
| 57 |
+
from app.quality import QualityVerdict, assess
|
| 58 |
+
from app.schemas import (
|
| 59 |
+
ConfidenceLabel,
|
| 60 |
+
InferenceLocation,
|
| 61 |
+
InferenceRequest,
|
| 62 |
+
InferenceResult,
|
| 63 |
+
Observation,
|
| 64 |
+
QualityCheck,
|
| 65 |
+
)
|
| 66 |
+
from app.tiling import Level, converged, pyramid, subject_count
|
| 67 |
+
|
| 68 |
+
#: How much the count may grow when the frame is cut finer before the frame is
|
| 69 |
+
#: called unreadable.
|
| 70 |
+
#:
|
| 71 |
+
#: **Every number below was re-measured on the shipped YOLOX-m artefact** over
|
| 72 |
+
#: the 61-image set (ADR 0018). They previously came from YOLO11m, and carrying
|
| 73 |
+
#: them across backends was wrong: the grid at which a frame settles depends on
|
| 74 |
+
#: how much the detector's whole-frame pass can resolve, which is exactly what
|
| 75 |
+
#: differs between detectors. Re-deriving them moved coverage from 0.71 to 0.935
|
| 76 |
+
#: without letting a single dense frame through.
|
| 77 |
+
#:
|
| 78 |
+
#: **The two classes overlap here, and the honest reading is that this rule does
|
| 79 |
+
#: not separate them on its own.** Last-refinement growth runs 0%β75% on frames a
|
| 80 |
+
#: human could count and 7%β162% on frames a human could not. At 20% it catches
|
| 81 |
+
#: seventeen of the twenty uncountable frames and wrongly withholds two countable
|
| 82 |
+
#: ones; the two dense frames that slip past it are caught by
|
| 83 |
+
#: `MAX_VALIDATED_COUNT`. Neither rule is sufficient alone, which is why they are
|
| 84 |
+
#: OR-ed rather than tuned against each other.
|
| 85 |
+
COUNT_GROWTH_TOLERANCE = 0.20
|
| 86 |
+
|
| 87 |
+
#: Median share of the frame a subject covers, below which no count is reported
|
| 88 |
+
#: *even after the count has settled*. A frame can settle simply because every
|
| 89 |
+
#: animal in it is a smudge the detector resolves the same way at every grid.
|
| 90 |
+
#:
|
| 91 |
+
#: **This rule never fires on the evaluation set, so it carries no evidence.**
|
| 92 |
+
#: Measured at the grid that ran: the smallest median on a countable frame is
|
| 93 |
+
#: 0.00077 (a hillside herd in the Turkish EΔribel pass) and the smallest on an
|
| 94 |
+
#: uncountable one is 0.00042 β but every frame it would have caught was already
|
| 95 |
+
#: withheld by convergence. 0.0006 sits in a gap two frames wide.
|
| 96 |
+
#:
|
| 97 |
+
#: It is kept as a backstop for a frame type this set does not contain, and it is
|
| 98 |
+
#: labelled unexercised rather than described as if it were doing work. Do not
|
| 99 |
+
#: cite it as a reason the guard is safe.
|
| 100 |
+
SATURATION_MEDIAN_AREA_FRACTION = 0.0006
|
| 101 |
+
|
| 102 |
+
#: Median detection score below which the count is reported but capped at `low`
|
| 103 |
+
#: confidence and a recapture is asked for. Measured on the fixtures: cattle
|
| 104 |
+
#: 0.72, sparse hens 0.49, dense hens 0.47. Sparse and dense are only 0.02
|
| 105 |
+
#: apart, so this cannot decide whether to publish a number β it can only decide
|
| 106 |
+
#: how much to trust one.
|
| 107 |
+
LOW_CONFIDENCE_MEDIAN_SCORE = 0.50
|
| 108 |
+
|
| 109 |
+
#: Confusable detections per subject, above which the frame is *reported* as
|
| 110 |
+
#: class-confused. **This is an observation, not a suppression**, and it used to
|
| 111 |
+
#: be the latter.
|
| 112 |
+
#:
|
| 113 |
+
#: As a suppression rule it was measured net-harmful: on the 61-image set it
|
| 114 |
+
#: withheld exactly one count β `cattle_ng_red_bororo`, three Red Bororo cattle,
|
| 115 |
+
#: where the detector found all three and also called some of them `horse` β and
|
| 116 |
+
#: it caught none of the twenty uncountable frames, because convergence and the
|
| 117 |
+
#: count ceiling had already taken all twenty. It cost a correct answer on a
|
| 118 |
+
#: Nigerian frame and bought nothing.
|
| 119 |
+
#:
|
| 120 |
+
#: The signal is still worth recording: it was real when the small backend called
|
| 121 |
+
#: seven of thirteen hens `sheep`, and a future artefact may bring it back. So it
|
| 122 |
+
#: rides along as `confusable_detections` and a warning, where it informs a
|
| 123 |
+
#: reader without silently deleting a number.
|
| 124 |
+
CONFUSION_RATIO = 1.0
|
| 125 |
+
|
| 126 |
+
#: Minimum subjects before that ratio means anything. Below this a single
|
| 127 |
+
#: mislabelled animal would trip it.
|
| 128 |
+
CONFUSION_MIN_SUBJECTS = 3
|
| 129 |
+
|
| 130 |
+
#: The largest count that has been checked against ground truth. Above it the
|
| 131 |
+
#: service reports no number β not because a larger count is necessarily wrong,
|
| 132 |
+
#: but because nobody has ever verified one.
|
| 133 |
+
#:
|
| 134 |
+
#: **Load-bearing, and tight.** It is what catches the two dense frames whose
|
| 135 |
+
#: counts settle anyway: a Swedish free-range yard that stabilises at 32 birds
|
| 136 |
+
#: and a Karamoja kraal that stabilises at 30, both holding many times that. The
|
| 137 |
+
#: largest correctly published count on the set is 20 animals against a human
|
| 138 |
+
#: count of 19, so the threshold sits directly on the edge of the evidence
|
| 139 |
+
#: rather than at a comfortable distance from it.
|
| 140 |
+
#:
|
| 141 |
+
#: Raise it by measuring more frames, not by deciding the detector is probably
|
| 142 |
+
#: fine up there.
|
| 143 |
+
MAX_VALIDATED_COUNT = 20
|
| 144 |
+
|
| 145 |
+
HIGH_CONFIDENCE_SCORE = 0.70
|
| 146 |
+
MEDIUM_CONFIDENCE_SCORE = 0.50
|
| 147 |
+
|
| 148 |
+
#: Share of the frame one subject must cover before the per-animal capabilities
|
| 149 |
+
#: β weight, body condition, skin β have something they could work with. At 15%
|
| 150 |
+
#: of a 1080-line capture the animal is roughly 400 px across, which is the point
|
| 151 |
+
#: below which a girth measured off it is noise rather than a measurement.
|
| 152 |
+
ISOLATION_MIN_AREA_FRACTION = 0.15
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
@dataclass(frozen=True)
|
| 156 |
+
class CountingProfile:
|
| 157 |
+
"""Everything that differs between counting cattle and counting birds."""
|
| 158 |
+
|
| 159 |
+
capability_key: str
|
| 160 |
+
subject_noun: str
|
| 161 |
+
count_observation: str
|
| 162 |
+
#: COCO classes that are the subject. Only these are counted.
|
| 163 |
+
subject_classes: tuple[str, ...]
|
| 164 |
+
#: COCO classes the detector reaches for when it can no longer tell what it
|
| 165 |
+
#: is looking at. **Never counted** β they are a signal, not a subject.
|
| 166 |
+
#: The shipped YOLOX-m does not confuse them on any of the 61 evaluation
|
| 167 |
+
#: images, so the signal is unexercised: see `CONFUSION_MIN_SUBJECTS`.
|
| 168 |
+
confusable_classes: tuple[str, ...]
|
| 169 |
+
unit: str
|
| 170 |
+
#: Said on every run, whatever the result. These are the claims the brief
|
| 171 |
+
#: forbids, stated before anyone can misread the number.
|
| 172 |
+
standing_warning: str
|
| 173 |
+
saturation_warning: str
|
| 174 |
+
#: Whether a single well-framed subject is what this capability is for.
|
| 175 |
+
reports_isolation: bool = False
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
CATTLE_DETECTION = CountingProfile(
|
| 179 |
+
capability_key="cattle_detection",
|
| 180 |
+
subject_noun="cattle",
|
| 181 |
+
count_observation="cattle_visible",
|
| 182 |
+
subject_classes=("cow",),
|
| 183 |
+
confusable_classes=("horse", "sheep"),
|
| 184 |
+
unit="animals",
|
| 185 |
+
standing_warning=(
|
| 186 |
+
"A count of the animals visible in this frame. It is not the herd size β "
|
| 187 |
+
"animals behind others, behind cover, or out of frame are not in it."
|
| 188 |
+
),
|
| 189 |
+
saturation_warning=(
|
| 190 |
+
"There are more cattle in this frame than can be counted from it. No "
|
| 191 |
+
"number is reported. Move closer, or frame a smaller part of the herd."
|
| 192 |
+
),
|
| 193 |
+
reports_isolation=True,
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
POULTRY_COUNT = CountingProfile(
|
| 197 |
+
capability_key="poultry_count",
|
| 198 |
+
subject_noun="birds",
|
| 199 |
+
count_observation="birds_visible",
|
| 200 |
+
subject_classes=("bird",),
|
| 201 |
+
confusable_classes=("sheep", "cat", "dog"),
|
| 202 |
+
unit="birds",
|
| 203 |
+
standing_warning=(
|
| 204 |
+
"A count of the birds visible in this frame. It is never the flock "
|
| 205 |
+
"population, and it must not be used to reconcile a house."
|
| 206 |
+
),
|
| 207 |
+
saturation_warning=(
|
| 208 |
+
"This flock is denser than a detector can count. No number is reported, "
|
| 209 |
+
"because a detector undercounts a crowded shed by an amount nobody can "
|
| 210 |
+
"estimate. Frame a smaller section, closer in."
|
| 211 |
+
),
|
| 212 |
+
)
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
class DetectionCountRunner:
|
| 216 |
+
"""Runs one counting capability against one frame."""
|
| 217 |
+
|
| 218 |
+
def __init__(self, profile: CountingProfile) -> None:
|
| 219 |
+
self.profile = profile
|
| 220 |
+
|
| 221 |
+
def run(
|
| 222 |
+
self,
|
| 223 |
+
*,
|
| 224 |
+
request: InferenceRequest,
|
| 225 |
+
capability: Capability,
|
| 226 |
+
artefact: ModelArtefact,
|
| 227 |
+
store: MediaStore,
|
| 228 |
+
request_id: UUID | None = None,
|
| 229 |
+
) -> InferenceResult:
|
| 230 |
+
profile = self.profile
|
| 231 |
+
request_id = request_id or uuid4()
|
| 232 |
+
warnings: list[str] = [profile.standing_warning]
|
| 233 |
+
|
| 234 |
+
# These capabilities read one frame. Saying so beats silently ignoring
|
| 235 |
+
# the rest, and beats pretending a count was aggregated across them.
|
| 236 |
+
if len(request.media_ids) > capability.frames_required:
|
| 237 |
+
warnings.append(
|
| 238 |
+
f"{len(request.media_ids)} frames were supplied; this capability "
|
| 239 |
+
f"reads {capability.frames_required}."
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
# `farm_id` and `captured_at` are what let a blob store find the object
|
| 243 |
+
# in one request instead of scanning a prefix (`app/media.py`). A local
|
| 244 |
+
# store ignores them.
|
| 245 |
+
image = store.open_image(MediaRef(
|
| 246 |
+
media_id=request.media_ids[0],
|
| 247 |
+
farm_id=request.farm_id,
|
| 248 |
+
captured_at=request.captured_at,
|
| 249 |
+
object_path=request.path_for(request.media_ids[0]),
|
| 250 |
+
))
|
| 251 |
+
verdict = assess(image)
|
| 252 |
+
|
| 253 |
+
if verdict.blocked:
|
| 254 |
+
return self._blocked(request, capability, artefact, verdict, warnings, request_id)
|
| 255 |
+
|
| 256 |
+
detector = build(artefact)
|
| 257 |
+
levels = pyramid(
|
| 258 |
+
detector, image, profile.subject_classes, COUNT_GROWTH_TOLERANCE,
|
| 259 |
+
)
|
| 260 |
+
final = levels[-1]
|
| 261 |
+
settled = len(levels) < 2 or converged(
|
| 262 |
+
levels[-2], levels[-1], profile.subject_classes, COUNT_GROWTH_TOLERANCE,
|
| 263 |
+
)
|
| 264 |
+
|
| 265 |
+
subjects = [d for d in final.detections if d.label in profile.subject_classes]
|
| 266 |
+
confusable = [d for d in final.detections if d.label in profile.confusable_classes]
|
| 267 |
+
|
| 268 |
+
checks = list(verdict.checks)
|
| 269 |
+
checks.append(self._framing_check(subjects))
|
| 270 |
+
checks.append(self._convergence_check(levels, profile, settled))
|
| 271 |
+
if profile.reports_isolation:
|
| 272 |
+
checks.append(self._isolation_check(subjects))
|
| 273 |
+
|
| 274 |
+
saturation = self._withhold_reason(subjects, confusable, settled)
|
| 275 |
+
observations: list[Observation] = [
|
| 276 |
+
# Emitted on every path, including the ones that publish no count,
|
| 277 |
+
# because these three are what let a threshold be re-derived later
|
| 278 |
+
# from stored results instead of re-run from photographs nobody kept.
|
| 279 |
+
Observation(type="counting_grid", value=float(final.grid), confidence=None),
|
| 280 |
+
Observation(
|
| 281 |
+
type="subjects_detected", value=float(len(subjects)),
|
| 282 |
+
unit=profile.unit, confidence=None,
|
| 283 |
+
),
|
| 284 |
+
]
|
| 285 |
+
if subjects:
|
| 286 |
+
observations.append(Observation(
|
| 287 |
+
type="median_subject_frame_fraction",
|
| 288 |
+
value=round(median(d.area_fraction for d in subjects), 5),
|
| 289 |
+
unit="fraction", confidence=None,
|
| 290 |
+
))
|
| 291 |
+
observations.append(Observation(
|
| 292 |
+
type="largest_subject_frame_fraction",
|
| 293 |
+
value=round(max(d.area_fraction for d in subjects), 5),
|
| 294 |
+
unit="fraction", confidence=None,
|
| 295 |
+
))
|
| 296 |
+
observations.append(Observation(
|
| 297 |
+
type="confusable_detections",
|
| 298 |
+
value=float(len(confusable)),
|
| 299 |
+
confidence=None,
|
| 300 |
+
))
|
| 301 |
+
|
| 302 |
+
if self._is_class_confused(subjects, confusable):
|
| 303 |
+
# Reported, never suppressing. Measured net-harmful as a guard; see
|
| 304 |
+
# `CONFUSION_RATIO`.
|
| 305 |
+
warnings.append(
|
| 306 |
+
f"The detector also labelled {len(confusable)} things in this "
|
| 307 |
+
f"frame as another animal, which is as many as it called "
|
| 308 |
+
f"{profile.subject_noun}. It may be struggling to tell what it "
|
| 309 |
+
f"is looking at, so treat the number as a rough indication."
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
if not subjects:
|
| 313 |
+
# **Not a count of zero.** "We could not find any birds" and "there
|
| 314 |
+
# are no birds" are different claims, and on a packed broiler house
|
| 315 |
+
# the detector produces the first while the second would be absurd.
|
| 316 |
+
# Emitting no count observation is what keeps the app from rendering
|
| 317 |
+
# a zero it would have to defend.
|
| 318 |
+
warnings.append(
|
| 319 |
+
f"No {profile.subject_noun} were found in this frame. That is not "
|
| 320 |
+
f"a count of zero β it means nothing recognisable was detected. "
|
| 321 |
+
f"Capture again, closer in and better lit."
|
| 322 |
+
)
|
| 323 |
+
confidence = ConfidenceLabel.LOW
|
| 324 |
+
recapture = True
|
| 325 |
+
elif saturation is not None:
|
| 326 |
+
warnings.append(profile.saturation_warning)
|
| 327 |
+
observations.append(Observation(
|
| 328 |
+
type="count_withheld",
|
| 329 |
+
value=saturation,
|
| 330 |
+
confidence=None,
|
| 331 |
+
))
|
| 332 |
+
# `subjects_detected` above is already the floor. It is deliberately
|
| 333 |
+
# not named as a count anywhere on this path.
|
| 334 |
+
confidence = ConfidenceLabel.LOW
|
| 335 |
+
recapture = True
|
| 336 |
+
else:
|
| 337 |
+
mean_score = sum(d.score for d in subjects) / len(subjects)
|
| 338 |
+
observations.append(Observation(
|
| 339 |
+
type=profile.count_observation,
|
| 340 |
+
value=float(len(subjects)),
|
| 341 |
+
unit=profile.unit,
|
| 342 |
+
confidence=round(mean_score, 3),
|
| 343 |
+
))
|
| 344 |
+
confidence = _label(mean_score)
|
| 345 |
+
recapture = False
|
| 346 |
+
|
| 347 |
+
if median(d.score for d in subjects) < LOW_CONFIDENCE_MEDIAN_SCORE:
|
| 348 |
+
warnings.append(
|
| 349 |
+
f"The detector was unsure about most of these {profile.subject_noun}. "
|
| 350 |
+
f"Treat the number as a rough indication and capture again closer in."
|
| 351 |
+
)
|
| 352 |
+
confidence = ConfidenceLabel.LOW
|
| 353 |
+
recapture = True
|
| 354 |
+
|
| 355 |
+
if verdict.degraded:
|
| 356 |
+
# A frame the gate flagged cannot produce a high-confidence claim,
|
| 357 |
+
# whatever the detector's own scores say about it.
|
| 358 |
+
confidence = ConfidenceLabel.LOW
|
| 359 |
+
recapture = True
|
| 360 |
+
|
| 361 |
+
return self._result(
|
| 362 |
+
request=request,
|
| 363 |
+
capability=capability,
|
| 364 |
+
artefact=artefact,
|
| 365 |
+
request_id=request_id,
|
| 366 |
+
observations=observations,
|
| 367 |
+
confidence=confidence,
|
| 368 |
+
checks=checks,
|
| 369 |
+
warnings=warnings,
|
| 370 |
+
recapture=recapture,
|
| 371 |
+
)
|
| 372 |
+
|
| 373 |
+
def _framing_check(self, subjects: list[Detection]) -> QualityCheck:
|
| 374 |
+
if subjects:
|
| 375 |
+
return QualityCheck(check="framing", passed=True)
|
| 376 |
+
return QualityCheck(
|
| 377 |
+
check="framing", passed=False,
|
| 378 |
+
detail=f"No {self.profile.subject_noun} found in this frame.",
|
| 379 |
+
)
|
| 380 |
+
|
| 381 |
+
def _isolation_check(self, subjects: list[Detection]) -> QualityCheck:
|
| 382 |
+
"""Whether the per-animal capabilities could use this frame.
|
| 383 |
+
|
| 384 |
+
`cattle_detection` runs before weight, body condition and skin, and each
|
| 385 |
+
of those needs one animal filling the frame. Reporting that here saves a
|
| 386 |
+
second capture attempt later.
|
| 387 |
+
"""
|
| 388 |
+
large = [d for d in subjects if d.area_fraction >= ISOLATION_MIN_AREA_FRACTION]
|
| 389 |
+
if len(large) == 1:
|
| 390 |
+
return QualityCheck(check="subject_isolation", passed=True)
|
| 391 |
+
return QualityCheck(
|
| 392 |
+
check="subject_isolation", passed=False,
|
| 393 |
+
detail=(
|
| 394 |
+
f"{len(subjects)} animals in frame and {len(large)} close enough to "
|
| 395 |
+
f"assess individually. Per-animal capabilities need one animal, "
|
| 396 |
+
f"filling the frame."
|
| 397 |
+
),
|
| 398 |
+
)
|
| 399 |
+
|
| 400 |
+
def _convergence_check(
|
| 401 |
+
self, levels: list[Level], profile: CountingProfile, settled: bool
|
| 402 |
+
) -> QualityCheck:
|
| 403 |
+
counts = [subject_count(level, profile.subject_classes) for level in levels]
|
| 404 |
+
trail = " β ".join(
|
| 405 |
+
f"{level.grid}x{level.grid}: {count}" for level, count in zip(levels, counts)
|
| 406 |
+
)
|
| 407 |
+
if settled:
|
| 408 |
+
return QualityCheck(check="count_convergence", passed=True, detail=trail)
|
| 409 |
+
return QualityCheck(
|
| 410 |
+
check="count_convergence", passed=False,
|
| 411 |
+
detail=(
|
| 412 |
+
f"The count kept rising as the frame was read more finely "
|
| 413 |
+
f"({trail}), so animals are still hidden behind other animals."
|
| 414 |
+
),
|
| 415 |
+
)
|
| 416 |
+
|
| 417 |
+
def _is_class_confused(
|
| 418 |
+
self, subjects: list[Detection], confusable: list[Detection]
|
| 419 |
+
) -> bool:
|
| 420 |
+
"""Whether the detector is reaching for neighbouring classes as often as
|
| 421 |
+
the right one. A reported signal, not a reason to withhold a count."""
|
| 422 |
+
return (
|
| 423 |
+
len(subjects) >= CONFUSION_MIN_SUBJECTS
|
| 424 |
+
and len(confusable) >= len(subjects) * CONFUSION_RATIO
|
| 425 |
+
)
|
| 426 |
+
|
| 427 |
+
def _withhold_reason(
|
| 428 |
+
self,
|
| 429 |
+
subjects: list[Detection],
|
| 430 |
+
confusable: list[Detection],
|
| 431 |
+
settled: bool,
|
| 432 |
+
) -> str | None:
|
| 433 |
+
"""Whether this frame has left the regime the detector was validated in.
|
| 434 |
+
|
| 435 |
+
Any one signal is enough. They are OR-ed rather than AND-ed on purpose: a
|
| 436 |
+
guard that needs every signal to agree is a guard that goes quiet as soon
|
| 437 |
+
as one of them drifts, and going quiet here means publishing a number
|
| 438 |
+
that is wrong by an unknown factor.
|
| 439 |
+
|
| 440 |
+
On the 61-image set the two working signals withhold every one of the
|
| 441 |
+
twenty uncountable frames and wrongly withhold two of the thirty-one
|
| 442 |
+
countable ones. The last two signals never fire; their comments say so.
|
| 443 |
+
"""
|
| 444 |
+
if not subjects:
|
| 445 |
+
return None
|
| 446 |
+
|
| 447 |
+
# Seventeen of the twenty uncountable frames stop here.
|
| 448 |
+
if not settled:
|
| 449 |
+
return "count_did_not_converge"
|
| 450 |
+
|
| 451 |
+
# The other two. A frame can settle and still be a shed: the detector
|
| 452 |
+
# runs out of things it can resolve, so the count stops moving for the
|
| 453 |
+
# wrong reason. A count larger than anything ever checked is the signal.
|
| 454 |
+
if len(subjects) > MAX_VALIDATED_COUNT:
|
| 455 |
+
return "beyond_validated_range"
|
| 456 |
+
|
| 457 |
+
if median(d.area_fraction for d in subjects) < SATURATION_MEDIAN_AREA_FRACTION:
|
| 458 |
+
return "subjects_too_small"
|
| 459 |
+
return None
|
| 460 |
+
|
| 461 |
+
def _blocked(
|
| 462 |
+
self, request, capability, artefact, verdict: QualityVerdict, warnings, request_id
|
| 463 |
+
) -> InferenceResult:
|
| 464 |
+
failure = verdict.first_failure
|
| 465 |
+
warnings.append(
|
| 466 |
+
failure.detail if failure and failure.detail
|
| 467 |
+
else "The capture was not usable."
|
| 468 |
+
)
|
| 469 |
+
return self._result(
|
| 470 |
+
request=request,
|
| 471 |
+
capability=capability,
|
| 472 |
+
artefact=artefact,
|
| 473 |
+
request_id=request_id,
|
| 474 |
+
observations=[],
|
| 475 |
+
confidence=None,
|
| 476 |
+
checks=list(verdict.checks),
|
| 477 |
+
warnings=warnings,
|
| 478 |
+
recapture=True,
|
| 479 |
+
)
|
| 480 |
+
|
| 481 |
+
def _result(
|
| 482 |
+
self, *, request, capability, artefact, request_id, observations,
|
| 483 |
+
confidence, checks, warnings, recapture,
|
| 484 |
+
) -> InferenceResult:
|
| 485 |
+
forbidden = [o.type for o in observations if o.type in FORBIDDEN_CLAIMS]
|
| 486 |
+
if forbidden:
|
| 487 |
+
# Belt and braces. The registry holds these as data precisely so a
|
| 488 |
+
# runner can be stopped by them rather than reviewed against them.
|
| 489 |
+
raise ValueError(f"{capability.key} tried to emit a forbidden claim: {forbidden}")
|
| 490 |
+
|
| 491 |
+
return InferenceResult(
|
| 492 |
+
request_id=request_id,
|
| 493 |
+
capability_key=capability.key,
|
| 494 |
+
model_id=artefact.model_id,
|
| 495 |
+
model_version=artefact.version,
|
| 496 |
+
inference_location=InferenceLocation.REMOTE,
|
| 497 |
+
subject_type=request.subject_type,
|
| 498 |
+
subject_id=request.subject_id,
|
| 499 |
+
observations=observations,
|
| 500 |
+
# A measurement carries no interpretation. A count of animals is a
|
| 501 |
+
# fact about the frame; what a farmer should do about it is not
|
| 502 |
+
# something this model knows (ADR 0006).
|
| 503 |
+
interpretations=[],
|
| 504 |
+
observation_confidence=confidence,
|
| 505 |
+
interpretation_confidence=None,
|
| 506 |
+
quality_checks=checks,
|
| 507 |
+
warnings=warnings,
|
| 508 |
+
recommended_recapture=recapture,
|
| 509 |
+
)
|
| 510 |
+
|
| 511 |
+
|
| 512 |
+
def _label(score: float) -> ConfidenceLabel:
|
| 513 |
+
if score >= HIGH_CONFIDENCE_SCORE:
|
| 514 |
+
return ConfidenceLabel.HIGH
|
| 515 |
+
if score >= MEDIUM_CONFIDENCE_SCORE:
|
| 516 |
+
return ConfidenceLabel.MEDIUM
|
| 517 |
+
return ConfidenceLabel.LOW
|
| 518 |
+
|
| 519 |
+
|
| 520 |
+
#: Capabilities with an implemented adapter. A capability that has a validated
|
| 521 |
+
#: artefact but is absent from here returns 501 rather than a plausible result β
|
| 522 |
+
#: see `models/README.md`.
|
| 523 |
+
RUNNERS: dict[str, DetectionCountRunner] = {
|
| 524 |
+
CATTLE_DETECTION.capability_key: DetectionCountRunner(CATTLE_DETECTION),
|
| 525 |
+
POULTRY_COUNT.capability_key: DetectionCountRunner(POULTRY_COUNT),
|
| 526 |
+
}
|
app/detectors/__init__.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Detector backends, chosen by the model card.
|
| 2 |
+
|
| 3 |
+
One COCO-pretrained detector serves every sparse-counting capability, because a
|
| 4 |
+
paddock of cattle and a yard of chickens are the same computer vision problem
|
| 5 |
+
with a different class index. What sits above these modules decides what a box
|
| 6 |
+
*means*; they only say where the boxes are.
|
| 7 |
+
|
| 8 |
+
**Which backend runs is a field on the model card, not a code path.** That is
|
| 9 |
+
what made ADR 0017 cheap: moving both counting capabilities off AGPL-3.0 and on
|
| 10 |
+
to the Apache-2.0 backend was an edit to two reviewed JSON files, not a rewrite.
|
| 11 |
+
|
| 12 |
+
Both backends are still registered here. Only one can answer a request β
|
| 13 |
+
`providers.discover()` refuses a card naming the `ultralytics` runtime, whatever
|
| 14 |
+
that card declares about its licence β and the Ultralytics adapter remains so
|
| 15 |
+
`evaluation/` can reproduce the comparison the decision rests on.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
from app.detectors.base import (
|
| 21 |
+
COCO_CLASSES,
|
| 22 |
+
Detection,
|
| 23 |
+
Detector,
|
| 24 |
+
DetectorError,
|
| 25 |
+
)
|
| 26 |
+
from app.detectors.ultralytics_yolo import UltralyticsDetector
|
| 27 |
+
from app.detectors.yolox_onnx import YoloxDetector
|
| 28 |
+
|
| 29 |
+
#: Card value β backend. A card naming anything else is refused rather than
|
| 30 |
+
#: defaulted, because defaulting would mean a typo silently changes which
|
| 31 |
+
#: licensed model produced a farmer's result.
|
| 32 |
+
RUNTIMES = {
|
| 33 |
+
"ultralytics": UltralyticsDetector,
|
| 34 |
+
"yolox-onnx": YoloxDetector,
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
__all__ = [
|
| 38 |
+
"COCO_CLASSES",
|
| 39 |
+
"Detection",
|
| 40 |
+
"Detector",
|
| 41 |
+
"DetectorError",
|
| 42 |
+
"RUNTIMES",
|
| 43 |
+
"UltralyticsDetector",
|
| 44 |
+
"YoloxDetector",
|
| 45 |
+
"build",
|
| 46 |
+
]
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def build(artefact) -> Detector:
|
| 50 |
+
"""Construct the backend a validated artefact asks for.
|
| 51 |
+
|
| 52 |
+
`artefact` is a `providers.ModelArtefact` β already checksummed against its
|
| 53 |
+
card. This function only decides which adapter reads it.
|
| 54 |
+
"""
|
| 55 |
+
runtime = getattr(artefact, "runtime", None)
|
| 56 |
+
if not runtime:
|
| 57 |
+
raise DetectorError(
|
| 58 |
+
f"{artefact.model_id} does not name a runtime on its card, so there "
|
| 59 |
+
f"is no way to know how to run it. Add `\"runtime\": \"β¦\"`, one of "
|
| 60 |
+
f"{', '.join(sorted(RUNTIMES))}."
|
| 61 |
+
)
|
| 62 |
+
backend = RUNTIMES.get(runtime)
|
| 63 |
+
if backend is None:
|
| 64 |
+
raise DetectorError(
|
| 65 |
+
f"{artefact.model_id} asks for runtime {runtime!r}, which does not "
|
| 66 |
+
f"exist. Known runtimes: {', '.join(sorted(RUNTIMES))}."
|
| 67 |
+
)
|
| 68 |
+
return backend(artefact.path)
|
app/detectors/base.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""What every detector backend agrees to return.
|
| 2 |
+
|
| 3 |
+
This file is why ADR 0017 was affordable, and it is not an abstract nicety.
|
| 4 |
+
Animap shipped an AGPL-3.0 model, decided not to, and swapped to an Apache-2.0
|
| 5 |
+
one β and because nothing above this file knows which backend produced a box,
|
| 6 |
+
that was an edit to two model cards rather than a rewrite.
|
| 7 |
+
|
| 8 |
+
The lesson is worth keeping rather than congratulating: the interface earned its
|
| 9 |
+
keep on the day it was used, and it only worked because the second backend had
|
| 10 |
+
been kept tested while it was still hypothetical. `tests/test_alternate_runtime.py`
|
| 11 |
+
still runs a second artefact through the whole counting layer for exactly that
|
| 12 |
+
reason.
|
| 13 |
+
|
| 14 |
+
**The thresholds above this file are not interface-neutral.** Swapping the
|
| 15 |
+
backend leaves `app/counting.py`'s numbers measuring the wrong detector; that
|
| 16 |
+
mistake was made once and cost 20 points of coverage (ADR 0018).
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
from __future__ import annotations
|
| 20 |
+
|
| 21 |
+
from dataclasses import dataclass
|
| 22 |
+
from typing import Protocol
|
| 23 |
+
|
| 24 |
+
from PIL import Image
|
| 25 |
+
|
| 26 |
+
#: COCO's 80 classes, in the order every COCO-trained head emits them. Order is
|
| 27 |
+
#: load-bearing: `cow` is index 19 and `bird` is index 14, and a silently
|
| 28 |
+
#: shifted list would turn cattle into horses without failing anything. Backends
|
| 29 |
+
#: that carry their own name table should prefer it over this one.
|
| 30 |
+
COCO_CLASSES: tuple[str, ...] = (
|
| 31 |
+
"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train",
|
| 32 |
+
"truck", "boat", "traffic light", "fire hydrant", "stop sign",
|
| 33 |
+
"parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
|
| 34 |
+
"elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag",
|
| 35 |
+
"tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite",
|
| 36 |
+
"baseball bat", "baseball glove", "skateboard", "surfboard",
|
| 37 |
+
"tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon",
|
| 38 |
+
"bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot",
|
| 39 |
+
"hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant",
|
| 40 |
+
"bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote",
|
| 41 |
+
"keyboard", "cell phone", "microwave", "oven", "toaster", "sink",
|
| 42 |
+
"refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
|
| 43 |
+
"hair drier", "toothbrush",
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
DEFAULT_SCORE_THRESHOLD = 0.30
|
| 47 |
+
DEFAULT_IOU_THRESHOLD = 0.45
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@dataclass(frozen=True)
|
| 51 |
+
class Detection:
|
| 52 |
+
label: str
|
| 53 |
+
score: float
|
| 54 |
+
#: Pixel coordinates in the *source* image, whatever the backend resized it to.
|
| 55 |
+
box: tuple[float, float, float, float]
|
| 56 |
+
#: Share of the frame the box covers. The counting layer uses this to decide
|
| 57 |
+
#: whether the subjects are close enough for a detector to be counting them
|
| 58 |
+
#: rather than guessing at them.
|
| 59 |
+
area_fraction: float
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class DetectorError(RuntimeError):
|
| 63 |
+
"""The artefact could not be loaded or does not have the expected shape."""
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class Detector(Protocol):
|
| 67 |
+
"""One image in, boxes out. Deliberately the whole interface.
|
| 68 |
+
|
| 69 |
+
Anything richer β batching, tracking, class filtering β belongs above this,
|
| 70 |
+
because every addition here is another thing a replacement backend has to
|
| 71 |
+
reimplement on the day the licence forces a swap.
|
| 72 |
+
"""
|
| 73 |
+
|
| 74 |
+
def detect(self, image: Image.Image) -> list[Detection]:
|
| 75 |
+
...
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def to_detections(
|
| 79 |
+
rows: list[tuple[str, float, tuple[float, float, float, float]]],
|
| 80 |
+
frame_size: tuple[int, int],
|
| 81 |
+
) -> list[Detection]:
|
| 82 |
+
"""Shared box bookkeeping, so each backend only produces label/score/box."""
|
| 83 |
+
width, height = frame_size
|
| 84 |
+
frame_area = float(width * height)
|
| 85 |
+
detections = [
|
| 86 |
+
Detection(
|
| 87 |
+
label=label,
|
| 88 |
+
score=score,
|
| 89 |
+
box=(x1, y1, x2, y2),
|
| 90 |
+
area_fraction=((x2 - x1) * (y2 - y1)) / frame_area,
|
| 91 |
+
)
|
| 92 |
+
for label, score, (x1, y1, x2, y2) in rows
|
| 93 |
+
]
|
| 94 |
+
detections.sort(key=lambda d: d.score, reverse=True)
|
| 95 |
+
return detections
|
app/detectors/ultralytics_yolo.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The Ultralytics YOLO backend. **AGPL-3.0.**
|
| 2 |
+
|
| 3 |
+
**This backend cannot serve a request, and that is deliberate.**
|
| 4 |
+
`providers.discover()` refuses any card naming the `ultralytics` runtime, so
|
| 5 |
+
nothing reachable from `POST /jobs` can reach this file. It exists for
|
| 6 |
+
`evaluation/run.py`, which measures YOLO11m against the shipped YOLOX-m β the
|
| 7 |
+
comparison ADR 0017's decision rests on. Deleting it would delete the evidence.
|
| 8 |
+
|
| 9 |
+
Read `docs/adr/0017-ultralytics-licence.md` before reinstating it. The short
|
| 10 |
+
version: the strict text of AGPL-3.0 is narrower than ADR 0014 assumed β Β§13's
|
| 11 |
+
source-offer duty is conditioned on modifying the program, and Β§0 puts network
|
| 12 |
+
interaction outside "convey" β but Ultralytics publishes the position that a
|
| 13 |
+
closed-source SaaS using its models needs an Enterprise licence, the question is
|
| 14 |
+
genuinely unsettled, and YOLOX costs nothing measurable. Animap chose not to have
|
| 15 |
+
the argument.
|
| 16 |
+
|
| 17 |
+
Running this requires `requirements-agpl.txt`, which the deployment never reads.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
import os
|
| 23 |
+
import threading
|
| 24 |
+
from functools import lru_cache
|
| 25 |
+
from pathlib import Path
|
| 26 |
+
|
| 27 |
+
from PIL import Image
|
| 28 |
+
|
| 29 |
+
from app.detectors.base import (
|
| 30 |
+
DEFAULT_IOU_THRESHOLD,
|
| 31 |
+
DEFAULT_SCORE_THRESHOLD,
|
| 32 |
+
Detection,
|
| 33 |
+
DetectorError,
|
| 34 |
+
to_detections,
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
INPUT_SIZE = 640
|
| 38 |
+
|
| 39 |
+
_lock = threading.Lock()
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@lru_cache(maxsize=4)
|
| 43 |
+
def _model(artefact_path: str):
|
| 44 |
+
"""Load a checkpoint once.
|
| 45 |
+
|
| 46 |
+
`ultralytics` is imported here, not at module scope, because it pulls in
|
| 47 |
+
torch β about 28 seconds and a gigabyte of resident memory on this machine.
|
| 48 |
+
Nineteen of twenty-one capabilities never touch a detector, and the refusal
|
| 49 |
+
path must not pay for one.
|
| 50 |
+
"""
|
| 51 |
+
try:
|
| 52 |
+
from ultralytics import YOLO
|
| 53 |
+
except ImportError as exc: # pragma: no cover - environment problem, not logic
|
| 54 |
+
raise DetectorError(
|
| 55 |
+
"The ultralytics package is not installed, so this artefact cannot "
|
| 56 |
+
"run. Install requirements.txt, or move the capability to the "
|
| 57 |
+
"yolox-onnx runtime."
|
| 58 |
+
) from exc
|
| 59 |
+
|
| 60 |
+
try:
|
| 61 |
+
return YOLO(artefact_path, task="detect")
|
| 62 |
+
except Exception as exc:
|
| 63 |
+
raise DetectorError(f"{Path(artefact_path).name} did not load: {exc}") from exc
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class UltralyticsDetector:
|
| 67 |
+
"""A checksummed Ultralytics checkpoint, ready to run."""
|
| 68 |
+
|
| 69 |
+
def __init__(
|
| 70 |
+
self,
|
| 71 |
+
artefact_path: Path,
|
| 72 |
+
score_threshold: float = DEFAULT_SCORE_THRESHOLD,
|
| 73 |
+
iou_threshold: float = DEFAULT_IOU_THRESHOLD,
|
| 74 |
+
) -> None:
|
| 75 |
+
self.artefact_path = Path(artefact_path)
|
| 76 |
+
self.score_threshold = score_threshold
|
| 77 |
+
self.iou_threshold = iou_threshold
|
| 78 |
+
# CPU by default. A GPU that is present on the dev box and absent in
|
| 79 |
+
# production is a difference that shows up as a crash on deploy day.
|
| 80 |
+
self.device = os.environ.get("ANIMAP_INFERENCE_DEVICE", "cpu")
|
| 81 |
+
|
| 82 |
+
with _lock:
|
| 83 |
+
self._model = _model(str(self.artefact_path))
|
| 84 |
+
|
| 85 |
+
names = getattr(self._model, "names", None)
|
| 86 |
+
if not names:
|
| 87 |
+
raise DetectorError(
|
| 88 |
+
f"{self.artefact_path.name} carries no class names, so its "
|
| 89 |
+
f"outputs cannot be mapped to a species."
|
| 90 |
+
)
|
| 91 |
+
# The checkpoint's own table, not our COCO tuple. A model fine-tuned on
|
| 92 |
+
# cattle would have three classes, and reading index 19 out of it would
|
| 93 |
+
# be nonsense that never raised.
|
| 94 |
+
self.names: dict[int, str] = dict(names)
|
| 95 |
+
|
| 96 |
+
def detect(self, image: Image.Image) -> list[Detection]:
|
| 97 |
+
rgb = image.convert("RGB")
|
| 98 |
+
try:
|
| 99 |
+
result = self._model.predict(
|
| 100 |
+
source=rgb,
|
| 101 |
+
conf=self.score_threshold,
|
| 102 |
+
iou=self.iou_threshold,
|
| 103 |
+
imgsz=INPUT_SIZE,
|
| 104 |
+
device=self.device,
|
| 105 |
+
verbose=False,
|
| 106 |
+
)[0]
|
| 107 |
+
except Exception as exc:
|
| 108 |
+
raise DetectorError(f"Prediction failed: {exc}") from exc
|
| 109 |
+
|
| 110 |
+
boxes = result.boxes
|
| 111 |
+
if boxes is None or len(boxes) == 0:
|
| 112 |
+
return []
|
| 113 |
+
|
| 114 |
+
rows = []
|
| 115 |
+
for class_id, score, box in zip(
|
| 116 |
+
boxes.cls.tolist(), boxes.conf.tolist(), boxes.xyxy.tolist()
|
| 117 |
+
):
|
| 118 |
+
label = self.names.get(int(class_id))
|
| 119 |
+
if label is None:
|
| 120 |
+
raise DetectorError(
|
| 121 |
+
f"{self.artefact_path.name} emitted class {int(class_id)}, "
|
| 122 |
+
f"which is not in its own name table."
|
| 123 |
+
)
|
| 124 |
+
x1, y1, x2, y2 = (float(v) for v in box)
|
| 125 |
+
rows.append((label, float(score), (x1, y1, x2, y2)))
|
| 126 |
+
|
| 127 |
+
return to_detections(rows, rgb.size)
|
app/detectors/yolox_onnx.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The YOLOX backend. Apache-2.0, and **the one that ships**.
|
| 2 |
+
|
| 3 |
+
This was the escape hatch under ADR 0014, kept working so the AGPL-3.0 decision
|
| 4 |
+
stayed reversible. ADR 0017 took the exit, so it is now the only backend that can
|
| 5 |
+
answer a request: `cattle_detection` and `poultry_count` both run YOLOX-m through
|
| 6 |
+
onnxruntime, with no torch dependency.
|
| 7 |
+
|
| 8 |
+
Keeping it tested while it was still the fallback is the reason the switch took
|
| 9 |
+
an afternoon rather than a quarter. An untested escape hatch is not one.
|
| 10 |
+
|
| 11 |
+
**Nothing is downloaded here.** The artefact arrives through
|
| 12 |
+
`scripts/install_models.py`, is checksummed against a committed model card, and
|
| 13 |
+
this file is handed a path that already passed those checks.
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import threading
|
| 19 |
+
from functools import lru_cache
|
| 20 |
+
from pathlib import Path
|
| 21 |
+
|
| 22 |
+
import numpy as np
|
| 23 |
+
import onnxruntime as ort
|
| 24 |
+
from PIL import Image
|
| 25 |
+
|
| 26 |
+
from app.detectors.base import (
|
| 27 |
+
COCO_CLASSES,
|
| 28 |
+
DEFAULT_IOU_THRESHOLD,
|
| 29 |
+
DEFAULT_SCORE_THRESHOLD,
|
| 30 |
+
Detection,
|
| 31 |
+
DetectorError,
|
| 32 |
+
to_detections,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
INPUT_SIZE = 640
|
| 36 |
+
|
| 37 |
+
#: YOLOX's own strides. The exported graph emits one flat tensor of 8,400
|
| 38 |
+
#: anchor-free predictions and leaves the grid arithmetic to the caller, so
|
| 39 |
+
#: these have to match the export or every box lands in the wrong place.
|
| 40 |
+
_STRIDES = (8, 16, 32)
|
| 41 |
+
|
| 42 |
+
#: Padding value from YOLOX's reference preprocessing. Mid-grey, so the letterbox
|
| 43 |
+
#: bars do not read as an object edge.
|
| 44 |
+
_PAD_VALUE = 114
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
_session_lock = threading.Lock()
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@lru_cache(maxsize=4)
|
| 51 |
+
def _session(artefact_path: str) -> ort.InferenceSession:
|
| 52 |
+
"""One session per artefact, built once.
|
| 53 |
+
|
| 54 |
+
Building a session costs about 200 ms and allocates the weights. A request
|
| 55 |
+
that pays that every time turns a 60 ms inference into a 260 ms one.
|
| 56 |
+
"""
|
| 57 |
+
return ort.InferenceSession(artefact_path, providers=["CPUExecutionProvider"])
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
@lru_cache(maxsize=4)
|
| 61 |
+
def _grid(size: int) -> tuple[np.ndarray, np.ndarray]:
|
| 62 |
+
grids, strides = [], []
|
| 63 |
+
for stride in _STRIDES:
|
| 64 |
+
cells = size // stride
|
| 65 |
+
xv, yv = np.meshgrid(np.arange(cells), np.arange(cells))
|
| 66 |
+
grid = np.stack((xv, yv), 2).reshape(1, -1, 2)
|
| 67 |
+
grids.append(grid)
|
| 68 |
+
strides.append(np.full((1, grid.shape[1], 1), stride))
|
| 69 |
+
return np.concatenate(grids, 1), np.concatenate(strides, 1)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _preprocess(image: Image.Image) -> tuple[np.ndarray, float]:
|
| 73 |
+
"""Letterbox to 640Γ640, BGR, 0β255, unnormalised.
|
| 74 |
+
|
| 75 |
+
Every part of that sentence is a YOLOX-specific choice and getting any of it
|
| 76 |
+
wrong degrades results quietly rather than raising. YOLOX folds the
|
| 77 |
+
mean/std normalisation into its first convolution, so feeding it 0β1 floats
|
| 78 |
+
halves the input range and the model simply detects less. Channel order is
|
| 79 |
+
BGR because the reference implementation reads frames with OpenCV and never
|
| 80 |
+
converts.
|
| 81 |
+
"""
|
| 82 |
+
rgb = image.convert("RGB")
|
| 83 |
+
width, height = rgb.size
|
| 84 |
+
ratio = min(INPUT_SIZE / height, INPUT_SIZE / width)
|
| 85 |
+
new_w, new_h = int(width * ratio), int(height * ratio)
|
| 86 |
+
|
| 87 |
+
resized = np.asarray(rgb.resize((new_w, new_h), Image.BILINEAR), dtype=np.uint8)
|
| 88 |
+
padded = np.full((INPUT_SIZE, INPUT_SIZE, 3), _PAD_VALUE, dtype=np.uint8)
|
| 89 |
+
padded[:new_h, :new_w] = resized
|
| 90 |
+
|
| 91 |
+
bgr = padded[:, :, ::-1]
|
| 92 |
+
blob = np.ascontiguousarray(bgr.transpose(2, 0, 1)[None].astype(np.float32))
|
| 93 |
+
return blob, ratio
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _nms(boxes: np.ndarray, scores: np.ndarray, iou_threshold: float) -> list[int]:
|
| 97 |
+
"""Greedy per-class NMS.
|
| 98 |
+
|
| 99 |
+
Written out rather than pulled from torchvision because the whole point of
|
| 100 |
+
the ONNX path is that this service does not carry a training framework into
|
| 101 |
+
production for one function.
|
| 102 |
+
"""
|
| 103 |
+
x1, y1, x2, y2 = boxes.T
|
| 104 |
+
areas = (x2 - x1) * (y2 - y1)
|
| 105 |
+
order = scores.argsort()[::-1]
|
| 106 |
+
keep: list[int] = []
|
| 107 |
+
while order.size:
|
| 108 |
+
best = order[0]
|
| 109 |
+
keep.append(int(best))
|
| 110 |
+
rest = order[1:]
|
| 111 |
+
if rest.size == 0:
|
| 112 |
+
break
|
| 113 |
+
xx1 = np.maximum(x1[best], x1[rest])
|
| 114 |
+
yy1 = np.maximum(y1[best], y1[rest])
|
| 115 |
+
xx2 = np.minimum(x2[best], x2[rest])
|
| 116 |
+
yy2 = np.minimum(y2[best], y2[rest])
|
| 117 |
+
overlap = np.maximum(0.0, xx2 - xx1) * np.maximum(0.0, yy2 - yy1)
|
| 118 |
+
iou = overlap / np.maximum(areas[best] + areas[rest] - overlap, 1e-9)
|
| 119 |
+
order = rest[iou <= iou_threshold]
|
| 120 |
+
return keep
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
class YoloxDetector:
|
| 124 |
+
"""A validated YOLOX artefact, ready to run.
|
| 125 |
+
|
| 126 |
+
Construct it from a path that `providers.load_card` has already checksummed.
|
| 127 |
+
It re-reads nothing and downloads nothing.
|
| 128 |
+
"""
|
| 129 |
+
|
| 130 |
+
def __init__(
|
| 131 |
+
self,
|
| 132 |
+
artefact_path: Path,
|
| 133 |
+
score_threshold: float = DEFAULT_SCORE_THRESHOLD,
|
| 134 |
+
iou_threshold: float = DEFAULT_IOU_THRESHOLD,
|
| 135 |
+
) -> None:
|
| 136 |
+
self.artefact_path = Path(artefact_path)
|
| 137 |
+
self.score_threshold = score_threshold
|
| 138 |
+
self.iou_threshold = iou_threshold
|
| 139 |
+
|
| 140 |
+
with _session_lock:
|
| 141 |
+
session = _session(str(self.artefact_path))
|
| 142 |
+
inputs = session.get_inputs()
|
| 143 |
+
if len(inputs) != 1 or list(inputs[0].shape[1:]) != [3, INPUT_SIZE, INPUT_SIZE]:
|
| 144 |
+
raise DetectorError(
|
| 145 |
+
f"{self.artefact_path.name} does not take a single "
|
| 146 |
+
f"3Γ{INPUT_SIZE}Γ{INPUT_SIZE} input, so it is not the artefact "
|
| 147 |
+
f"this adapter was written for."
|
| 148 |
+
)
|
| 149 |
+
self._session = session
|
| 150 |
+
self._input_name = inputs[0].name
|
| 151 |
+
|
| 152 |
+
def detect(self, image: Image.Image) -> list[Detection]:
|
| 153 |
+
width, height = image.size
|
| 154 |
+
blob, ratio = _preprocess(image)
|
| 155 |
+
|
| 156 |
+
raw = self._session.run(None, {self._input_name: blob})[0]
|
| 157 |
+
if raw.ndim != 3 or raw.shape[2] != len(COCO_CLASSES) + 5:
|
| 158 |
+
raise DetectorError(
|
| 159 |
+
f"Expected an anchor-free head emitting {len(COCO_CLASSES) + 5} "
|
| 160 |
+
f"values per prediction, found {raw.shape}."
|
| 161 |
+
)
|
| 162 |
+
|
| 163 |
+
grid, strides = _grid(INPUT_SIZE)
|
| 164 |
+
centres = (raw[..., :2] + grid) * strides
|
| 165 |
+
sizes = np.exp(raw[..., 2:4]) * strides
|
| 166 |
+
# Scores are objectness Γ class probability, which is what makes a
|
| 167 |
+
# confident box of a wrong class score low rather than high.
|
| 168 |
+
scores = raw[..., 4:5] * raw[..., 5:]
|
| 169 |
+
|
| 170 |
+
centres, sizes, scores = centres[0], sizes[0], scores[0]
|
| 171 |
+
class_ids = scores.argmax(1)
|
| 172 |
+
best = scores.max(1)
|
| 173 |
+
|
| 174 |
+
above = best > self.score_threshold
|
| 175 |
+
if not above.any():
|
| 176 |
+
return []
|
| 177 |
+
centres, sizes = centres[above], sizes[above]
|
| 178 |
+
class_ids, best = class_ids[above], best[above]
|
| 179 |
+
|
| 180 |
+
half = sizes / 2.0
|
| 181 |
+
boxes = np.concatenate([centres - half, centres + half], axis=1) / ratio
|
| 182 |
+
boxes[:, 0::2] = boxes[:, 0::2].clip(0, width)
|
| 183 |
+
boxes[:, 1::2] = boxes[:, 1::2].clip(0, height)
|
| 184 |
+
|
| 185 |
+
rows = []
|
| 186 |
+
for class_id in np.unique(class_ids):
|
| 187 |
+
members = np.flatnonzero(class_ids == class_id)
|
| 188 |
+
for local in _nms(boxes[members], best[members], self.iou_threshold):
|
| 189 |
+
index = members[local]
|
| 190 |
+
x1, y1, x2, y2 = (float(v) for v in boxes[index])
|
| 191 |
+
rows.append(
|
| 192 |
+
(COCO_CLASSES[int(class_id)], float(best[index]), (x1, y1, x2, y2))
|
| 193 |
+
)
|
| 194 |
+
return to_detections(rows, (width, height))
|
app/dispositions.py
ADDED
|
@@ -0,0 +1,1178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The evidence behind each capability, and what the product may say because of it.
|
| 2 |
+
|
| 3 |
+
ADR 0021. Every capability in the registry has one of these.
|
| 4 |
+
|
| 5 |
+
**This file used to hand down verdicts. It now hands down evidence.** The
|
| 6 |
+
earlier pass asked whether one model could do each job from one unconstrained
|
| 7 |
+
RGB photograph, recorded honestly what it found, and then wrote four verdicts β
|
| 8 |
+
six of them `not_viable`. The founder's directive rejects that last step and
|
| 9 |
+
keeps the rest. So does this file: the measurements are unchanged, and the
|
| 10 |
+
conclusions drawn from them are not.
|
| 11 |
+
|
| 12 |
+
**Nothing measured was deleted or edited.** `blocker`, `data_needed` and
|
| 13 |
+
`evidence` are carried across word for word, including the figures that argued
|
| 14 |
+
for the old verdict β 58.1% observer agreement on body condition, 8.5% of birds
|
| 15 |
+
found in a commercial broiler house, kappa 0.57 on iOS and 0.38 on Android for
|
| 16 |
+
the same hoof model. Where a verdict changed, `superseded` records what it was
|
| 17 |
+
and why it moved, so the reframing is auditable rather than invisible. Deleting
|
| 18 |
+
those numbers would repeat the original mistake pointing the other way.
|
| 19 |
+
|
| 20 |
+
The two figures the earlier pass corrected in the *previous* registry are also
|
| 21 |
+
still here, in the entries for `cattle_weight` and `cattle_bcs`: sample size was
|
| 22 |
+
never the binding constraint for either, and that finding survives the
|
| 23 |
+
reframing. It is now an argument about the shape of the claim rather than about
|
| 24 |
+
whether to build.
|
| 25 |
+
|
| 26 |
+
## What each field is for
|
| 27 |
+
|
| 28 |
+
Directive Β§37 asks for three things to be kept apart, and this file keeps them
|
| 29 |
+
in three fields:
|
| 30 |
+
|
| 31 |
+
- **Measured accuracy** β what a benchmark demonstrated. It lives in `blocker`
|
| 32 |
+
and `evidence`, with a URL for every number.
|
| 33 |
+
- **Product uncertainty** β what Animap shows a person. `stated_uncertainty`.
|
| 34 |
+
- **Model confidence** β what a model reports about itself. Not here; it comes
|
| 35 |
+
back on the result, per capability run.
|
| 36 |
+
|
| 37 |
+
`group` is directive Β§31's activation order, which is the sequencing the founder
|
| 38 |
+
asked for and not a re-derivation of it.
|
| 39 |
+
"""
|
| 40 |
+
|
| 41 |
+
from __future__ import annotations
|
| 42 |
+
|
| 43 |
+
from dataclasses import dataclass, field
|
| 44 |
+
from enum import Enum
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class ActivationGroup(str, Enum):
|
| 48 |
+
"""Directive Β§31. The order the work should be attempted in.
|
| 49 |
+
|
| 50 |
+
Not a difficulty ranking. Group A is the set that needs little or no custom
|
| 51 |
+
ML work; Group B needs an engineering spike with pretrained parts; Group C
|
| 52 |
+
needs hardware or a fixed installation. No registry capability is in Group C
|
| 53 |
+
β the hardware-gated forms are individual claims, and they are listed in
|
| 54 |
+
`capabilities.REJECTED_CLAIMS` with the accessory that would unlock them.
|
| 55 |
+
"""
|
| 56 |
+
|
| 57 |
+
A_IMPLEMENT_NOW = "a_implement_now"
|
| 58 |
+
B_ENGINEERING_SPIKE = "b_engineering_spike"
|
| 59 |
+
C_HARDWARE_OR_FIXED = "c_hardware_or_fixed"
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
@dataclass(frozen=True)
|
| 63 |
+
class Source:
|
| 64 |
+
claim: str
|
| 65 |
+
url: str
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
@dataclass(frozen=True)
|
| 69 |
+
class Superseded:
|
| 70 |
+
"""The verdict this capability used to carry, and why it moved.
|
| 71 |
+
|
| 72 |
+
Kept so that nobody has to take the reframing on trust. If the directive is
|
| 73 |
+
ever wrong about one of these, this is the field that makes the argument
|
| 74 |
+
recoverable.
|
| 75 |
+
"""
|
| 76 |
+
|
| 77 |
+
verdict: str
|
| 78 |
+
summary: str
|
| 79 |
+
reframed_because: str
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
@dataclass(frozen=True)
|
| 83 |
+
class Disposition:
|
| 84 |
+
capability_key: str
|
| 85 |
+
group: ActivationGroup
|
| 86 |
+
#: One sentence, in the words the founder would use. This is the line that
|
| 87 |
+
#: decides whether the capability appears on a roadmap.
|
| 88 |
+
summary: str
|
| 89 |
+
#: What limits the claim. Not a list of difficulties β the one constraint
|
| 90 |
+
#: that shapes what may be said. Carries the measured figures inline.
|
| 91 |
+
blocker: str
|
| 92 |
+
#: What the product shows a person alongside a result (Β§37). This is the
|
| 93 |
+
#: sentence that turns a measurement into an honest claim, and it is where
|
| 94 |
+
#: an unflattering number does its work rather than where it is buried.
|
| 95 |
+
stated_uncertainty: str
|
| 96 |
+
#: How much labelled data, and of what exactly. `None` when data is not what
|
| 97 |
+
#: is missing β which, after the reframing, is most of them.
|
| 98 |
+
data_needed: str | None = None
|
| 99 |
+
evidence: tuple[Source, ...] = field(default_factory=tuple)
|
| 100 |
+
superseded: Superseded | None = None
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _d(key, group, summary, blocker, stated_uncertainty,
|
| 104 |
+
data_needed=None, evidence=(), superseded=None) -> Disposition:
|
| 105 |
+
return Disposition(
|
| 106 |
+
capability_key=key, group=group, summary=summary, blocker=blocker,
|
| 107 |
+
stated_uncertainty=stated_uncertainty, data_needed=data_needed,
|
| 108 |
+
evidence=tuple(Source(c, u) for c, u in evidence),
|
| 109 |
+
superseded=superseded,
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
_A = ActivationGroup.A_IMPLEMENT_NOW
|
| 114 |
+
_B = ActivationGroup.B_ENGINEERING_SPIKE
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
DISPOSITIONS: dict[str, Disposition] = {
|
| 118 |
+
d.capability_key: d
|
| 119 |
+
for d in [
|
| 120 |
+
# ---- the two that run -------------------------------------------
|
| 121 |
+
_d("cattle_detection", _A,
|
| 122 |
+
"Runs today, and the directive's stack should replace the detector "
|
| 123 |
+
"rather than the capability. SAM 3.1 with a Grounding DINO fallback "
|
| 124 |
+
"(Β§6.1); YOLOX-m is what is wired up, not what this should end at.",
|
| 125 |
+
"Nothing blocks it. What limits it is validation: the only cattle "
|
| 126 |
+
"counting sets with usable licences do not exist, so its accuracy "
|
| 127 |
+
"rests on 29 photographs one non-expert annotator counted. The "
|
| 128 |
+
"Bristol cattle datasets are Non-Commercial and WAID carries no "
|
| 129 |
+
"licence at all, so neither may be used to check it (ADR 0018).",
|
| 130 |
+
"Experimental. MAE 0.62 animals and MAPE 11.1% over 16 published "
|
| 131 |
+
"frames of 18, labelled by one non-expert. Animals visible in one "
|
| 132 |
+
"frame β never the herd.",
|
| 133 |
+
evidence=(
|
| 134 |
+
("Bristol Cows2021, Non-Commercial Government Licence",
|
| 135 |
+
"https://data.bris.ac.uk/data/dataset/4vnrca7qw1642qlwxjadp87h7"),
|
| 136 |
+
("MAE 0.62 animals, MAPE 11.1%, over 16 published frames of 18",
|
| 137 |
+
"evaluation/reports/yolox-m.json"),
|
| 138 |
+
)),
|
| 139 |
+
# Β§31 Group A item 4 names "poultry visible count with CountGD" as
|
| 140 |
+
# implement-immediately. Whole-house controlled counting is Group B and
|
| 141 |
+
# is the same registry key in a stronger form; the key follows the form
|
| 142 |
+
# that ships, and `exact_house_population` carries the rejection.
|
| 143 |
+
_d("poultry_count", _A,
|
| 144 |
+
"Build now as Experimental, on CountGD (Β§6.3). **The 8.5% figure "
|
| 145 |
+
"below measures YOLOX-m, and the directive's reading of it is the "
|
| 146 |
+
"correct one: it proves that model is wrong for the domain, not that "
|
| 147 |
+
"counting is impossible.** CountGD has not been tried.",
|
| 148 |
+
"COCO's `bird` class is wild birds. On PIO's 452 annotated frames "
|
| 149 |
+
"from two commercial broiler houses the detector found nothing at "
|
| 150 |
+
"all in 254 of them, and across the whole split it detected 8.5% of "
|
| 151 |
+
"the birds that were there β a 92% undercount, with a median of "
|
| 152 |
+
"zero. The guard withholds a number on essentially every frame, "
|
| 153 |
+
"which is correct and is also the whole capability failing to "
|
| 154 |
+
"deliver in a commercial house. A poultry-trained detector fixes "
|
| 155 |
+
"this and PIO is the data for it: 1,035 training images, 253,429 "
|
| 156 |
+
"boxes, CC BY 4.0.",
|
| 157 |
+
"Experimental. 'Approximately 327 birds visible', and **the number "
|
| 158 |
+
"to show is Animap's own, not the directive's placeholder**: on the "
|
| 159 |
+
"61-frame evaluation set the shipped detector reaches MAPE 20.7% "
|
| 160 |
+
"with MAE 1.80 and a bias of -1.80 over 15 published poultry frames "
|
| 161 |
+
"of 16, which is a systematic undercount roughly double the 10-20% "
|
| 162 |
+
"band Β§6.3 suggests as an interim. Β§6.3 says to use the measured "
|
| 163 |
+
"benchmark once it exists; it exists. In a commercial house the "
|
| 164 |
+
"guard withholds a number instead, which is the honest output and "
|
| 165 |
+
"not a count. PIO's 452 frames are the set CountGD must be measured "
|
| 166 |
+
"on before any figure here is restated. Of Β§6.3's three quantities "
|
| 167 |
+
"the model may claim two β visible count and unique birds observed "
|
| 168 |
+
"during a scan. **Reconciled flock population is not a vision "
|
| 169 |
+
"claim at all**: reconciling needs the placement count, the "
|
| 170 |
+
"mortality log and the previous scan, none of which is in the "
|
| 171 |
+
"photograph. It is `derived_claims`, the app computes it, and a "
|
| 172 |
+
"watchdog showed why that matters by getting a model to publish "
|
| 173 |
+
"'reconciled_flock_population: 3200' from a partial pan while the "
|
| 174 |
+
"identifier was still in `allowed_claims`.",
|
| 175 |
+
data_needed="None to buy. PIO is published under CC BY 4.0 and is "
|
| 176 |
+
"enough to fine-tune a house detector; the work is a "
|
| 177 |
+
"training run and an Apache-2.0 training stack, not a "
|
| 178 |
+
"labelling budget.",
|
| 179 |
+
evidence=(
|
| 180 |
+
("PIO, CC BY 4.0. The archive holds 1,035 train + 452 val "
|
| 181 |
+
"images and 327,289 boxes; the record describes 1,435 images "
|
| 182 |
+
"from a commercial farm and a prototype house. Unreconciled.",
|
| 183 |
+
"https://doi.org/10.5281/zenodo.16686320"),
|
| 184 |
+
("DFCCNet density map: MAE 12.07 at ~166 birds/frame",
|
| 185 |
+
"https://pmc.ncbi.nlm.nih.gov/articles/PMC10705762/"),
|
| 186 |
+
("Poultry, through the whole service: coverage 0.938, MAE 1.80, "
|
| 187 |
+
"MAPE 20.7%, bias -1.80, over 15 published frames of 16",
|
| 188 |
+
"evaluation/reports/yolox-m.json"),
|
| 189 |
+
),
|
| 190 |
+
superseded=Superseded(
|
| 191 |
+
"buildable_now",
|
| 192 |
+
"Runs today for yard flocks. **In a commercial house it does not "
|
| 193 |
+
"work, and the guard is what stands between that and a wrong "
|
| 194 |
+
"number.**",
|
| 195 |
+
"The verdict was right about YOLOX-m and was being read as a "
|
| 196 |
+
"verdict about counting. Β§6.2: a detector failing on commercial "
|
| 197 |
+
"broiler-house imagery proves the model is wrong for the domain. "
|
| 198 |
+
"Β§40.3 asks for CountGD benchmarked against these same 452 "
|
| 199 |
+
"frames before any capability decision, so the capability moves "
|
| 200 |
+
"to an engineering spike rather than resting on the old result.",
|
| 201 |
+
)),
|
| 202 |
+
|
| 203 |
+
# ---- cattle, per animal -------------------------------------------
|
| 204 |
+
_d("cattle_weight", _B,
|
| 205 |
+
"Build the spike (Β§22). A guided 2-4 second side sweep with camera "
|
| 206 |
+
"pose and optional ARCore depth is not 'an arbitrary single "
|
| 207 |
+
"photograph', which is the only thing the scale argument rules out.",
|
| 208 |
+
"A photograph carries no scale. Every low-error result in the "
|
| 209 |
+
"literature comes from a camera fixed above a race, where the "
|
| 210 |
+
"camera-to-animal distance is constant and the network learns the "
|
| 211 |
+
"scale implicitly. The one study using freehand photos at varying "
|
| 212 |
+
"angles reached 16.8% MAPE β and only by sticking a circular sticker "
|
| 213 |
+
"of known size on every animal and normalising to it. `gates.py` "
|
| 214 |
+
"asks for 7%, which freehand capture does not reach.",
|
| 215 |
+
"Experimental, and the range is allowed to be wide: '350-430 kg'. "
|
| 216 |
+
"The nearest measured comparator is 16.8% MAPE from freehand photos "
|
| 217 |
+
"with metric depth and a size sticker, which on a 400 kg animal is "
|
| 218 |
+
"about +/-67 kg. A scale reading outranks the estimate and is what "
|
| 219 |
+
"verifies it. `gates.py` still asks for 7% before this reaches "
|
| 220 |
+
"`production`, and nothing here claims to have reached it.",
|
| 221 |
+
data_needed="1,200 weighbridge-paired samples is the right order and "
|
| 222 |
+
"the wrong lever. Published learning curve, fixed rig: "
|
| 223 |
+
"58 animals / 211 images gives 8-9% MAPE, 215 / 2,116 "
|
| 224 |
+
"gives 6.2%, 1,201 / 13,357 gives 3.0%. Pretraining on "
|
| 225 |
+
"another farm's herd took a 58-animal farm from 9.3% to "
|
| 226 |
+
"5.6%. So ~200 animals is the useful first target β but "
|
| 227 |
+
"only once a marker board or a phone mount is part of "
|
| 228 |
+
"the capture.",
|
| 229 |
+
evidence=(
|
| 230 |
+
("Freehand photos + metric depth + size sticker: MAPE 16.8%",
|
| 231 |
+
"https://cris.unibo.it/bitstream/11585/1027956/1/1-s2.0-S2772375525003326-main.pdf"),
|
| 232 |
+
("Fixed RealSense, RGB-only: MAPE 3.79%, MAE 15.9 kg, 1,289 pairs",
|
| 233 |
+
"https://pmc.ncbi.nlm.nih.gov/articles/PMC10971323/"),
|
| 234 |
+
("Learning curve, 58 to 1,201 cows: MAPE 9.3% to 3.0%",
|
| 235 |
+
"https://arxiv.org/pdf/2601.01044"),
|
| 236 |
+
("Zebu heart-girth tape, 703 animals, R^2 0.98 β the incumbent",
|
| 237 |
+
"https://pmc.ncbi.nlm.nih.gov/articles/PMC3552367/"),
|
| 238 |
+
),
|
| 239 |
+
superseded=Superseded(
|
| 240 |
+
"needs_capture_change",
|
| 241 |
+
"Reachable to about 5-8% error with a fixed capture. **Not "
|
| 242 |
+
"reachable at all from a freehand photograph**, at any sample "
|
| 243 |
+
"size.",
|
| 244 |
+
"The capture does change β that part was right, and it is now "
|
| 245 |
+
"the declared protocol rather than a reason to wait. Β§22 says "
|
| 246 |
+
"plainly not to disable cattle weight because no custom weight "
|
| 247 |
+
"model exists, and the 16.8% study is evidence that a marker "
|
| 248 |
+
"plus depth already works badly-but-usefully, which is what an "
|
| 249 |
+
"experimental band is for.",
|
| 250 |
+
)),
|
| 251 |
+
_d("cattle_bcs", _A,
|
| 252 |
+
"Build now as Experimental plus Human Confirmation (Β§7). **The "
|
| 253 |
+
"directive rejects the previous 'not viable' explicitly, and it is "
|
| 254 |
+
"right:** 58.1% observer agreement is an argument for a broader band "
|
| 255 |
+
"and a trend line, not for removing the feature.",
|
| 256 |
+
"The label is the ceiling. Four trained observers scoring 225 cows "
|
| 257 |
+
"agreed exactly 58.1% of the time, and practising vets span kappa "
|
| 258 |
+
"0.22 to 0.78. The gate asks for QWK 0.70 against a consensus, which "
|
| 259 |
+
"is roughly what two humans manage with each other β so the model "
|
| 260 |
+
"would be measured against a ruler with the same error it is being "
|
| 261 |
+
"asked to beat. Worse for Nigeria: the published 5- and 9-point "
|
| 262 |
+
"grids are temperate-breed instruments, the scale stops resolving "
|
| 263 |
+
"below 2.5, and the one public dataset contains no animal thinner "
|
| 264 |
+
"than 3.25.",
|
| 265 |
+
"Experimental. A half-point band β 'Body condition: 2.5-3.0' β with "
|
| 266 |
+
"'Looks right / Correct score / Retake', and the previous band shown "
|
| 267 |
+
"beside it so the trend carries the meaning. The band is half a point "
|
| 268 |
+
"wide because four trained observers agree exactly 58.1% of the time; "
|
| 269 |
+
"a point score would claim resolution the ground truth does not have. "
|
| 270 |
+
"**Never 'BCS 2.63'.** Below 2.5 the published grids stop resolving "
|
| 271 |
+
"and the public dataset holds no animal thinner than 3.25, so a thin "
|
| 272 |
+
"animal is reported as thin rather than scored.",
|
| 273 |
+
data_needed="2,000 samples is the right order for the coarse-band "
|
| 274 |
+
"version β published work reaches 81% on five bands from "
|
| 275 |
+
"1,270 phone images. It does nothing for the half-point "
|
| 276 |
+
"version, because more of a noisy label is still a noisy "
|
| 277 |
+
"label.",
|
| 278 |
+
evidence=(
|
| 279 |
+
("Ferguson 1994: 58.1% exact agreement, 4 observers, n=225",
|
| 280 |
+
"https://www.journalofdairyscience.org/article/S0022-0302(94)77212-X/fulltext"),
|
| 281 |
+
("DeLaval camera over-estimated 44% of cows below BCS 3.0, n=343",
|
| 282 |
+
"https://pmc.ncbi.nlm.nih.gov/articles/PMC6616514/"),
|
| 283 |
+
("Depth camera, 53 cows, 4 coarse groups: 70%, and 0% on one class",
|
| 284 |
+
"https://academic.oup.com/tas/article/7/1/txad085/7230216"),
|
| 285 |
+
("Tropical breeds need a different grid entirely",
|
| 286 |
+
"https://doi.org/10.1007/s11250-025-04328-4"),
|
| 287 |
+
),
|
| 288 |
+
superseded=Superseded(
|
| 289 |
+
"not_viable",
|
| 290 |
+
"Not viable as a 1-5 score. A coarse thin / fit / fat band is "
|
| 291 |
+
"defensible; half-point resolution is not.",
|
| 292 |
+
"The measurement was right and the conclusion inverted the "
|
| 293 |
+
"product decision. Β§7: 'The previous conclusion that BCS is not "
|
| 294 |
+
"viable is rejected. Human disagreement means the system should "
|
| 295 |
+
"use broader uncertainty and emphasize trend.' The old sentence "
|
| 296 |
+
"already contained the answer β a band is defensible β and then "
|
| 297 |
+
"removed the capability anyway. The half-point *point score* "
|
| 298 |
+
"stays rejected, as `bcs_point_score`.",
|
| 299 |
+
)),
|
| 300 |
+
_d("cattle_identity", _A,
|
| 301 |
+
"Build immediately, high priority (Β§6.4). Frozen MegaDescriptor and "
|
| 302 |
+
"DINOv3 embeddings with nearest-neighbour retrieval, benchmarked "
|
| 303 |
+
"against each other. Do not wait for a custom muzzle model.",
|
| 304 |
+
"Nothing blocks the muzzle route: two CC BY 4.0 datasets are public "
|
| 305 |
+
"(268 and 459 animals) and four or five images per animal reaches "
|
| 306 |
+
">90% closed-set accuracy. Two things need saying anyway. Accuracy "
|
| 307 |
+
"falls from 96% to 72% when a fifth of the animals presented are not "
|
| 308 |
+
"enrolled, which is the only regime a pastoralist herd is ever in. "
|
| 309 |
+
"And coat-pattern re-identification β the method with the best public "
|
| 310 |
+
"data β works on Holstein markings and has nothing to key on with "
|
| 311 |
+
"White Fulani, Sokoto Gudali or N'Dama.",
|
| 312 |
+
"Experimental. 'This looks like Kofi', with Confirm / Not Kofi / "
|
| 313 |
+
"Choose another animal / Register new animal. The open-set number is "
|
| 314 |
+
"why confirmation is mandatory rather than a courtesy: closed-set "
|
| 315 |
+
"accuracy of 96.3% falls to 72.5% once a fifth of the animals shown "
|
| 316 |
+
"are not enrolled, and a working herd is always in that regime. An "
|
| 317 |
+
"unconfirmed match is never written as an identity.",
|
| 318 |
+
data_needed="4-5 muzzle photographs per animal at enrolment. No "
|
| 319 |
+
"external labelling budget; the enrolment is the label.",
|
| 320 |
+
evidence=(
|
| 321 |
+
("Beef cattle muzzle, 4,923 images, 268 animals, CC BY 4.0",
|
| 322 |
+
"https://zenodo.org/records/6324361"),
|
| 323 |
+
("Over four images per animal gives >90% accuracy",
|
| 324 |
+
"https://pmc.ncbi.nlm.nih.gov/articles/PMC9179917/"),
|
| 325 |
+
("Open-set drops 96.3% to 72.5% with 20% unseen animals",
|
| 326 |
+
"https://doi.org/10.3168/jds.2024-26069"),
|
| 327 |
+
)),
|
| 328 |
+
# ---- added at the coordinator's instruction, Β§31 Group A ------------
|
| 329 |
+
# These four sections named capabilities the registry never held. Their
|
| 330 |
+
# absence was a gap rather than a decision, and it had a second cost:
|
| 331 |
+
# three claims the directive rejects had no capability to attach to.
|
| 332 |
+
_d("cattle_breed", _A,
|
| 333 |
+
"Build now as Human Confirmation (Β§6.5). A hosted multimodal model "
|
| 334 |
+
"plus DINOv3 reference retrieval, and a crossbred animal is allowed "
|
| 335 |
+
"to stay crossbred.",
|
| 336 |
+
"No measurement was made for this capability, by the earlier pass or "
|
| 337 |
+
"by this one, and none should be implied. What is known is "
|
| 338 |
+
"structural: Nigerian herds are substantially crossbred, the "
|
| 339 |
+
"reference photographs that retrieval would key on are the same "
|
| 340 |
+
"temperate-breed-heavy sets that made `cattle_bcs` unreliable below "
|
| 341 |
+
"2.5, and the phenotypes that matter here β White Fulani, Sokoto "
|
| 342 |
+
"Gudali, N'Dama β are the ones with the least public imagery. The "
|
| 343 |
+
"constraint is therefore the claim rather than the model: a breed "
|
| 344 |
+
"named confidently on a crossbred animal is wrong in a way the user "
|
| 345 |
+
"cannot easily correct, because it looks like knowledge.",
|
| 346 |
+
"Experimental. 'Likely White Fulani', or 'White-Fulani-like "
|
| 347 |
+
"phenotype' where the animal does not sit cleanly in one breed, and "
|
| 348 |
+
"the user confirms either way. **There is no accuracy figure for "
|
| 349 |
+
"this and the product must not imply one.** `crossbred_or_uncertain` "
|
| 350 |
+
"is a first-class answer rather than a failure, per Β§6.5's "
|
| 351 |
+
"instruction not to force a breed.",
|
| 352 |
+
evidence=()),
|
| 353 |
+
_d("cattle_sex", _A,
|
| 354 |
+
"Build now as Human Confirmation (Β§6.6), and never let it hold up "
|
| 355 |
+
"registering an animal.",
|
| 356 |
+
"No measurement was made and none is claimed. The visible cue is "
|
| 357 |
+
"external genitalia and, in some breeds, conformation and horn "
|
| 358 |
+
"shape, none of which is reliably in frame in a side photograph "
|
| 359 |
+
"taken at working distance β which is why Β§6.6 asks for confirmation "
|
| 360 |
+
"rather than a decision, and why the honest failure mode is "
|
| 361 |
+
"'not determinable from this view' rather than a guess. The binding "
|
| 362 |
+
"constraint is a workflow one: an animal that cannot be registered "
|
| 363 |
+
"because a model would not commit is a worse outcome than an "
|
| 364 |
+
"unrecorded sex.",
|
| 365 |
+
"Experimental. 'Likely male', confirmed by the user. **It never "
|
| 366 |
+
"blocks registration** β declared as `never_blocks` on the "
|
| 367 |
+
"acquisition protocol rather than left to a screen β and "
|
| 368 |
+
"'not determinable from this view' is an allowed answer.",
|
| 369 |
+
evidence=()),
|
| 370 |
+
_d("cattle_feces", _A,
|
| 371 |
+
"Build now as Experimental (Β§12). SAM 3.1 segmentation, multimodal "
|
| 372 |
+
"visual reasoning and DINOv3 reference retrieval, reporting "
|
| 373 |
+
"appearance only.",
|
| 374 |
+
"The poultry evidence does not transfer, and that is the whole "
|
| 375 |
+
"point. The two CC BY 4.0 dropping datasets under `poultry_fecal` "
|
| 376 |
+
"are chicken droppings photographed in Tanzania; no comparable "
|
| 377 |
+
"public set of cattle feces was found by the earlier pass or this "
|
| 378 |
+
"one. So there is no retrieval index to build against yet and no "
|
| 379 |
+
"accuracy figure to quote. What is defensible without one is the "
|
| 380 |
+
"list Β§12 gives β normal, loose, watery, visible blood, visible "
|
| 381 |
+
"mucus, unusual colour β because each is an appearance a photograph "
|
| 382 |
+
"carries, and blood and mucus are worth surfacing on their own "
|
| 383 |
+
"whatever caused them.",
|
| 384 |
+
"Experimental, and appearance only: normal / loose / watery / "
|
| 385 |
+
"visible blood / visible mucus / unusual colour. **No accuracy "
|
| 386 |
+
"figure exists for cattle feces and none is implied.** Β§12 is "
|
| 387 |
+
"explicit that a strong disease claim must not be made from cattle "
|
| 388 |
+
"feces alone, so blood or mucus escalates to a vet rather than "
|
| 389 |
+
"naming a parasite or an infection.",
|
| 390 |
+
evidence=()),
|
| 391 |
+
_d("cattle_age_dentition", _A,
|
| 392 |
+
"Build now as an age *band* (Β§8). The teeth carry a band and the old "
|
| 393 |
+
"entry proved it β five states across five years is exactly what an "
|
| 394 |
+
"age band is made of.",
|
| 395 |
+
"Incisor eruption resolves five states across five years and nothing "
|
| 396 |
+
"finer, with 50% transition points around 23, 30, 37 and 42 months β "
|
| 397 |
+
"and males erupt about three weeks earlier, dairy types two to three "
|
| 398 |
+
"months earlier, and Bos indicus crosses keep their incisors longer. "
|
| 399 |
+
"Past five years you are reading wear, which is diet-confounded. A "
|
| 400 |
+
"published claim of 0.06 years RMSE from a tooth photograph is not "
|
| 401 |
+
"physically achievable and should be read as label leakage. It also "
|
| 402 |
+
"needs the animal restrained and its mouth opened.",
|
| 403 |
+
"Experimental. 'Estimated age: 3-4 years. Four permanent incisors "
|
| 404 |
+
"visible. Medium confidence.' The band is a year wide below five "
|
| 405 |
+
"years because the 50% eruption points sit around 23, 30, 37 and 42 "
|
| 406 |
+
"months and shift with sex and breed type; past five years the signal "
|
| 407 |
+
"is wear, which is diet-confounded, so the band widens to 'over five "
|
| 408 |
+
"years' rather than narrowing. A known birth date always wins. "
|
| 409 |
+
"**Never a month.**",
|
| 410 |
+
evidence=(
|
| 411 |
+
("Whiting et al., ~60,000 cattle: eruption timing and its spread",
|
| 412 |
+
"https://doi.org/10.1017/S1751731112001656"),
|
| 413 |
+
("Sheep analogue, the honest benchmark: 540 photos, 3 classes, 96.9%",
|
| 414 |
+
"https://doi.org/10.1080/09540091.2025.2506456"),
|
| 415 |
+
),
|
| 416 |
+
superseded=Superseded(
|
| 417 |
+
"not_viable",
|
| 418 |
+
"Not viable as an age. The teeth do not carry one.",
|
| 419 |
+
"True of a chronological age and false of the capability. Β§8: "
|
| 420 |
+
"'Do not attempt exact chronological age. Age from dentition "
|
| 421 |
+
"should be an age-band feature.' The five resolvable states and "
|
| 422 |
+
"their transition months are the rule table Β§8 asks for, so the "
|
| 423 |
+
"old blocker is now the specification. `exact_age_from_teeth` "
|
| 424 |
+
"stays rejected.",
|
| 425 |
+
)),
|
| 426 |
+
_d("cattle_gait", _B,
|
| 427 |
+
"Build the spike (Β§24). DeepLabCut SuperAnimal-Quadruped is zero-shot "
|
| 428 |
+
"on quadruped pose, and a 5-10 metre side-on walk is a capture "
|
| 429 |
+
"protocol a person can follow without a lane.",
|
| 430 |
+
"Every published method films one animal at a time walking single "
|
| 431 |
+
"file past a camera on a fixed mount β 2 m up and 4.5 m back, or 6 m "
|
| 432 |
+
"back down a 44 m walkway. The single-farm accuracies of 95-99% do "
|
| 433 |
+
"not survive contact with a second farm: two at-scale validations of "
|
| 434 |
+
"a commercial system across 3 and 7 farms reached kappa 0.23-0.41, "
|
| 435 |
+
"with 40% sensitivity against painful-lesion ground truth. Human "
|
| 436 |
+
"observers reach kappa 0.28-0.84 with each other.",
|
| 437 |
+
"Experimental, and screening only: 'Possible gait asymmetry.' The "
|
| 438 |
+
"cross-farm numbers are why there is no score β a commercial system "
|
| 439 |
+
"validated across 3 and 7 farms reached kappa 0.23-0.41 at 40% "
|
| 440 |
+
"sensitivity, and human observers only reach kappa 0.28-0.84 with "
|
| 441 |
+
"each other. A missed asymmetry is expected at that sensitivity and "
|
| 442 |
+
"the wording must not imply a clear result. **Never 'lameness score "
|
| 443 |
+
"3 caused by left rear hoof disease'.**",
|
| 444 |
+
evidence=(
|
| 445 |
+
("Fixed ZED camera, 2 m up, 4.5 m from the passageway",
|
| 446 |
+
"https://arxiv.org/pdf/2401.05202"),
|
| 447 |
+
("244 articles, 25 scoring systems: inter-rater kappa 0.28-0.84",
|
| 448 |
+
"https://doi.org/10.1016/j.prevetmed.2014.06.006"),
|
| 449 |
+
),
|
| 450 |
+
superseded=Superseded(
|
| 451 |
+
"needs_capture_change",
|
| 452 |
+
"Needs a walking lane and a fixed side-on camera. Not a phone "
|
| 453 |
+
"feature.",
|
| 454 |
+
"Β§24: 'Do not require a permanently mounted camera in v0.' The "
|
| 455 |
+
"fixed rigs in the literature exist to support a *score*, and "
|
| 456 |
+
"the v0 output is a screen β 'possible gait asymmetry' β which "
|
| 457 |
+
"the cross-farm kappa figures argue for rather than against. The "
|
| 458 |
+
"fixed-camera version remains the stronger form.",
|
| 459 |
+
)),
|
| 460 |
+
_d("cattle_ticks", _B,
|
| 461 |
+
"Build the spike (Β§25). Guided close-ups of four regions with phone "
|
| 462 |
+
"zoom, tiled high-resolution inference, SAM 3.1 exemplar prompting "
|
| 463 |
+
"and a multimodal verifier β none of which the thermal study used.",
|
| 464 |
+
"Every tick computer-vision paper is laboratory work on detached "
|
| 465 |
+
"material β eggs in a dish, larval mortality in a container. The best "
|
| 466 |
+
"attempt on live animals used thermal imaging and correlated with "
|
| 467 |
+
"manual counts at 0.62 on the hind end and 0.29 on the neck, because "
|
| 468 |
+
"ticks could not be told apart from hair. The standard phenotype "
|
| 469 |
+
"counts only adult females of 4.5 mm or more on one whole side of the "
|
| 470 |
+
"animal, and the sites that matter β dewlap, escutcheon, udder, inner "
|
| 471 |
+
"thigh β are folded, shaded, and not in any photograph a farmer takes.",
|
| 472 |
+
"Experimental. '17 probable ticks visible across sampled regions', "
|
| 473 |
+
"with every detection shown for confirmation or removal, plus a "
|
| 474 |
+
"none / low / moderate / high band. This is **sampled burden across "
|
| 475 |
+
"four regions, not a total-body count** β the standard phenotype "
|
| 476 |
+
"counts adult females of 4.5 mm or more over one whole side, and the "
|
| 477 |
+
"folded sites are not photographable.\n\n"
|
| 478 |
+
"**There is an accuracy figure now, and it supports a confirmation "
|
| 479 |
+
"queue and nothing else.** An earlier version of this paragraph said "
|
| 480 |
+
"no figure existed and the product must not imply one; the first half "
|
| 481 |
+
"has stopped being true and the second half is why the first half "
|
| 482 |
+
"mattered. On 24 composite frames holding 322 real ticks pasted onto "
|
| 483 |
+
"real Nigerian and East African cattle, scored with an exemplar bank "
|
| 484 |
+
"sharing no photograph with the pasted ticks: **precision 0.798, "
|
| 485 |
+
"recall 0.258**, F1 0.390, count MAE 9.2, bias -9.08. Four detections "
|
| 486 |
+
"in five are ticks and roughly three ticks in four are missed, so the "
|
| 487 |
+
"number under-reports β the safer direction for a screen a farmer "
|
| 488 |
+
"confirms, and still not a number to subtract from. **The holdout is "
|
| 489 |
+
"the load-bearing word**: with a bank that had seen the paste sources "
|
| 490 |
+
"recall reads 0.578, and more than half of that was near-duplicate "
|
| 491 |
+
"retrieval of pixels the method had been shown in advance. On twelve "
|
| 492 |
+
"real photographs it lands inside the annotator's interval on **0 of "
|
| 493 |
+
"12, and 0 of 5 cattle**, returning zero detections on a neck holding "
|
| 494 |
+
"70-160 ticks. Composite figures are an upper bound: the ticks are "
|
| 495 |
+
"copies of two specimens, none of them overlap, and all of them were "
|
| 496 |
+
"pasted on smooth pixels inside a detected cow.",
|
| 497 |
+
evidence=(
|
| 498 |
+
("Thermal on live cattle: r 0.619 hind end, 0.285 neck",
|
| 499 |
+
"https://www.embrapa.br/busca-de-publicacoes/-/publicacao/1069406/"),
|
| 500 |
+
("The standard count excludes anything under 4.5 mm",
|
| 501 |
+
"https://doi.org/10.3389/fimmu.2021.620847"),
|
| 502 |
+
("Holdout exemplar bank, 24 composite frames, 322 real ticks: "
|
| 503 |
+
"precision 0.798, recall 0.258, F1 0.390, count MAE 9.2. Inside "
|
| 504 |
+
"the annotator's interval on 0 of 12 field photographs and 0 of "
|
| 505 |
+
"5 cattle",
|
| 506 |
+
"experiments/cattle_ticks/metrics.json"),
|
| 507 |
+
),
|
| 508 |
+
superseded=Superseded(
|
| 509 |
+
"not_viable",
|
| 510 |
+
"Do not build. There is no prior art on live animals and the "
|
| 511 |
+
"geometry is against it.",
|
| 512 |
+
"Β§36 sets the bar for 'do not build': the observable signal must "
|
| 513 |
+
"be shown absent or impractical, and a failed first model is not "
|
| 514 |
+
"sufficient. One thermal study at r 0.62 and 0.29 is one model, "
|
| 515 |
+
"not the signal. Β§25 is explicit β 'Do not remove' β and the "
|
| 516 |
+
"geometry objection is answered by guiding the capture to the "
|
| 517 |
+
"four regions instead of hoping they appear. The total-body "
|
| 518 |
+
"count stays rejected as `total_body_tick_count`.",
|
| 519 |
+
)),
|
| 520 |
+
_d("cattle_wound", _A,
|
| 521 |
+
"Build now as Experimental (Β§9). SAM 3.1 for the mask, a multimodal "
|
| 522 |
+
"model for the observations, and OpenCV with a reference marker for "
|
| 523 |
+
"an area when one is in frame.",
|
| 524 |
+
"There is no public wound dataset. What exists is the lumpy-skin "
|
| 525 |
+
"image set, and it is not a wound set. Anything shipped here must "
|
| 526 |
+
"refuse to name a condition, for the reason recorded under "
|
| 527 |
+
"`cattle_skin`.",
|
| 528 |
+
"Experimental, and descriptive: 'Open wound visible. Moderate "
|
| 529 |
+
"surrounding swelling. No obvious visible discharge.' With a "
|
| 530 |
+
"reference marker in frame, 'Approximate visible area: 12-16 cmΒ²'. "
|
| 531 |
+
"There is no public wound dataset, so there is no accuracy figure and "
|
| 532 |
+
"the product must not imply one β the value is the follow-up "
|
| 533 |
+
"comparison, not the first reading. **Never a cause.**",
|
| 534 |
+
data_needed="No public dataset found. A triage classifier needs on "
|
| 535 |
+
"the order of 1,000 photographs of intact and injured "
|
| 536 |
+
"skin from working Nigerian herds, labelled by a "
|
| 537 |
+
"veterinarian as 'needs attention' or not β a binary "
|
| 538 |
+
"referral label, not a diagnosis.",
|
| 539 |
+
evidence=(),
|
| 540 |
+
superseded=Superseded(
|
| 541 |
+
"needs_labelled_data",
|
| 542 |
+
"Buildable as triage β 'something here needs a vet' β and only "
|
| 543 |
+
"that.",
|
| 544 |
+
"The verdict was nearly right and priced a classifier nobody "
|
| 545 |
+
"needs yet. Β§9 asks for visible observations from a multimodal "
|
| 546 |
+
"model plus a SAM mask, which needs no training set at all; the "
|
| 547 |
+
"1,000-photograph budget is what a *trained* triage classifier "
|
| 548 |
+
"would cost later.",
|
| 549 |
+
)),
|
| 550 |
+
_d("cattle_skin", _A,
|
| 551 |
+
"Build now as visual screening with mandatory escalation (Β§10). The "
|
| 552 |
+
"legal exposure changes the wording and the workflow. It does not "
|
| 553 |
+
"change whether a nodule is visible in a photograph.",
|
| 554 |
+
"Lumpy skin disease and foot-and-mouth are notifiable. Nigeria's "
|
| 555 |
+
"Animal Diseases (Control) Act 1988 s.8(1) obliges the person in "
|
| 556 |
+
"charge of an animal *suspected* to be infected to give notice and "
|
| 557 |
+
"isolate it, and s.8(4) lets a veterinary officer order slaughter; "
|
| 558 |
+
"the First Schedule lists FMD, lumpy skin disease **and "
|
| 559 |
+
"streptothricosis**, which is the commonest look-alike in West "
|
| 560 |
+
"Africa. So a false positive creates a legal duty that can end with a "
|
| 561 |
+
"healthy animal destroyed. The evidence underneath the published "
|
| 562 |
+
"models does not support that risk: every high accuracy traces to one "
|
| 563 |
+
"Mendeley set of 324 lumpy and 700 normal images with no stated "
|
| 564 |
+
"collection method and no veterinary or PCR confirmation, and the "
|
| 565 |
+
"same model drops from 96% to 85% as soon as the negatives include "
|
| 566 |
+
"other skin diseases. **The statute should be confirmed by counsel "
|
| 567 |
+
"before anyone relies on this paragraph.**",
|
| 568 |
+
"Experimental, described not named: 'Multiple raised nodular lesions "
|
| 569 |
+
"visible. Abnormal skin pattern. Veterinary review recommended.' The "
|
| 570 |
+
"96%-to-85% drop once other skin diseases enter the negatives is why "
|
| 571 |
+
"no disease is named β streptothricosis is the commonest West African "
|
| 572 |
+
"look-alike and is on the same notifiable schedule. **Never 'Lumpy "
|
| 573 |
+
"skin disease confirmed'**, because under s.8(1) a suspicion creates "
|
| 574 |
+
"a legal duty and under s.8(4) that can end with a healthy animal "
|
| 575 |
+
"destroyed.",
|
| 576 |
+
evidence=(
|
| 577 |
+
("The whole literature's dataset: 324 lumpy, 700 normal, no provenance",
|
| 578 |
+
"https://data.mendeley.com/datasets/w36hpf86j2/1"),
|
| 579 |
+
("96% falls to 85.45% once other skin diseases are in the negatives",
|
| 580 |
+
"https://pmc.ncbi.nlm.nih.gov/articles/PMC11512320/"),
|
| 581 |
+
),
|
| 582 |
+
superseded=Superseded(
|
| 583 |
+
"not_viable",
|
| 584 |
+
"The model is easy and the claim is the liability. Not viable as "
|
| 585 |
+
"anything that names a disease.",
|
| 586 |
+
"The two halves of that sentence point opposite ways and the "
|
| 587 |
+
"second one won. Β§10: 'Do not remove this feature because certain "
|
| 588 |
+
"skin diseases are legally important. The legal/regulatory issue "
|
| 589 |
+
"changes the workflow and wording, not the computer-vision "
|
| 590 |
+
"feasibility.' The naming stays rejected β `lsd_diagnosis` and "
|
| 591 |
+
"`fmd_diagnosis` β and the escalation workflow is now part of the "
|
| 592 |
+
"acquisition protocol.",
|
| 593 |
+
)),
|
| 594 |
+
_d("cattle_hoof", _A,
|
| 595 |
+
"Build now with guided presentation (Β§11). 'Lift and clean the hoof "
|
| 596 |
+
"before photographing' is a capture instruction, and the sole not "
|
| 597 |
+
"being visible while the animal stands is what it exists to fix.",
|
| 598 |
+
"Sole ulcer and white line disease are on the sole, invisible while "
|
| 599 |
+
"the animal is standing, and no public dataset covers them at all. "
|
| 600 |
+
"Digital dermatitis is photographable, and the honest number for it "
|
| 601 |
+
"is a 2024 field trial: mAP 0.95 offline, then kappa 0.57 on iOS and "
|
| 602 |
+
"0.38 on Android with the same model β the phone changed the result "
|
| 603 |
+
"more than the lesion did. Capture protocol is a restrained animal, "
|
| 604 |
+
"feet sprayed with water, camera 35 cm perpendicular.",
|
| 605 |
+
"Experimental. Visible crack, lesion, swelling, erosion, overgrowth, "
|
| 606 |
+
"or normal β never a named condition. The field numbers are the "
|
| 607 |
+
"reason: the same model that scored mAP 0.95 offline reached kappa "
|
| 608 |
+
"0.57 on iOS and 0.38 on Android, so the device changes the answer "
|
| 609 |
+
"more than the lesion does and the result must read as an observation "
|
| 610 |
+
"a vet acts on. Sole ulcer and white line disease have no public "
|
| 611 |
+
"dataset at all and are not claimed.",
|
| 612 |
+
evidence=(
|
| 613 |
+
("mAP 0.95 offline, kappa 0.57 / 0.38 in the field on two phones",
|
| 614 |
+
"https://pmc.ncbi.nlm.nih.gov/articles/PMC11829201/"),
|
| 615 |
+
),
|
| 616 |
+
superseded=Superseded(
|
| 617 |
+
"needs_capture_change",
|
| 618 |
+
"Needs the foot lifted, washed and held. Not reachable on a herd "
|
| 619 |
+
"with no crush.",
|
| 620 |
+
"Β§11: 'The fact that the sole is not visible while the animal is "
|
| 621 |
+
"standing is a capture requirement, not a reason to remove the "
|
| 622 |
+
"capability.' The lifting and washing are now `reject_if` "
|
| 623 |
+
"conditions and a capture prompt. A herd with no crush cannot "
|
| 624 |
+
"use this, which is a coverage limit rather than a feasibility "
|
| 625 |
+
"one.",
|
| 626 |
+
)),
|
| 627 |
+
_d("cattle_respiratory", _A,
|
| 628 |
+
"Build now as a respiratory *rate*, from video (Β§14). This is "
|
| 629 |
+
"segmentation, optical flow and an FFT β signal processing, with no "
|
| 630 |
+
"training and no audio. **The capability was re-scoped:** the cough "
|
| 631 |
+
"evidence below belongs to continuous surveillance, which Β§27 makes a "
|
| 632 |
+
"separate, fixed-microphone capability.",
|
| 633 |
+
"The disease signal is a multi-day rise in cough rate against that "
|
| 634 |
+
"house's own baseline, read off a trend line. A single recording "
|
| 635 |
+
"cannot produce it. The best published cattle result β 62 calves, "
|
| 636 |
+
"205 minutes of labelled audio, 385 coughs β is 50.3% sensitive at "
|
| 637 |
+
"99.2% specificity, and the authors say plainly that algorithms do "
|
| 638 |
+
"not transfer between set-ups. Room acoustics alone swing precision "
|
| 639 |
+
"from over 80% to 54% between compartments of one building.",
|
| 640 |
+
"Experimental. 'Estimated respiratory rate: 44-50 breaths/min', with "
|
| 641 |
+
"the capture quality shown beside it. A range rather than a number "
|
| 642 |
+
"because the measurement is a peak-detection over a 30-60 second "
|
| 643 |
+
"window and both the animal and the camera move. **This is a spot "
|
| 644 |
+
"measurement and says so.** The 50.3%-sensitive cough figure applies "
|
| 645 |
+
"to the continuous form and is not this capability's number.",
|
| 646 |
+
data_needed="No public event-labelled cattle cough dataset exists. "
|
| 647 |
+
"Building one means continuous audio from a fixed "
|
| 648 |
+
"microphone with every cough time-stamped by a person: "
|
| 649 |
+
"the reference study needed 205 minutes to collect 385 "
|
| 650 |
+
"events.",
|
| 651 |
+
evidence=(
|
| 652 |
+
("62 calves, 385 labelled coughs: SE 50.3%, SP 99.2%",
|
| 653 |
+
"https://t-stor.teagasc.ie/handle/11019/1751"),
|
| 654 |
+
),
|
| 655 |
+
superseded=Superseded(
|
| 656 |
+
"needs_capture_change",
|
| 657 |
+
"Needs a microphone left in the house, not thirty seconds from a "
|
| 658 |
+
"phone.",
|
| 659 |
+
"True, and about a different capability. Β§14 asks for "
|
| 660 |
+
"respiratory rate from 30-60 seconds of flank video; Β§27 keeps "
|
| 661 |
+
"continuous cough monitoring as a separate fixed-microphone mode "
|
| 662 |
+
"and says the requirement for it must not block phone-based spot "
|
| 663 |
+
"screening. The old entry measured the second and cancelled the "
|
| 664 |
+
"first. `continuous_surveillance_from_spot_recording` stays "
|
| 665 |
+
"rejected.",
|
| 666 |
+
)),
|
| 667 |
+
|
| 668 |
+
# ---- poultry -------------------------------------------------------
|
| 669 |
+
_d("poultry_respiratory", _B,
|
| 670 |
+
"Build the spike as a spot screen (Β§26). SAM Audio to pull bird "
|
| 671 |
+
"sound out of fan and machinery noise, then audio embeddings or "
|
| 672 |
+
"hosted multimodal audio reasoning over 30 seconds.",
|
| 673 |
+
"Same structural problem β the signal is a rate against a baseline β "
|
| 674 |
+
"but sneezes are more frequent than cattle coughs and the reference "
|
| 675 |
+
"study reached 66.7% sensitivity at 88.4% precision on 51 chickens. "
|
| 676 |
+
"The published labels are bird-level rather than event-level, which "
|
| 677 |
+
"is what a detector needs.\n\n"
|
| 678 |
+
"**Animap's own detector has now been measured and it is below "
|
| 679 |
+
"chance.** Over 6,346 clips from two CC BY 4.0 commercial-farm "
|
| 680 |
+
"datasets, the shipped spectral-flux path in `app/adapters/audio/` "
|
| 681 |
+
"scores **AUC 0.4141** on sick against healthy, where chance is 0.5. "
|
| 682 |
+
"The direction says why: healthy clips average **6.90 events a "
|
| 683 |
+
"minute** and sick clips **1.16**. A healthy poultry house is a noisy "
|
| 684 |
+
"one β birds move, peck and scratch, and every one of those is a "
|
| 685 |
+
"broadband transient an onset detector is built to find β while a "
|
| 686 |
+
"sick flock is lethargic and quiet. The detector measures activity "
|
| 687 |
+
"and activity runs the wrong way. No threshold repairs it: `ONSET_K` "
|
| 688 |
+
"moves the event rate and the ordering between the classes is what is "
|
| 689 |
+
"wrong. Two caveats travel with the number and neither rescues it β "
|
| 690 |
+
"74% of the sick-by-healthy pairs are ties, so it is a mostly inert "
|
| 691 |
+
"detector rather than a strongly anti-correlated one; and on the 141 "
|
| 692 |
+
"clips long enough to meet Β§26's own 30-second protocol it is 0.3253, "
|
| 693 |
+
"which agrees with the larger result.",
|
| 694 |
+
"**No number.** 'Cough/sneeze-like events detected. Spot respiratory "
|
| 695 |
+
"screen only.' β Β§26's own wording, which carries no digit, and the "
|
| 696 |
+
"registry now agrees: the event count is declared as carrying no "
|
| 697 |
+
"quantity, so a figure has nowhere to be published from. The reason "
|
| 698 |
+
"is Animap's own measurement rather than caution: the only "
|
| 699 |
+
"implementation scores AUC 0.4141 against a chance line of 0.5 and "
|
| 700 |
+
"fires six times more often on healthy flocks than on sick ones, so "
|
| 701 |
+
"the count is not an imprecise reading of the events but an inverted "
|
| 702 |
+
"one, and a farm cannot tell those apart. The external comparator is "
|
| 703 |
+
"unchanged and is the better of the two β 66.7% sensitivity at 88.4% "
|
| 704 |
+
"precision on 51 chickens, roughly a third of events missed β so a "
|
| 705 |
+
"quiet result is not a clear result even where the method works. "
|
| 706 |
+
"**The capability survives, because the signal does**: frozen CLAP "
|
| 707 |
+
"embeddings on the identical clips, on a split where no recording "
|
| 708 |
+
"node appears on both sides, reach 0.5651 against a 0.5182 majority "
|
| 709 |
+
"baseline, and 0.6032 against 0.3621 over three classes. That is a "
|
| 710 |
+
"method failure, not a capability failure, which is exactly the "
|
| 711 |
+
"distinction Β§36 turns on. **Never presented as continuous "
|
| 712 |
+
"surveillance.**",
|
| 713 |
+
data_needed="An event-labelled set: the reference work annotated 763 "
|
| 714 |
+
"sneezes across 480 minutes from 51 birds. Bowen "
|
| 715 |
+
"University's 346-file Nigerian set (139 healthy, 121 "
|
| 716 |
+
"unhealthy, 86 noise, CC BY 4.0) is a starting point but "
|
| 717 |
+
"carries bird-level labels only.",
|
| 718 |
+
evidence=(
|
| 719 |
+
("Nigerian poultry vocalisation set, 346 files, CC BY 4.0",
|
| 720 |
+
"https://data.mendeley.com/datasets/zp4nf2dxbh/1"),
|
| 721 |
+
("51 chickens, 763 sneezes: SE 66.7%, precision 88.4%",
|
| 722 |
+
"https://doi.org/10.1016/j.compag.2018.12.028"),
|
| 723 |
+
),
|
| 724 |
+
superseded=Superseded(
|
| 725 |
+
"needs_labelled_data",
|
| 726 |
+
"Closer than the cattle version, and there is a Nigerian dataset "
|
| 727 |
+
"to start from.",
|
| 728 |
+
"Not overturned β re-sequenced. Β§26 asks for the zero-training "
|
| 729 |
+
"audio stack to be tried before a labelling budget is committed, "
|
| 730 |
+
"so this is a spike now and a data question afterwards. The "
|
| 731 |
+
"event-labelled set is still what a trained detector would need.",
|
| 732 |
+
)),
|
| 733 |
+
_d("poultry_fecal", _A,
|
| 734 |
+
"One of the strongest early capabilities (Β§13). DINOv3 embeddings "
|
| 735 |
+
"over the existing labelled sets, nearest-neighbour retrieval, then a "
|
| 736 |
+
"structured multimodal review β no training run in the path.",
|
| 737 |
+
"Two CC BY 4.0 datasets exist, collected in Tanzania on ordinary "
|
| 738 |
+
"smartphones, one of them PCR-validated. The published 98% is carried "
|
| 739 |
+
"by the three common classes: Newcastle is 376 of 6,812 images, and a "
|
| 740 |
+
"model that never predicts it still scores 94.5%. Reported Newcastle "
|
| 741 |
+
"recall is 62.7%. Newcastle is also notifiable, so the same "
|
| 742 |
+
"constraint as `cattle_skin` applies to what may be said on screen.",
|
| 743 |
+
"Experimental, flock-level, over 4-6 samples: 'Elevated GI-health "
|
| 744 |
+
"risk', not a diagnosis from one dropping. **The published 98% is not "
|
| 745 |
+
"the number to quote** β a model that never predicts Newcastle still "
|
| 746 |
+
"scores 94.5% on that set, and reported Newcastle recall is 62.7%, so "
|
| 747 |
+
"more than a third of Newcastle cases would be missed. Newcastle is "
|
| 748 |
+
"notifiable, so it is named only as a qualified visual pattern and "
|
| 749 |
+
"escalates to a vet. **Coccidiosis may be named as a visual pattern "
|
| 750 |
+
"and Newcastle may not, and the reason is recorded rather than "
|
| 751 |
+
"assumed**: Β§13 names the coccidiosis wording and names nothing "
|
| 752 |
+
"comparable for Newcastle; coccidiosis is one of the three common "
|
| 753 |
+
"classes carrying that set's headline accuracy while Newcastle is "
|
| 754 |
+
"376 of 6,812 at 62.7% recall; and Newcastle is notifiable in "
|
| 755 |
+
"Nigeria while coccidiosis is not. Neither may be *diagnosed* β "
|
| 756 |
+
"`coccidiosis_diagnosis` is forbidden alongside "
|
| 757 |
+
"`newcastle_diagnosis`, so the real line is between an appearance "
|
| 758 |
+
"and a conclusion, not between two diseases.",
|
| 759 |
+
data_needed="Newcastle images specifically. The class is 5.5% of the "
|
| 760 |
+
"largest public set; a usable screen needs it at "
|
| 761 |
+
"something like 20%, which is roughly 1,000 more "
|
| 762 |
+
"PCR-confirmed Newcastle droppings.",
|
| 763 |
+
evidence=(
|
| 764 |
+
("Machuve et al., 6,812 farm-labelled images, CC BY 4.0",
|
| 765 |
+
"https://zenodo.org/records/4628934"),
|
| 766 |
+
("1,255 PCR-validated images, CC BY 4.0",
|
| 767 |
+
"https://zenodo.org/records/5801834"),
|
| 768 |
+
("Nigerian set, 14,618 images, binary labels, CC BY 4.0",
|
| 769 |
+
"https://data.mendeley.com/datasets/8pnbzpt2k9/1"),
|
| 770 |
+
),
|
| 771 |
+
superseded=Superseded(
|
| 772 |
+
"needs_labelled_data",
|
| 773 |
+
"The best-evidenced poultry feature, and its headline accuracy is "
|
| 774 |
+
"not the number to quote.",
|
| 775 |
+
"Both halves survive; only the sequencing moved. Β§13 specifies a "
|
| 776 |
+
"retrieval pipeline over the datasets that already exist, which "
|
| 777 |
+
"needs no new labels to start. The 1,000 PCR-confirmed Newcastle "
|
| 778 |
+
"droppings remain what a Newcastle *claim* would cost, and that "
|
| 779 |
+
"claim stays rejected until then.",
|
| 780 |
+
)),
|
| 781 |
+
_d("poultry_inactive_birds", _A,
|
| 782 |
+
"Build now as Experimental plus Human Confirmation (Β§15). Use time "
|
| 783 |
+
"rather than a single frame: track per-bird motion over a 15-30 "
|
| 784 |
+
"second section scan and surface candidates.",
|
| 785 |
+
"A dead bird and a sleeping bird are the same photograph. The methods "
|
| 786 |
+
"that work add thermal imaging or track the bird over time; the "
|
| 787 |
+
"single-frame result on real houses is mAP@0.5 of 80.1% at 79% "
|
| 788 |
+
"recall from 2,299 RGB-infrared pairs. The 98% paper composited 19 "
|
| 789 |
+
"photographs of dead chickens into 223 empty backgrounds, which "
|
| 790 |
+
"measures compositing.",
|
| 791 |
+
"Experimental. '5 birds need review', each answered Dead / Sick / "
|
| 792 |
+
"Resting / Fine. The honest single-frame comparator is mAP@0.5 of "
|
| 793 |
+
"80.1% at 79% recall on real houses β about one in five missed β so "
|
| 794 |
+
"the output is a review queue and not a mortality count. The 98% "
|
| 795 |
+
"figure in the literature composited 19 photographs into 223 empty "
|
| 796 |
+
"backgrounds and measures compositing; it is not quoted. **Never "
|
| 797 |
+
"dead-versus-sleeping certainty from one image.**",
|
| 798 |
+
data_needed="~2,300 paired frames with roughly 8,000 boxed instances "
|
| 799 |
+
"is what the credible study used. On RGB alone, expect "
|
| 800 |
+
"less.",
|
| 801 |
+
evidence=(
|
| 802 |
+
("2,299 real RGB-IR pairs: mAP@0.5 80.1%, recall 79.0%",
|
| 803 |
+
"https://pmc.ncbi.nlm.nih.gov/articles/PMC13072331/"),
|
| 804 |
+
),
|
| 805 |
+
superseded=Superseded(
|
| 806 |
+
"needs_labelled_data",
|
| 807 |
+
"Buildable, and the honest accuracy is about 80%, not the 98% in "
|
| 808 |
+
"the literature.",
|
| 809 |
+
"Β§15 replaces the classifier with tracking plus an inactivity "
|
| 810 |
+
"threshold, which needs no labelled set: the output is a review "
|
| 811 |
+
"queue, and the farmer's four answers are the labels. The 2,300 "
|
| 812 |
+
"paired frames are what a trained dead-bird detector would need "
|
| 813 |
+
"later.",
|
| 814 |
+
)),
|
| 815 |
+
_d("poultry_weight", _B,
|
| 816 |
+
"Build the spike as a sample workflow (Β§23). Ten to twenty held, "
|
| 817 |
+
"isolated birds with SAM masks and ARCore or VGGT geometry against "
|
| 818 |
+
"published allometric relationships. Not every bird in a flock.",
|
| 819 |
+
"The only figure collected on a commercial flock with a held-out set "
|
| 820 |
+
"of birds is 7.8% mean relative error, from a Kinect fixed above a "
|
| 821 |
+
"house of 48,000. The lab studies reporting R^2 0.98 used 30 birds "
|
| 822 |
+
"photographed 2,520 times, and the same group reported 21.5% MAPE on "
|
| 823 |
+
"the same animals in a second paper. One recent study reaching 7.27% "
|
| 824 |
+
"states in its own text that birds appear in both its training and "
|
| 825 |
+
"test sets.",
|
| 826 |
+
"Experimental, and a sample statistic: **a mean and a range** over "
|
| 827 |
+
"10-20 birds, each estimate shown as a range. The credible "
|
| 828 |
+
"comparator is 7.8% mean relative error from a *fixed overhead* "
|
| 829 |
+
"Kinect; the lab figures of R^2 0.98 come from 30 birds photographed "
|
| 830 |
+
"2,520 times, and the same group reported 21.5% MAPE on those "
|
| 831 |
+
"animals elsewhere, so a handheld phone should be expected nearer the "
|
| 832 |
+
"worse end. A scale reading verifies it. **Never every bird in a "
|
| 833 |
+
"crowded flock from arbitrary video.**\n\n"
|
| 834 |
+
"**Β§23 asks for four statistics and only two of them survive this "
|
| 835 |
+
"error.** Animap's own simulation, at that same 7.8%: the sample "
|
| 836 |
+
"mean is out by 0.33% +/- 3.5 against 0.18% +/- 3.2 with exact "
|
| 837 |
+
"weights, which is no difference worth reporting β measurement error "
|
| 838 |
+
"averages out of a mean. It squares into a variance, so the "
|
| 839 |
+
"coefficient of variation is inflated by +2.40 points at fifteen "
|
| 840 |
+
"birds and the derived uniformity reads 6.96 points low, neither of "
|
| 841 |
+
"which improves with sample size. Both are refused: see "
|
| 842 |
+
"`poultry_uniformity` and ADR 0023. **The range's own inflation was "
|
| 843 |
+
"not measured** β the experiment covered the mean and the CV β so "
|
| 844 |
+
"keeping it is a judgement about the shape of the claim (a band in "
|
| 845 |
+
"kilograms, shown as a band, per Β§38) and not a measurement of it.",
|
| 846 |
+
data_needed="Paired image-and-scale readings under a fixed camera. "
|
| 847 |
+
"The credible reference used ~13,000 annotated frames "
|
| 848 |
+
"against 83 individually weighed birds.",
|
| 849 |
+
evidence=(
|
| 850 |
+
("Commercial house, held-out birds: 7.8% mean relative error",
|
| 851 |
+
"https://doi.org/10.1016/j.compag.2016.02.011"),
|
| 852 |
+
("Broiler weight set, CDLA-Permissive-1.0",
|
| 853 |
+
"https://www.kaggle.com/datasets/lucasheilbuthh/inferring-broiler-chicken-weight"),
|
| 854 |
+
),
|
| 855 |
+
superseded=Superseded(
|
| 856 |
+
"needs_capture_change",
|
| 857 |
+
"Same shape as cattle weight: a fixed overhead camera works, a "
|
| 858 |
+
"handheld phone does not.",
|
| 859 |
+
"Β§23 concedes the hard case and keeps the easy one: 'Do not "
|
| 860 |
+
"attempt weight of every bird in a crowded flock from arbitrary "
|
| 861 |
+
"video. Attempt a representative sample-bird workflow.' A held "
|
| 862 |
+
"bird against a clear background is a different geometry problem "
|
| 863 |
+
"from a bird in a crowd, and the fixed-camera figures do not "
|
| 864 |
+
"bound it.",
|
| 865 |
+
)),
|
| 866 |
+
# **The one capability in this file whose claim was refused rather than
|
| 867 |
+
# corrected**, and the one whose disposition moved on Animap's own
|
| 868 |
+
# measurement instead of on somebody else's paper. ADR 0023.
|
| 869 |
+
_d("poultry_uniformity", _B,
|
| 870 |
+
"**Refused as a vision claim (Β§23, Β§36).** The arithmetic was never "
|
| 871 |
+
"in doubt and is not what failed β the weights underneath it are. "
|
| 872 |
+
"`unsupported_claim`, not `experimental` with an unmet dependency: "
|
| 873 |
+
"the input it was waiting for is the input that makes the answer "
|
| 874 |
+
"wrong, so waiting was never going to end well.",
|
| 875 |
+
"Uniformity is a coefficient of variation over individually weighed "
|
| 876 |
+
"birds, and Aviagen's own protocol says to weigh 1% or 100 birds, "
|
| 877 |
+
"whichever is larger β 65 birds at CV 8%, 140 at CV 12%, for +/-2% "
|
| 878 |
+
"accuracy. That framing survives and is not the blocker. **The "
|
| 879 |
+
"blocker is that a measured weight carries its own error into the "
|
| 880 |
+
"variance.** `experiments/poultry_weight/` simulated 200 draws at "
|
| 881 |
+
"each of six sample sizes and six true CVs, at the best published "
|
| 882 |
+
"per-bird error for this method β 7.8% mean relative error, "
|
| 883 |
+
"Mortensen et al. 2016, from a *fixed* Kinect depth camera over a "
|
| 884 |
+
"commercial house of 48,000, which is a floor on a phone's error and "
|
| 885 |
+
"not an estimate of it. Against a commercial flock's own 11-18% "
|
| 886 |
+
"spread (Vasdal et al. 2019, 45 Ross 308 flocks, mean 13%), a "
|
| 887 |
+
"fifteen-bird sample at a true CV of 12% returns an estimated CV of "
|
| 888 |
+
"14.38% β **a bias of +2.40 points**, within two points of the truth "
|
| 889 |
+
"40.0% of the time. **More birds does not fix it**: +1.91 at ten, "
|
| 890 |
+
"+2.40 at fifteen, +2.24 at twenty, +2.16 at thirty, while the "
|
| 891 |
+
"exact-weight control converges towards zero over the same sweep "
|
| 892 |
+
"(-0.50, -0.37, -0.01, -0.05). The bias is systematic, not sampling "
|
| 893 |
+
"noise.",
|
| 894 |
+
"**Nothing.** No uniformity percentage and no coefficient of "
|
| 895 |
+
"variation may be shown from camera-estimated weights, and the "
|
| 896 |
+
"capability publishes no number at all. Converted through "
|
| 897 |
+
"`2*Phi(10/CV) - 1` the CV inflation reads the flock **6.96 points "
|
| 898 |
+
"low** at a true CV of 12% β a flock at 60% uniformity reports as "
|
| 899 |
+
"53% β and across true CVs of 8-18% at fifteen birds the error runs "
|
| 900 |
+
"-1.46 to -13.94 points, **worst where the flock is most uniform**. "
|
| 901 |
+
"Uniformity is a decision variable: a farm culls, re-feeds or delays "
|
| 902 |
+
"a harvest on it, and every one of those errors runs in the "
|
| 903 |
+
"direction that makes a bad flock look acceptable. This is why the "
|
| 904 |
+
"answer is not a wider band β `experimental` offers visible "
|
| 905 |
+
"uncertainty and a band does not move a centre. **The sample mean "
|
| 906 |
+
"and the range survive on `poultry_weight`**: at the same fifteen "
|
| 907 |
+
"birds and the same error the mean is out by 0.33% +/- 3.5 against "
|
| 908 |
+
"0.18% +/- 3.2 with exact weights, statistically indistinguishable, "
|
| 909 |
+
"because error averages out of a mean and squares into a variance. "
|
| 910 |
+
"**Uniformity from scale weights is exact and is what Β§23's 'no ML "
|
| 911 |
+
"once weights exist' was always true of** β over exact weights the "
|
| 912 |
+
"same arithmetic is unbiased and lands within two points of the true "
|
| 913 |
+
"CV 63.5% of the time at fifteen birds. It is a derived claim the app "
|
| 914 |
+
"computes, not a capability that runs.",
|
| 915 |
+
data_needed="A per-bird error well under 4%, and 'under 4%' is not "
|
| 916 |
+
"the answer a first pass gives. At a true CV of 12% a 4% "
|
| 917 |
+
"error puts the uniformity bias at -0.79 points with "
|
| 918 |
+
"fifteen birds, which looks like enough; at thirty it is "
|
| 919 |
+
"-1.07 and at fifty -1.88, because the small figure at "
|
| 920 |
+
"n=15 is two biases cancelling rather than an error small "
|
| 921 |
+
"enough to ignore. The measurement bias alone is about -3 "
|
| 922 |
+
"points at 4% and about -9 at 7.8%. **Those 4% figures "
|
| 923 |
+
"are the experiment README's working and have no run "
|
| 924 |
+
"record** β `config.yaml` declares no 4% arm, so unlike "
|
| 925 |
+
"every other figure in this entry they cannot be "
|
| 926 |
+
"recomputed from `results/`. Checked independently "
|
| 927 |
+
"against the closed form "
|
| 928 |
+
"`CV_obs = sqrt(CV^2 + e^2 + CV^2 e^2)`, which puts the "
|
| 929 |
+
"4% measurement bias at -2.49 uniformity points rather "
|
| 930 |
+
"than -3: the conclusion holds and the README's figure is "
|
| 931 |
+
"the more pessimistic. Nothing else is missing: not "
|
| 932 |
+
"labels, not compute, not a model.",
|
| 933 |
+
evidence=(
|
| 934 |
+
("Aviagen Ross handbook: weigh 1% or 100 birds, individually",
|
| 935 |
+
"https://aviagen.com/assets/Tech_Center/Ross_Broiler/Aviagen-ROSS-Broiler-Handbook-EN.pdf"),
|
| 936 |
+
("CV bias +2.40 points and uniformity error -6.96 points at true "
|
| 937 |
+
"CV 12%, 15 birds, 7.8% per-bird error; +2.16 at 30 birds",
|
| 938 |
+
"experiments/poultry_weight/metrics.json"),
|
| 939 |
+
("The per-bird error the simulation uses: 7.8% mean relative "
|
| 940 |
+
"error, 83 held-out broilers, fixed Kinect, commercial house",
|
| 941 |
+
"https://doi.org/10.1016/j.compag.2016.02.011"),
|
| 942 |
+
("The flock spread it runs against: uniformity 11-18%, mean 13%, "
|
| 943 |
+
"over 45 Ross 308 flocks",
|
| 944 |
+
"https://doi.org/10.3382/ps/pez252"),
|
| 945 |
+
),
|
| 946 |
+
superseded=Superseded(
|
| 947 |
+
"needs_capture_change",
|
| 948 |
+
"The best-founded poultry claim available, because the manual "
|
| 949 |
+
"method it replaces is already a sample.",
|
| 950 |
+
"**This entry has now moved twice, and the second move reverses "
|
| 951 |
+
"the first.** The original verdict was about `poultry_weight`'s "
|
| 952 |
+
"capture and this entry inherited it. It was then classified "
|
| 953 |
+
"`coming_soon`, which an auditor called the weakest "
|
| 954 |
+
"classification in the registry β rightly, because `state` is a "
|
| 955 |
+
"claim ceiling and availability is `is_runnable`'s to report β "
|
| 956 |
+
"so it became `experimental` with an unmet `depends_on`. Every "
|
| 957 |
+
"step of that was correct reasoning about the wrong question. "
|
| 958 |
+
"**Nobody had asked what the input would be worth when it "
|
| 959 |
+
"arrived**, and the sentence above β 'the manual method it "
|
| 960 |
+
"replaces is already a sample' β is the error in miniature: it is "
|
| 961 |
+
"true, and it compares Animap against the wrong baseline. The "
|
| 962 |
+
"manual method is a sample *weighed on a scale*, and the thing "
|
| 963 |
+
"that separates it from a camera is not the sampling. It is that "
|
| 964 |
+
"one of them has measurement error and the other does not. Β§36 "
|
| 965 |
+
"asks for the observable signal to be shown impractical before a "
|
| 966 |
+
"capability is called unavailable, and error propagation over a "
|
| 967 |
+
"published error floor is that showing β not a failed first "
|
| 968 |
+
"model, of which there is none here.",
|
| 969 |
+
)),
|
| 970 |
+
_d("poultry_footpad", _A,
|
| 971 |
+
"Build now as sampling (Β§18). A multimodal model against an explicit "
|
| 972 |
+
"0-4 rubric plus DINOv3 reference retrieval, over 10-20 held birds. "
|
| 973 |
+
"A slaughter line existing does not make farm sampling pointless.",
|
| 974 |
+
"ChickenCheck runs at 12,000-15,000 birds an hour and reaches kappa "
|
| 975 |
+
"0.70 against the median human scorer, with the humans themselves at "
|
| 976 |
+
"0.33-0.47. On a live bird the only published method stands each hen "
|
| 977 |
+
"on a transparent box over an upward-facing camera. A phone "
|
| 978 |
+
"photograph of a bird standing in litter does not show the footpad.",
|
| 979 |
+
"Experimental. 'Approximate grade: 2 / 4', correctable, aggregated as "
|
| 980 |
+
"'20 birds sampled, 4 moderate or worse, 20%'. The word approximate "
|
| 981 |
+
"is doing real work: the commercial slaughter-line system reaches "
|
| 982 |
+
"kappa 0.70 with a presented, washed foot, and the human scorers it "
|
| 983 |
+
"was measured against only reach 0.33-0.47 with each other. A held "
|
| 984 |
+
"bird photographed on a farm is a harder capture than either, so no "
|
| 985 |
+
"figure is claimed for it. The prevalence is a sample, not the flock.",
|
| 986 |
+
evidence=(
|
| 987 |
+
("Slaughterline system, 500 images / 1,000 feet: kappa 0.70",
|
| 988 |
+
"https://doi.org/10.1016/j.psj.2020.05.052"),
|
| 989 |
+
),
|
| 990 |
+
superseded=Superseded(
|
| 991 |
+
"needs_capture_change",
|
| 992 |
+
"Solved commercially β on a slaughter line, where the foot "
|
| 993 |
+
"arrives presented and washed.",
|
| 994 |
+
"Β§18: 'The existence of slaughter-line automation does not make "
|
| 995 |
+
"farm sampling pointless.' The capture does change β the farmer "
|
| 996 |
+
"holds the bird and photographs the underside of the foot β and "
|
| 997 |
+
"that is now the declared protocol. A farm has no slaughter line "
|
| 998 |
+
"to use instead.",
|
| 999 |
+
)),
|
| 1000 |
+
# Β§19's four, split into four keys because the capture differs even
|
| 1001 |
+
# though the model stack does not. See `app/capabilities.py` for the
|
| 1002 |
+
# argument. They share a disposition shape for the same reason they
|
| 1003 |
+
# share a stack: the evidence question is identical for all four.
|
| 1004 |
+
_d("poultry_hock", _A,
|
| 1005 |
+
"Build now as a visual observation (Β§19). SAM 3.1 for the region, a "
|
| 1006 |
+
"multimodal model for the description, DINOv3 retrieval where "
|
| 1007 |
+
"reference images exist.",
|
| 1008 |
+
"No public hock-lesion dataset was found by the earlier pass or this "
|
| 1009 |
+
"one, and Β§19 asks for no named diagnosis, so there is nothing here "
|
| 1010 |
+
"that a benchmark would currently be measuring. The nearest "
|
| 1011 |
+
"evidenced neighbour is `poultry_footpad`, where a commercial "
|
| 1012 |
+
"slaughter-line system reaches kappa 0.70 against the median human "
|
| 1013 |
+
"scorer while the human scorers themselves reach only 0.33-0.47 β "
|
| 1014 |
+
"which is the honest prior for how well any lesion grade is agreed "
|
| 1015 |
+
"on, before a phone and a live bird are added.",
|
| 1016 |
+
"Experimental. 'Moderate visible hock lesion', for the bird in "
|
| 1017 |
+
"frame. **No accuracy figure exists and none is implied.** Human "
|
| 1018 |
+
"scorers agree with each other at kappa 0.33-0.47 on the comparable "
|
| 1019 |
+
"footpad grade, so a severity word is a description and not a score, "
|
| 1020 |
+
"and one bird is never a flock rate.",
|
| 1021 |
+
evidence=(
|
| 1022 |
+
("Slaughterline system, 500 images / 1,000 feet: kappa 0.70, "
|
| 1023 |
+
"human scorers 0.33-0.47 β the comparable lesion grade",
|
| 1024 |
+
"https://doi.org/10.1016/j.psj.2020.05.052"),
|
| 1025 |
+
)),
|
| 1026 |
+
_d("poultry_feather", _A,
|
| 1027 |
+
"Build now as a visual observation (Β§19). Feather coverage is a "
|
| 1028 |
+
"surface property and a photograph carries it.",
|
| 1029 |
+
"No public feather-coverage dataset was found, and no accuracy "
|
| 1030 |
+
"figure is available. There is an argument that the observation "
|
| 1031 |
+
"suits the method β coverage is an area on a surface, which is what "
|
| 1032 |
+
"a segmentation mask measures directly, and unlike a lesion grade it "
|
| 1033 |
+
"asks no scorer to judge severity β but that is reasoning and not "
|
| 1034 |
+
"evidence, and nothing here has measured it. What the observation "
|
| 1035 |
+
"cannot carry is cause: reduced coverage on the back and tail is "
|
| 1036 |
+
"produced by moult, by pecking, by rubbing and by disease alike, and "
|
| 1037 |
+
"Β§19 asks for none of them to be named.",
|
| 1038 |
+
"Experimental. 'Reduced feather coverage on back and tail.' Where "
|
| 1039 |
+
"the coverage is missing is the observation; why it is missing is "
|
| 1040 |
+
"not claimed, because moult, pecking, rubbing and disease all look "
|
| 1041 |
+
"the same in a photograph. **No accuracy figure exists and none is "
|
| 1042 |
+
"implied**, and one bird is never a flock rate.",
|
| 1043 |
+
evidence=()),
|
| 1044 |
+
_d("poultry_wound", _A,
|
| 1045 |
+
"Build now as a visual observation (Β§19), on the same footing as "
|
| 1046 |
+
"`cattle_wound`: SAM 3.1 mask, multimodal description, and an area "
|
| 1047 |
+
"when a reference marker is in frame.",
|
| 1048 |
+
"There is no public poultry wound dataset, exactly as there is no "
|
| 1049 |
+
"public cattle wound dataset β the entry under `cattle_wound` "
|
| 1050 |
+
"records that, and the same answer applies. The value is not in the "
|
| 1051 |
+
"first reading but in the comparison: an area and an appearance "
|
| 1052 |
+
"recorded today are what make next week's capture mean something. "
|
| 1053 |
+
"Naming a cause is refused for the same reason as everywhere else in "
|
| 1054 |
+
"this file.",
|
| 1055 |
+
"Experimental, and descriptive: 'Open wound visible on left flank', "
|
| 1056 |
+
"with 'Approximate visible area: 12-16 cmΒ²' when a reference marker "
|
| 1057 |
+
"is in frame. **No accuracy figure exists and none is implied** β "
|
| 1058 |
+
"there is no public wound dataset for poultry any more than for "
|
| 1059 |
+
"cattle. The follow-up comparison is the value. Never a cause.",
|
| 1060 |
+
evidence=()),
|
| 1061 |
+
_d("poultry_eye_head", _A,
|
| 1062 |
+
"Build now as a visual observation (Β§19), escalating rather than "
|
| 1063 |
+
"naming. Ocular and head signs are where the notifiable diseases "
|
| 1064 |
+
"show, which changes the wording and not the feasibility.",
|
| 1065 |
+
"Discharge and swelling around the eye are visible, and they are "
|
| 1066 |
+
"also among the presenting signs of Newcastle disease and infectious "
|
| 1067 |
+
"coryza. The constraint is therefore the same one recorded under "
|
| 1068 |
+
"`poultry_fecal` and `cattle_skin`: Newcastle is notifiable, the "
|
| 1069 |
+
"published Newcastle recall in the best public dropping dataset is "
|
| 1070 |
+
"62.7%, and a false positive on a notifiable disease creates "
|
| 1071 |
+
"obligations that a farmer did not ask a phone to create. No public "
|
| 1072 |
+
"dataset of poultry ocular signs was found, so there is additionally "
|
| 1073 |
+
"no figure to quote.",
|
| 1074 |
+
"Experimental. 'Visible discharge around left eye', plus swelling "
|
| 1075 |
+
"and location, escalating to veterinary review. **No accuracy figure "
|
| 1076 |
+
"exists and none is implied.** Nothing is named: these are among the "
|
| 1077 |
+
"presenting signs of Newcastle disease, which is notifiable, and the "
|
| 1078 |
+
"best public recall figure for Newcastle anywhere in this file is "
|
| 1079 |
+
"62.7%. One bird is never a flock rate.",
|
| 1080 |
+
evidence=(
|
| 1081 |
+
("1,255 PCR-validated images, CC BY 4.0 β the nearest validated "
|
| 1082 |
+
"poultry set, and it is droppings rather than heads",
|
| 1083 |
+
"https://zenodo.org/records/5801834"),
|
| 1084 |
+
)),
|
| 1085 |
+
_d("poultry_heat_stress", _A,
|
| 1086 |
+
"Build now as behaviour screening (Β§17). A probe measures the house; "
|
| 1087 |
+
"vision measures the birds' response to it. They are complementary, "
|
| 1088 |
+
"and the probe cannot tell you the birds are panting.",
|
| 1089 |
+
"Panting is detectable in RGB β mAP@50 0.927 on 1,000 images β but "
|
| 1090 |
+
"the demonstration was on caged tiers at 45 degrees, not overhead in "
|
| 1091 |
+
"a floor house. Heat stress is a house condition, and a temperature "
|
| 1092 |
+
"and humidity probe measures it directly, continuously, and for the "
|
| 1093 |
+
"price of a phone call. Build the sensor integration instead.",
|
| 1094 |
+
"Experimental, in three bands: no obvious visual signs / some "
|
| 1095 |
+
"heat-associated behaviours / heat-stress-associated behaviour "
|
| 1096 |
+
"elevated. The only panting figure available β mAP@50 0.927 β was "
|
| 1097 |
+
"measured on caged tiers at 45 degrees, not overhead in a floor "
|
| 1098 |
+
"house, so it does not transfer and is not quoted as Animap's. "
|
| 1099 |
+
"Temperature and humidity are optional inputs, not outputs. **Never "
|
| 1100 |
+
"an exact physiological claim, and never a fever.**",
|
| 1101 |
+
evidence=(
|
| 1102 |
+
("YOLOv8n on caged tiers, 1,000 images: mAP@50 0.927",
|
| 1103 |
+
"https://doi.org/10.3390/agriculture14071066"),
|
| 1104 |
+
),
|
| 1105 |
+
superseded=Superseded(
|
| 1106 |
+
"not_viable",
|
| 1107 |
+
"Technically workable and commercially pointless. A five-dollar "
|
| 1108 |
+
"sensor does it better.",
|
| 1109 |
+
"'Technically workable' was the finding; 'commercially "
|
| 1110 |
+
"pointless' was a product opinion that Β§17 overrules: 'A "
|
| 1111 |
+
"temperature sensor measures the environment. Vision measures the "
|
| 1112 |
+
"birds' response. They are complementary.' The sensor integration "
|
| 1113 |
+
"is still worth building and is now an optional input here.",
|
| 1114 |
+
)),
|
| 1115 |
+
_d("poultry_litter", _A,
|
| 1116 |
+
"Build now with the claim renamed (Β§16). Litter *condition* is "
|
| 1117 |
+
"visible in RGB β caking, wet-looking areas, the share of a scanned "
|
| 1118 |
+
"region affected, where it is worst. Litter *moisture* is not.",
|
| 1119 |
+
"The target variable is moisture, and moisture has no photometric "
|
| 1120 |
+
"signature: a dark patch is wet litter, or shadow, or manure. No "
|
| 1121 |
+
"litter dataset appears in any survey of poultry computer-vision "
|
| 1122 |
+
"datasets, and 'litter condition' does not occur in a review of 82 "
|
| 1123 |
+
"YOLO-in-poultry papers. The only work that functions uses UWB radar, "
|
| 1124 |
+
"in a laboratory.",
|
| 1125 |
+
"Experimental. 'Caking: High. Wet-looking areas: 18% of scanned "
|
| 1126 |
+
"region. Worst near drinker line 3.' Every word of that is an "
|
| 1127 |
+
"appearance claim about a scanned area, because no litter dataset "
|
| 1128 |
+
"exists in any published survey and there is no accuracy figure to "
|
| 1129 |
+
"quote. **Never a moisture percentage without a probe** β a dark "
|
| 1130 |
+
"patch is wet litter, or shadow, or manure, and RGB cannot separate "
|
| 1131 |
+
"them.",
|
| 1132 |
+
evidence=(),
|
| 1133 |
+
superseded=Superseded(
|
| 1134 |
+
"not_viable",
|
| 1135 |
+
"Nothing exists, and the thing being asked for is not visible.",
|
| 1136 |
+
"Right about moisture, wrong about the capability. Β§16: 'Do not "
|
| 1137 |
+
"claim exact litter moisture from normal RGB. Do build Litter "
|
| 1138 |
+
"Condition.' Caking, soiling and wet-looking areas are "
|
| 1139 |
+
"appearances, and appearances are what a photograph carries. "
|
| 1140 |
+
"`exact_litter_moisture` stays rejected and Β§29 names the probe.",
|
| 1141 |
+
)),
|
| 1142 |
+
_d("egg_quality", _A,
|
| 1143 |
+
"Build now for external quality (Β§20): count, shape, obvious dirt, "
|
| 1144 |
+
"discolouration and visible damage, from SAM plus OpenCV geometry "
|
| 1145 |
+
"plus multimodal reasoning. Fine cracks need candling.",
|
| 1146 |
+
"Geometry from a photograph is easy β 99.4% mAP on 844 images β and "
|
| 1147 |
+
"mechanical graders already do it for less. Cracks are what a farmer "
|
| 1148 |
+
"would pay for, and a hairline crack has almost no photometric "
|
| 1149 |
+
"signature: dedicated hyperspectral work reaches F1 75.5%, while "
|
| 1150 |
+
"tapping the shell and listening reaches 100%, because the crack "
|
| 1151 |
+
"changes the resonance and not the picture. Competing against that in "
|
| 1152 |
+
"RGB is competing against physics.",
|
| 1153 |
+
"Experimental, for what is on the outside of the shell. Geometry is "
|
| 1154 |
+
"the well-measured half β 99.4% mAP on 844 images β and it is also "
|
| 1155 |
+
"the half worth least, so it is reported plainly and not dressed up. "
|
| 1156 |
+
"**Hairline cracks are not claimed from an ambient photograph**: "
|
| 1157 |
+
"dedicated hyperspectral work reaches only F1 75.5% while acoustic "
|
| 1158 |
+
"tapping reaches 100%, because the crack changes the resonance and "
|
| 1159 |
+
"not the picture. Candling is the route, and it needs a backlight.",
|
| 1160 |
+
evidence=(
|
| 1161 |
+
("Acoustic crack detection outperforms every optical method",
|
| 1162 |
+
"https://doi.org/10.1016/j.compag.2020.105716"),
|
| 1163 |
+
),
|
| 1164 |
+
superseded=Superseded(
|
| 1165 |
+
"needs_capture_change",
|
| 1166 |
+
"Size and shape grading works and is worth little. Crack "
|
| 1167 |
+
"detection is the valuable half and RGB cannot do it.",
|
| 1168 |
+
"Both sentences survive and neither justifies removal. Β§20 keeps "
|
| 1169 |
+
"external quality now and puts fine cracks behind a candling "
|
| 1170 |
+
"accessory, which is a capture change with a five-dollar answer. "
|
| 1171 |
+
"`hairline_crack_from_ambient_photo` stays rejected.",
|
| 1172 |
+
)),
|
| 1173 |
+
]
|
| 1174 |
+
}
|
| 1175 |
+
|
| 1176 |
+
|
| 1177 |
+
def get(key: str) -> Disposition | None:
|
| 1178 |
+
return DISPOSITIONS.get(key)
|
app/identification.py
ADDED
|
@@ -0,0 +1,699 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Matching a capture against a farm's enrolled animals, and refusing to name one.
|
| 2 |
+
|
| 3 |
+
Directive Β§6.4, and the counterpart to `app/counting.py`: that module turns boxes
|
| 4 |
+
into a claim, this one turns a nearest neighbour into a candidate. Both exist
|
| 5 |
+
because the hard part is not the model β it is being honest about the cases the
|
| 6 |
+
model cannot carry.
|
| 7 |
+
|
| 8 |
+
**The measurement this runner ships against is the open-set one, not the
|
| 9 |
+
closed-set one.** Run `8db9e0bd1b30`
|
| 10 |
+
(`experiments/cattle_identity/metrics.json`) enrolled 169 animals from the
|
| 11 |
+
muzzle268 database and measured, on 789 probes of animals that *were* enrolled:
|
| 12 |
+
top-1 0.9772, top-3 0.9937, MRR 0.9853, against a chance rate of 0.005917. That
|
| 13 |
+
is the number worth quoting and it is not the number that decides the product.
|
| 14 |
+
|
| 15 |
+
On 416 probes of animals nobody enrolled, **the unthresholded false-accept rate
|
| 16 |
+
is 1.000**. Every unenrolled animal comes back as somebody, because every query
|
| 17 |
+
has a nearest neighbour and nothing about a nearest neighbour knows the right
|
| 18 |
+
answer was absent. The two similarity distributions overlap badly β enrolled
|
| 19 |
+
probes median 0.9707, unenrolled median 0.9042 with an unenrolled *maximum* of
|
| 20 |
+
0.9776 β so the cutoff that admits no impostor at all sits at 0.98 and accepts
|
| 21 |
+
**24.08%** of the correct matches.
|
| 22 |
+
|
| 23 |
+
`MEASURED_POLICY` is that operating point, and choosing it is the whole design:
|
| 24 |
+
|
| 25 |
+
- Three quarters of the time an enrolled animal is photographed, this runner
|
| 26 |
+
claims `no_confident_match` **and still returns the ranked candidates**, so a
|
| 27 |
+
person sees the names and picks one. That is a worse headline and the same
|
| 28 |
+
information.
|
| 29 |
+
- The alternative β no threshold, always name the top candidate β is the one
|
| 30 |
+
that reads better and puts a neighbour's cow in a farm's records 100% of the
|
| 31 |
+
time an unenrolled animal is photographed.
|
| 32 |
+
|
| 33 |
+
`Β§6.4`'s confirm step is therefore load-bearing rather than decorative.
|
| 34 |
+
|
| 35 |
+
**What carries that requirement to a device, precisely.** This paragraph used to
|
| 36 |
+
say `IdentityResult.to_json` states `requires_confirmation: True` on every path.
|
| 37 |
+
It does β and this runner never calls it. The served result is
|
| 38 |
+
`schemas.InferenceResult`, which has no such field, so **the flag reaches
|
| 39 |
+
nobody** and citing it here described a guarantee that was not on the wire.
|
| 40 |
+
|
| 41 |
+
What actually crosses is the capability row. `Requirement.HUMAN_CONFIRMATION` is
|
| 42 |
+
on the registry entry, travels verbatim to the API, and is published as
|
| 43 |
+
`requirements: ["guided_capture", "human_confirmation"]` by `GET /capabilities`,
|
| 44 |
+
where a database constraint forces Β§6.4's four buttons to travel beside it. That
|
| 45 |
+
is a real mechanism and it is a **different endpoint from the result**: a client
|
| 46 |
+
that renders a result without having read `/capabilities` has nothing in the
|
| 47 |
+
payload telling it to ask. Closing that gap means a field on `InferenceResult`,
|
| 48 |
+
which is a contract change for every capability and is not made here.
|
| 49 |
+
|
| 50 |
+
What this runner does guarantee on every path is narrower and worth stating
|
| 51 |
+
exactly: no result carries an interpretation, an `observation_confidence`, or a
|
| 52 |
+
non-`None` confidence on any observation; `identity_candidate` is emitted only
|
| 53 |
+
when the measured policy accepted; and `_result` raises on
|
| 54 |
+
`identity_without_confirmation`. A result from here cannot *look* settled. It
|
| 55 |
+
relies on the client having read the capability to know it must ask.
|
| 56 |
+
|
| 57 |
+
**What is not measured here.** The database is US beef breeds β Angus, Angus x
|
| 58 |
+
Hereford, Continental x British crosses. No Nigerian and no zebu animal has been
|
| 59 |
+
through this, and a White Fulani is white all over, so the coat-pattern signal
|
| 60 |
+
the published re-identification literature leans on is absent for the herds this
|
| 61 |
+
product is for. The registry carries `UNVALIDATED_GEOGRAPHY`, which is what stops
|
| 62 |
+
`may_be_promoted_to_production` mechanically.
|
| 63 |
+
|
| 64 |
+
## The cold-gallery cost, which is a real limit and not a solved problem
|
| 65 |
+
|
| 66 |
+
Measured on 2026-08-23 over this service's own HTTP path by
|
| 67 |
+
`scripts/verify_identity_wiring.py`, on an Apple M-series laptop, single process,
|
| 68 |
+
enrolling five muzzle photographs per animal:
|
| 69 |
+
|
| 70 |
+
| Register | Enrolment photographs | Cold request | Per photograph | Warm median |
|
| 71 |
+
|---|---|---|---|---|
|
| 72 |
+
| 20 animals | 100 | **11.13 s** | 0.11 s | **0.059 s** |
|
| 73 |
+
| 60 animals | 300 | **60.35 s** | 0.20 s | **0.07 s** |
|
| 74 |
+
|
| 75 |
+
`_VECTOR_CACHE` is what separates the cold column from the warm one, and it only
|
| 76 |
+
helps the warm one. **A farm's first identification pays for embedding its whole
|
| 77 |
+
register.**
|
| 78 |
+
|
| 79 |
+
**Both per-photograph rates are recorded because they disagree**, and the honest
|
| 80 |
+
reading is that this is a loaded shared laptop rather than a controlled
|
| 81 |
+
measurement β the 0.20 s run had a test suite beside it. Extrapolate on the
|
| 82 |
+
slower one; a latency figure from a shared machine is not a latency figure, which
|
| 83 |
+
is a rule this repository already writes down in
|
| 84 |
+
`experiments/cattle_respiratory/README.md`. At 0.20 s a 169-animal register is
|
| 85 |
+
about **169 seconds** cold.
|
| 86 |
+
|
| 87 |
+
Neither rate is the 59 ms in `backbones.py`. That is the embed call alone on a
|
| 88 |
+
frame somebody already decoded; this path also reads the blob and decodes a JPEG.
|
| 89 |
+
An earlier version of this paragraph extrapolated on 59 ms and so put a
|
| 90 |
+
169-animal farm at "about a minute" β roughly three times better than measured,
|
| 91 |
+
in a paragraph whose entire purpose is to be honest about a limit.
|
| 92 |
+
|
| 93 |
+
**Even the small register does not fit the inline job model.** `app/main.py` runs
|
| 94 |
+
jobs synchronously, and this service declined to wire an eighteen-second
|
| 95 |
+
capability inline at all; eleven seconds for twenty animals is the same order and
|
| 96 |
+
a hundred and sixty-nine seconds is not close. So the honest statement is not
|
| 97 |
+
"wired for the farm sizes Animap has" β it is that **every** cold request is slow
|
| 98 |
+
and only the warm path is fast, so a deployment has to keep the cache warm rather
|
| 99 |
+
than treat the cold case as an edge.
|
| 100 |
+
|
| 101 |
+
The fix is not a bigger cache. It is for the API to store each enrolment vector
|
| 102 |
+
beside its `MediaAsset` and send vectors instead of media ids, which moves the
|
| 103 |
+
cost to enrolment time where a person is already waiting. That is a change to the
|
| 104 |
+
API's schema and is not made here.
|
| 105 |
+
|
| 106 |
+
**Where these numbers come from, since they cite no run record.** They are a
|
| 107 |
+
session measurement, not a benchmark, and the distinction matters in a repository
|
| 108 |
+
whose whole discipline is that a figure names a run id. The script is
|
| 109 |
+
`scripts/verify_identity_wiring.py`; it is reproducible and it has not been
|
| 110 |
+
through the experiment harness, so nothing here may be quoted as a measured
|
| 111 |
+
accuracy. What it does establish is that the wiring reproduces the benchmark
|
| 112 |
+
rather than merely citing it: over 122 held-out probes against a 60-animal
|
| 113 |
+
gallery, top-1 similarity ran min 0.8336, median 0.9764, max 0.9930 against the
|
| 114 |
+
run record's min 0.8336, median 0.9707, max 0.9930, and rank-1 was right 121
|
| 115 |
+
times out of 122. The accept rate was 41% against the benchmark's 24.08%, which
|
| 116 |
+
is the direction a 60-animal gallery moves it against 169.
|
| 117 |
+
"""
|
| 118 |
+
|
| 119 |
+
from __future__ import annotations
|
| 120 |
+
|
| 121 |
+
import threading
|
| 122 |
+
from collections import OrderedDict
|
| 123 |
+
from uuid import UUID, uuid4
|
| 124 |
+
|
| 125 |
+
import numpy as np
|
| 126 |
+
from PIL import Image
|
| 127 |
+
|
| 128 |
+
from app.adapters.embedding import DINOV3_SPEC, OnnxEmbeddingAdapter
|
| 129 |
+
from app.adapters.embedding.identity import (
|
| 130 |
+
ENROLMENT_VIEWS,
|
| 131 |
+
PRIMARY_VIEW,
|
| 132 |
+
Embedding,
|
| 133 |
+
IdentityIndex,
|
| 134 |
+
IndexMismatch,
|
| 135 |
+
OpenSetPolicy,
|
| 136 |
+
)
|
| 137 |
+
from app.capabilities import FORBIDDEN_CLAIMS, Capability
|
| 138 |
+
from app.media import MediaRef, MediaStore
|
| 139 |
+
from app.providers import ModelArtefact
|
| 140 |
+
from app.quality import assess
|
| 141 |
+
from app.schemas import (
|
| 142 |
+
EnrolledAnimal,
|
| 143 |
+
InferenceLocation,
|
| 144 |
+
InferenceRequest,
|
| 145 |
+
InferenceResult,
|
| 146 |
+
Observation,
|
| 147 |
+
QualityCheck,
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
#: The operating point measured in run `8db9e0bd1b30`, and the only policy this
|
| 151 |
+
#: runner ships.
|
| 152 |
+
#:
|
| 153 |
+
#: `accept_similarity` 0.98 is the lowest cutoff in the published sweep whose
|
| 154 |
+
#: false-accept rate is 0.0. `accept_margin` 0.0 is the measured companion and it
|
| 155 |
+
#: never rejects anything β a margin can only be non-negative β which is
|
| 156 |
+
#: deliberate rather than an oversight: `open_set_margin_sweep` in the same run
|
| 157 |
+
#: shows the margin rule buys nothing here. At margin 0.07 it holds the
|
| 158 |
+
#: false-accept rate to 0.0072 while accepting 23.45% of correct matches, which
|
| 159 |
+
#: is *worse on both axes* than the similarity rule alone. The rule stays wired
|
| 160 |
+
#: because a future backbone may separate the two distributions differently, and
|
| 161 |
+
#: it is set where the evidence puts it.
|
| 162 |
+
#:
|
| 163 |
+
#: **Both numbers cite the run that produced them and `OpenSetPolicy` will not
|
| 164 |
+
#: accept them otherwise.** Its `__post_init__` raises `UnmeasuredThreshold` on a
|
| 165 |
+
#: threshold with no run id, which is what stops a cutoff being tuned by whoever
|
| 166 |
+
#: is looking at a demo that morning.
|
| 167 |
+
MEASURED_POLICY = OpenSetPolicy.measured(
|
| 168 |
+
accept_similarity=0.98,
|
| 169 |
+
accept_margin=0.0,
|
| 170 |
+
run_id="8db9e0bd1b30",
|
| 171 |
+
measured_on="muzzle268-169enrolled",
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
#: Said on every run, whatever the result, in the shape `CountingProfile`
|
| 175 |
+
#: established. Β§6.4's product statement, cut to what a person reads.
|
| 176 |
+
STANDING_WARNING = (
|
| 177 |
+
"A suggestion, not a record. Animap proposes an animal and you confirm it β "
|
| 178 |
+
"an unconfirmed match is never written to this animal's history."
|
| 179 |
+
)
|
| 180 |
+
|
| 181 |
+
#: Said whenever candidates are returned at all, because the open-set result is
|
| 182 |
+
#: not something a reader can infer from a ranked list.
|
| 183 |
+
UNENROLLED_WARNING = (
|
| 184 |
+
"An animal that has never been enrolled will still produce a nearest match. "
|
| 185 |
+
"Measured on 416 photographs of unenrolled cattle, every one of them came "
|
| 186 |
+
"back as somebody. If this animal is new, register it rather than picking "
|
| 187 |
+
"the closest name."
|
| 188 |
+
)
|
| 189 |
+
|
| 190 |
+
#: How many candidates a result carries. Β§6.4 shows a ranked list rather than one
|
| 191 |
+
#: answer, and `evidence_correction.selected_interpretation` is the reason: a
|
| 192 |
+
#: farmer picking the second name records *"rank 2 was right"*, which its own
|
| 193 |
+
#: docstring calls the highest-value training signal in the table. Three, because
|
| 194 |
+
#: `closed_set_top3_accuracy` is 0.9937 and a fourth carries no measurement.
|
| 195 |
+
TOP_K = 3
|
| 196 |
+
|
| 197 |
+
#: Enrolment vectors held between requests, keyed by media id and artefact.
|
| 198 |
+
#:
|
| 199 |
+
#: **Without this the design does not fit the inline job model.** The gallery
|
| 200 |
+
#: travels on the request so the service holds no farm state
|
| 201 |
+
#: (`schemas.EnrolledAnimal` says why), and the cost of that is re-embedding
|
| 202 |
+
#: every enrolled photograph on every identification. At the **measured 0.20 s**
|
| 203 |
+
#: per enrolment photograph β end to end, including decode, not the 59 ms embed
|
| 204 |
+
#: call `backbones.py` reports on a frame somebody already decoded β a farm with
|
| 205 |
+
#: a hundred animals and the five-shot enrolment the accuracy was measured on is
|
| 206 |
+
#: 500 photographs, or **about a hundred seconds a request**. That is far past
|
| 207 |
+
#: the inline ceiling `app/main.py` describes and far worse than the
|
| 208 |
+
#: eighteen-second capability this service declined to wire inline.
|
| 209 |
+
#:
|
| 210 |
+
#: **A media id is immutable, which is what makes caching it correct rather than
|
| 211 |
+
#: merely fast.** `MediaAsset` rows are write-once and the blob behind one is
|
| 212 |
+
#: never rewritten, so the same id is the same bytes forever and its vector
|
| 213 |
+
#: cannot go stale. The artefact digest is in the key because a re-export with
|
| 214 |
+
#: different pooling produces a vector of the same width and a different meaning
|
| 215 |
+
#: β the case `IdentityIndex` pins its backbone id to catch β so vectors from two
|
| 216 |
+
#: artefacts must never collide here.
|
| 217 |
+
#:
|
| 218 |
+
#: **The farm id is in the key, and it was not for one commit.** The argument for
|
| 219 |
+
#: leaving it out was that nothing can be read from the cache without already
|
| 220 |
+
#: holding the media id. A watchdog showed that is not quite the property that
|
| 221 |
+
#: matters: `AzureBlobMediaStore` scopes an unpathed lookup to `farm/{farm_id}/`,
|
| 222 |
+
#: so *without* a cache a request naming another farm's media id fails to resolve
|
| 223 |
+
#: β and *with* one it would hit and score. What leaked was a similarity against
|
| 224 |
+
#: a foreign photograph rather than the photograph or the name, and it needed the
|
| 225 |
+
#: service token plus a known foreign UUID, so it was narrow. It was also free to
|
| 226 |
+
#: close, and a cache must not be the reason a farm boundary that the media store
|
| 227 |
+
#: enforces stops being enforced.
|
| 228 |
+
_VECTOR_CACHE: OrderedDict[tuple[str, str, str], np.ndarray] = OrderedDict()
|
| 229 |
+
|
| 230 |
+
#: Entries kept. 768 float32 values is 3 KB, so this is about 12 MB β small
|
| 231 |
+
#: against the container's 4 GiB, and enough for a few hundred animals at five
|
| 232 |
+
#: shots each. Least-recently-used is evicted, so the farms being worked today
|
| 233 |
+
#: stay warm.
|
| 234 |
+
_CACHE_LIMIT = 4096
|
| 235 |
+
|
| 236 |
+
#: `submit()` runs in FastAPI's threadpool, so two captures can be scored at
|
| 237 |
+
#: once and an `OrderedDict` is not safe under that on its own.
|
| 238 |
+
_CACHE_LOCK = threading.Lock()
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def cache_clear() -> None:
|
| 242 |
+
"""Empty the vector cache. For tests, and for a deployment that swaps an
|
| 243 |
+
artefact without a restart."""
|
| 244 |
+
with _CACHE_LOCK:
|
| 245 |
+
_VECTOR_CACHE.clear()
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def _cached_vector(key: tuple[str, str, str]) -> np.ndarray | None:
|
| 249 |
+
with _CACHE_LOCK:
|
| 250 |
+
vector = _VECTOR_CACHE.get(key)
|
| 251 |
+
if vector is not None:
|
| 252 |
+
_VECTOR_CACHE.move_to_end(key)
|
| 253 |
+
return vector
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
def _remember(key: tuple[str, str, str], vector: np.ndarray) -> None:
|
| 257 |
+
with _CACHE_LOCK:
|
| 258 |
+
_VECTOR_CACHE[key] = vector
|
| 259 |
+
_VECTOR_CACHE.move_to_end(key)
|
| 260 |
+
while len(_VECTOR_CACHE) > _CACHE_LIMIT:
|
| 261 |
+
_VECTOR_CACHE.popitem(last=False)
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
#: Muzzle photographs per animal in the protocol the accuracy was measured under.
|
| 265 |
+
#:
|
| 266 |
+
#: Run `8db9e0bd1b30` enrolled each animal from five, all under the `muzzle` view
|
| 267 |
+
#: name, and `IdentityIndex.candidates` scores an animal by its **best** view β
|
| 268 |
+
#: so the five are five chances to match rather than five samples averaged
|
| 269 |
+
#: together. An animal enrolled from one photograph is being asked a harder
|
| 270 |
+
#: question than the benchmark asked, and measured that way on twelve animals,
|
| 271 |
+
#: **0 of 23 held-out queries cleared the 0.98 cutoff**.
|
| 272 |
+
#:
|
| 273 |
+
#: Used only to report a thin enrolment, never to refuse one.
|
| 274 |
+
MEASURED_ENROLMENT_SHOTS = 5
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
class IdentityRunner:
|
| 278 |
+
"""Runs `cattle_identity` against one capture and one farm's register."""
|
| 279 |
+
|
| 280 |
+
#: The view a single capture is compared against. Β§6.4 leads with the muzzle
|
| 281 |
+
#: and the re-identification measurement is entirely muzzle-to-muzzle, so
|
| 282 |
+
#: comparing a muzzle print against a side-body vector would contribute
|
| 283 |
+
#: nothing but a chance of a spurious high score. Restricting the pool is
|
| 284 |
+
#: what keeps the runtime behaviour inside the measured one.
|
| 285 |
+
#:
|
| 286 |
+
#: **Nothing checks that the capture actually is a muzzle, and a watchdog was
|
| 287 |
+
#: right to call that a gap.** The registry declares `muzzle_not_visible` in
|
| 288 |
+
#: `reject_if`, but `app/quality.py` implements resolution, illumination and
|
| 289 |
+
#: motion blur and nothing that looks for an anatomical part β so the
|
| 290 |
+
#: condition is declared and never evaluated. A caller that sends the five
|
| 291 |
+
#: enrolment views in the order the registry lists them sends `front_face`
|
| 292 |
+
#: first, and it would be scored against a gallery of muzzles.
|
| 293 |
+
#:
|
| 294 |
+
#: What that costs is bounded in the safe direction: a front-face photograph
|
| 295 |
+
#: is *less* similar to every enrolled muzzle, so it drives the top score down
|
| 296 |
+
#: and towards `no_confident_match`. It degrades to a refusal rather than to a
|
| 297 |
+
#: wrong name. But it is a refusal for a reason nobody is told, and the fix is
|
| 298 |
+
#: a real one β either the capture flow states which view it took, or
|
| 299 |
+
#: something evaluates `muzzle_not_visible`. Neither exists today.
|
| 300 |
+
query_view = PRIMARY_VIEW
|
| 301 |
+
|
| 302 |
+
def run(
|
| 303 |
+
self,
|
| 304 |
+
*,
|
| 305 |
+
request: InferenceRequest,
|
| 306 |
+
capability: Capability,
|
| 307 |
+
artefact: ModelArtefact,
|
| 308 |
+
store: MediaStore,
|
| 309 |
+
request_id: UUID | None = None,
|
| 310 |
+
) -> InferenceResult:
|
| 311 |
+
request_id = request_id or uuid4()
|
| 312 |
+
warnings: list[str] = [STANDING_WARNING]
|
| 313 |
+
|
| 314 |
+
if len(request.media_ids) > capability.frames_required:
|
| 315 |
+
warnings.append(
|
| 316 |
+
f"{len(request.media_ids)} frames were supplied; this capability "
|
| 317 |
+
f"reads {capability.frames_required}."
|
| 318 |
+
)
|
| 319 |
+
|
| 320 |
+
image = store.open_image(self._ref(request, request.media_ids[0]))
|
| 321 |
+
verdict = assess(image)
|
| 322 |
+
checks = list(verdict.checks)
|
| 323 |
+
|
| 324 |
+
if verdict.blocked:
|
| 325 |
+
failure = verdict.first_failure
|
| 326 |
+
warnings.append(
|
| 327 |
+
failure.detail if failure and failure.detail
|
| 328 |
+
else "The capture was not usable."
|
| 329 |
+
)
|
| 330 |
+
return self._result(
|
| 331 |
+
request=request, capability=capability, artefact=artefact,
|
| 332 |
+
request_id=request_id, observations=[], checks=checks,
|
| 333 |
+
warnings=warnings, recapture=True,
|
| 334 |
+
)
|
| 335 |
+
|
| 336 |
+
adapter = OnnxEmbeddingAdapter(
|
| 337 |
+
artefact, DINOV3_SPEC,
|
| 338 |
+
input_size=artefact.input_size or 224,
|
| 339 |
+
mean=artefact.image_mean or (0.485, 0.456, 0.406),
|
| 340 |
+
std=artefact.image_std or (0.229, 0.224, 0.225),
|
| 341 |
+
dimensions=artefact.embedding_dimensions or 768,
|
| 342 |
+
).load()
|
| 343 |
+
|
| 344 |
+
index = IdentityIndex(
|
| 345 |
+
farm_id=str(request.farm_id),
|
| 346 |
+
backbone_id=DINOV3_SPEC.adapter_id,
|
| 347 |
+
dimensions=adapter.dimensions,
|
| 348 |
+
artefact_sha256=artefact.sha256,
|
| 349 |
+
)
|
| 350 |
+
names: dict[str, str] = {}
|
| 351 |
+
enrolled_views, skipped = self._enrol(
|
| 352 |
+
index, adapter, artefact, request, store, names,
|
| 353 |
+
)
|
| 354 |
+
warnings.extend(skipped)
|
| 355 |
+
|
| 356 |
+
query = self._embed(adapter, artefact, image)
|
| 357 |
+
result = index.match(
|
| 358 |
+
query,
|
| 359 |
+
policy=MEASURED_POLICY,
|
| 360 |
+
top_k=TOP_K,
|
| 361 |
+
# Compared against muzzles only. See `query_view`.
|
| 362 |
+
restrict_to_views=(self.query_view,),
|
| 363 |
+
names=names,
|
| 364 |
+
)
|
| 365 |
+
|
| 366 |
+
checks.append(self._enrolment_check(index, enrolled_views))
|
| 367 |
+
|
| 368 |
+
observations: list[Observation] = [
|
| 369 |
+
# Emitted on every path, refusals included, for the reason
|
| 370 |
+
# `counting.py` emits its grid: a threshold that has to be re-derived
|
| 371 |
+
# later is re-derived from stored results or not at all.
|
| 372 |
+
Observation(
|
| 373 |
+
type="enrolled_animals",
|
| 374 |
+
value=float(result.enrolled_animals),
|
| 375 |
+
unit="animals",
|
| 376 |
+
confidence=None,
|
| 377 |
+
),
|
| 378 |
+
Observation(
|
| 379 |
+
type="open_set_verified",
|
| 380 |
+
value=1.0 if result.open_set_verified else 0.0,
|
| 381 |
+
confidence=None,
|
| 382 |
+
),
|
| 383 |
+
]
|
| 384 |
+
|
| 385 |
+
top = result.top
|
| 386 |
+
if top is not None:
|
| 387 |
+
observations.append(Observation(
|
| 388 |
+
# **Uncalibrated, and named so.** Nothing mapped cosine
|
| 389 |
+
# similarity onto a probability that the animal is Kofi, so this
|
| 390 |
+
# is a diagnostic and the release declares no sentence for it β
|
| 391 |
+
# exactly as `counting_grid` carries none. Β§37's whole complaint
|
| 392 |
+
# is an uncalibrated model score reaching a person as a promise.
|
| 393 |
+
type="top_candidate_similarity",
|
| 394 |
+
value=round(top.similarity, 4),
|
| 395 |
+
confidence=None,
|
| 396 |
+
))
|
| 397 |
+
|
| 398 |
+
accepted = result.claim == "identity_candidate"
|
| 399 |
+
if not accepted:
|
| 400 |
+
observations.append(Observation(
|
| 401 |
+
type="no_confident_match", value=None, confidence=None,
|
| 402 |
+
))
|
| 403 |
+
|
| 404 |
+
# **The ranked list is emitted on both paths, under different names.**
|
| 405 |
+
# `IdentityResult` returns candidates whether or not the policy accepted
|
| 406 |
+
# the top one, and dropping them on the refusal path would throw away the
|
| 407 |
+
# thing Β§6.4 is shaped around: `evidence_correction.selected_
|
| 408 |
+
# interpretation` records *"rank 2 was right"*, and its own docstring
|
| 409 |
+
# calls that the highest-value training signal in the table. At the
|
| 410 |
+
# measured operating point three quarters of correct matches are refused,
|
| 411 |
+
# so the refusal path is where most of that signal lives β emitting
|
| 412 |
+
# nothing there would collect it almost nowhere.
|
| 413 |
+
#
|
| 414 |
+
# The type differs because the claim differs. `identity_candidate` is
|
| 415 |
+
# Β§6.4's *"This looks like Kofi"*; `closest_candidate` is *"not a
|
| 416 |
+
# confident match, and these are the nearest on your register"*. One
|
| 417 |
+
# observation type carrying both would leave the device deciding which
|
| 418 |
+
# sentence to show from a field that does not say, and the release
|
| 419 |
+
# declares a separate sentence for each.
|
| 420 |
+
#
|
| 421 |
+
# ## Exactly one settled name, ever
|
| 422 |
+
#
|
| 423 |
+
# An accepted run used to publish `identity_candidate` for **all three**
|
| 424 |
+
# ranked candidates, so `deploy/semantics` rendered *"This looks like
|
| 425 |
+
# Cow 0100"*, *"This looks like Cow 0200"* and *"This looks like Cow
|
| 426 |
+
# 0300"* β three settled names for one animal, on one `captured_at`,
|
| 427 |
+
# which no ordering on the device can pick between. The policy accepted
|
| 428 |
+
# **the top candidate**; it said nothing whatever about ranks 2 and 3.
|
| 429 |
+
#
|
| 430 |
+
# So rank 1 is the proposal and the rest are what Β§6.4's screen calls
|
| 431 |
+
# *"Also considered"* β the same list, under the type that does not
|
| 432 |
+
# assert. Picking one of them is `choose_another_animal`, which is a
|
| 433 |
+
# correction and the signal the ledger is built to keep.
|
| 434 |
+
for position, candidate in enumerate(result.candidates):
|
| 435 |
+
settled = accepted and position == 0
|
| 436 |
+
observations.append(Observation(
|
| 437 |
+
type="identity_candidate" if settled else "closest_candidate",
|
| 438 |
+
# The animal's own name. `identity_candidate` is one of three
|
| 439 |
+
# claims in the whole registry that carries free text, and
|
| 440 |
+
# `app/capabilities.py` says why: a registry cannot enumerate a
|
| 441 |
+
# farm's animals. The value comes from the farm's own register
|
| 442 |
+
# via the request, never from this service.
|
| 443 |
+
value=candidate.display_name,
|
| 444 |
+
# **`confidence` stays None on both paths.**
|
| 445 |
+
# `Candidate.confidence` is documented as never set by the
|
| 446 |
+
# matcher, and promoting a similarity into it here would defeat
|
| 447 |
+
# that in one line.
|
| 448 |
+
confidence=None,
|
| 449 |
+
))
|
| 450 |
+
|
| 451 |
+
if result.candidates:
|
| 452 |
+
warnings.append(UNENROLLED_WARNING)
|
| 453 |
+
|
| 454 |
+
warnings.extend(result.warnings)
|
| 455 |
+
|
| 456 |
+
if index.unverified_queries:
|
| 457 |
+
# Should be zero on this path: every vector is built as an
|
| 458 |
+
# `Embedding` below. Reported rather than asserted, because the count
|
| 459 |
+
# existing at all is the gap between what the index checks and what
|
| 460 |
+
# it would like to.
|
| 461 |
+
warnings.append(
|
| 462 |
+
f"{index.unverified_queries} vectors were compared without "
|
| 463 |
+
f"provenance. Every vector this service builds carries it, so "
|
| 464 |
+
f"this is a defect rather than a capture problem."
|
| 465 |
+
)
|
| 466 |
+
|
| 467 |
+
return self._result(
|
| 468 |
+
request=request, capability=capability, artefact=artefact,
|
| 469 |
+
request_id=request_id, observations=observations, checks=checks,
|
| 470 |
+
warnings=warnings,
|
| 471 |
+
# A refusal here is not a bad photograph. The animal may simply not
|
| 472 |
+
# be enrolled, and asking for a recapture would send a worker back to
|
| 473 |
+
# a pen to re-photograph a cow the system has never seen.
|
| 474 |
+
recapture=verdict.degraded,
|
| 475 |
+
)
|
| 476 |
+
|
| 477 |
+
def _ref(self, request: InferenceRequest, media_id: UUID,
|
| 478 |
+
object_path: str | None = None) -> MediaRef:
|
| 479 |
+
return MediaRef(
|
| 480 |
+
media_id=media_id,
|
| 481 |
+
farm_id=request.farm_id,
|
| 482 |
+
captured_at=request.captured_at,
|
| 483 |
+
object_path=object_path or request.path_for(media_id),
|
| 484 |
+
)
|
| 485 |
+
|
| 486 |
+
def _embed(
|
| 487 |
+
self, adapter: OnnxEmbeddingAdapter, artefact: ModelArtefact,
|
| 488 |
+
image: Image.Image,
|
| 489 |
+
) -> Embedding:
|
| 490 |
+
"""A vector that can prove where it came from.
|
| 491 |
+
|
| 492 |
+
Always an `Embedding`, never a bare array. `IdentityIndex._accept` can
|
| 493 |
+
only check the *width* of an array, and a watchdog scored a foreign
|
| 494 |
+
vector against a `dinov3-vits16` index at similarity 1.0 through exactly
|
| 495 |
+
that gap.
|
| 496 |
+
"""
|
| 497 |
+
return Embedding(
|
| 498 |
+
vector=adapter.embed(image),
|
| 499 |
+
backbone_id=DINOV3_SPEC.adapter_id,
|
| 500 |
+
artefact_sha256=artefact.sha256,
|
| 501 |
+
)
|
| 502 |
+
|
| 503 |
+
def _enrol(
|
| 504 |
+
self,
|
| 505 |
+
index: IdentityIndex,
|
| 506 |
+
adapter: OnnxEmbeddingAdapter,
|
| 507 |
+
artefact: ModelArtefact,
|
| 508 |
+
request: InferenceRequest,
|
| 509 |
+
store: MediaStore,
|
| 510 |
+
names: dict[str, str],
|
| 511 |
+
) -> tuple[int, list[str]]:
|
| 512 |
+
"""Build this farm's gallery from the request. Returns views enrolled.
|
| 513 |
+
|
| 514 |
+
A gallery entry that cannot be read is **skipped with a warning rather
|
| 515 |
+
than failing the job**: one unreadable enrolment photograph out of two
|
| 516 |
+
hundred should not stop a farmer identifying an animal, and the animals
|
| 517 |
+
that did load are still a gallery. The count that reaches the result is
|
| 518 |
+
the count that actually enrolled, so a farm cannot be told it was
|
| 519 |
+
compared against more animals than it was.
|
| 520 |
+
"""
|
| 521 |
+
enrolled_views = 0
|
| 522 |
+
skipped: list[str] = []
|
| 523 |
+
|
| 524 |
+
for entry in request.enrolled:
|
| 525 |
+
usable = self._views_for(entry, skipped)
|
| 526 |
+
if not usable:
|
| 527 |
+
continue
|
| 528 |
+
who = entry.display_name or entry.animal_id
|
| 529 |
+
stored = 0
|
| 530 |
+
|
| 531 |
+
for view, media_ids in usable.items():
|
| 532 |
+
for position, media_id in enumerate(media_ids):
|
| 533 |
+
key = (str(request.farm_id), str(media_id), artefact.sha256)
|
| 534 |
+
cached = _cached_vector(key)
|
| 535 |
+
if cached is not None:
|
| 536 |
+
vector = Embedding(
|
| 537 |
+
vector=cached,
|
| 538 |
+
backbone_id=DINOV3_SPEC.adapter_id,
|
| 539 |
+
artefact_sha256=artefact.sha256,
|
| 540 |
+
)
|
| 541 |
+
else:
|
| 542 |
+
try:
|
| 543 |
+
image = store.open_image(self._ref(
|
| 544 |
+
request, media_id,
|
| 545 |
+
entry.path_for_view(view, position),
|
| 546 |
+
))
|
| 547 |
+
except Exception as exc: # MediaError, or any store error
|
| 548 |
+
skipped.append(
|
| 549 |
+
f"An enrolled {view} photograph of {who} could "
|
| 550 |
+
f"not be read ({exc}), so it was left out of "
|
| 551 |
+
f"the comparison."
|
| 552 |
+
)
|
| 553 |
+
continue
|
| 554 |
+
vector = self._embed(adapter, artefact, image)
|
| 555 |
+
_remember(key, vector.vector)
|
| 556 |
+
try:
|
| 557 |
+
# **One call per photograph, under the one view name.**
|
| 558 |
+
# `enrol` takes a mapping of view to vector, so a single
|
| 559 |
+
# call can only hold one shot per view β and the measured
|
| 560 |
+
# protocol is five muzzles per animal, scored by maximum.
|
| 561 |
+
# Calling it per photograph is how those five reach the
|
| 562 |
+
# index, and it is what run 8db9e0bd1b30 did.
|
| 563 |
+
index.enrol(
|
| 564 |
+
entry.animal_id, {view: vector},
|
| 565 |
+
media_ids={view: str(media_id)},
|
| 566 |
+
)
|
| 567 |
+
except (IndexMismatch, ValueError) as exc:
|
| 568 |
+
skipped.append(
|
| 569 |
+
f"A {view} photograph of {who} was left out of the "
|
| 570 |
+
f"comparison: {exc}"
|
| 571 |
+
)
|
| 572 |
+
continue
|
| 573 |
+
stored += 1
|
| 574 |
+
|
| 575 |
+
if not stored:
|
| 576 |
+
continue
|
| 577 |
+
names[entry.animal_id] = who
|
| 578 |
+
enrolled_views += stored
|
| 579 |
+
|
| 580 |
+
return enrolled_views, skipped
|
| 581 |
+
|
| 582 |
+
def _views_for(
|
| 583 |
+
self, entry: EnrolledAnimal, skipped: list[str]
|
| 584 |
+
) -> dict[str, list[UUID]]:
|
| 585 |
+
"""The views of one animal this runner will compare against.
|
| 586 |
+
|
| 587 |
+
Unknown view names are dropped here with a warning rather than left for
|
| 588 |
+
`enrol` to raise on, because one typo in one animal's record should cost
|
| 589 |
+
that animal's view and not the whole request.
|
| 590 |
+
"""
|
| 591 |
+
usable: dict[str, list[UUID]] = {}
|
| 592 |
+
for view, media_ids in entry.views.items():
|
| 593 |
+
if view not in ENROLMENT_VIEWS:
|
| 594 |
+
skipped.append(
|
| 595 |
+
f"{entry.display_name or entry.animal_id} has a view called "
|
| 596 |
+
f"{view!r}, which is not one of the five Β§6.4 enrols "
|
| 597 |
+
f"({', '.join(ENROLMENT_VIEWS)}). It was ignored."
|
| 598 |
+
)
|
| 599 |
+
continue
|
| 600 |
+
if media_ids:
|
| 601 |
+
usable[view] = list(media_ids)
|
| 602 |
+
return usable
|
| 603 |
+
|
| 604 |
+
def _enrolment_check(
|
| 605 |
+
self, index: IdentityIndex, enrolled_views: int
|
| 606 |
+
) -> QualityCheck:
|
| 607 |
+
"""Whether this farm had anything to compare the capture against.
|
| 608 |
+
|
| 609 |
+
A quality check rather than a warning, because the answer is about the
|
| 610 |
+
*farm's register* rather than about the photograph, and a capture flow
|
| 611 |
+
that can distinguish the two can say *"register this animal"* instead of
|
| 612 |
+
*"take it again"*.
|
| 613 |
+
"""
|
| 614 |
+
muzzles = sum(1 for v in index.views if v.view == self.query_view)
|
| 615 |
+
animals = len(index.animal_ids)
|
| 616 |
+
if muzzles:
|
| 617 |
+
detail = (
|
| 618 |
+
f"{animals} animals enrolled, {enrolled_views} views, "
|
| 619 |
+
f"{muzzles} of them muzzles."
|
| 620 |
+
)
|
| 621 |
+
thin = [
|
| 622 |
+
a for a in index.animal_ids
|
| 623 |
+
if sum(1 for v in index.views
|
| 624 |
+
if v.animal_id == a and v.view == self.query_view)
|
| 625 |
+
< MEASURED_ENROLMENT_SHOTS
|
| 626 |
+
]
|
| 627 |
+
if thin:
|
| 628 |
+
# **Reported, never a refusal.** A thin enrolment still matches,
|
| 629 |
+
# and one good muzzle photograph is worth more than a refused
|
| 630 |
+
# enrolment β `IdentityIndex.enrol` takes that position and this
|
| 631 |
+
# agrees with it. What it is not is the protocol the accuracy was
|
| 632 |
+
# measured under, and a farm whose animals never quite match
|
| 633 |
+
# deserves to know the reason is its register rather than its
|
| 634 |
+
# camera.
|
| 635 |
+
detail += (
|
| 636 |
+
f" {len(thin)} of them carry fewer than "
|
| 637 |
+
f"{MEASURED_ENROLMENT_SHOTS} muzzle photographs, which is "
|
| 638 |
+
f"the enrolment the accuracy was measured on."
|
| 639 |
+
)
|
| 640 |
+
return QualityCheck(check="gallery", passed=True, detail=detail)
|
| 641 |
+
return QualityCheck(
|
| 642 |
+
check="gallery",
|
| 643 |
+
passed=False,
|
| 644 |
+
detail=(
|
| 645 |
+
f"No enrolled {self.query_view} to compare against. "
|
| 646 |
+
f"{len(index.animal_ids)} animals were sent and none carries "
|
| 647 |
+
f"the view this capability matches on."
|
| 648 |
+
),
|
| 649 |
+
)
|
| 650 |
+
|
| 651 |
+
def _result(
|
| 652 |
+
self, *, request, capability, artefact, request_id, observations,
|
| 653 |
+
checks, warnings, recapture,
|
| 654 |
+
) -> InferenceResult:
|
| 655 |
+
forbidden = [o.type for o in observations if o.type in FORBIDDEN_CLAIMS]
|
| 656 |
+
if forbidden:
|
| 657 |
+
raise ValueError(
|
| 658 |
+
f"{capability.key} tried to emit a forbidden claim: {forbidden}"
|
| 659 |
+
)
|
| 660 |
+
banned = set(capability.acquisition.forbidden_claims)
|
| 661 |
+
offending = [o.type for o in observations if o.type in banned]
|
| 662 |
+
if offending:
|
| 663 |
+
# `identity_without_confirmation` is the one thing Β§6.4 forbids, and
|
| 664 |
+
# it is checked here as well as in the registry because this runner
|
| 665 |
+
# is the only thing that could emit it.
|
| 666 |
+
raise ValueError(
|
| 667 |
+
f"{capability.key} tried to emit {offending}, which its own "
|
| 668 |
+
f"registry entry forbids by name."
|
| 669 |
+
)
|
| 670 |
+
|
| 671 |
+
return InferenceResult(
|
| 672 |
+
request_id=request_id,
|
| 673 |
+
capability_key=capability.key,
|
| 674 |
+
model_id=artefact.model_id,
|
| 675 |
+
model_version=artefact.version,
|
| 676 |
+
inference_location=InferenceLocation.REMOTE,
|
| 677 |
+
subject_type=request.subject_type,
|
| 678 |
+
subject_id=request.subject_id,
|
| 679 |
+
observations=observations,
|
| 680 |
+
# **No interpretation, and none is possible.** A name is a fact about
|
| 681 |
+
# the register, not a judgement about the animal, and there is
|
| 682 |
+
# nothing for a vet to review. `observation_confidence` is likewise
|
| 683 |
+
# None: the three-level label would be read as a confidence in the
|
| 684 |
+
# name, and nothing calibrated one.
|
| 685 |
+
interpretations=[],
|
| 686 |
+
observation_confidence=None,
|
| 687 |
+
interpretation_confidence=None,
|
| 688 |
+
quality_checks=checks,
|
| 689 |
+
warnings=warnings,
|
| 690 |
+
recommended_recapture=recapture,
|
| 691 |
+
)
|
| 692 |
+
|
| 693 |
+
|
| 694 |
+
#: Capabilities with an implemented adapter, in the shape `app/counting.py`
|
| 695 |
+
#: publishes. `app/main.py` merges the two: a capability with a validated
|
| 696 |
+
#: artefact and no entry in either returns 501 rather than a plausible result.
|
| 697 |
+
RUNNERS: dict[str, IdentityRunner] = {
|
| 698 |
+
"cattle_identity": IdentityRunner(),
|
| 699 |
+
}
|
app/main.py
ADDED
|
@@ -0,0 +1,597 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The inference service.
|
| 2 |
+
|
| 3 |
+
Brief Β§30. FastAPI, and deliberately small: submit a job, ask about a job, list
|
| 4 |
+
what the service can actually do.
|
| 5 |
+
|
| 6 |
+
The service's most important property is what it refuses. A capability with no
|
| 7 |
+
validated artefact returns `unavailable` β not a plausible-looking result, not a
|
| 8 |
+
placeholder, and not an error. `unavailable` is the honest answer for
|
| 9 |
+
twenty-five of the twenty-eight capabilities today, and the API's job is to say
|
| 10 |
+
so plainly enough that a client cannot mistake it for anything else.
|
| 11 |
+
|
| 12 |
+
**`state` and `runnable` are different questions and the client must render
|
| 13 |
+
both** (ADR 0021). `state` is the strongest claim a capability is entitled to
|
| 14 |
+
make once its stack is wired; `runnable` is whether an artefact and an adapter
|
| 15 |
+
exist today. Most capabilities are `experimental` and `runnable: false`, which
|
| 16 |
+
means *"this is a real feature and it is not switched on yet"* β not *"this
|
| 17 |
+
works."* A client that renders `state` alone will overclaim.
|
| 18 |
+
|
| 19 |
+
Three run for real. `cattle_detection` and `poultry_count` execute a checksummed
|
| 20 |
+
YOLOX artefact against the referenced media and return a result built from what
|
| 21 |
+
the model actually produced (ADR 0018). `cattle_identity` embeds the capture
|
| 22 |
+
with a checksummed DINOv3 artefact and ranks it against the enrolled animals the
|
| 23 |
+
request carries (`app/identification.py`).
|
| 24 |
+
|
| 25 |
+
**Twenty-five is not a backlog of twenty-five equal items**, and reading it as
|
| 26 |
+
one is how this service gets misreported. Several of the unwired capabilities
|
| 27 |
+
have complete benchmarks whose result is that the method does not work:
|
| 28 |
+
`cattle_gait` separates lame from sound at AUC 0.3117 (p = 0.91) and
|
| 29 |
+
`poultry_respiratory` at AUC 0.4141, both at or below chance. Those are measured
|
| 30 |
+
negatives, not missing work, and registering either would wire a screen that is
|
| 31 |
+
anti-correlated with the thing it screens for.
|
| 32 |
+
|
| 33 |
+
Jobs run inline. A frame that settles at the first grid is well under a second on
|
| 34 |
+
a CPU; a dense one that runs all three grids takes a few seconds (ADR 0019). A
|
| 35 |
+
queue becomes worth its moving parts when a capability arrives that takes tens of
|
| 36 |
+
seconds, and none does.
|
| 37 |
+
|
| 38 |
+
**Two endpoints are open and two are not.** `/health` and `/capabilities` carry
|
| 39 |
+
no farm data and the platform's probes need `/health`, so both are unauthenticated.
|
| 40 |
+
`/jobs` runs a model against a farm's photographs, so it needs a bearer token
|
| 41 |
+
whenever one is configured.
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
from __future__ import annotations
|
| 45 |
+
|
| 46 |
+
import hmac
|
| 47 |
+
import logging
|
| 48 |
+
import os
|
| 49 |
+
from uuid import UUID, uuid4
|
| 50 |
+
|
| 51 |
+
from fastapi import Depends, FastAPI, Header, HTTPException
|
| 52 |
+
from pydantic import BaseModel
|
| 53 |
+
|
| 54 |
+
from app import dispositions
|
| 55 |
+
from app.capabilities import (
|
| 56 |
+
REGISTRY,
|
| 57 |
+
Capability,
|
| 58 |
+
CapabilityState,
|
| 59 |
+
Requirement,
|
| 60 |
+
licence_exposure,
|
| 61 |
+
)
|
| 62 |
+
from app.counting import RUNNERS as _COUNTING_RUNNERS
|
| 63 |
+
from app.detectors import DetectorError
|
| 64 |
+
from app.identification import RUNNERS as _IDENTITY_RUNNERS
|
| 65 |
+
from app.media import MediaError, build_store
|
| 66 |
+
from app.providers import (
|
| 67 |
+
CompositeProvider,
|
| 68 |
+
HostedModelProvider,
|
| 69 |
+
LocalArtefactProvider,
|
| 70 |
+
)
|
| 71 |
+
from app.reasoning import RUNNERS as _REASONING_RUNNERS
|
| 72 |
+
from app.schemas import InferenceRequest, JobState, JobStatus
|
| 73 |
+
|
| 74 |
+
logger = logging.getLogger(__name__)
|
| 75 |
+
|
| 76 |
+
app = FastAPI(
|
| 77 |
+
title="Animap inference",
|
| 78 |
+
version="0.1.0",
|
| 79 |
+
summary="Runs validated livestock models, and refuses to invent results.",
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
#: Both kinds of capability, asked in order.
|
| 83 |
+
#:
|
| 84 |
+
#: `LocalArtefactProvider` serves the three with a checksummed file on disk;
|
| 85 |
+
#: `HostedModelProvider` serves the sixteen whose model is somebody else's and
|
| 86 |
+
#: which can never have a file. The sets are disjoint and `CompositeProvider`
|
| 87 |
+
#: says why that is a rule rather than a coincidence.
|
| 88 |
+
provider = CompositeProvider(LocalArtefactProvider(), HostedModelProvider())
|
| 89 |
+
media = build_store()
|
| 90 |
+
|
| 91 |
+
#: Every capability with an implemented adapter, from both modules that hold
|
| 92 |
+
#: one. Merged here rather than in either, so neither has to import the other and
|
| 93 |
+
#: the set `/health` counts is the set `/jobs` dispatches on.
|
| 94 |
+
#:
|
| 95 |
+
#: **A key in exactly one of them.** The two dicts are disjoint by construction β
|
| 96 |
+
#: counting owns the two detector capabilities, identification owns
|
| 97 |
+
#: `cattle_identity` β and a collision would silently take whichever was merged
|
| 98 |
+
#: last, so it is refused rather than resolved.
|
| 99 |
+
_MODULES = {
|
| 100 |
+
"counting": _COUNTING_RUNNERS,
|
| 101 |
+
"identification": _IDENTITY_RUNNERS,
|
| 102 |
+
"reasoning": _REASONING_RUNNERS,
|
| 103 |
+
}
|
| 104 |
+
_SEEN: dict[str, str] = {}
|
| 105 |
+
for _module, _runners in _MODULES.items():
|
| 106 |
+
for _key in _runners:
|
| 107 |
+
if _key in _SEEN:
|
| 108 |
+
raise RuntimeError(
|
| 109 |
+
f"{_SEEN[_key]} and {_module} both claim {_key}. One capability "
|
| 110 |
+
f"has one runner, and a silent winner here is a capability "
|
| 111 |
+
f"answered by a model nobody chose."
|
| 112 |
+
)
|
| 113 |
+
_SEEN[_key] = _module
|
| 114 |
+
|
| 115 |
+
RUNNERS: dict[str, object] = {
|
| 116 |
+
**_COUNTING_RUNNERS, **_IDENTITY_RUNNERS, **_REASONING_RUNNERS,
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
#: In-memory for now. Jobs are re-submittable and carry no record of their own β
|
| 120 |
+
#: the durable state lives in the API's `model_runs`, and a lost job means a
|
| 121 |
+
#: retry rather than lost data. A queue replaces this when a model exists to run.
|
| 122 |
+
_jobs: dict[UUID, JobStatus] = {}
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def _configured_token() -> str:
|
| 126 |
+
"""Read at call time, not import time, so a test can set it and so a secret
|
| 127 |
+
rotation takes effect on restart rather than needing a rebuild."""
|
| 128 |
+
return os.environ.get("ANIMAP_INFERENCE_TOKEN", "")
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def require_token(authorization: str = Header(default="")) -> None:
|
| 132 |
+
"""Bearer auth on the endpoints that touch a farm's media.
|
| 133 |
+
|
| 134 |
+
Absent `ANIMAP_INFERENCE_TOKEN` this is a no-op, which is what lets the
|
| 135 |
+
tests and a local uvicorn run without ceremony. That default is only safe
|
| 136 |
+
because it is *visible*: `/health` reports `authenticated`, so a deployment
|
| 137 |
+
that reached the internet without a token says so to anyone who asks,
|
| 138 |
+
including the person checking it after a deploy.
|
| 139 |
+
"""
|
| 140 |
+
expected = _configured_token()
|
| 141 |
+
if not expected:
|
| 142 |
+
return
|
| 143 |
+
|
| 144 |
+
scheme, _, presented = authorization.partition(" ")
|
| 145 |
+
# Constant-time, because a token compared with `==` leaks its prefix to
|
| 146 |
+
# anyone willing to time a few thousand requests.
|
| 147 |
+
if scheme.lower() != "bearer" or not hmac.compare_digest(presented, expected):
|
| 148 |
+
raise HTTPException(401, "A valid bearer token is required.")
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
class HealthResponse(BaseModel):
|
| 152 |
+
status: str
|
| 153 |
+
capabilities_registered: int
|
| 154 |
+
capabilities_runnable: int
|
| 155 |
+
artefacts_loaded: int
|
| 156 |
+
#: Capabilities that have both an artefact and an adapter. The gap between
|
| 157 |
+
#: this and `capabilities_runnable` is the set that returns 501.
|
| 158 |
+
adapters_implemented: int
|
| 159 |
+
#: Where captures are read from. Reported because a service that is healthy
|
| 160 |
+
#: and pointed at an empty local directory fails every job for a reason no
|
| 161 |
+
#: probe would otherwise show.
|
| 162 |
+
media_provider: str
|
| 163 |
+
#: The licence of every artefact that is actually loaded. A deployment that
|
| 164 |
+
#: has picked up a copyleft model is a deployment in breach, so it is visible
|
| 165 |
+
#: from outside rather than only in a log (ADR 0017).
|
| 166 |
+
artefact_licenses: list[str]
|
| 167 |
+
#: Whether `/jobs` requires a bearer token. False on a public deployment is
|
| 168 |
+
#: a misconfiguration, and this is what makes it findable without reading
|
| 169 |
+
#: the Container App's environment.
|
| 170 |
+
authenticated: bool
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
class SupersededView(BaseModel):
|
| 174 |
+
"""The verdict a capability used to carry, and why it moved (ADR 0021).
|
| 175 |
+
|
| 176 |
+
Served rather than kept internal so that a reframing nobody agrees with is
|
| 177 |
+
arguable from outside this service.
|
| 178 |
+
"""
|
| 179 |
+
|
| 180 |
+
verdict: str
|
| 181 |
+
summary: str
|
| 182 |
+
reframed_because: str
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
class DispositionView(BaseModel):
|
| 186 |
+
"""The evidence behind a capability, and what the product may say about it.
|
| 187 |
+
|
| 188 |
+
Served alongside the state because a client that can only render `Coming
|
| 189 |
+
soon` has nothing to say when a farmer asks *when*. `stated_uncertainty` is
|
| 190 |
+
the line a result screen shows beside a number (directive Β§37).
|
| 191 |
+
"""
|
| 192 |
+
|
| 193 |
+
group: str
|
| 194 |
+
summary: str
|
| 195 |
+
blocker: str
|
| 196 |
+
stated_uncertainty: str
|
| 197 |
+
data_needed: str | None
|
| 198 |
+
evidence: list[str]
|
| 199 |
+
superseded: SupersededView | None
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
class QuantityView(BaseModel):
|
| 203 |
+
"""What one claim's observation value may carry, and on whose authority.
|
| 204 |
+
|
| 205 |
+
`basis` is published rather than kept internal because the three kinds are
|
| 206 |
+
not equally strong: a `scale` bound is what the unit means, and a
|
| 207 |
+
`guardrail` is an engineering judgement with no measurement behind it. A
|
| 208 |
+
client that shows a farmer a refused reading should be able to say which.
|
| 209 |
+
"""
|
| 210 |
+
|
| 211 |
+
claim: str
|
| 212 |
+
#: `null` when the claim carries no quantity at all.
|
| 213 |
+
unit: str | None
|
| 214 |
+
carries_number: bool
|
| 215 |
+
minimum: float | None
|
| 216 |
+
maximum: float | None
|
| 217 |
+
step: float | None
|
| 218 |
+
basis: str | None
|
| 219 |
+
why: str
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
class OutputView(BaseModel):
|
| 223 |
+
type: str
|
| 224 |
+
unit: str | None
|
| 225 |
+
show_range: bool
|
| 226 |
+
#: The range outside which a value is broken rather than merely surprising,
|
| 227 |
+
#: and the granularity the rubric supports. All `null` for the twelve
|
| 228 |
+
#: capabilities whose output carries no number. See `OutputSpec` for what
|
| 229 |
+
#: these are and, more importantly, what they are not: they bound the
|
| 230 |
+
#: impossible, and say nothing about where a real answer usually falls.
|
| 231 |
+
#:
|
| 232 |
+
#: **Enforced as well as published**, which they were not for one commit:
|
| 233 |
+
#: `app/adapters/claims.py::schema_for` and `check_numeric_bounds` both read
|
| 234 |
+
#: them, so a client that pre-checks against these gets the same answer the
|
| 235 |
+
#: service does.
|
| 236 |
+
plausible_min: float | None
|
| 237 |
+
plausible_max: float | None
|
| 238 |
+
step: float | None
|
| 239 |
+
#: Which of `allowed_claims` report the quantity the three fields above
|
| 240 |
+
#: bound. Empty for a categorical output. A client showing an observation
|
| 241 |
+
#: knows from this whether the bounds apply to it β a footpad grade is 0 to
|
| 242 |
+
#: 4, and the sampled prevalence beside it is a percentage.
|
| 243 |
+
measured_claims: list[str]
|
| 244 |
+
#: Every claim in the vocabulary and what number it may carry, which is what
|
| 245 |
+
#: the three fields above could not say. They bound one quantity, so an
|
| 246 |
+
#: observation on any other claim went unbounded β 73 of the registry's 93,
|
| 247 |
+
#: and the route a lameness score took to a farm.
|
| 248 |
+
#:
|
| 249 |
+
#: `carries_number: false` means a figure beside this claim is the defect,
|
| 250 |
+
#: not a finer reading of it, and the service refuses every number there. The
|
| 251 |
+
#: claim still carries a word: `Observation.value` takes a string, and
|
| 252 |
+
#: 'moderate' or 'left flank' is what such a claim is for.
|
| 253 |
+
quantities: list[QuantityView]
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
class AcquisitionView(BaseModel):
|
| 257 |
+
"""Directive Β§34. How this capability's signal is acquired, and what may be
|
| 258 |
+
claimed from it. The Android capture flow reads this."""
|
| 259 |
+
|
| 260 |
+
protocol: str
|
| 261 |
+
modality: list[str]
|
| 262 |
+
output: OutputView
|
| 263 |
+
minimum_capture_seconds: int | None
|
| 264 |
+
preferred_capture_seconds: int | None
|
| 265 |
+
minimum_distance_m: float | None
|
| 266 |
+
minimum_samples: int | None
|
| 267 |
+
preferred_samples: int | None
|
| 268 |
+
required_views: list[str]
|
| 269 |
+
optional_inputs: list[str]
|
| 270 |
+
confirmation_options: list[str]
|
| 271 |
+
escalation: str | None
|
| 272 |
+
#: Workflows this capability must never hold up (Β§6.6).
|
| 273 |
+
never_blocks: list[str]
|
| 274 |
+
reject_if: list[str]
|
| 275 |
+
#: What a model may emit. This is the closed vocabulary a reasoner's schema
|
| 276 |
+
#: is built from, so a client rendering it is showing what Animap can say.
|
| 277 |
+
allowed_claims: list[str]
|
| 278 |
+
forbidden_claims: list[str]
|
| 279 |
+
#: Quantities the result screen may show that no model produces β the app
|
| 280 |
+
#: computes them from a capability output plus the farm's own records.
|
| 281 |
+
#: `poultry_count` is the only holder, carrying Β§6.3's third quantity.
|
| 282 |
+
derived_claims: list[str]
|
| 283 |
+
#: **What a model may say in words a farmer reads** (ADR 0024). `evidence`
|
| 284 |
+
#: used to be free text; it is a closed enum of these phrases now, per
|
| 285 |
+
#: capability, exactly as `allowed_claims` is. The strings are reader-facing
|
| 286 |
+
#: rather than identifiers, so a client renders them as they are β and a
|
| 287 |
+
#: client that pre-checks a response against this list gets the same answer
|
| 288 |
+
#: the service does.
|
| 289 |
+
#:
|
| 290 |
+
#: Empty for `poultry_uniformity`, which may claim nothing (ADR 0023).
|
| 291 |
+
allowed_evidence: list[str]
|
| 292 |
+
#: What a model may say about the **capture** rather than about the animal.
|
| 293 |
+
#: A shared set plus one phrase per condition in `reject_if`, so the
|
| 294 |
+
#: rejection a client already renders and the sentence a farmer reads for it
|
| 295 |
+
#: cannot drift apart.
|
| 296 |
+
allowed_limits: list[str]
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
class CapabilityView(BaseModel):
|
| 300 |
+
key: str
|
| 301 |
+
species: str
|
| 302 |
+
#: The single-word spelling the API's `capabilities` table stores. The full
|
| 303 |
+
#: list is `acquisition.modality`.
|
| 304 |
+
modality: str
|
| 305 |
+
state: CapabilityState
|
| 306 |
+
#: Any combination of `guided_capture`, `human_confirmation`,
|
| 307 |
+
#: `hardware_required`, `fixed_installation` β or none. Sorted, so a client
|
| 308 |
+
#: can compare two responses.
|
| 309 |
+
requirements: list[str]
|
| 310 |
+
#: What a farmer gets with no signal. Nineteen capabilities are `required`
|
| 311 |
+
#: because their first stage is a hosted model β which is a hosting fact,
|
| 312 |
+
#: never a reason for a lower `state`.
|
| 313 |
+
connectivity: str
|
| 314 |
+
#: Whether this could plausibly move to the phone later. An engineering
|
| 315 |
+
#: property, not a promise and not a badge.
|
| 316 |
+
on_device_candidate: bool
|
| 317 |
+
output_kind: str
|
| 318 |
+
depends_on: list[str]
|
| 319 |
+
#: The runtimes this capability's declared stack leans on, named as
|
| 320 |
+
#: `app/adapters/licences.py` names them.
|
| 321 |
+
model_stack: list[str]
|
| 322 |
+
#: Runtimes in that stack Animap may not currently serve, cannot fetch
|
| 323 |
+
#: unattended, or has never checked the terms of. **Never a reason for a
|
| 324 |
+
#: lower `state`** β a licence is an attribute of a model, not a property of
|
| 325 |
+
#: a capability. Empty for most.
|
| 326 |
+
licence_exposure: list[str]
|
| 327 |
+
acquisition: AcquisitionView
|
| 328 |
+
#: Whether a result can be produced **today**. Independent of `state`.
|
| 329 |
+
runnable: bool
|
| 330 |
+
reason: str
|
| 331 |
+
disposition: DispositionView | None
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
@app.get("/health", response_model=HealthResponse)
|
| 335 |
+
def health() -> HealthResponse:
|
| 336 |
+
runnable = [c for c in REGISTRY.values() if provider.can_run(c)]
|
| 337 |
+
return HealthResponse(
|
| 338 |
+
status="ok",
|
| 339 |
+
capabilities_registered=len(REGISTRY),
|
| 340 |
+
capabilities_runnable=len(runnable),
|
| 341 |
+
artefacts_loaded=len(provider.artefacts),
|
| 342 |
+
adapters_implemented=sum(1 for c in runnable if c.key in RUNNERS),
|
| 343 |
+
media_provider=media.provider,
|
| 344 |
+
artefact_licenses=sorted({a.license for a in provider.artefacts.values()}),
|
| 345 |
+
authenticated=bool(_configured_token()),
|
| 346 |
+
)
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
@app.get("/capabilities", response_model=list[CapabilityView])
|
| 350 |
+
def capabilities() -> list[CapabilityView]:
|
| 351 |
+
"""What the service can do, what it cannot, and what it may say either way.
|
| 352 |
+
|
| 353 |
+
The single source of what a user is told about a capability. A client shows
|
| 354 |
+
`state` and `runnable` together, `acquisition` to drive the capture, and
|
| 355 |
+
`disposition.stated_uncertainty` beside any number.
|
| 356 |
+
"""
|
| 357 |
+
# Computed once for the whole response rather than per capability: it reads
|
| 358 |
+
# the adapters' licence ledger, and 28 lookups of the same table is waste.
|
| 359 |
+
exposure = licence_exposure()
|
| 360 |
+
return [_view(c, exposure.get(c.key, [])) for c in REGISTRY.values()]
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
@app.post("/jobs", response_model=JobStatus, status_code=202,
|
| 364 |
+
dependencies=[Depends(require_token)])
|
| 365 |
+
def submit(request: InferenceRequest) -> JobStatus:
|
| 366 |
+
capability = REGISTRY.get(request.capability_key)
|
| 367 |
+
if capability is None:
|
| 368 |
+
raise HTTPException(404, f"Unknown capability: {request.capability_key}")
|
| 369 |
+
|
| 370 |
+
if not provider.can_run(capability):
|
| 371 |
+
# Not an error. The capture is already saved on the device and in the
|
| 372 |
+
# API; this only says no interpretation is available for it yet.
|
| 373 |
+
status = JobStatus(
|
| 374 |
+
job_id=uuid4(),
|
| 375 |
+
state=JobState.UNAVAILABLE,
|
| 376 |
+
capability_key=capability.key,
|
| 377 |
+
detail=_unavailable_reason(capability),
|
| 378 |
+
)
|
| 379 |
+
_jobs[status.job_id] = status
|
| 380 |
+
return status
|
| 381 |
+
|
| 382 |
+
runner = RUNNERS.get(capability.key)
|
| 383 |
+
if runner is None:
|
| 384 |
+
# An artefact without an adapter stays unimplemented rather than
|
| 385 |
+
# stubbed: an adapter that returns something plausible is precisely the
|
| 386 |
+
# failure ADR 0005 exists to prevent.
|
| 387 |
+
raise HTTPException(
|
| 388 |
+
501,
|
| 389 |
+
f"{capability.key} has a validated artefact but no adapter is "
|
| 390 |
+
f"implemented for it yet.",
|
| 391 |
+
)
|
| 392 |
+
|
| 393 |
+
artefact = provider.artefact_for(capability)
|
| 394 |
+
job_id = uuid4()
|
| 395 |
+
try:
|
| 396 |
+
result = runner.run(
|
| 397 |
+
request=request,
|
| 398 |
+
capability=capability,
|
| 399 |
+
artefact=artefact,
|
| 400 |
+
store=media,
|
| 401 |
+
request_id=job_id,
|
| 402 |
+
)
|
| 403 |
+
except MediaError as exc:
|
| 404 |
+
# The referenced media is missing or unreadable. The capture itself is
|
| 405 |
+
# safe on the device, so this is a job to retry, not data to discard.
|
| 406 |
+
status = JobStatus(
|
| 407 |
+
job_id=job_id, state=JobState.FAILED,
|
| 408 |
+
capability_key=capability.key, detail=str(exc),
|
| 409 |
+
)
|
| 410 |
+
except (DetectorError, ValueError) as exc:
|
| 411 |
+
logger.exception("%s failed on %s", capability.key, request.media_ids[0])
|
| 412 |
+
status = JobStatus(
|
| 413 |
+
job_id=job_id, state=JobState.FAILED,
|
| 414 |
+
capability_key=capability.key,
|
| 415 |
+
detail=f"The model could not be run: {exc}",
|
| 416 |
+
)
|
| 417 |
+
else:
|
| 418 |
+
status = JobStatus(
|
| 419 |
+
job_id=job_id, state=JobState.COMPLETE,
|
| 420 |
+
capability_key=capability.key, result=result,
|
| 421 |
+
)
|
| 422 |
+
|
| 423 |
+
_jobs[job_id] = status
|
| 424 |
+
return status
|
| 425 |
+
|
| 426 |
+
|
| 427 |
+
@app.get("/jobs/{job_id}", response_model=JobStatus,
|
| 428 |
+
dependencies=[Depends(require_token)])
|
| 429 |
+
def job(job_id: UUID) -> JobStatus:
|
| 430 |
+
status = _jobs.get(job_id)
|
| 431 |
+
if status is None:
|
| 432 |
+
raise HTTPException(404, "No such job.")
|
| 433 |
+
return status
|
| 434 |
+
|
| 435 |
+
|
| 436 |
+
def _view(c: Capability, exposure: list[str] | None = None) -> CapabilityView:
|
| 437 |
+
runnable = provider.can_run(c)
|
| 438 |
+
return CapabilityView(
|
| 439 |
+
key=c.key,
|
| 440 |
+
species=c.species,
|
| 441 |
+
modality=c.modality,
|
| 442 |
+
state=c.state,
|
| 443 |
+
requirements=sorted(r.value for r in c.requirements),
|
| 444 |
+
connectivity=c.connectivity.value,
|
| 445 |
+
on_device_candidate=c.on_device_candidate,
|
| 446 |
+
output_kind=c.output_kind,
|
| 447 |
+
depends_on=list(c.depends_on),
|
| 448 |
+
model_stack=list(c.model_stack),
|
| 449 |
+
licence_exposure=list(exposure or []),
|
| 450 |
+
acquisition=_acquisition_view(c),
|
| 451 |
+
runnable=runnable,
|
| 452 |
+
reason="" if runnable else _unavailable_reason(c),
|
| 453 |
+
disposition=_disposition_view(c.key),
|
| 454 |
+
)
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def _acquisition_view(c: Capability) -> AcquisitionView:
|
| 458 |
+
a = c.acquisition
|
| 459 |
+
return AcquisitionView(
|
| 460 |
+
protocol=a.protocol,
|
| 461 |
+
modality=[m.value for m in a.modality],
|
| 462 |
+
output=OutputView(
|
| 463 |
+
type=a.output.type, unit=a.output.unit, show_range=a.output.show_range,
|
| 464 |
+
plausible_min=a.output.plausible_min,
|
| 465 |
+
plausible_max=a.output.plausible_max,
|
| 466 |
+
step=a.output.step,
|
| 467 |
+
measured_claims=list(a.output.measured_claims),
|
| 468 |
+
quantities=[
|
| 469 |
+
QuantityView(
|
| 470 |
+
claim=q.claim, unit=q.unit, carries_number=q.carries_number,
|
| 471 |
+
minimum=q.minimum, maximum=q.maximum, step=q.step,
|
| 472 |
+
basis=q.basis.value if q.basis else None, why=q.why,
|
| 473 |
+
)
|
| 474 |
+
for q in a.output.quantities
|
| 475 |
+
],
|
| 476 |
+
),
|
| 477 |
+
minimum_capture_seconds=a.minimum_capture_seconds,
|
| 478 |
+
preferred_capture_seconds=a.preferred_capture_seconds,
|
| 479 |
+
minimum_distance_m=a.minimum_distance_m,
|
| 480 |
+
minimum_samples=a.minimum_samples,
|
| 481 |
+
preferred_samples=a.preferred_samples,
|
| 482 |
+
required_views=list(a.required_views),
|
| 483 |
+
optional_inputs=list(a.optional_inputs),
|
| 484 |
+
confirmation_options=list(a.confirmation_options),
|
| 485 |
+
escalation=a.escalation,
|
| 486 |
+
never_blocks=list(a.never_blocks),
|
| 487 |
+
reject_if=list(a.reject_if),
|
| 488 |
+
allowed_claims=list(a.allowed_claims),
|
| 489 |
+
forbidden_claims=list(a.forbidden_claims),
|
| 490 |
+
derived_claims=list(a.derived_claims),
|
| 491 |
+
allowed_evidence=list(a.evidence_phrases),
|
| 492 |
+
allowed_limits=list(a.limit_phrases),
|
| 493 |
+
)
|
| 494 |
+
|
| 495 |
+
|
| 496 |
+
def _disposition_view(key: str) -> DispositionView | None:
|
| 497 |
+
disposition = dispositions.get(key)
|
| 498 |
+
if disposition is None:
|
| 499 |
+
return None
|
| 500 |
+
superseded = disposition.superseded
|
| 501 |
+
return DispositionView(
|
| 502 |
+
group=disposition.group.value,
|
| 503 |
+
summary=disposition.summary,
|
| 504 |
+
blocker=disposition.blocker,
|
| 505 |
+
stated_uncertainty=disposition.stated_uncertainty,
|
| 506 |
+
data_needed=disposition.data_needed,
|
| 507 |
+
evidence=[f"{s.claim} β {s.url}" for s in disposition.evidence],
|
| 508 |
+
superseded=None if superseded is None else SupersededView(
|
| 509 |
+
verdict=superseded.verdict,
|
| 510 |
+
summary=superseded.summary,
|
| 511 |
+
reframed_because=superseded.reframed_because,
|
| 512 |
+
),
|
| 513 |
+
)
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
def _unavailable_reason(c: Capability) -> str:
|
| 517 |
+
"""Why a capability cannot run today, in words a client can show a person.
|
| 518 |
+
|
| 519 |
+
**The distinction this string used to draw has moved into `state`.** It once
|
| 520 |
+
had to separate "late" from "never", because six capabilities were
|
| 521 |
+
`coming_soon` in the registry and `not_viable` in the research, and a client
|
| 522 |
+
rendering `status` alone showed the same `Coming soon` for both. After
|
| 523 |
+
ADR 0021 nothing is `not_viable`: the claims that were rejected are rejected
|
| 524 |
+
as claims, in `capabilities.REJECTED_CLAIMS`, and every capability survives
|
| 525 |
+
in a corrected form.
|
| 526 |
+
|
| 527 |
+
What is left to say is narrower and more useful β this feature is real, the
|
| 528 |
+
model behind it is not installed on this deployment yet, and here is what it
|
| 529 |
+
is waiting for.
|
| 530 |
+
"""
|
| 531 |
+
if c.state == CapabilityState.UNSUPPORTED_CLAIM:
|
| 532 |
+
# **One capability holds this state**: `poultry_uniformity`, since
|
| 533 |
+
# ADR 0023. The branch existed so that one arriving could not be
|
| 534 |
+
# rendered as merely late, and the arrival showed it was half a branch.
|
| 535 |
+
#
|
| 536 |
+
# Β§39 says not to change a capability to "Not planned", and a flat
|
| 537 |
+
# refusal here would do exactly that to a claim that survives in another
|
| 538 |
+
# form. So the two cases are split. A capability whose corrected form is
|
| 539 |
+
# something the app computes says what that is; only a claim with
|
| 540 |
+
# nothing behind it at all reads as unplanned.
|
| 541 |
+
if c.acquisition.derived_claims:
|
| 542 |
+
return (
|
| 543 |
+
"The capture is saved. Animap will not work this out from the "
|
| 544 |
+
"capture this capability declares β measured against the real "
|
| 545 |
+
"answer it is wrong by more than the number is worth, and by "
|
| 546 |
+
"more birds rather than fewer. What survives is computed from "
|
| 547 |
+
"entered values instead: "
|
| 548 |
+
+ ", ".join(c.acquisition.derived_claims)
|
| 549 |
+
+ "."
|
| 550 |
+
)
|
| 551 |
+
return (
|
| 552 |
+
"This is not planned. The claim behind it is not scientifically "
|
| 553 |
+
"supportable in any capture protocol."
|
| 554 |
+
)
|
| 555 |
+
# Checked before `state`, not inside a `coming_soon` branch. A capability
|
| 556 |
+
# waiting on another capability is waiting whatever its own state says.
|
| 557 |
+
#
|
| 558 |
+
# **Nothing declares a dependency today.** `poultry_uniformity` was the only
|
| 559 |
+
# one and ADR 0023 removed it, because the input it waited on is the input
|
| 560 |
+
# that makes its answer wrong β a dependency that resolves into a refusal is
|
| 561 |
+
# a worse pointer than none.
|
| 562 |
+
if c.depends_on:
|
| 563 |
+
return (
|
| 564 |
+
"The capture is saved. This needs "
|
| 565 |
+
+ ", ".join(c.depends_on)
|
| 566 |
+
+ " first β there is nothing to compute until those results exist."
|
| 567 |
+
)
|
| 568 |
+
if c.state == CapabilityState.COMING_SOON:
|
| 569 |
+
return "Not built yet. The engineering path exists and nobody has walked it."
|
| 570 |
+
if Requirement.HARDWARE_REQUIRED in c.requirements:
|
| 571 |
+
return "This capability needs compatible hardware."
|
| 572 |
+
if Requirement.FIXED_INSTALLATION in c.requirements:
|
| 573 |
+
return "This capability needs a permanently placed camera or microphone."
|
| 574 |
+
artefact = provider.artefact_for(c)
|
| 575 |
+
if artefact is None:
|
| 576 |
+
return (
|
| 577 |
+
"The capture is saved. No validated model for this capability is "
|
| 578 |
+
"installed on this deployment yet, so the analysis is still to come."
|
| 579 |
+
)
|
| 580 |
+
# **These two were one string, and it named the wrong thing.** The fallback
|
| 581 |
+
# read "The model artefact is present but has not been validated." for
|
| 582 |
+
# `cattle_identity`, whose artefact is present *and* validated β it passes
|
| 583 |
+
# its checksum, `discover()` loads it, and its licence is in
|
| 584 |
+
# `/health.artefact_licenses`. What was missing was on the capability, not on
|
| 585 |
+
# the model: the registry entry named no `model_provider`, so
|
| 586 |
+
# `Capability.is_runnable` was False. A reason that sends somebody to inspect
|
| 587 |
+
# a model card when the model card is fine costs an afternoon, and
|
| 588 |
+
# `DEPLOY.md` had to carry a paragraph warning readers not to believe it.
|
| 589 |
+
if not artefact.is_validated:
|
| 590 |
+
return (
|
| 591 |
+
"The model artefact is installed but its card attests nothing about "
|
| 592 |
+
"whether it works, so this deployment will not run it."
|
| 593 |
+
)
|
| 594 |
+
return (
|
| 595 |
+
"The capture is saved. The model for this capability is installed and "
|
| 596 |
+
"checksummed, and the capability is not switched on in this build yet."
|
| 597 |
+
)
|