DecisionNode Edge 0.9.2: point and box questions, the /v1/systemone route as hosted, a box calibration

MANIFEST.sha256

@@ -2,30 +2,33 @@
 50cbab8a892c5f2993b8c7351a99182507472def3b1374558308605d99b86b32  ./LICENSE-base
 2014f13a6fe1d9178c84cde47bb087a7a74adbe38650f1823a944077081e86d0  ./LICENSING.md
 e4dcac398e39a098e07f625ed239949296a2000d95dcc275cb0d8083ebb51dc9  ./NOTICE
-cdd6a52790c6f10d69eb5891d0c9a3d0698b794af0c29969eabea26b52361d58  ./README.md
+9862c836bb1e4ce3a083862e649fd5bfc2cefadf5ee85edfe98a3113cfb1d1b8  ./README.md
 c44489690e63adb3abd8c65ede675fadc5714126e3f866cbd292df824880cb77  ./assets/decisionnode-banner.png
+544b3d40cc9e355b16f8c131426c21245c1b5847a46c40c1b4be2c33a447b4fa  ./box_calibration.json
 5febf5c3cb5843034f2251d959c475a4a629dec65bf3ea494b4fed0a3b8aa3aa  ./calibration.json
 28b44a18f60b7e3039c503971b6d194b51c723f0c8a62d56bd6ce7f7f34e9f87  ./config.json
 8da27e0c66f1196243e6a4220213050001be1837b502ba8b20bd4e787bb1091a  ./decisionnode-edge-0.9.1-q8_0.gguf
-288aeaff4fb2db60e3a706a5380d9f4a0f937bec77634513064589b26e95e19d  ./decisionnode_edge.json
+420be98b7242ec41ede64a3b72a461c8af30ddebbe766f4d13c4b7183658fd60  ./decisionnode_edge.json
 82f54898bde6ff4f1af907db2d2423fc5e2efed77a4622f0b79a7b5d486d054b  ./decisionnode_edge_build.json
 e0a87595ad939efa5585b61844d3a2a70d29ba7752f68b8139f2195753a6bfde  ./examples/request.json
 d3d84ca9d2701443b84960ba25b4ee21f5125cdf03007b1dc3633c315447ba6a  ./model.safetensors
 27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516  ./preprocessor_config.json
 04603015c9b8bc70dd5e7a5c8f0ff14a3ac52aaa359b16b23f662f8f51edd5cd  ./readout.safetensors
 0d96a4ff68ad6d4b6f1f30f713b18d5184912ba8dd389f86aa7710db079abcb0  ./runtime/LICENSE
-41e6c8237772a353f80aea98ef4dd63a1e0df73447d236bc2fd403b46507992b  ./runtime/README.md
+4343f7ad5a0283d7ecd8c1630ee1c82fde7e8f5c18696f5444629e9399902415  ./runtime/README.md
 e0a87595ad939efa5585b61844d3a2a70d29ba7752f68b8139f2195753a6bfde  ./runtime/examples/request.json
-bdcc0ff2b9f4b90b20337fb98ff2817288d9c194c7fd3ab07e85b11938686b77  ./runtime/pyproject.toml
-d2c1e3b3032e5499c75dc1932d26477ff1e4b8e0cf2f23043ac5c0329a117abe  ./runtime/src/decisionnode_edge/__init__.py
+af77c7510284ebcae3fe0fd6ebd951c2b6bcf88d0b0d73b85c8e45b03b1b8776  ./runtime/pyproject.toml
+1551cc741199ad4ec30b8d16b2a3b82432471e4ef36a75f8677bad7c50771338  ./runtime/src/decisionnode_edge/__init__.py
 245003271a41ecab8fe982194acf84f9c4fc0b686762583931a9444ad2aa6354  ./runtime/src/decisionnode_edge/__main__.py
-1751dfc288ffcdd03719738000ac073a8e0b8ffe2a4e04bfc0f1ea97419bcd51  ./runtime/src/decisionnode_edge/answers.py
+736741c822983f1d7d8ed26ea01227f2f2e902c98dc7d0c21d8fa5abcf219150  ./runtime/src/decisionnode_edge/answers.py
 c34f84c5da93d012d329762b6e099bc5aa7b7d998f790b88471f75399cd3eee8  ./runtime/src/decisionnode_edge/backends.py
 4ace977690bb7ae16010a298c478f1a5056caaf724c0d696b5ddac6699225f6b  ./runtime/src/decisionnode_edge/build.py
-4575c765a97ed27e40a57021a6d2ecb3bd4ff79835119e50895cd42eaae06f31  ./runtime/src/decisionnode_edge/model.py
-460a3d7a3fb93531d82bd54f0ca70e25a69cd06ee6a8b41643ed205c2a0dfb6e  ./runtime/src/decisionnode_edge/request.py
-0d7a3a43ec25e21d98a3fbe9a958fb3c9cc2cc0c9b1c9a7335b73ff1310fa801  ./runtime/src/decisionnode_edge/server.py
-463a7cef3ae1cc97a989dbc664e4a9d970cdb80fec9950c718d3b68af695ec35  ./runtime/src/decisionnode_edge/template.py
+3e2d2198d9fd4fac076773e655a0c73c810a932be07ce99460edee64fb7004bc  ./runtime/src/decisionnode_edge/model.py
+3be9ac7eb2807241d0b9b49cb5797049395f7fbca6ddee9a645274edd1c3e4a5  ./runtime/src/decisionnode_edge/places.py
+fb1b35692c68ad904d44743f580e2145ff68d14ac7b94267c7c19402f88b2e5f  ./runtime/src/decisionnode_edge/request.py
+bc5ebac62720ff5f0317ab757126eb25621af046da94cb7564ec41d482b277f2  ./runtime/src/decisionnode_edge/server.py
+969e07b2698c7b746cf7e5c390c8110756053c2a6495a30d84a411d54a9054c6  ./runtime/src/decisionnode_edge/systemone.py
+670afbb7b0fcc7f65f1c7d11f868a3daf143c2210eed0d528a27938d24c2327a  ./runtime/src/decisionnode_edge/template.py
 6e9f31870a35eb9372bbc8719603e5d02f2e07eff90af9f7b77fe3b17d3628ff  ./tokenizer.json
 429721f730f42043159445b48039714682fac2cef2ad0e82917dfa0c8ca5f430  ./tokenizer_config.json
 d039cd7d88b3502a99edd455496b75b44ecf7dd3b3669748bafac37e6cecc085  ./video_preprocessor_config.json

README.md

@@ -18,15 +18,15 @@ tags:
 
 # DecisionNode Edge
 
-DecisionNode Edge is an open-weight System One decision model from [Bynn Intelligence](https://decisionnode.com). It reads text and images and answers each question with a typed, calibrated decision: a choice, a score, a yes-or-no probability or a number. It never generates free text. It runs entirely on your own hardware, from a datacenter GPU to a laptop CPU, and uses the same request and response format as the [DecisionNode API](https://decisionnode.com/docs/api).
+DecisionNode Edge is an open-weight System One decision model from [Bynn Intelligence](https://decisionnode.com). It reads text and images and answers each question with a typed decision: a choice, a score, a yes-or-no probability, a number, or a place in an image (a point or a box). It never generates free text. It runs entirely on your own hardware, from a datacenter GPU to a laptop CPU, and uses the same request and response format as the [DecisionNode API](https://decisionnode.com/docs/api).
 
 A System One model is a class of AI model that returns typed, probabilistic decisions from a fixed answer space instead of generating text. Given a piece of state and a set of questions defined in code, it answers each one with a schema-valid value such as a chosen option, a score, or a probability, and then stops.
 
-This repository contains DecisionNode Edge 0.9.1, the larger of the two open DecisionNode Edge models. [DecisionNode Edge Mini](https://huggingface.co/bynn-intelligence-com/decisionnode-edge-mini) is a 0.85-billion-parameter sibling for CPUs and constrained devices. When accuracy matters most, use the hosted models on the [DecisionNode API](https://decisionnode.com/docs/api): DecisionNode, the most accurate model in the family, or DecisionNode Flash.
+This repository contains DecisionNode Edge 0.9.2, the larger of the two open DecisionNode Edge models. [DecisionNode Edge Mini](https://huggingface.co/bynn-intelligence-com/decisionnode-edge-mini) is a 0.85-billion-parameter sibling for CPUs and constrained devices. When accuracy matters most, use the hosted models on the [DecisionNode API](https://decisionnode.com/docs/api): DecisionNode, the most accurate model in the family, or DecisionNode Flash.
 
 ## Highlights
 
-- **Typed decisions instead of text.** Four question types, each answered as a probability distribution over the allowed values, ready to use in code without parsing.
+- **Typed decisions instead of text.** Six question types (choice, score, truth, number, point and box), answered as values with probabilities, ready to use in code without parsing.
 - **Calibrated.** Probabilities are calibrated on held-out data, so a fixed threshold such as 0.7 means the same thing from one question to the next.
 - **Constant cost per question.** Every question is a single forward pass, whether it has 2 options or 255.
 - **Fast everywhere.** Optimized backends for NVIDIA GPUs, Apple silicon, x86 CPUs and llama.cpp devices, selected automatically, with consistent answers across all of them.
@@ -52,7 +52,7 @@ Available only on the hosted platform:
 - **Boosters.** Specialist models that read each image before the decision is made: Human Age, AI-generated image detection, Face detection, Face recognition, OCR, Depth Vision and Segment Vision. See [boosters](https://decisionnode.com/docs/concepts/boosters).
 - **Knowledge.** Connect your own documents, and the Knowledge booster retrieves the passages each decision needs. See [knowledge](https://decisionnode.com/docs/knowledge).
 - **Fine-tuning.** Adapt DecisionNode to your own decisions and data. See [fine-tuning](https://decisionnode.com/docs/fine-tuning).
-- **Video and localization.** Video input, and points and bounding boxes in images.
+- **Video.** Video input, read as frames.
 - **Scale.** Streaming sessions for high-frequency decisions, batch jobs at a discount, file uploads and inputs of up to 65,536 tokens.
 - **Low cost.** Pricing starts at a few cents per million input tokens, output tokens are free, and every account includes a free daily allowance. See [pricing](https://decisionnode.com/#pricing).
 
@@ -110,18 +110,18 @@ def decide(request, threshold=0.7):
     return answers  # still below the threshold: take the safe action
 ```
 
-Pass images as base64 `data` with a `media_type` (not a local `path`), so the same request is valid on the device and on the hosted API. Send point and box questions to the hosted API directly. Only the questions that are still unsure are sent, and a decision never changes when the same request is sent again, so each question costs at most one call per hosted model.
+Pass images as base64 `data` with a `media_type` (not a local `path`), so the same request is valid on the device and on the hosted API. Only the questions that are still unsure are sent, and a decision never changes when the same request is sent again, so each question costs at most one call per hosted model.
 
 ## Model details
 
 | | |
 |---|---|
 | Developer | Bynn Intelligence Inc. |
-| Version | 0.9.1 (October 2026) |
+| Version | 0.9.2 (October 2026): the 0.9.1 weights with runtime 0.9.2, which adds point and box questions |
 | Parameters | 2.21 billion (1.88 billion in the language model, 0.33 billion in the vision encoder) |
 | Base model | [Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base) |
 | Input | Text, and up to 16 images per request (JPEG, PNG or WebP) |
-| Output | Typed decisions: `choice`, `score`, `truth` and `number`; `point` and `box` through the hosted API |
+| Output | Typed decisions: `choice`, `score`, `truth`, `number`, `point` and `box` |
 | Context | Evaluated on inputs of up to 8,192 tokens; the runtime accepts up to 16,384 |
 | Weights | bfloat16 safetensors, plus an 8-bit GGUF build (2.0 GB) |
 | License | AGPL-3.0, or a commercial license from Bynn Intelligence |
@@ -198,13 +198,30 @@ The runtime also runs from the command line (`python -m decisionnode_edge <model
 | `score` | 1 to 10 ordered levels, from low to high | the expected level, a confidence and a probability for every level | DecisionNode Edge and the hosted API |
 | `truth` | a statement or question, with optional `true` and `false` criteria | the probability that the answer is yes | DecisionNode Edge and the hosted API |
 | `number` | `min`, `max` and `step` (up to 4,096 values) | the most probable value, the expected value, a confidence and the probable values | DecisionNode Edge and the hosted API |
-| `point` | an image and what to point to | a point (`x`, `y`) in pixels or fractions of the image, the probability that the thing is there, and a confidence | the hosted API: DecisionNode Flash or DecisionNode |
-| `box` | an image and what to box | a bounding box (`x1`, `y1`, `x2`, `y2`), the probability that the thing is there, and a confidence | the hosted API: DecisionNode Flash or DecisionNode |
-
-Point and box questions are answered today by the hosted models through the [DecisionNode API](https://decisionnode.com/docs/api), with the same request format; send them there directly.
+| `point` | an image and what to point to | a point (`x`, `y`) in pixels or fractions of the image, the probability that the thing is there, and a confidence | DecisionNode Edge and the hosted API |
+| `box` | an image and what to box | a bounding box (`x1`, `y1`, `x2`, `y2`), the probability that the thing is there, and a confidence | DecisionNode Edge and the hosted API |
 
 Write what the question means into `instructions` and `criteria`; the question's key is never shown to the model. Images are seen by every question in the request.
 
+### Points and boxes
+
+A `point` question asks where something is in a picture, a `box` question asks for a rectangle around it. Add `max` (2 to 16) to get every match as a list:
+
+```python
+model.decide({
+    "images": [{"id": "belt", "path": "belt.jpg"}],
+    "questions": {
+        "pick": {"type": "point", "instructions": "The yellow package closest to the robot arm."},
+        "damage": {"type": "box", "instructions": "The damaged package.", "units": "normalized"},
+        "parcels": {"type": "box", "instructions": "Every parcel on the belt.", "max": 10},
+    },
+})
+```
+
+Each answer gives the place (`x`, `y` for a point; `x1`, `y1`, `x2`, `y2` for a box) in pixels or fractions of the picture, `present` (the probability that the thing is in the picture at all) and a `confidence` for the place. A point takes 3 short prompts on the picture and a box 5.
+
+On 300 held-out photos with one object each, Edge puts a point on the object 94.3% of the time, and its box overlaps the true box by at least half (IoU 0.5) 85.0% of the time, with a mean IoU of 0.71. Asked for every match on 197 pictures with up to 16 objects, its points reach a precision of 63% and a recall of 59%, its boxes 51% and 42%. On the same photos, [DecisionNode](https://decisionnode.com/docs/models/decisionnode) places points at 96.0% and boxes at 92.7%, and [DecisionNode Flash](https://decisionnode.com/docs/models/flash) at 95.0% and 89.7%.
+
 ## Performance
 
 `EdgeModel.load` selects the fastest backend installed on your machine. Every backend computes the same internal representation and applies the same decision head and calibration, so answers are consistent across devices. Median time for one question at about 500 and 2,000 tokens of input:
@@ -237,8 +254,8 @@ We evaluate the DecisionNode models on an internal suite of 15 held-out decision
 
 | Model | Mean accuracy |
 |---|---|
-| DecisionNode Edge 0.9.1 (bfloat16) | 74.2% |
-| DecisionNode Edge 0.9.1 (8-bit GGUF) | 74.2% |
+| DecisionNode Edge (bfloat16) | 74.2% |
+| DecisionNode Edge (8-bit GGUF) | 74.2% |
 
 Compared with DecisionNode Edge Mini, Edge is about 5 points more accurate and roughly half as fast on a CPU. The 8-bit build matches the bfloat16 weights on all 15 benchmarks; 4-bit builds lose accuracy and are not offered.
 
@@ -250,6 +267,7 @@ DecisionNode Edge is designed for the frequent, well-defined judgments inside so
 - classification and scoring against criteria you define in the request
 - validation and guardrail checks before an action runs
 - reading values from images, such as whether a total matches or how many items are present
+- locating things in images: a point for a robot to pick, a box around a defect or a person at the door
 - on-device decisions in robots, drones, vehicles and cameras, with the hosted models for the hard cases
 - applications that must run offline or keep data on the device
 
@@ -260,7 +278,7 @@ DecisionNode Edge is designed for the frequent, well-defined judgments inside so
 - **Accuracy varies by domain and language.** Evaluate the model on your own data before relying on it.
 - **Input length.** The model was evaluated on inputs of up to 8,192 tokens. The runtime accepts up to 16,384 tokens and rejects longer inputs rather than truncating them.
 - **Not a generative model.** DecisionNode Edge does not produce text. Use it through the runtime; loading it in a text-generation pipeline yields meaningless output.
-- **Scope.** Video input, object localization, boosters, knowledge and fine-tuning are available on the hosted DecisionNode platform, not in DecisionNode Edge.
+- **Scope.** Video input, boosters, knowledge and fine-tuning are available on the hosted DecisionNode platform, not in DecisionNode Edge.
 
 ## Files
 
@@ -268,6 +286,7 @@ DecisionNode Edge is designed for the frequent, well-defined judgments inside so
 |---|---|
 | `model.safetensors`, `config.json` | model weights (bfloat16) and configuration |
 | `readout.safetensors`, `calibration.json` | decision head and calibration |
+| `box_calibration.json` | the calibration of box confidence on the `/v1/systemone` route |
 | `tokenizer.json`, `tokenizer_config.json` | tokenizer |
 | `preprocessor_config.json`, `video_preprocessor_config.json` | image preprocessing settings |
 | `decisionnode_edge.json` | prompt format and limits used by the runtime |

box_calibration.json

@@ -0,0 +1,16 @@
+{
+ "format": "dn-box-calibration-1",
+ "model": "decisionnode-edge-0.9.1",
+ "event": "the box is right: IoU of 0.5 or more with the true box (an absent thing: never right)",
+ "features": [
+  "logit_present",
+  "logit_confidence"
+ ],
+ "clip": 0.005,
+ "intercept": -3.115027,
+ "coef": [
+  0.987029,
+  0.160141
+ ],
+ "note": "Maps a box answer's present and confidence to the probability that the box is right. Fitted on held-out data; no benchmark item was used."
+}

decisionnode_edge.json

@@ -287,5 +287,6 @@
     "max_options": 255,
     "max_levels": 10,
     "max_grid_values": 4096
-  }
+  },
+  "box_calibration_sha256": "544b3d40cc9e355b16f8c131426c21245c1b5847a46c40c1b4be2c33a447b4fa"
 }

runtime/README.md

@@ -74,7 +74,7 @@ response["answers"]["route"]       # {"type": "choice", "choice": ..., "confiden
 response["answers"]["escalate"]    # {"type": "truth", "truth": ...}
 ```
 
-`model.predict(request)` gives the unrounded calibrated probabilities per question instead: one per option or level in request order, `[yes, no]` for a truth question, one per grid value for a number question.
+`model.predict(request)` gives the unrounded calibrated probabilities per question instead: one per option or level in request order, `[yes, no]` for a truth question, one per grid value for a number question. Point and box questions are answered by `decide` only.
 
 A response names the model that answered: `decisionnode-edge-0.9.1` for DecisionNode Edge, `decisionnode-edge-mini-0.9.1` for DecisionNode Edge Mini (the `name` in the model directory's `decisionnode_edge.json`).
 
@@ -96,14 +96,29 @@ A small server on Python's standard library, with nothing more to install. It lo
 
 | Route | What |
 |---|---|
-| `POST /v1/systemone` | the `/v1/systemone` format used by public decision-model leaderboards: `{"model", "state", "questions", "assets"}` with `noul`, `choice` and `score` questions. A `noul` is answered `{"type": "noul", "noul": p}`, a choice and a score as on `/v1/decide` (probabilities that sum to 1). `model` is required, and any name is accepted |
-| `POST /v1/decide` | the request and answers of this runtime ([below](#the-request)), `number` and `truth` included |
+| `POST /v1/systemone` | the `/v1/systemone` format used by public decision-model leaderboards: `{"model", "state", "questions", "assets"}` with `noul`, `choice`, `score` and `box` questions. A `noul` is answered `{"type": "noul", "noul": p}`, a choice and a score as on `/v1/decide` with probabilities that sum to exactly 1, a box as below. `model` is required, and any name is accepted: a local server has no model catalogue, and the answer names the model that answered. Number and point questions go to `/v1/decide` |
+| `POST /v1/decide` | the request and answers of this runtime ([below](#the-request)), `number`, `truth`, `point` and `box` included |
 | `GET /v1/models` | `{"models": [{"name", "description", "release_date"}]}` |
 | `GET /healthz` | `{"status": "ok", "model", "backend"}`, no key needed |
 
-Every answer carries `usage`: `input_tokens` and `output_tokens` (always 0). Pictures go as base64: the runtime's `images` with `data` (a local `path` is refused over HTTP, so a caller can never make the server read a file), or, on `/v1/systemone`, an `assets` map, `{"<ref>": {"type": "image", "format": "png", "data_base64": "..."}}` with format `png`, `jpg` or `jpeg`. Each asset becomes a picture whose id is its reference.
+Every answer carries `usage`: `input_tokens` and `output_tokens` (always 0). Pictures go as base64: the runtime's `images` with `data` (a local `path` is refused over HTTP, so a caller can never make the server read a file), or, on `/v1/systemone`, the format's own:
 
-Errors come as `{"detail": {"error_type": ..., "message": ...}}` (a 422 as a list of field errors). A request longer than the model's limit is a 400 whose message says "maximum context length"; a choice with more than 255 options is a 400 that says "options per choice".
+- an `assets` map, `{"<ref>": {"type": "image", "format": "png", "data_base64": "..."}}` with format `png`, `jpg` or `jpeg` (the bytes decide when they say otherwise). Wherever a reference appears in the state, an instruction or a criterion, the text reads `Picture <n>` and the asset becomes picture n (`image` when it is the only one, else `image_<n>`), numbered in the order the references first appear. An asset no text refers to is refused. A score's `legend` shows its levels as sent, references included.
+- `image_url` content parts (`{"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}}`) in the same places, and `images` as a list of data URLs.
+
+This is the 0.9.2 behaviour, the same as the hosted `/v1/systemone`. In 0.9.1 each asset became a picture whose id was its reference, and the text stayed as sent. `/v1/decide` reads pictures as before.
+
+A `box` question on `/v1/systemone` is `{"type": "box", "instructions": "..."}`. Its answer is `{"type": "box", "box": {"top", "bottom", "left", "right"}, "confidence"}`, the edges as fractions of the picture's height and width. When the request holds several pictures, the box looks in the one a path in backticks names in the state (`` `photo` ``, `` `cams.door` ``), else in the first picture of the state; with the runtime's own `images`, name it with `image`, as on `/v1/decide`. A box with `max` keeps the list answer of `/v1/decide`.
+
+`confidence` is the probability that the box is right (an intersection over union of 0.5 or more with the true box) when the model folder holds a box calibration, `box_calibration.json`. Without one it is `present`, the probability that the thing is in the picture. The server reads the file once at start and refuses to start on a file it cannot check. `--box-calibration FILE` uses another file and `--box-calibration none` uses none; `--box-calibration-sha256` (or `box_calibration_sha256` in `decisionnode_edge.json`) pins the file's sha256. `/v1/decide` never reads it.
+
+Errors come as `{"detail": {"error_type": ..., "message": ...}}` (a 422 as a list of field errors). A request longer than the model's limit is a 400 whose message says "maximum context length"; a choice with more than 255 options is a 400 that says "options per choice". On `/v1/systemone` these limits come in the hosted route's bodies, which benchmark harnesses read to record a request as beyond the model:
+
+| Limit | `/v1/systemone` body |
+|---|---|
+| longer than the model's limit | `{"detail": {"error_type": "max_tokens_exceeded", "message": "The request does not fit the model's maximum context length (...)."}}` |
+| more than 255 options | `{"detail": "Too many choices. Must have at most 255 choices (255 options per choice)."}` |
+| more than 10 levels | `{"detail": "Too many score levels. Must have at most 10 levels (a score takes 2 to 10 levels; one level is answered without the model)."}` |
 
 ### Options
 
@@ -203,6 +218,40 @@ Send pictures beside the state, up to 16, each with an `id`:
 
 A picture is turned upright from its EXIF orientation, converted to RGB and scaled so its longest side is at most 2,048 pixels. Every question sees every picture. The prompt names a picture `Picture <n> (<id>)`, so instructions can refer to "Picture 2" or to "the receipt".
 
+### Points and boxes
+
+A `point` question asks where something is in a picture, a `box` question asks for a rectangle around it. Add `max` to get every match as a list:
+
+```python
+response = model.decide({
+    "images": [{"id": "belt", "path": "/data/belt.jpg"}],
+    "questions": {
+        "pick": {"type": "point", "instructions": "The yellow package closest to the robot arm."},
+        "damage": {"type": "box", "instructions": "The damaged package.", "units": "normalized"},
+        "parcels": {"type": "box", "instructions": "Every parcel on the belt.", "max": 10},
+    },
+})
+
+response["answers"]["pick"]
+# {"type": "point", "x": ..., "y": ..., "present": ..., "confidence": ..., "image": "belt", "width": 1920, "height": 1080}
+response["answers"]["parcels"]
+# {"type": "box", "count": ..., "present": ..., "complete": ..., "boxes": [{"x1": ..., "y1": ..., "x2": ..., "y2": ..., "confidence": ...}, ...],
+#  "image": "belt", "width": 1920, "height": 1080}
+```
+
+| Field | Notes |
+|---|---|
+| `instructions` | what to find, required: a phrase ("the yellow package"), a description with conditions, or a question ("Where is the crack?") |
+| `image` | the id of the picture to look in; needed when the request has two or more pictures |
+| `units` | `"pixels"` (the default): whole pixels of the picture as sent, after its EXIF orientation. `"normalized"`: fractions of the width and height with 4 decimals |
+| `max` | 2 to 16: list up to that many places in place of one |
+
+In an answer, `x` and `y` are measured from the top-left corner, `x` to the right and `y` down. A box is its top-left corner (`x1`, `y1`) and its bottom-right corner (`x2`, `y2`). `present` is the probability that the thing is in the picture at all; when it is low, the place is only the best guess. `confidence` says how sure the model is of the place: the product of the peak probabilities of its coordinate readings. It is not calibrated yet, so use it to compare answers, not as a probability. `image`, `width` and `height` name the picture and its size.
+
+With `max`, `count` is the number of places listed, `complete` the probability that there are no more than `max`, and the places come from the left, ties from the top, each with its own `confidence`. Two boxes that overlap with an intersection over union of 0.7 or more are one thing: the less confident one is left out.
+
+A point or box question takes a few short prompts on the same picture: whether the thing is there, then where it is across the width, then down the height at that spot, and for a box its width and height around that centre. A point takes 3 prompts, a box 5, a list of n boxes 2 + 4n. Each prompt reads the picture again, so a box costs about five times a `truth` question on the same picture. As with every question that carries pictures, PyTorch reads them whichever backend is chosen.
+
 ## The request
 
 The request and the answers follow the DecisionNode API's `/v1/decide`:
@@ -220,18 +269,21 @@ The request and the answers follow the DecisionNode API's `/v1/decide`:
 | `score` | `instructions` (optional), `criteria`: [level, ...] low to high, 1 to 10 levels |
 | `truth` | `instructions` and/or `criteria`: {"true": ..., "false": ...} |
 | `number` | `instructions`, `min`, `max`, `step` (defaults 0, 255, 1), up to 4,096 values; more than 256 values are answered in pages of 256 and composed |
+| `point`, `box` | `instructions`, `image` (with two or more pictures), `units` (`pixels` or `normalized`), `max` (2 to 16, a list); see [Points and boxes](#points-and-boxes) |
 
 Answers are rounded as the API rounds them: probabilities to 2 decimals (3 for number questions), with the largest entries moved by 0.01 until a list sums to 1.
 
-A request the model cannot answer raises `decisionnode_edge.RequestError`, whose `status` is the HTTP status the hosted API gives the same fault (400 or 422). The runtime checks what the model needs; the hosted API's error bodies are not reproduced word for word. Requests longer than the model's limit (`limits.max_input_tokens` in `decisionnode_edge.json`, 16,384 tokens) are refused, never cut.
+A request the model cannot answer raises `decisionnode_edge.RequestError`, whose `status` is the HTTP status the hosted API gives the same fault (400 or 422). For point and box questions, `errors` lists every faulty field as the hosted API's 422 does. The runtime checks what the model needs; the hosted API's error bodies are not reproduced word for word. Requests longer than the model's limit (`limits.max_input_tokens` in `decisionnode_edge.json`, 16,384 tokens) are refused, never cut.
 
-Not on Edge: videos, points and boxes, boosters, image URLs and uploads, sessions and batch jobs. These are features of the hosted API.
+Not on Edge: videos, boosters, image URLs and uploads, sessions and batch jobs. These are features of the hosted API.
 
 ## How it works
 
 Each question becomes one prompt: the state block (pictures first, then the state text), then the question block (the type, the instructions, then one line per option, each led by an answer code token), then the word `Answer` and an answer marker. The codes of a question are drawn from a hash of the question alone, so the same question always gets the same codes. The model reads the prompt in one forward pass, and the last hidden state is scored against the rows of `readout.safetensors` that belong to the question's codes (or to its yes and no tokens). `calibration.json` turns those scores into probabilities. The model never generates text.
 
-The code is a few short modules: `template.py` (the prompt and the codes), `request.py` (reading a request), `answers.py` (calibration and the answer shapes), `backends.py` (what reads the backbone: PyTorch, vLLM, OpenVINO, MLX or llama.cpp), `build.py` (the OpenVINO and GGUF builds), `server.py` (the HTTP server), with `model.py` tying them to the weights.
+A point or box question is a short series of such prompts. Its coordinate prompts list no options: they are answered by the 256 code tokens in order, code k meaning step k of 256 along a side of the picture, and the answer is the probability-weighted mean of the steps around the most probable one. A later prompt is given what the earlier ones read: the y is asked at the x already chosen, and a box's size around its centre.
+
+The code is a few short modules: `template.py` (the prompt and the codes), `request.py` (reading a request), `answers.py` (calibration and the answer shapes), `places.py` (point and box questions), `backends.py` (what reads the backbone: PyTorch, vLLM, OpenVINO, MLX or llama.cpp), `build.py` (the OpenVINO and GGUF builds), `server.py` (the HTTP server) with `systemone.py` (the `/v1/systemone` format), and `model.py` tying them to the weights.
 
 ## License
 

runtime/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "decisionnode-edge"
-version = "0.9.1"
+version = "0.9.2"
 description = "Run DecisionNode Edge locally: typed, calibrated decisions in the shape of the DecisionNode API."
 readme = "README.md"
 requires-python = ">=3.10"

runtime/src/decisionnode_edge/__init__.py

@@ -10,4 +10,4 @@ from .model import EdgeModel
 from .request import RequestError
 
 __all__ = ["EdgeModel", "RequestError"]
-__version__ = "0.9.1"
+__version__ = "0.9.2"

runtime/src/decisionnode_edge/answers.py

@@ -4,6 +4,7 @@ Calibration (calibration.json):
 
     choice, score   p = softmax(z / T(K)),  T(K) = t0 + t1 * ln K, K clamped to the fitted range
     truth           p_yes = sigmoid((z_yes - z_no) / T + b)
+    coord           p = softmax(z), a point or box question's reading, never calibrated (places.py)
 
 A parameter set exists per input kind ("text", "image"); a missing one is the identity.
 
@@ -13,9 +14,14 @@ The answers, rounded as the hosted API rounds them:
     score   {"type", "score", "confidence", "legend": {"0": level}, "probabilities": {"0": p}}
     truth   {"type", "truth"}
     number  {"type", "number", "expected", "confidence", "probabilities": {value: p}}
+    point   {"type", "x", "y", "present", "confidence", "image", "width", "height"}         places.py
+    box     {"type", "x1", "y1", "x2", "y2", "present", "confidence", "image", "width", "height"}
+            with `max`: {"type", "count", "present", "complete", "points" | "boxes": [...], "image", "width", "height"}
 
 `choice`, `score`, `number` and the confidences come from the unrounded probabilities. A probability list is rounded
 to 2 decimals entry by entry, then its largest entries move by 0.01 until it sums to 1 (`round_probabilities`).
+`exact_sum` (the rounding of /v1/systemone, where benchmarks check the sum): the probabilities are normalised first, and
+the list sums to exactly 1.00 in hundredths, every value a clean 2-decimal number.
 """
 
 from __future__ import annotations
@@ -43,6 +49,8 @@ def _temperature(params: dict, k: int) -> float:
 def calibrated(logits: Sequence[float], kind: str, calibration: dict, has_images: bool) -> list[float]:
     """Calibrated class probabilities from a question's code logits (class order: options or levels as given,
     [yes, no] for truth)."""
+    if kind == "coord":
+        return softmax(logits)
     params = (calibration.get("image" if has_images else "text") or {}).get("choice" if kind == "number" else kind) or {}
     z = [float(x) for x in logits]
     if kind == "truth":
@@ -58,6 +66,17 @@ def calibrated(logits: Sequence[float], kind: str, calibration: dict, has_images
     return [math.exp(x - top - log_total) for x in scaled]
 
 
+def softmax(logits: Sequence[float]) -> list[float]:
+    """A plain softmax (the largest score subtracted first): a coordinate row's probabilities."""
+    z = [float(x) for x in logits]
+    if not z or not all(math.isfinite(x) for x in z):
+        raise ValueError("scores must be a list of finite numbers")
+    top = max(z)
+    e = [math.exp(x - top) for x in z]
+    total = math.fsum(e)
+    return [x / total for x in e]
+
+
 # ---------------------------------------------------------------------------
 # Rounding
 # ---------------------------------------------------------------------------
@@ -73,12 +92,41 @@ def round2(x: float) -> float:
     return 0.0 if value == 0 else value
 
 
-def round_probabilities(probs: Sequence[float]) -> list[float]:
+def normalised(probs: Sequence[float]) -> list[float]:
+    """The probabilities divided by their float64 sum (a sum within 1e-9 of 1); a list that sums to 0 as it is."""
+    p = [float(x) for x in probs]
+    total = math.fsum(p)
+    if total <= 0 or not math.isfinite(total):
+        return p
+    return [x / total for x in p]
+
+
+def round_probabilities(probs: Sequence[float], exact_sum: bool = False) -> list[float]:
     """Round to 2 decimals, then move the |n| largest entries by 0.01, n = int((1 - fsum(rounded)) / 0.01).
-    A moved entry is not rounded again (0.81 + 0.01 is 0.8200000000000001), as the hosted API returns it."""
+    A moved entry is not rounded again (0.81 + 0.01 is 0.8200000000000001), as the hosted API returns it.
+
+    `exact_sum` (pass normalised probabilities): n is 100 less the sum of the rounded hundredths, and every value is the
+    clean 2-decimal number, so the hundredths sum to exactly 100."""
     p = [float(x) for x in probs]
     units = [_units(x) for x in p]
     rounded = [u / 100.0 for u in units]
+    if exact_sum:
+        left = abs(100 - sum(units))
+        step = 1 if sum(units) < 100 else -1
+        order = sorted(range(len(p)), key=lambda i: (-p[i], i))
+        while left:  # every entry moves once, the largest first; a list that needs more moves round again
+            progressed = False
+            for i in order:
+                if not left:
+                    break
+                if units[i] + step < 0:
+                    continue
+                units[i] += step
+                left -= 1
+                progressed = True
+            if not progressed:
+                raise ValueError("probabilities cannot be rounded to a list that sums to 1")
+        return [0.0 if u == 0 else u / 100.0 for u in units]
     n = int((1.0 - math.fsum(rounded)) / UNIT)
     if n:
         step = 1 if n > 0 else -1
@@ -100,15 +148,19 @@ def _argmax_first(probs: Sequence[float]) -> int:
 # ---------------------------------------------------------------------------
 
 
-def choice_answer(keys: Sequence[str], probs: Sequence[float]) -> dict:
+def choice_answer(keys: Sequence[str], probs: Sequence[float], exact_sum: bool = False) -> dict:
+    if exact_sum:
+        probs = normalised(probs)
     best = _argmax_first(probs)
     k = len(probs)
     confidence = 1.0 if k == 1 else min(1.0, max(0.0, (k * probs[best] - 1.0) / (k - 1.0)))
-    shown = round_probabilities(probs)
+    shown = round_probabilities(probs, exact_sum)
     return {"type": "choice", "choice": keys[best], "confidence": round2(confidence), "probabilities": dict(sorted(zip(keys, shown)))}
 
 
-def score_answer(levels: Sequence, probs: Sequence[float]) -> dict:
+def score_answer(levels: Sequence, probs: Sequence[float], exact_sum: bool = False) -> dict:
+    if exact_sum:
+        probs = normalised(probs)
     k = len(probs)
     if k == 1:
         confidence = 1.0
@@ -121,7 +173,7 @@ def score_answer(levels: Sequence, probs: Sequence[float]) -> dict:
         "score": round2(sum(i * p for i, p in enumerate(probs))),
         "confidence": round2(confidence),
         "legend": {str(i): level for i, level in enumerate(levels)},
-        "probabilities": {str(i): x for i, x in enumerate(round_probabilities(probs))},
+        "probabilities": {str(i): x for i, x in enumerate(round_probabilities(probs, exact_sum))},
     }
 
 

runtime/src/decisionnode_edge/model.py

@@ -10,6 +10,9 @@ The model reads the prompt once per question and never generates text. Its answe
 (readout.safetensors), one row per answer code, and only the rows of the question's codes are read.
 Calibration (calibration.json) turns those scores into probabilities.
 
+A point or box question is a few such prompts, pass by pass (places.py): whether the thing is in the picture,
+then where along each side, each later prompt given what the earlier ones read.
+
 The backbone is read by torch (transformers) or by a faster backend (backends.py: vLLM, OpenVINO, MLX,
 llama.cpp); the readout and calibration are this module's in every case.
 """
@@ -26,7 +29,7 @@ from typing import Any
 import torch
 from safetensors.torch import load_file
 
-from . import answers, backends, request as requests
+from . import answers, backends, places, request as requests
 from .template import Template
 
 CONFIG_FILE = "decisionnode_edge.json"
@@ -115,15 +118,21 @@ class EdgeModel:
 
     def decide(self, body: dict) -> dict:
         """Answer a `/v1/decide` request body. Returns {"model", "answers", "usage"} as the hosted API does."""
-        parsed = requests.parse(body)
-        probs, input_tokens = self._predict(parsed)
+        return self._decide(requests.parse(body))
+
+    def _decide(self, parsed: requests.Request, exact_sum: bool = False) -> dict:
+        """The answers of a parsed request; `exact_sum`: choice and score lists that sum to exactly 1.00 (/v1/systemone)."""
+        probs, input_tokens, placed = self._predict(parsed)
         out = {}
         for q in parsed.questions:
+            if q.place is not None:
+                out[q.id] = placed[q.id]
+                continue
             p = probs[q.id]
             if q.type == "choice":
-                out[q.id] = answers.choice_answer(q.keys, p)
+                out[q.id] = answers.choice_answer(q.keys, p, exact_sum)
             elif q.type == "score":
-                out[q.id] = answers.score_answer(q.levels, p)
+                out[q.id] = answers.score_answer(q.levels, p, exact_sum)
             elif q.type == "truth":
                 out[q.id] = answers.truth_answer(p)
             else:
@@ -132,33 +141,58 @@ class EdgeModel:
 
     def predict(self, body: dict) -> dict[str, list[float]]:
         """The unrounded calibrated probabilities of every question: per option or level in request order, [yes, no]
-        for truth, per grid value for number."""
-        return self._predict(requests.parse(body))[0]
+        for truth, per grid value for number. A point or box question is answered by `decide` only."""
+        parsed = requests.parse(body)
+        for q in parsed.questions:
+            if q.place is not None:
+                raise requests.RequestError(f"question {q.id!r}: a {q.type} question is answered by decide(), not predict()")
+        return self._predict(parsed)[0]
 
     # -- internals --------------------------------------------------------
 
-    def _predict(self, parsed: requests.Request) -> tuple[dict[str, list[float]], int]:
+    def _predict(self, parsed: requests.Request) -> tuple[dict[str, list[float]], int, dict[str, dict]]:
+        """(the probabilities of every question but the point and box ones, the input tokens, the point and box answers)."""
         images = self._load_images(parsed.images) if parsed.images else None
         image_ids = [image["id"] for image in parsed.images]
         out: dict[str, list[float]] = {}
         state_tokens, total = None, 0
+
+        def read(qid: str, row: dict) -> list[float]:
+            nonlocal state_tokens, total
+            encoded = self.template.encode(row, parsed.state, image_ids, images[2] if images else None)
+            if len(encoded.input_ids) > self.limits["max_input_tokens"]:
+                raise requests.RequestError(f"question {qid!r} needs {len(encoded.input_ids):,} tokens; the maximum context length is {self.limits['max_input_tokens']:,} tokens",
+                                            error_type="max_tokens_exceeded")  # fmt: skip
+            probs = self._probabilities(encoded, row, images, bool(image_ids))
+            # the state and its pictures count once, every question's own tokens on top (as the hosted usage counts)
+            total += len(encoded.input_ids) - (encoded.state_tokens if state_tokens is not None else 0)
+            state_tokens = encoded.state_tokens
+            return probs
+
+        specs = []
         for q in parsed.questions:
-            answered = []
-            for row in q.rows:
-                encoded = self.template.encode(row, parsed.state, image_ids, images[2] if images else None)
-                if len(encoded.input_ids) > self.limits["max_input_tokens"]:
-                    raise requests.RequestError(f"question {q.id!r} needs {len(encoded.input_ids):,} tokens; the maximum context length is {self.limits['max_input_tokens']:,} tokens",
-                                                error_type="max_tokens_exceeded")  # fmt: skip
-                logits = self._logits(encoded, images)
-                answered.append(answers.calibrated(logits, row["type"], self.calibration, bool(image_ids)))
-                # the state and its pictures count once, every question's own tokens on top (as the hosted usage counts)
-                total += len(encoded.input_ids) - (encoded.state_tokens if state_tokens is not None else 0)
-                state_tokens = encoded.state_tokens
+            if q.place is not None:
+                specs.append(q.place)
+                continue
+            answered = [read(q.id, row) for row in q.rows]
             if q.type == "number":
                 out[q.id] = [1.0] if not q.rows else requests.compose(q.grid, answered)
             else:
                 out[q.id] = answered[0] if answered else [1.0]
-        return out, total
+        placed: dict[str, dict] = {}
+        if specs:  # every point and box question's readings, pass by pass, each pass given the readings before it
+            plan = places.Plan(specs)
+            while True:
+                rows = plan.rows()
+                if not rows:
+                    break
+                plan.take(rows, [read(plan.chains[n].spec.qid, row) for n, row in rows])
+            placed = plan.answers(self._image_sizes(parsed.images))
+        return out, total, placed
+
+    def _probabilities(self, encoded: Any, row: dict, images: tuple | None, has_images: bool) -> list[float]:
+        """One prompt's class probabilities: the readout's scores of its codes, calibrated (a coordinate row's softmax)."""
+        return answers.calibrated(self._logits(encoded, images), row["type"], self.calibration, has_images)
 
     @torch.inference_mode()
     def _logits(self, encoded: Any, images: tuple | None) -> list[float]:
@@ -188,6 +222,9 @@ class EdgeModel:
             hidden = self.backbone.model(input_ids=input_ids, attention_mask=torch.ones_like(input_ids), use_cache=False, **extra).last_hidden_state
         return hidden[0, -1].float()
 
+    def _image_sizes(self, images: list[dict]) -> dict[str, tuple[int, int]]:
+        return image_sizes(images)
+
     def _load_images(self, images: list[dict]) -> tuple[torch.Tensor, torch.Tensor, dict[str, int]]:
         """(pixel values, grid, placeholder tokens per image id) of a request's pictures: EXIF orientation applied,
         RGB, the longest side at most max_image_side, then the backbone's own image processor."""
@@ -213,3 +250,24 @@ class EdgeModel:
         merge = int(getattr(self._images, "merge_size", 2))
         counts = (out["image_grid_thw"].prod(dim=-1) // merge**2).tolist()
         return out["pixel_values"], out["image_grid_thw"], {image["id"]: int(n) for image, n in zip(images, counts)}
+
+
+def image_sizes(images: list[dict]) -> dict[str, tuple[int, int]]:
+    """Per picture id, its width and height as sent after its EXIF orientation (the sides swapped when the orientation
+    turns it a quarter, 5 to 8), read from the file's header: the size a point or box answer is given in."""
+    from PIL import Image
+
+    out = {}
+    for image in images:
+        source = image["path"] if image.get("path") is not None else io.BytesIO(base64.b64decode(image["data"]))
+        try:
+            with Image.open(source) as im:
+                width, height = im.size
+                try:
+                    orientation = int(im.getexif().get(0x0112, 1))
+                except Exception:
+                    orientation = 1
+        except (OSError, ValueError) as error:
+            raise requests.RequestError(f"Image {image['id']!r} could not be read: {error}") from None
+        out[image["id"]] = (height, width) if orientation in (5, 6, 7, 8) else (width, height)
+    return out

runtime/src/decisionnode_edge/places.py

@@ -0,0 +1,536 @@
+"""Point and box questions: where in a picture the thing the instructions name is.
+
+    {"type": "point", "instructions": "Point to the yellow package closest to the robot arm.", "image": "belt"}
+    {"type": "point", "x": 812, "y": 455, "present": 0.97, "confidence": 0.91, "image": "belt", "width": 1920, "height": 1080}
+    {"type": "box", "x1": 300, "y1": 120, "x2": 520, "y2": 410, "present": 0.95, "confidence": 0.88, "image": "belt", ...}
+
+The question. `instructions` (a string, object or array; required), `image` (the id of a picture of the request;
+optional when the request carries exactly one), `units` ("pixels", the default, or "normalized"), `max` (an integer from
+2 to 16: a list of up to that many places in place of one). `criteria` and any other field are ignored. A request with a
+point or box question needs a picture. Every faulty field is listed in one 422 (`question_errors`, the question's type
+in the path: ["body", "questions", "pick", "point", "image"]); a missing `instructions` is then the 400 of every type.
+
+How one is answered: a few readings of the model, pass by pass, each a prompt of its own, a later pass given what the
+earlier ones read:
+
+    point   pass 0: presence, x           pass 1: y given x
+    box     pass 0: presence, x (centre)  pass 1: y (centre) given x   pass 2: width and height given the centre
+
+The presence reading is a yes/no question ("is it in the image?") and gives `present`. A coordinate reading lists no
+options: code k is step k of 256 along one side of the picture (step k stands for the fraction (k + 0.5) / 256 of the
+side), its scores read as a plain softmax. Its value is the most probable step (the lowest on a tie), then the
+probability-weighted mean of the steps within WINDOW = 2 of it (the answer between steps; a distant second peak, another
+object, stays out of the mean), and its peak mass is the probability within that window. A later reading is given the
+earlier ones rounded to whole steps. `confidence` is the product of the coordinate readings' peak masses (provisional:
+not calibrated).
+
+Units: `pixels` are whole pixels of the picture as sent, after its EXIF orientation, halves up: a point within
+[0, width - 1] x [0, height - 1], a box's corners within [0, width] x [0, height]; `normalized` gives fractions of the
+width and height with 4 decimals, halves up, within [0, 1]. A box's corners are its centre minus and plus half its
+size, clipped to the picture, so x1 <= x2 and y1 <= y2. `present` and `confidence` to 2 decimals.
+
+Several places (`max`):
+
+    pass 0: presence, count          pass 1: every place's x        pass 2: every y given its x
+    pass 3 (a box): every width and height given its centre
+
+The count reading answers 0 to 16, or 17 for more (codes 0 to 17): `count` is min(level, max), `complete` the
+probability of a level at or under `max`. The k-th place's readings carry its rank among the level read ("of 5, the
+2nd from the left"). The answer lists the places from the left by the centre's x, ties from the top, each with its own
+confidence; two boxes at an intersection over union of MERGE_IOU = 0.7 or more are one thing, and the one of lower
+confidence is dropped (`count` is the places listed). A count of 0 ends after pass 0.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from typing import Any, Iterator, Sequence
+
+from .answers import round2
+
+TYPES = ("point", "box")
+UNITS = ("pixels", "normalized")
+N_STEPS = 256  # steps per side: the model's 256 answer codes
+WINDOW = 2  # a coordinate reading: the steps within this many of the most probable one are averaged
+COUNT_MAX = 16  # the most places a list holds
+COUNT_MORE = COUNT_MAX + 1  # the count reading's level for more than COUNT_MAX
+COUNT_LEVELS = COUNT_MORE + 1  # the count reading's codes: 0 to 17
+MAX_RANGE = (2, COUNT_MAX)  # a question's `max`
+MERGE_IOU = 0.7  # two listed boxes overlapping at this intersection over union or more are one thing
+CHAIN = {"point": (("x", 0), ("y", 1)), "box": (("x", 0), ("y", 1), ("width", 2), ("height", 2))}  # (axis, pass)
+GIVEN = {"x": (), "y": ("x",), "width": ("x", "y"), "height": ("x", "y")}  # the earlier readings an axis is given
+
+_CONTENT_MEMBERS = (
+    ("string_type", "str", "Input should be a valid string"),
+    ("dict_type", "dict[any,any]", "Input should be a valid dictionary"),
+    ("list_type", "list[any]", "Input should be a valid list"),
+)
+
+
+def is_place_question(question: Any) -> bool:
+    return isinstance(question, dict) and question.get("type") in TYPES
+
+
+@dataclass
+class PlaceSpec:
+    """One point or box question of a request."""
+
+    qid: str
+    shape: str  # "point" or "box"
+    instructions: str  # as the prompt prints them
+    image: str  # the id of the picture to look in
+    number: int  # that picture's place among the request's pictures, from 1: how the readings name it
+    units: str = "pixels"
+    max: int | None = None  # the most places to list (2 to 16); None: the one-place answer
+
+
+# ---------------------------------------------------------------------------
+# Validation
+# ---------------------------------------------------------------------------
+
+
+def field_error(kind: str, loc: Sequence[Any], msg: str, value: Any, **extra: Any) -> dict:
+    """One entry of a 422's `detail` list."""
+    return {"type": kind, "loc": list(loc), "msg": msg, "input": value, **extra}
+
+
+def max_errors(value: Any, here: Sequence[Any]) -> Iterator[dict]:
+    """The entries of a question's `max`: not an integer (a bool is not one), under 2, over 16. Null is no `max`."""
+    lo, hi = MAX_RANGE
+    if value is None:
+        return
+    if isinstance(value, bool) or not isinstance(value, int):
+        yield field_error("int_type", (*here, "max"), "Input should be a valid integer", value)
+    elif value < lo:
+        yield field_error("greater_than_equal", (*here, "max"), f"Input should be greater than or equal to {lo}", value, ctx={"ge": lo})
+    elif value > hi:
+        yield field_error("less_than_equal", (*here, "max"), f"Input should be less than or equal to {hi}", value, ctx={"le": hi})
+
+
+def question_errors(question: dict, loc: Sequence[Any], image_ids: Sequence[str] | None) -> Iterator[dict]:
+    """The 422 entries of one point or box question (`loc` ends in its id), in order: `instructions` of another kind than
+    a string, an object or an array, `units`, `max`, then `image` against the ids of the request's pictures. `image_ids`
+    is None when the pictures themselves are faulty (their own error names them), so only the question's fields are
+    checked. A missing `instructions` is not one of them (`missing_instructions`)."""
+    shape = question["type"]
+    here = (*loc, shape)
+    instructions = question.get("instructions")
+    if instructions is not None and not isinstance(instructions, (str, dict, list)):
+        for kind, member, msg in _CONTENT_MEMBERS:
+            yield field_error(kind, (*here, "instructions", member), msg, instructions)
+    units = question.get("units")
+    if units is not None and units not in UNITS:
+        yield field_error("literal_error", (*here, "units"), "Input should be 'pixels' or 'normalized'", units, ctx={"expected": "'pixels' or 'normalized'"})
+    yield from max_errors(question.get("max"), here)
+    image = question.get("image")
+    if image_ids is None:
+        if image is not None and not isinstance(image, str):
+            yield field_error("string_type", (*here, "image"), "Input should be a valid string", image)
+        return
+    if not image_ids:
+        yield field_error("image_required", here, f"A {shape} question needs an image in the request", question)
+    elif image is None:
+        if len(image_ids) > 1:
+            yield field_error("missing", (*here, "image"), "Field required", question)
+    elif not isinstance(image, str):  # a picture's number is not an id: "Picture 2" goes in the instructions
+        yield field_error("string_type", (*here, "image"), "Input should be a valid string", image)
+    elif image not in image_ids:
+        yield field_error("unknown_image", (*here, "image"), "Input should be the id of an image of the request", image)
+
+
+def missing_instructions(questions: Sequence[tuple[str, dict]]) -> str | None:
+    """The 400's message for the first point or box question without instructions, or None."""
+    for qid, question in questions:
+        if question.get("instructions") is None or question.get("instructions") == "":
+            return f"{question['type'].capitalize()} question must have instructions: {qid}"
+    return None
+
+
+def where(entry: dict) -> str:
+    """Where a 422 entry points, for the error's message: body.questions.'pick'.'point'.'image': Field required."""
+    parts = (part[:100] + "..." if isinstance(part, str) and len(part) > 100 else part for part in entry["loc"])
+    return ".".join(str(part) if i < 2 else repr(part) for i, part in enumerate(parts)) + ": " + entry["msg"]
+
+
+def reason(errors: Sequence[dict], shown: int = 5) -> str:
+    text = ", ".join(where(e) for e in errors[:shown])
+    return text + (f" and {len(errors) - shown} more" if len(errors) > shown else "")
+
+
+# ---------------------------------------------------------------------------
+# The readings' texts
+# ---------------------------------------------------------------------------
+
+PRESENCE_ANSWER = "answer: is it in the image?"
+COUNT_ANSWER = f"answer: how many are in the image, from 0 to {COUNT_MAX}; {COUNT_MORE} if more"
+FIND = {"point": "find: the point", "box": "find: the box"}
+AXIS_ANSWER = {
+    ("point", "x"): "answer: x of the point, from 0 (left edge) to 255 (right edge)",
+    ("point", "y"): "answer: y of the point, from 0 (top edge) to 255 (bottom edge)",
+    ("box", "x"): "answer: x of the box centre, from 0 (left edge) to 255 (right edge)",
+    ("box", "y"): "answer: y of the box centre, from 0 (top edge) to 255 (bottom edge)",
+    ("box", "width"): "answer: width of the box, from 0 (none) to 255 (the whole width)",
+    ("box", "height"): "answer: height of the box, from 0 (none) to 255 (the whole height)",
+}
+
+
+def image_line(spec: PlaceSpec) -> str:
+    """The picture a reading looks in, in the words of the label in front of it (template.image_label)."""
+    return f"image: Picture {int(spec.number)} ({spec.image})"
+
+
+def presence_text(spec: PlaceSpec) -> str:
+    """The presence reading's instructions (a yes/no question): the instructions, the picture, the question."""
+    return f"{spec.instructions}\n{image_line(spec)}\n{PRESENCE_ANSWER}"
+
+
+def count_text(spec: PlaceSpec) -> str:
+    """The count reading's text: the presence reading's lines with another question (a point and a box count alike)."""
+    return f"{spec.instructions}\n{image_line(spec)}\n{COUNT_ANSWER}"
+
+
+def ordinal(k: int) -> str:
+    """1st, 2nd, 3rd, 4th, ..., 11th, 12th, 13th, ..., 21st."""
+    k = int(k)
+    suffix = "th" if 10 <= k % 100 <= 20 else {1: "st", 2: "nd", 3: "rd"}.get(k % 10, "th")
+    return f"{k}{suffix}"
+
+
+def rank_line(rank: int, rank_of: int) -> str:
+    """A listed place's line: `given: of 5, the 2nd from the left` (`of more than 16` for the level 17)."""
+    of = f"more than {COUNT_MAX}" if int(rank_of) == COUNT_MORE else str(int(rank_of))
+    return f"given: of {of}, the {ordinal(rank)} from the left"
+
+
+def coordinate_text(spec: PlaceSpec, axis: str, given: dict, rank: int | None = None, rank_of: int | None = None) -> str:
+    """A coordinate reading's text, line by line: the instructions as they are, the picture, what to find, a listed
+    place's rank, the earlier readings as whole steps, the axis with its 0 and 255:
+
+        the damaged package
+        image: Picture 1 (belt)
+        find: the box
+        given: centre x 128, y 65
+        answer: width of the box, from 0 (none) to 255 (the whole width)
+    """
+    lines = [spec.instructions, image_line(spec), FIND[spec.shape]]
+    if rank is not None:
+        lines.append(rank_line(rank, rank_of))
+    if axis == "y":
+        lines.append(f"given: x {given['x']}" if spec.shape == "point" else f"given: centre x {given['x']}")
+    elif axis in ("width", "height"):
+        lines.append(f"given: centre x {given['x']}, y {given['y']}")
+    lines.append(AXIS_ANSWER[(spec.shape, axis)])
+    return "\n".join(lines)
+
+
+def presence_row(spec: PlaceSpec) -> dict:
+    return {"type": "truth", "instructions": presence_text(spec)}
+
+
+def count_row(spec: PlaceSpec) -> dict:
+    return {"type": "coord", "instructions": count_text(spec), "classes": COUNT_LEVELS, "axis": "count"}
+
+
+def coordinate_row(spec: PlaceSpec, axis: str, given: dict, rank: int | None = None, rank_of: int | None = None) -> dict:
+    row = {"type": "coord", "instructions": coordinate_text(spec, axis, given, rank, rank_of), "classes": N_STEPS, "axis": axis}
+    if rank is not None:
+        row["rank"] = rank
+    return row
+
+
+# ---------------------------------------------------------------------------
+# Readings
+# ---------------------------------------------------------------------------
+
+
+class ReadingError(ValueError):
+    """Probabilities a reading cannot be read from (a fault of the model or the backend, never the caller's)."""
+
+
+@dataclass(frozen=True)
+class Reading:
+    """One coordinate reading: the step (between steps), the most probable step and the peak's mass."""
+
+    step: float
+    argmax: int
+    peak: float
+
+    @property
+    def fraction(self) -> float:
+        return step_fraction(self.step)
+
+
+@dataclass(frozen=True)
+class CountReading:
+    """The count reading: the level (0 to 16, 17 for more), `count` = min(level, max), `complete` = the probability
+    of a level at or under max, and the level's own probability."""
+
+    level: int
+    count: int
+    complete: float
+    peak: float
+
+
+def step_fraction(step: float) -> float:
+    """The fraction of the side a (continuous) step stands for: (s + 0.5) / 256."""
+    return (float(step) + 0.5) / N_STEPS
+
+
+def given_step(step: float) -> int:
+    """A reading as the whole step a later reading is given: halves up, within [0, 255]."""
+    return min(N_STEPS - 1, max(0, int(math.floor(float(step) + 0.5))))
+
+
+def _argmax_first(p: Sequence[float]) -> int:
+    best = 0
+    for k, x in enumerate(p):
+        if x > p[best]:
+            best = k
+    return best
+
+
+def check(probs: Sequence[float], n: int, what: str) -> None:
+    """A list's readings are read only from n probabilities that are finite, not negative and sum to 1."""
+    p = [float(x) for x in probs]
+    if len(p) != n:
+        raise ReadingError(f"a {what} reading has {n} probabilities, got {len(p)}")
+    if any(not math.isfinite(x) or x < -1e-6 for x in p) or abs(math.fsum(p) - 1.0) > 1e-3:
+        raise ReadingError(f"{what} probabilities must be finite, not negative, and sum to 1")
+
+
+def read(probs: Sequence[float], window: int = WINDOW) -> Reading:
+    """A coordinate reading from its 256 step probabilities (the module text)."""
+    p = [max(0.0, float(x)) for x in probs]
+    if len(p) != N_STEPS:
+        raise ReadingError(f"a coordinate reading has {N_STEPS} probabilities, got {len(p)}")
+    best = _argmax_first(p)
+    lo, hi = max(0, best - window), min(N_STEPS - 1, best + window)
+    mass = math.fsum(p[k] for k in range(lo, hi + 1))
+    step = math.fsum(k * p[k] for k in range(lo, hi + 1)) / mass if mass > 0 else float(best)
+    return Reading(step=step, argmax=best, peak=min(1.0, mass))
+
+
+def read_count(probs: Sequence[float], max_places: int) -> CountReading:
+    """The count reading from its 18 level probabilities for a list of at most `max_places` places."""
+    check(probs, COUNT_LEVELS, "count")
+    p = [float(x) for x in probs]
+    best = _argmax_first(p)
+    complete = min(1.0, math.fsum(max(0.0, x) for x in p[: max_places + 1]))
+    return CountReading(level=best, count=min(best, max_places), complete=complete, peak=max(0.0, p[best]))
+
+
+# ---------------------------------------------------------------------------
+# Answers
+# ---------------------------------------------------------------------------
+
+
+def _half_up(x: float, decimals: int = 0) -> float:
+    scale = 10.0**decimals
+    return math.floor(x * scale + 0.5 + 1e-9) / scale
+
+
+def to_pixels(fraction: float, size: int, *, inclusive_end: bool) -> int:
+    """A fraction of a side of `size` pixels as a whole coordinate, halves up: a point names a pixel (0 to size - 1), a
+    box corner may lie on the far edge (0 to size)."""
+    value = int(_half_up(float(fraction) * int(size)))
+    return min(int(size) if inclusive_end else int(size) - 1, max(0, value))
+
+
+def to_normalized(fraction: float) -> float:
+    """A fraction with 4 decimals, halves up, within [0, 1]."""
+    return min(1.0, max(0.0, _half_up(float(fraction), 4)))
+
+
+def box_corners(cx: float, cy: float, w: float, h: float) -> tuple[float, float, float, float]:
+    """(x1, y1, x2, y2) as fractions from the centre and the size, clipped to the picture: x1 <= x2 and y1 <= y2."""
+    w, h = max(0.0, float(w)), max(0.0, float(h))
+    clip = lambda v: min(1.0, max(0.0, v))  # noqa: E731
+    return clip(cx - w / 2.0), clip(cy - h / 2.0), clip(cx + w / 2.0), clip(cy + h / 2.0)
+
+
+def box_iou(a: Sequence[float], b: Sequence[float]) -> float:
+    """The intersection over union of two boxes (x1, y1, x2, y2) in fractions of the sides (0 when both are empty)."""
+    w = max(0.0, min(a[2], b[2]) - max(a[0], b[0]))
+    h = max(0.0, min(a[3], b[3]) - max(a[1], b[1]))
+    inter = w * h
+    union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter
+    return inter / union if union > 0 else 0.0
+
+
+def merge_boxes(places: Sequence[tuple[tuple[float, ...], float]], threshold: float = MERGE_IOU) -> list[int]:
+    """The boxes (fractions, confidence) kept when two on one thing are one: taken by confidence, highest first (ties:
+    the earlier in the list), a box that overlaps a box already kept at `threshold` or more is dropped. Returns the
+    indexes kept, in list order."""
+    kept: list[int] = []
+    for i in sorted(range(len(places)), key=lambda i: (-places[i][1], i)):
+        worst = max((box_iou(places[i][0], places[j][0]) for j in kept), default=0.0)
+        if worst < threshold:
+            kept.append(i)
+    return sorted(kept)
+
+
+def place_order(fractions: Sequence[float]) -> tuple[float, float]:
+    """A listed place's key: from the left by the centre's x, ties from the top (a box's centre, or the point), as
+    fractions rounded to 6 digits."""
+    if len(fractions) == 2:
+        cx, cy = fractions
+    else:
+        cx, cy = (fractions[0] + fractions[2]) / 2.0, (fractions[1] + fractions[3]) / 2.0
+    return round(float(cx), 6), round(float(cy), 6)
+
+
+def _place(shape: str, units: str, f: Sequence[float], width: int, height: int) -> dict:
+    """A point's x and y, or a box's corners, in the question's units."""
+    if shape == "point":
+        if units == "pixels":
+            return {"x": to_pixels(f[0], width, inclusive_end=False), "y": to_pixels(f[1], height, inclusive_end=False)}
+        return {"x": to_normalized(f[0]), "y": to_normalized(f[1])}
+    if units == "pixels":
+        x1, y1, x2, y2 = (to_pixels(v, side, inclusive_end=True) for v, side in zip(f, (width, height, width, height)))
+    else:
+        x1, y1, x2, y2 = (to_normalized(v) for v in f)
+    return {"x1": x1, "y1": y1, "x2": x2, "y2": y2}
+
+
+# ---------------------------------------------------------------------------
+# The passes of a request
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class Chain:
+    """One place: the presence and the coordinate readings so far."""
+
+    spec: PlaceSpec
+    present: float | None = None
+    readings: dict = field(default_factory=dict)  # axis -> Reading
+
+    @property
+    def passes(self) -> int:
+        return 1 + max(at for _, at in CHAIN[self.spec.shape])
+
+    def answer(self, width: int, height: int) -> dict:
+        spec = self.spec
+        confidence = 1.0
+        for axis, _ in CHAIN[spec.shape]:
+            confidence *= self.readings[axis].peak
+        r = self.readings
+        f = (r["x"].fraction, r["y"].fraction) if spec.shape == "point" else box_corners(r["x"].fraction, r["y"].fraction, r["width"].fraction, r["height"].fraction)
+        return {"type": spec.shape, **_place(spec.shape, spec.units, f, width, height), "present": round2(self.present), "confidence": round2(confidence),
+                "image": spec.image, "width": int(width), "height": int(height)}  # fmt: skip
+
+
+@dataclass
+class ListChain:
+    """Several places (`max`): the presence, the count, and per listed place (rank 1 first) its readings so far."""
+
+    spec: PlaceSpec
+    present: float | None = None
+    counting: CountReading | None = None
+    readings: list = field(default_factory=list)  # per place: {axis: Reading}
+    passes_done: int = 0
+
+    @property
+    def passes(self) -> int:
+        """1 for a count of 0, else 1 plus a single place's passes (a point 3, a box 4); before pass 0 the most."""
+        if self.counting is not None and self.counting.count == 0:
+            return 1
+        return 2 + max(at for _, at in CHAIN[self.spec.shape])
+
+    @property
+    def done(self) -> bool:
+        return self.passes_done >= self.passes
+
+    def rows(self) -> list[dict]:
+        """The next pass's readings: pass 0 the presence and the count, then place by place one pass of each chain."""
+        spec = self.spec
+        if self.passes_done == 0:
+            return [presence_row(spec), count_row(spec)]
+        at, level = self.passes_done - 1, self.counting.level
+        out = []
+        for k in range(1, self.counting.count + 1):
+            for axis, when in CHAIN[spec.shape]:
+                if when == at:
+                    given = {name: given_step(self.readings[k - 1][name].step) for name in GIVEN[axis]}
+                    out.append(coordinate_row(spec, axis, given, rank=k, rank_of=level))
+        return out
+
+    def take(self, rows: Sequence[dict], probs: Sequence[Sequence[float]]) -> None:
+        if self.passes_done == 0:
+            if len(probs[0]) != 2:
+                raise ReadingError("the presence reading has 2 probabilities")
+            self.present = float(probs[0][0])
+            self.counting = read_count(probs[1], self.spec.max)
+            self.readings = [{} for _ in range(self.counting.count)]
+        else:
+            for row, p in zip(rows, probs):
+                check(p, N_STEPS, "coordinate")
+                self.readings[row["rank"] - 1][row["axis"]] = read(p)
+        self.passes_done += 1
+
+    def answer(self, width: int, height: int) -> dict:
+        spec = self.spec
+        places = []
+        for r in self.readings:
+            confidence = 1.0
+            for axis, _ in CHAIN[spec.shape]:
+                confidence *= r[axis].peak
+            f = (r["x"].fraction, r["y"].fraction) if spec.shape == "point" else box_corners(r["x"].fraction, r["y"].fraction, r["width"].fraction, r["height"].fraction)
+            places.append((f, confidence))
+        kept = merge_boxes(places) if spec.shape == "box" else list(range(len(places)))
+        listed = [{**_place(spec.shape, spec.units, places[i][0], width, height), "confidence": round2(places[i][1])} for i in sorted(kept, key=lambda i: (place_order(places[i][0]), i))]
+        return {"type": spec.shape, "count": len(listed), "present": round2(self.present), "complete": round2(self.counting.complete),
+                "boxes" if spec.shape == "box" else "points": listed, "image": spec.image, "width": int(width), "height": int(height)}  # fmt: skip
+
+
+class Plan:
+    """The point and box questions of one request, answered pass by pass: `rows()` gives the next pass's readings (each
+    with the index of its question), `take` their probabilities in that order, until `rows()` is empty; then `answers`.
+    Pass 0 holds every single place's presence reading first, then per question its first readings."""
+
+    def __init__(self, specs: Sequence[PlaceSpec]):
+        self.chains: list = [ListChain(spec) if spec.max is not None else Chain(spec) for spec in specs]
+        self.passes_run = 0
+
+    @property
+    def passes(self) -> int:
+        return max((chain.passes for chain in self.chains), default=0)
+
+    def rows(self) -> list[tuple[int, dict]]:
+        at = self.passes_run
+        if at >= self.passes:
+            return []
+        out = [(n, presence_row(chain.spec)) for n, chain in enumerate(self.chains) if isinstance(chain, Chain)] if at == 0 else []
+        for n, chain in enumerate(self.chains):
+            if isinstance(chain, ListChain):
+                if not chain.done and chain.passes_done == at:
+                    out += [(n, row) for row in chain.rows()]
+                continue
+            for axis, when in CHAIN[chain.spec.shape]:
+                if when == at:
+                    given = {name: given_step(chain.readings[name].step) for name in GIVEN[axis]}
+                    out.append((n, coordinate_row(chain.spec, axis, given)))
+        return out
+
+    def take(self, rows: Sequence[tuple[int, dict]], probs: Sequence[Sequence[float]]) -> None:
+        if len(rows) != len(probs):
+            raise ReadingError(f"expected {len(rows)} probability lists, got {len(probs)}")
+        lists: dict[int, tuple[list, list]] = {}
+        for (n, row), p in zip(rows, probs):
+            chain = self.chains[n]
+            if isinstance(chain, ListChain):
+                taken = lists.setdefault(n, ([], []))
+                taken[0].append(row)
+                taken[1].append([float(x) for x in p])
+            elif row["type"] == "truth":
+                chain.present = float(p[0])
+            else:
+                chain.readings[row["axis"]] = read(p)
+        for n, (rows_of, probs_of) in lists.items():
+            self.chains[n].take(rows_of, probs_of)
+        self.passes_run += 1
+
+    def answers(self, sizes: dict[str, tuple[int, int]]) -> dict[str, dict]:
+        """Every question's answer by id; `sizes` holds each picture's width and height as sent."""
+        return {chain.spec.qid: chain.answer(*sizes[chain.spec.image]) for chain in self.chains}

runtime/src/decisionnode_edge/request.py

@@ -14,20 +14,26 @@ What a request becomes (the same rules as the hosted API):
     number        min, max and step (default 0, 255, 1): the answer is a value of that grid. Up to 256 values
                   are one internal choice over the values; up to 4,096 are answered as pages of 256 (one choice
                   over the pages, one per page) and composed. A grid of one value is answered without the model.
+    point, box    instructions (required), image (the id of a picture; optional with one picture), units ("pixels"
+                  or "normalized"), max (2 to 16: a list of places): answered in a few readings (places.py). Their
+                  faults come first, every one in one 422 (RequestError.errors).
     images        [{"id", "data" (base64) and "media_type"}] or, on Edge, [{"id", "path"}] for a local file.
                   Every question sees every picture, in request order.
 
-Edge has no videos, boosters, uploads, image URLs, points or boxes: a request that names them is refused.
+Edge has no videos, boosters, uploads or image URLs: a request that names them is refused.
 """
 
 from __future__ import annotations
 
+import itertools
 import json
 import math
 from dataclasses import dataclass, field
 from decimal import Decimal, localcontext
 from typing import Any
 
+from . import places
+
 MAX_OPTIONS = 255
 MAX_LEVELS = 10
 MAX_IMAGES = 16
@@ -38,16 +44,21 @@ MAX_GRID_VALUES = 4096
 MAX_DECIMALS = 6
 NUMBER_DEFAULTS = {"min": 0, "max": 255, "step": 1}
 NOT_ON_EDGE = ("videos", "boosters")
+MAX_ERRORS = 1000  # the entries a 422 lists at most
 
 
 class RequestError(ValueError):
     """A request the model cannot answer. `status` is the HTTP status the hosted API gives the same fault, `error_type`
-    the kind of fault its error body names (the local server's, server.py)."""
+    the kind of fault its error body names (the local server's, server.py), `errors` a 422's field errors when it lists
+    them ({"type", "loc", "msg", "input"}, as the hosted API's body), `detail` the error body's detail when it is not the
+    server's own shape (the hosted API's plain-text details on /v1/systemone)."""
 
-    def __init__(self, message: str, status: int = 400, error_type: str = "api_usage_error"):
+    def __init__(self, message: str, status: int = 400, error_type: str = "api_usage_error", errors: list[dict] | None = None, detail: Any = None):
         super().__init__(message)
         self.status = status
         self.error_type = error_type
+        self.errors = errors
+        self.detail = detail
 
 
 def text_of(value: Any) -> str:
@@ -186,6 +197,7 @@ class Question:
     levels: list | None = None  # score: the levels as sent, for `legend`
     keys: list[str] | None = None  # choice: the options in request order
     grid: Grid | None = None  # number
+    place: places.PlaceSpec | None = None  # point and box (answered by places.Plan, not by rows of its own)
 
 
 @dataclass
@@ -202,16 +214,45 @@ def parse(body: Any) -> Request:
     for name in NOT_ON_EDGE:
         if body.get(name):
             raise RequestError(f"`{name}` is not available on DecisionNode Edge")
+    specs = _place_specs(body)
     state = body.get("state")
     _content(state, "state")
     questions = body.get("questions")
     if not isinstance(questions, dict) or not questions:
         raise RequestError("questions must be an object with at least one question", 422)
     state_text = text_of(state)
-    out = [_question(qid, q) for qid, q in questions.items()]
+    out = [Question(qid, specs[qid].shape, [], place=specs[qid]) if qid in specs else _question(qid, q) for qid, q in questions.items()]
     return Request(state=state_text, questions=out, images=_images(body.get("images")))
 
 
+def _place_specs(body: dict) -> dict[str, places.PlaceSpec]:
+    """The point and box questions by id, read before the rest of the request: their 422 (every faulty field), then
+    the 400 of one without instructions."""
+    questions = body.get("questions")
+    if not isinstance(questions, dict):
+        return {}
+    found = [(qid, q) for qid, q in questions.items() if places.is_place_question(q)]
+    if not found:
+        return {}
+    images = body.get("images")
+    image_ids = [image.get("id") for image in images if isinstance(image, dict) and isinstance(image.get("id"), str)] if isinstance(images, list) else []
+    # pictures the image check refuses (not a list, an id that is not a string, an id used twice) are its fault to name
+    faulty = images is not None and (not isinstance(images, list) or len(image_ids) != len(images) or len(set(image_ids)) != len(image_ids))
+    entries = (entry for qid, q in found for entry in places.question_errors(q, ("body", "questions", qid), None if faulty else image_ids))
+    errors = list(itertools.islice(entries, MAX_ERRORS))
+    if errors:
+        raise RequestError(places.reason(errors), 422, errors=errors)
+    missing = places.missing_instructions(found)
+    if missing:
+        raise RequestError(missing)
+    out = {}
+    for qid, q in found:
+        image = q.get("image") if q.get("image") is not None else (image_ids[0] if image_ids else "")
+        number = image_ids.index(image) + 1 if image in image_ids else 0
+        out[qid] = places.PlaceSpec(str(qid), q["type"], text_of(q.get("instructions")), image, number, q.get("units") or "pixels", q.get("max"))
+    return out
+
+
 def _question(qid: str, q: Any) -> Question:
     if not isinstance(q, dict) or "type" not in q:
         raise RequestError(f"question {qid!r} must be an object with a type", 422)

runtime/src/decisionnode_edge/server.py

@@ -3,20 +3,28 @@
     python -m decisionnode_edge serve path/to/model [--host 127.0.0.1] [--port 8000] [--api-key FILE] [--backend ...]
 
     POST /v1/systemone   the /v1/systemone format used by public decision-model leaderboards:
-                         {"model", "state", "questions", "assets"}; noul, choice and score questions; a noul is
-                         answered {"type": "noul", "noul": p}, a choice and a score as on /v1/decide
+                         {"model", "state", "questions", "assets"}; noul, choice, score and box questions; a noul is
+                         answered {"type": "noul", "noul": p}, a choice and a score as on /v1/decide but with lists
+                         that sum to exactly 1.00, a box as {"type": "box", "box": {"top", "bottom", "left", "right"},
+                         "confidence"} (systemone.py)
     POST /v1/decide      the DecisionNode API's call: the runtime's own request and answers (EdgeModel.decide)
     GET  /v1/models      {"models": [{"name", "description", "release_date"}]}
     GET  /healthz        {"status": "ok", "model", "backend"} (no key needed)
 
 Pictures: `images` as the runtime takes them, base64 `data` only (a local `path` is refused over HTTP: a caller must
-never make the server read its files), and on /v1/systemone the `assets` map of that format, {"<ref>": {"type": "image",
-"format": "png" | "jpg" | "jpeg", "data_base64": ...}}, each asset a picture whose id is its reference (the state names
-it), in the map's order.
+never make the server read its files), and on /v1/systemone the format's own: the `assets` map, {"<ref>": {"type":
+"image", "format": "png" | "jpg" | "jpeg", "data_base64": ...}}, whose references in the text become `Picture <n>`,
+`image_url` content parts and `images` as data URLs (systemone.translate).
+
+Point and box questions are taken on /v1/decide. On /v1/systemone a box is taken (in the format's shape, its
+`confidence` the model folder's box calibration where there is one, box_calibration.json, else `present`), and point
+and number questions are refused.
 
 Errors carry a `detail` body: {"detail": {"error_type", "message"}} (a 422 as a list of field errors). A
 request over the model's length limit says "maximum context length", a choice over 255 options "options per choice",
-the words leaderboard harnesses look for to count a request as beyond the model rather than as a failure.
+and on /v1/systemone a score over 10 levels "a score takes 2 to 10 levels": the words leaderboard harnesses look for to
+count a request as beyond the model rather than as a failure. On /v1/systemone these three come in the hosted route's
+bodies (systemone.mark_capacity).
 
 The model answers one request at a time (a lock); the server listens on the loopback address unless told otherwise.
 `model` is required on /v1/systemone, as on the hosted API, and any name is accepted: the answer names the model that
@@ -29,6 +37,7 @@ import argparse
 import hmac
 import json
 import logging
+import os
 import sys
 import threading
 import time
@@ -36,11 +45,11 @@ import traceback
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from typing import Any
 
+from . import request as requests, systemone
 from .request import RequestError
 
 MAX_BODY_BYTES = 64 << 20  # 16 pictures as base64 with room to spare
-SYSTEMONE_TYPES = ("noul", "choice", "score")
-ASSET_MEDIA_TYPES = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg"}
+SYSTEMONE_TYPES = ("noul", "choice", "score", "box")
 AUTH_MESSAGE = "Cannot authenticate with the server. Please check your API key and try again."
 ROUTES = {"/v1/systemone": "POST", "/v1/decide": "POST", "/v1/models": "GET", "/healthz": "GET"}
 
@@ -53,9 +62,9 @@ class HTTPError(Exception):
         self.status, self.error_type, self.message, self.headers = status, error_type, message, headers or {}
 
 
-def error_body(status: int, error_type: str, message: str) -> dict:
+def error_body(status: int, error_type: str, message: str, errors: list[dict] | None = None) -> dict:
     if status == 422:
-        return {"detail": [{"type": "value_error", "loc": ["body"], "msg": message, "input": None}]}
+        return {"detail": errors or [{"type": "value_error", "loc": ["body"], "msg": message, "input": None}]}
     return {"detail": {"error_type": error_type, "message": message}}
 
 
@@ -70,34 +79,33 @@ def _images_over_http(images: Any) -> list:
     return list(images)
 
 
-def _asset_images(assets: Any) -> list[dict]:
-    """The /v1/systemone `assets` map as the runtime's images: the reference is the picture's id."""
-    if assets is None:
-        return []
-    if not isinstance(assets, dict):
-        raise RequestError("assets must be an object of references", 422)
-    out = []
-    for ref, asset in assets.items():
-        if not isinstance(asset, dict) or asset.get("type", "image") not in ("image", None):
-            raise RequestError(f"Asset {ref!r}: only image assets are taken.")
-        media_type = ASSET_MEDIA_TYPES.get(str(asset.get("format", "")).lower())
-        if media_type is None:
-            raise RequestError(f"Asset {ref!r}: format must be one of png, jpg, jpeg.")
-        if not isinstance(asset.get("data_base64"), str) or not asset["data_base64"]:
-            raise RequestError(f"Asset {ref!r} needs data_base64.", 422)
-        out.append({"id": ref, "data": asset["data_base64"], "media_type": media_type})
-    return out
+def box_calibration(model: Any, path: str | None = None, sha256: str | None = None) -> systemone.BoxCalibration | None:
+    """The box calibration /v1/systemone reads: `path` (or "none" for none), else the model folder's
+    box_calibration.json when there is one. `sha256` (else `box_calibration_sha256` in decisionnode_edge.json), when
+    set, must be the file's. A file that does not check is a BoxCalibrationError: the server does not start on it."""
+    if path == "none":
+        return None
+    sha256 = sha256 if sha256 is not None else model.config.get("box_calibration_sha256") or ""
+    if path is None:
+        path = os.path.join(model.path, systemone.BOX_CALIBRATION_FILE)
+        if not os.path.isfile(path):
+            if sha256:
+                raise systemone.BoxCalibrationError(f"{path}: the model names a box calibration that is not there")
+            return None
+    return systemone.BoxCalibration.load(path, sha256)
 
 
 class Service:
-    """The routes, without the socket: handle(method, path, headers, body) gives (status, JSON body, extra headers)."""
+    """The routes, without the socket: handle(method, path, headers, body) gives (status, JSON body, extra headers).
+    `calibration`: the box calibration of /v1/systemone (default: box_calibration(model))."""
 
-    def __init__(self, model: Any, api_key: str | None = None, description: str | None = None, release_date: str | None = None):
+    def __init__(self, model: Any, api_key: str | None = None, description: str | None = None, release_date: str | None = None, calibration: Any = "model"):
         self.model = model
         self.api_key = api_key
         self.name = model.config["name"]
         self.description = description or model.config.get("description") or f"{self.name}: typed, calibrated decisions, run locally"
         self.release_date = release_date or model.config.get("release_date")
+        self.calibration = box_calibration(model) if calibration == "model" else calibration
         self.lock = threading.Lock()
 
     def handle(self, method: str, path: str, headers: dict[str, str], body: bytes) -> tuple[int, dict, dict]:
@@ -106,7 +114,11 @@ class Service:
         except HTTPError as error:
             return error.status, error_body(error.status, error.error_type, error.message), error.headers
         except RequestError as error:
-            return error.status, error_body(error.status, error.error_type, str(error)), {}
+            if path.split("?", 1)[0] == "/v1/systemone":
+                error = systemone.mark_capacity(error)
+            if error.detail is not None:
+                return error.status, {"detail": error.detail}, {}
+            return error.status, error_body(error.status, error.error_type, str(error), error.errors), {}
         except Exception:  # a fault of the model or the backend, never the caller's
             log.error("the model could not answer:\n%s", traceback.format_exc())
             return 500, error_body(500, "internal_error", "The model could not answer this request."), {}
@@ -149,20 +161,25 @@ class Service:
         if isinstance(questions, dict):
             for qid, question in questions.items():
                 kind = question.get("type") if isinstance(question, dict) else None
-                if kind == "number":
-                    raise RequestError(f"question {qid!r}: the number type is not available on /v1/systemone; send it to /v1/decide")
+                if kind in ("number", "point"):
+                    raise RequestError(f"question {qid!r}: the {kind} type is not available on /v1/systemone; send it to /v1/decide")
                 if isinstance(question, dict) and "type" in question and kind not in SYSTEMONE_TYPES:
                     raise RequestError(f"question {qid!r}: unknown question type {kind!r}")
-        images = _images_over_http(request.get("images")) + _asset_images(request.get("assets"))
-        body = {"state": request.get("state"), "questions": questions}
-        if images:
-            body["images"] = images
+        _images_over_http(request.get("images"))
+        body = {name: request[name] for name in ("state", "questions", "images", "assets") if name in request}
+        body, done = systemone.translate(body, max_images=requests.MAX_IMAGES)
+        parsed = requests.parse(body)
+        systemone.normalised_boxes(q.place for q in parsed.questions if q.place is not None)  # the format's edges are fractions
         with self.lock:
-            response = self.model.decide(body)
+            response = self.model._decide(parsed, exact_sum=True)
         answers = {}
         for qid, answer in response["answers"].items():  # a noul is answered under its own name, as the format has it
-            answers[qid] = {"type": "noul", "noul": answer["truth"]} if answer["type"] == "truth" else answer
-        return {"model": response["model"], "answers": answers, "usage": response["usage"]}
+            if answer["type"] == "truth":
+                answer = {"type": "noul", "noul": answer["truth"]}
+            elif answer["type"] == "score" and qid in done.legends and len(done.legends[qid]) == len(answer["legend"]):
+                answer = {**answer, "legend": {str(i): level for i, level in enumerate(done.legends[qid])}}  # the levels as sent
+            answers[qid] = answer
+        return systemone.box_answers({"model": response["model"], "answers": answers, "usage": response["usage"]}, self.calibration)
 
 
 class Handler(BaseHTTPRequestHandler):
@@ -235,6 +252,8 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--artifacts", default=None, help="the directory python -m decisionnode_edge.build wrote")
     parser.add_argument("--description", default=None, help="the description GET /v1/models gives")
     parser.add_argument("--release-date", default=None, help="the release date GET /v1/models gives (YYYY-MM-DD)")
+    parser.add_argument("--box-calibration", default=None, help="the box calibration of /v1/systemone (default: the model folder's box_calibration.json when there is one; none for none)")
+    parser.add_argument("--box-calibration-sha256", default=None, help="the sha256 the box calibration file must have")
     args = parser.parse_args(argv)
     logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(message)s")
     api_key = None
@@ -244,9 +263,12 @@ def main(argv: list[str] | None = None) -> int:
         if not api_key:
             raise SystemExit(f"{args.api_key} is empty")
     model = EdgeModel.load(args.model, device=args.device, dtype=args.dtype, backend=args.backend, artifacts=args.artifacts)
+    calibration = box_calibration(model, args.box_calibration, args.box_calibration_sha256)
     model.decide({"state": "The color is red.", "questions": {"warmup": {"type": "choice", "instructions": "Which color is named?", "criteria": {"red": "red", "blue": "blue"}}}})
-    server = make_server(model, args.host, args.port, api_key, description=args.description, release_date=args.release_date)
+    server = make_server(model, args.host, args.port, api_key, description=args.description, release_date=args.release_date, calibration=calibration)
     log.info("serving %s (%s backend) on http://%s:%d%s", model.config["name"], model.backend, args.host, server.server_address[1], ", key required" if api_key else "")
+    if calibration is not None:
+        log.info("/v1/systemone box confidence: the box calibration %s (sha256 %s)", calibration.model or "of this model", calibration.sha256)
     try:
         server.serve_forever()
     except KeyboardInterrupt:

runtime/src/decisionnode_edge/systemone.py

@@ -0,0 +1,597 @@
+"""POST /v1/systemone: the request and answer format public decision-model benchmarks use, read onto the runtime's own.
+
+    translate(body)         the format's pictures onto the runtime's, before the parse:
+                            - `assets`: {reference: {"type": "image", "format": "png" | "jpg" | "jpeg", "data_base64"}}.
+                              Every place a reference appears (as a whole string value or inside a longer string, in the
+                              state, a question's instructions or its criteria; never in an object key) is the picture
+                              "in its place". The model reads every picture of a request before the state, labelled
+                              `Picture <n> (<id>):`, so each reference is replaced in the text by `Picture <n>` and the
+                              asset becomes picture n. A picture is read once however many times it is referenced, and
+                              every question sees every picture.
+                            - chat content parts, `{"type": "image_url", "image_url": {"url": "data:..."}}` (or
+                              `"image_url": "data:..."`), anywhere in the state, the instructions or the criteria: the
+                              same, the part replaced by `Picture <n>`; an array made only of such parts and
+                              `{"type": "text", "text"}` parts is read as its text, the parts joined by a line break.
+                            - `images` as a list of data URLs.
+                            - `box` questions: in the picture the instructions name (`image_for_box`) when the request
+                              holds several of the format's pictures; answered in fractions (`normalised_boxes`, after
+                              the checks). A box with `max` keeps the runtime's list answer.
+                            The legend of a score whose levels held a reference echoes the levels as sent.
+    box_answer(answer)      a one-box answer in the format's shape: {"type": "box", "box": {"top", "bottom", "left",
+                            "right"}, "confidence"}, the edges as fractions of the picture's height and width, and
+                            `confidence` the probability that the box is right where the model folder has a box
+                            calibration (BoxCalibration), else `present`.
+    mark_capacity(...)      the words benchmark harnesses look for in an error ("maximum context length", "options per
+                            choice", "a score takes 2 to 10 levels"), so that a limit of the model is recorded as one.
+
+Choice and score probabilities on this route are normalised first and rounded to lists that sum to exactly 1.00
+(answers.round_probabilities with exact_sum). Headers the format defines for pictures and long inputs are accepted and
+never required. A multipart body (`part` assets) is not taken: an asset with `part` is a 400 that says to send
+`data_base64`.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import hashlib
+import json
+import math
+import re
+from dataclasses import dataclass, field
+from typing import Any, Iterable
+
+from .answers import round2
+from .request import IMAGE_MEDIA_TYPES, MAX_LEVELS, MAX_OPTIONS, RequestError
+
+ASSET_FORMATS = {"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg"}  # the format's Asset.format
+MAX_REFERENCE_BYTES = 1024  # one reference, UTF-8
+MAX_REFERENCES_BYTES = 65536  # every reference of a request together
+MAX_NESTING = 128  # a body nested deeper is left as it is (the parse refuses it)
+PICTURE = "Picture {n}"  # what the text reads in a reference's place: the model's own label of picture n
+BOX_CALIBRATION_FILE = "box_calibration.json"  # in the model folder, optional
+BOX_CALIBRATION_FORMAT = "dn-box-calibration-1"
+BOX_CALIBRATION_FEATURES = ["logit_present", "logit_confidence"]
+_WHITESPACE = re.compile(r"[\t\n\x0b\x0c\r ]")
+_BACKTICKED = re.compile(r"`([^`\n]{1,512})`")
+_PATH_PART = re.compile(r"([^.\[\]]+)|\[(\d+)\]")
+
+
+@dataclass
+class Translation:
+    """What translate() did to a body, for the parse and the answer."""
+
+    images: list[dict] = field(default_factory=list)  # the image objects it added, in picture order
+    legends: dict[str, list] = field(default_factory=dict)  # score question id -> its levels as sent
+    boxes: list[str] = field(default_factory=list)  # the box questions, answered in the format's shape
+    references: int = 0  # the references replaced in the text
+
+
+# ---------------------------------------------------------------------------
+# Errors
+# ---------------------------------------------------------------------------
+
+
+def field_error(kind: str, loc: Iterable[Any], msg: str, value: Any, **extra: Any) -> dict:
+    return {"type": kind, "loc": list(loc), "msg": msg, "input": value, **extra}
+
+
+def _invalid(entry: dict) -> RequestError:
+    return RequestError(f"{'.'.join(str(part) for part in entry['loc'])}: {entry['msg']}", 422, errors=[entry])
+
+
+def _asset_error(ref: str, problem: str) -> RequestError:
+    shown = ref if len(ref) <= 80 else ref[:77] + "..."
+    return RequestError(f"Asset {shown!r}: {problem}.")
+
+
+def _too_large(message: str) -> RequestError:
+    return RequestError(message, 413, "request_too_large_error")
+
+
+# ---------------------------------------------------------------------------
+# Pictures
+# ---------------------------------------------------------------------------
+
+
+def _sniff(text: str) -> str | None:
+    """The type of a base64 picture from its first bytes (PNG, JPEG, WebP), or None."""
+    head = text[:64]
+    try:
+        blob = base64.b64decode(head[: len(head) - len(head) % 4])
+    except (binascii.Error, ValueError):
+        return None
+    if blob.startswith(b"\x89PNG\r\n\x1a\n"):
+        return "image/png"
+    if blob.startswith(b"\xff\xd8\xff"):
+        return "image/jpeg"
+    if blob[:4] == b"RIFF" and blob[8:12] == b"WEBP":
+        return "image/webp"
+    return None
+
+
+def _clean_base64(text: str) -> str:
+    """Base64 text as the decoder reads it: a `data:` prefix and any ASCII whitespace dropped, missing padding added."""
+    if text[:5].lower() == "data:":
+        text = text.partition(",")[2]
+    if _WHITESPACE.search(text):
+        text = _WHITESPACE.sub("", text)
+    if len(text) % 4 in (2, 3) and "=" not in text:
+        text += "=" * (4 - len(text) % 4)
+    return text
+
+
+def _asset_image(ref: str, asset: Any) -> dict:
+    """The runtime's image object (without its id) for one asset. Raises RequestError."""
+    if asset.get("type") is not None and asset.get("type") != "image":
+        raise _invalid(field_error("literal_error", ("body", "assets", ref, "type"), "Input should be 'image'", asset.get("type"), ctx={"expected": "'image'"}))
+    if asset.get("part") is not None:
+        raise _asset_error(ref, "multipart image parts are not taken by this server; send the image base64-encoded in data_base64 in a JSON body")
+    data = asset.get("data_base64")
+    if data is None:
+        raise _asset_error(ref, "data_base64 is required")
+    if not isinstance(data, str):
+        raise _invalid(field_error("string_type", ("body", "assets", ref, "data_base64"), "Input should be a valid string", data))
+    fmt = asset.get("format")
+    declared = None
+    if fmt is not None:
+        if not isinstance(fmt, str) or fmt not in ASSET_FORMATS:
+            raise _asset_error(ref, "format must be png, jpg or jpeg")
+        declared = ASSET_FORMATS[fmt]
+    data = _clean_base64(data)
+    if not data:
+        raise _asset_error(ref, "data_base64 is empty")
+    media = _sniff(data) or declared  # the format is read from the bytes
+    if media is None:
+        raise _asset_error(ref, "the data is not a PNG or JPEG image")
+    return {"media_type": media, "data": data}
+
+
+def _part_image(part: dict) -> dict:
+    """The runtime's image object (without its id) for an `image_url` content part. Raises RequestError."""
+    url = part.get("image_url")
+    if isinstance(url, dict):
+        url = url.get("url")
+    if not isinstance(url, str) or not url:
+        raise RequestError("An image_url part needs a url: a data:image/...;base64 URL.")
+    scheme = url[:8].lower()
+    if scheme.startswith("data:"):
+        head, comma, data = url.partition(",")
+        fields = head[5:].split(";")
+        if not comma or len(fields) < 2 or fields[-1].strip().lower() != "base64":
+            raise RequestError("An image_url data URL must be data:<media type>;base64,<data>.")
+        data = _clean_base64(data)
+        declared = fields[0].strip().lower()
+        declared = "image/jpeg" if declared == "image/jpg" else declared
+        media = _sniff(data) or (declared if declared in IMAGE_MEDIA_TYPES else None)
+        if media is None:
+            raise RequestError("An image_url part must hold a PNG, JPEG or WebP image.")
+        return {"media_type": media, "data": data}
+    if scheme == "https://":
+        raise RequestError("Image URLs are not fetched on this deployment: send the image as a base64 data URL.")
+    raise RequestError("An image_url part must be a base64 data URL.")
+
+
+def _is_image_part(value: Any) -> bool:
+    return isinstance(value, dict) and value.get("type") == "image_url" and "image_url" in value
+
+
+def _is_text_part(value: Any) -> bool:
+    return isinstance(value, dict) and value.get("type") == "text" and isinstance(value.get("text"), str) and set(value) <= {"type", "text"}
+
+
+# ---------------------------------------------------------------------------
+# The walk
+# ---------------------------------------------------------------------------
+
+
+class _Walk:
+    """Replaces references and image parts in the free-form parts of a request, numbering the pictures as it meets them."""
+
+    def __init__(self, refs: Iterable[str], first: int):
+        self.refs = list(refs)
+        self.first = first  # the picture number of the first picture this walk adds, less one (the body's own images)
+        self.order: list[tuple[str, Any]] = []  # ("ref", reference) or ("part", image object), in picture order
+        self.by_ref: dict[str, int] = {}
+        self.by_url: dict[str, int] = {}
+        self.replaced = 0
+        ordered = sorted(self.refs, key=len, reverse=True)  # at one position the longest reference wins
+        self.pattern = re.compile("|".join(re.escape(ref) for ref in ordered)) if ordered else None
+        self.whole = set(self.refs)
+
+    def _number_of_ref(self, ref: str) -> int:
+        if ref not in self.by_ref:
+            self.order.append(("ref", ref))
+            self.by_ref[ref] = self.first + len(self.order)
+        return self.by_ref[ref]
+
+    def _number_of_part(self, part: dict) -> int:
+        image = _part_image(part)
+        key = image.get("data") or ""
+        if key not in self.by_url:
+            self.order.append(("part", image))
+            self.by_url[key] = self.first + len(self.order)
+        return self.by_url[key]
+
+    def text(self, value: str) -> str:
+        if self.pattern is None:
+            return value
+        if value in self.whole:  # a whole value first
+            self.replaced += 1
+            return PICTURE.format(n=self._number_of_ref(value))
+        if not any(ref in value for ref in self.refs):
+            return value
+
+        def one(match: "re.Match[str]") -> str:
+            self.replaced += 1
+            return PICTURE.format(n=self._number_of_ref(match.group(0)))
+
+        return self.pattern.sub(one, value)
+
+    def value(self, value: Any) -> Any:
+        """`value` with every reference and image part replaced (a new object where anything changed)."""
+        if isinstance(value, str):
+            return self.text(value)
+        if _is_image_part(value):
+            self.replaced += 1
+            return PICTURE.format(n=self._number_of_part(value))
+        if isinstance(value, list):
+            if value and any(_is_image_part(item) for item in value) and all(_is_image_part(item) or _is_text_part(item) for item in value):
+                # a content-part array: its text, the pictures in their places
+                return "\n".join(self.value(item) if _is_image_part(item) else self.text(item["text"]) for item in value)
+            out = [self.value(item) for item in value]
+            return out if any(a is not b for a, b in zip(out, value)) else value
+        if isinstance(value, dict):
+            out = {key: self.value(item) for key, item in value.items()}  # keys stay text
+            return out if any(out[key] is not value[key] for key in value) else value
+        return value
+
+
+def _question(walk: _Walk, question: Any) -> Any:
+    """A question with its instructions and criteria walked (its type and every other field as sent)."""
+    if not isinstance(question, dict):
+        return question
+    out = dict(question)
+    if "instructions" in out:
+        out["instructions"] = walk.value(out["instructions"])
+    criteria = out.get("criteria")
+    if isinstance(criteria, dict):
+        out["criteria"] = {key: walk.value(item) for key, item in criteria.items()}  # an option's name is a key: never replaced
+    elif isinstance(criteria, list):
+        out["criteria"] = [walk.value(item) for item in criteria]
+    return out
+
+
+# ---------------------------------------------------------------------------
+# Box questions
+# ---------------------------------------------------------------------------
+
+
+def _resolve(state: Any, path: str) -> Any:
+    """The value a backticked path names in the state (`photo`, `frames[2].frame`, `faces.happy`), or None."""
+    value = state
+    for name, index in _PATH_PART.findall(path.strip()):
+        if name:
+            if not isinstance(value, dict) or name not in value:
+                return None
+            value = value[name]
+        else:
+            if not isinstance(value, list) or int(index) >= len(value):
+                return None
+            value = value[int(index)]
+    return value
+
+
+def _first_ref(value: Any, refs: set[str]) -> str | None:
+    """The first reference held whole by a string inside `value` (depth first, in order)."""
+    if isinstance(value, str):
+        return value if value in refs else None
+    items = value.values() if isinstance(value, dict) else value if isinstance(value, list) else ()
+    for item in items:
+        found = _first_ref(item, refs)
+        if found is not None:
+            return found
+    return None
+
+
+def image_for_box(instructions: Any, state: Any, refs: set[str], numbers: dict[str, int], in_state: list[int], own: set[int], count: int) -> int:
+    """The picture number a box question looks in when the request holds several pictures (the path in backticks names
+    which one to search): the picture a backticked path of the instructions names in the state (a reference held there,
+    whole); else the first picture of the state; else the first picture the instructions do not hold themselves (a
+    picture in a box's instructions shows the target, it is not the picture to search); else picture 1."""
+    text = instructions if isinstance(instructions, str) else "" if instructions is None else str(instructions)
+    for path in _BACKTICKED.findall(text):
+        found = _first_ref(_resolve(state, path), refs)
+        if found is not None and found in numbers:
+            return numbers[found]
+    if in_state:
+        return in_state[0]
+    for n in range(1, count + 1):
+        if n not in own:
+            return n
+    return 1
+
+
+def question_refs(question: dict, refs: Iterable[str]) -> set[str]:
+    """The references a question's own instructions and criteria hold (whole or inside a string)."""
+    found: set[str] = set()
+    wanted = list(refs)
+
+    def visit(value: Any) -> None:
+        if isinstance(value, str):
+            found.update(ref for ref in wanted if ref in value)
+        elif isinstance(value, dict):
+            for item in value.values():
+                visit(item)
+        elif isinstance(value, list):
+            for item in value:
+                visit(item)
+
+    visit(question.get("instructions"))
+    visit(question.get("criteria"))
+    return found
+
+
+# ---------------------------------------------------------------------------
+# The request
+# ---------------------------------------------------------------------------
+
+
+def _within_nesting(value: Any, limit: int = MAX_NESTING) -> bool:
+    """At most `limit` containers inside each other (walked without recursion)."""
+    stack = [(value, 0)]
+    while stack:
+        item, depth = stack.pop()
+        if isinstance(item, (dict, list)):
+            if depth + 1 > limit:
+                return False
+            stack.extend((child, depth + 1) for child in (item.values() if isinstance(item, dict) else item))
+    return True
+
+
+def _check_assets(assets: Any) -> dict:
+    if not isinstance(assets, dict):
+        raise _invalid(field_error("dict_type", ("body", "assets"), "Input should be a valid dictionary", assets))
+    if not assets:
+        raise _invalid(field_error("too_short", ("body", "assets"), "Dictionary should have at least 1 item after validation, not 0", assets, ctx={"field_type": "Dictionary", "min_length": 1, "actual_length": 0}))
+    total = 0
+    for ref, asset in assets.items():
+        if not isinstance(asset, dict):
+            raise _invalid(field_error("model_attributes_type", ("body", "assets", ref), "Input should be a valid dictionary or object to extract fields from", asset))
+        if not ref.strip():
+            raise RequestError("An asset reference cannot be empty or only whitespace.")
+        size = len(ref.encode("utf-8", "surrogatepass"))
+        if size > MAX_REFERENCE_BYTES:
+            raise _too_large(f"An asset reference is at most {MAX_REFERENCE_BYTES} bytes of UTF-8.")
+        total += size
+    if total > MAX_REFERENCES_BYTES:
+        raise _too_large(f"The asset references of a request are at most {MAX_REFERENCES_BYTES} bytes of UTF-8 together.")
+    return assets
+
+
+def _may_hold_parts(value: Any) -> bool:
+    """Cheap: whether a value may hold an image_url part (any dict with that type somewhere), walked without recursion."""
+    stack = [value]
+    while stack:
+        item = stack.pop()
+        if isinstance(item, dict):
+            if item.get("type") == "image_url":
+                return True
+            stack.extend(child for child in item.values() if isinstance(child, (dict, list)))
+        elif isinstance(item, list):
+            stack.extend(child for child in item if isinstance(child, (dict, list)))
+    return False
+
+
+def translate(body: Any, *, max_images: int) -> tuple[Any, Translation]:
+    """A /v1/systemone body with the format's pictures on the runtime's (module docstring), and what was done. A body
+    that is not an object with a question map is returned as it is (the parse refuses it). Raises RequestError for an
+    asset or an image part the runtime cannot take."""
+    done = Translation()
+    if not isinstance(body, dict) or not isinstance(body.get("questions"), dict):
+        return body, done
+    assets = body.get("assets")
+    has_assets = assets is not None
+    questions = body["questions"]
+    has_parts = _may_hold_parts(body.get("state")) or any(_may_hold_parts(q.get("instructions")) or _may_hold_parts(q.get("criteria")) for q in questions.values() if isinstance(q, dict))
+    boxes = [qid for qid, q in questions.items() if isinstance(q, dict) and q.get("type") == "box"]
+    url_images = isinstance(body.get("images"), list) and any(isinstance(image, str) for image in body["images"])
+    if not has_assets and not has_parts and not boxes and not url_images:
+        if "assets" in body:  # `"assets": null` equals no assets
+            body = {name: value for name, value in body.items() if name != "assets"}
+        return body, done
+    if not _within_nesting(body):
+        return body, done  # the parse refuses it as it refuses any body that deep
+    out = {name: value for name, value in body.items() if name != "assets"}
+    refs: list[str] = list(_check_assets(assets)) if has_assets else []
+    if len(refs) > max_images:
+        raise RequestError(f"Too many images: {len(refs)}, the limit is {max_images}.")
+    own_images = body.get("images") if isinstance(body.get("images"), list) else []
+    if url_images:
+        # `images` as a list of data URLs: each an image object, in the list's order
+        if len(own_images) > max_images:
+            raise RequestError(f"Too many images: {len(own_images)}, the limit is {max_images}.")
+        listed = []
+        for i, image in enumerate(own_images):
+            if isinstance(image, str):
+                image = {"id": "image" if len(own_images) == 1 else f"image_{i + 1}", **_part_image({"image_url": image})}
+            listed.append(image)
+        own_images = listed
+        out["images"] = listed
+    walk = _Walk(refs, len(own_images))
+    state = body.get("state")
+    out["state"] = walk.value(state)
+    in_state = sorted(set(walk.by_ref.values()) | set(walk.by_url.values()))
+    walked: dict[str, Any] = {}
+    for qid, question in questions.items():
+        walked[qid] = _question(walk, question)
+        if isinstance(question, dict) and question.get("type") == "score" and walked[qid].get("criteria") != question.get("criteria"):
+            done.legends[str(qid)] = question.get("criteria")  # the legend echoes the levels as sent
+    unused = [ref for ref in refs if ref not in walk.by_ref]
+    if unused:
+        raise _asset_error(unused[0], "it is not referenced in the state, the instructions or the criteria")
+    added = len(walk.order)
+    total = len(own_images) + added
+    if total > max_images:
+        raise RequestError(f"Too many images: {total}, the limit is {max_images}.")
+    taken = {image.get("id") for image in own_images if isinstance(image, dict)}
+    images = []
+    for i, (kind, item) in enumerate(walk.order):
+        n = len(own_images) + i + 1
+        image_id = "image" if total == 1 else f"image_{n}"
+        while image_id in taken:
+            image_id += "_"
+        taken.add(image_id)
+        image = _asset_image(item, assets[item]) if kind == "ref" else dict(item)
+        images.append({"id": image_id, **image})
+    if images:
+        out["images"] = list(own_images) + images
+    done.images = images
+    done.references = walk.replaced
+    ids = [image.get("id") for image in out.get("images") or [] if isinstance(image, dict)]
+    numbers = dict(walk.by_ref)
+    for qid in boxes:
+        # the picture is chosen only for the format's pictures (assets, image parts, data URLs), which no `image` field
+        # can name; with image objects alone a box without `image` among several stays the 422 of the runtime's parse
+        if walked[qid].get("image") is None and len(ids) > 1 and (added or url_images):
+            question = dict(walked[qid])
+            own = {numbers[ref] for ref in question_refs(questions[qid], refs) if ref in numbers}
+            n = image_for_box(questions[qid].get("instructions"), state, set(refs), numbers, in_state, own, len(ids))
+            question["image"] = ids[n - 1]
+            walked[qid] = question
+        done.boxes.append(str(qid))
+    out["questions"] = walked
+    return out, done
+
+
+def normalised_boxes(specs: Iterable[Any]) -> None:
+    """The box questions of a /v1/systemone request answered in fractions of the picture (the format's edges), set on the
+    parsed specs (places.PlaceSpec) after their checks. A box with `max` keeps the units the caller sent."""
+    for spec in specs:
+        if spec.shape == "box" and spec.max is None:
+            spec.units = "normalized"
+
+
+# ---------------------------------------------------------------------------
+# The answer
+# ---------------------------------------------------------------------------
+
+
+class BoxCalibrationError(ValueError):
+    """A box calibration file the runtime does not take (the server does not start on it)."""
+
+
+@dataclass(frozen=True)
+class BoxCalibration:
+    """The `confidence` of a box on /v1/systemone: the probability that the box is right (an intersection over union of
+    0.5 or more with the true box), read from the one-box answer's own two numbers, `present` and `confidence`, as the
+    answer writes them (2 decimals). A logistic map, fitted per model and kept beside it as box_calibration.json:
+
+        {"format": "dn-box-calibration-1", "features": ["logit_present", "logit_confidence"], "clip": 0.005,
+         "intercept": a, "coef": [b, c], "model": ..., "fit": {...}}
+
+    p = sigmoid(a + b logit(present) + c logit(confidence)), each input clipped to [clip, 1 - clip] first, the result
+    rounded to 2 decimals. Monotone: no coefficient below 0, so neither number can lower the probability by rising."""
+
+    intercept: float
+    coef: tuple[float, float]
+    clip: float
+    sha256: str
+    model: str = ""
+
+    @classmethod
+    def load(cls, path: str, sha256: str = "") -> "BoxCalibration":
+        """Read and check a calibration file; `sha256`, when given, must be the file's."""
+        try:
+            with open(path, "rb") as f:
+                data = f.read()
+        except OSError as error:
+            raise BoxCalibrationError(f"box calibration {path}: {error}") from None
+        digest = hashlib.sha256(data).hexdigest()
+        if sha256 and sha256.lower() != digest:
+            raise BoxCalibrationError(f"box calibration {path}: sha256 {digest}, expected {sha256}")
+        try:
+            spec = json.loads(data)
+        except ValueError as error:
+            raise BoxCalibrationError(f"box calibration {path}: not JSON ({error})") from None
+        if not isinstance(spec, dict) or spec.get("format") != BOX_CALIBRATION_FORMAT:
+            raise BoxCalibrationError(f"box calibration {path}: the format is not {BOX_CALIBRATION_FORMAT}")
+        if spec.get("features") != BOX_CALIBRATION_FEATURES:
+            raise BoxCalibrationError(f"box calibration {path}: features must be {BOX_CALIBRATION_FEATURES}")
+        coef, intercept, clip = spec.get("coef"), spec.get("intercept"), spec.get("clip")
+        numbers = [intercept, clip] + (list(coef) if isinstance(coef, list) else [])
+        if not isinstance(coef, list) or len(coef) != 2 or any(isinstance(x, bool) or not isinstance(x, (int, float)) or x != x or abs(x) == float("inf") for x in numbers):
+            raise BoxCalibrationError(f"box calibration {path}: intercept, coef (two numbers) and clip must be finite numbers")
+        if any(c < 0 for c in coef):
+            raise BoxCalibrationError(f"box calibration {path}: a coefficient below 0 is not monotone")
+        if not 0 < clip < 0.5:
+            raise BoxCalibrationError(f"box calibration {path}: clip must be above 0 and below 0.5")
+        return cls(float(intercept), (float(coef[0]), float(coef[1])), float(clip), digest, str(spec.get("model") or ""))
+
+    def probability(self, present: float, confidence: float) -> float:
+        """The unrounded probability that the box is right."""
+
+        def logit(p: float) -> float:
+            p = min(1.0 - self.clip, max(self.clip, float(p)))
+            return math.log(p / (1.0 - p))
+
+        z = self.intercept + self.coef[0] * logit(present) + self.coef[1] * logit(confidence)
+        return 1.0 / (1.0 + math.exp(-z)) if z >= 0 else math.exp(z) / (1.0 + math.exp(z))
+
+    def __call__(self, present: float, confidence: float) -> float:
+        return round2(self.probability(present, confidence))
+
+
+def box_answer(answer: dict, calibration: BoxCalibration | None = None) -> dict:
+    """A one-box answer (normalized units) in the format's shape. `confidence`: the calibrated probability that the box
+    is right where there is a calibration, else `present`."""
+    return {
+        "type": "box",
+        "box": {"top": answer["y1"], "bottom": answer["y2"], "left": answer["x1"], "right": answer["x2"]},
+        "confidence": calibration(answer["present"], answer["confidence"]) if calibration is not None else answer["present"],
+    }
+
+
+def box_answers(body: dict, calibration: BoxCalibration | None = None) -> dict:
+    """The response body with every one-box answer in the format's shape (a list answer, `max`, stays the runtime's)."""
+    answers = body.get("answers")
+    if isinstance(answers, dict):
+        for qid, answer in answers.items():
+            if isinstance(answer, dict) and answer.get("type") == "box" and "x1" in answer:
+                answers[qid] = box_answer(answer, calibration)
+    return body
+
+
+# ---------------------------------------------------------------------------
+# Errors with the capacity words
+# ---------------------------------------------------------------------------
+
+CONTEXT_MARKER = "maximum context length"
+OPTIONS_MARKER = "options per choice"
+LEVELS_MARKER = "a score takes 2 to 10 levels"
+TOO_MANY_LEVELS = f"Too many score levels. Must have at most {MAX_LEVELS} levels."
+
+
+def mark_capacity(error: RequestError) -> RequestError:
+    """A refusal of /v1/systemone for a limit of the model, in the words and bodies benchmark harnesses read:
+
+        over the length limit   {"detail": {"error_type": "max_tokens_exceeded", "message": "The request does not fit
+                                the model's maximum context length (<the count and the limit>)."}}
+        more than 255 options   {"detail": "Too many choices. Must have at most 255 choices (255 options per choice)."}
+        more than 10 levels     {"detail": "Too many score levels. Must have at most 10 levels (a score takes 2 to 10
+                                levels; one level is answered without the model)."}
+
+    Every other refusal is returned as it is."""
+    message = str(error)
+    if error.status != 400:
+        return error
+    if error.error_type == "max_tokens_exceeded":
+        text = f"The request does not fit the model's {CONTEXT_MARKER} ({message})."
+        return RequestError(text, 400, error.error_type, detail={"error_type": error.error_type, "message": text})
+    if message.startswith("Too many choices"):
+        text = f"Too many choices. Must have at most {MAX_OPTIONS} choices ({MAX_OPTIONS} {OPTIONS_MARKER})."
+        return RequestError(text, 400, error.error_type, detail=text)
+    if message == TOO_MANY_LEVELS:
+        text = f"Too many score levels. Must have at most {MAX_LEVELS} levels ({LEVELS_MARKER}; one level is answered without the model)."
+        return RequestError(text, 400, error.error_type, detail=text)
+    return error

runtime/src/decisionnode_edge/template.py

@@ -4,16 +4,20 @@ One prompt per question. The state block comes first, then the question block, t
 word "Answer" and the <|answer|> token, whose position the answer is read from:
 
     <|state|>\\n{Picture n (id): <|vision_start|>pads<|vision_end|>\\n ...}{state}\\n
-    <|question|>\\ntype: {choice|score|noul}\\n{instructions}\\n{option lines}Answer<|answer|>
+    <|question|>\\ntype: {choice|score|noul|coord}\\n{instructions}\\n{option lines}Answer<|answer|>
 
 A choice option line is `<|opt_NNN|> key: description\\n` (`<|opt_NNN|> key\\n` when the
 description is empty), a score level line `<|opt_NNN|> index: level\\n`. A truth question
 has no option lines (the prompt calls it `noul`) and carries
 its criteria as `yes: ...\\n` and `no: ...\\n` lines. It is answered by <|yes|> and <|no|>.
 
-Each option gets one of 256 code tokens. The codes are drawn from a stream seeded by a
-SHA-256 of the question alone (never the state), so the same question always gets the
-same codes. The seed's first part is a fixed string from the model directory
+A coordinate row (one reading of a point or box question, places.py) is `type: coord`, its
+text and no option lines. It is answered by the codes 0 to n - 1 in order (code k is step k
+of a side of the picture, or a count of k), the same codes for every such row, never drawn.
+
+Each option of a choice or a score gets one of 256 code tokens. The codes are drawn from a
+stream seeded by a SHA-256 of the question alone (never the state), so the same question
+always gets the same codes. The seed's first part is a fixed string from the model directory
 (`code_seed_hex` in decisionnode_edge.json): the model gives its published answers only
 with that string.
 
@@ -39,7 +43,7 @@ CODES = tuple(f"<|opt_{i:03d}|>" for i in range(N_CODES))
 ADDED_TOKENS = (STATE, QUESTION, ANSWER, YES, NO) + CODES  # 261 tokens, in id order
 ANSWER_CUE = "Answer"
 # The type word each question type carries in the prompt.
-TYPE_WORDS = {"choice": "choice", "score": "score", "truth": "noul"}
+TYPE_WORDS = {"choice": "choice", "score": "score", "truth": "noul", "coord": "coord"}
 
 IMAGE_PAD = "<|image_pad|>"
 VISION_START = "<|vision_start|>"
@@ -115,6 +119,8 @@ def n_classes(question: dict) -> int:
         return len(question["keys"])
     if question["type"] == "score":
         return len(question["levels"])
+    if question["type"] == "coord":
+        return int(question["classes"])
     return 2
 
 
@@ -200,7 +206,10 @@ class Template:
         return [self.first_id + 5 + c for c in codes]
 
     def encode(self, question: dict, state: str, image_ids: Sequence[str] = (), image_tokens: dict[str, int] | None = None) -> Encoded:
-        codes = assign_codes(self.seed, question) if question["type"] in ("choice", "score") else []
+        if question["type"] in ("choice", "score"):
+            codes = assign_codes(self.seed, question)
+        else:  # a coordinate row's codes are fixed, a truth question has none
+            codes = list(range(n_classes(question))) if question["type"] == "coord" else []
         segments = build_segments(question, state, image_ids, codes)
         table = self._tokenize([value for kind, value in segments if kind in (TEXT, STATE_TEXT)])
         ids: list[int] = []