d3_runtime.py

"""d3 runtime: System One typed decisions over a code-readout v1 checkpoint.

One question is decided per forward pass. The prompt lists every option under a single-token answer
code; a 255-way readout scores those codes at the last prompt position and the answer is a
probability for every option. No text is generated and no input is truncated.

    from d3_runtime import D3
    model = D3.from_pretrained("<package dir or Hub id>", device="cuda:0")
    model.system_one(state="...", questions={"route": {"type": "choice", ...}})
    # {"model": ..., "answers": {"route": {"type": "choice", "choice": ..., "probabilities": {...},
    #                                     "confidence": ...}}, "usage": {"input_tokens": n, "output_tokens": 0}}

    model.system_one(state="...", questions={...}, images=["photo.png", "label.jpg"])
    model.system_one(state="...", questions={...}, videos=["clip.mp4"])

The checkpoint directory holds ``config.json`` + ``model*.safetensors`` (a transformers
``Qwen3_5Model``, or a ``Qwen3VLModel`` when ``config.json`` says ``model_type: qwen3_vl``),
``readout.safetensors`` (``{"weight": [255, hidden]}``), ``decision_config.json``
(prompt family, answer codes, attention mode, pooling, temperature, input limit) and the tokenizer.
``d3_format.py`` next to this file is the prompt and answer-code contract of the model.

Images: a request takes any number of images (PIL images, local paths, http(s) URLs or base64
``data:image/...`` URLs), shared by all of its questions. They go before the text in the user turn,
one image placeholder per image in list order, and the checkpoint's own processor (``AutoProcessor``,
torchvision backend) resizes each to at most 1,638,400 pixels (1.6 MP) and at least 65,536, keeping
the aspect ratio; each 32 x 32 pixel patch is one input token. The prompt is the text prompt with the
images in front; a request without images takes exactly the text path. Image inputs need a checkpoint
with the vision tower (``visual.*`` weights).

Videos: a request also takes any number of videos (local paths, http(s) URLs, base64 ``data:video/...``
URLs, or frame arrays: a uint8 ``[frames, height, width, 3]`` RGB array or a list of PIL images, read as
frames sampled at 2 per second), shared by all of its questions and placed after the images. Files are
decoded with OpenCV (FFmpeg): 2 frames per second, at least 4 and at most 32 frames spread evenly over the
whole video, each read by the checkpoint's own video processor at up to 200,704 pixels (0.2 MP); every two
frames are one group of input tokens (one token per 32 x 32 pixels) after a timestamp. All videos of one
request take at most 16,384 input tokens. A request without videos takes exactly the text or image path.

Numerics: BF16 backbone with SDPA attention, FP32 readout and softmax (unless the checkpoint says
otherwise). A request's questions run in request order, ``batch_size`` per forward pass, each batch
left-padded to its longest prompt. ``noncausal_full_attention`` (Qwen3.5 backbones only) lets the
full-attention layers see the whole prompt while the Gated DeltaNet layers stay causal. The Gated
DeltaNet kernels are the ones transformers binds at import: flash-linear-attention (and causal-conv1d)
when installed, its PyTorch reference implementation otherwise. Qwen3-VL backbones have no linear-attention
layers and run causal attention only.

``permutation_average=True`` (off by default) also scores every choice question with two or more options
with its options in reversed order, in the same forward passes as the original order, and answers with
the per-option mean of the two distributions. Noul and score questions are scored once. A pass carrying both
orders that would exceed ``MERGE_TOKENS`` padded tokens runs the reversed prompts in a pass of their own, so
peak memory stays that of the original order.
"""

from __future__ import annotations

import base64
import binascii
import hashlib
import io
import json
import math
import os
import time
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

try:
    from .d3_format import (
        MAX_OPTIONS,
        SYSTEM_PROMPT,
        answer_codes,
        describe,
        normalized,
        options,
        render as render_d3,
        to_answer,
        user_prompt,
    )
except ImportError:
    from d3_format import (
        MAX_OPTIONS,
        SYSTEM_PROMPT,
        answer_codes,
        describe,
        normalized,
        options,
        render as render_d3,
        to_answer,
        user_prompt,
    )
try:
    from . import d3_fast
except ImportError:
    try:
        import d3_fast
    except ImportError:
        d3_fast = None

RUNTIME = "d3-runtime/1"
FORMAT_VERSION = 1
PROMPTS = ("d3",)
ATTENTION_MODES = ("causal", "noncausal_full_attention")
BACKBONES = ("qwen3_5", "qwen3_vl")
DEFAULT_BATCH_SIZE = 8
SCORE_LEVELS = (2, 10)
MANIFEST = "MODEL_MANIFEST.json"
MANIFEST_SCHEMA = "d3-package-manifest/1"
VERIFY_MODES = ("fast", "full", "none")
# "fast" verification hashes every file up to this size and checks the size of larger ones.
FAST_HASH_BYTES = 64 << 20
ERRORS = ("invalid_question", "max_length_exceeded", "invalid_model_output")
IMAGE_MIN_PIXELS = 65_536
IMAGE_MAX_PIXELS = 1_638_400
IMAGE_TOKEN = "<|image_pad|>"
VIDEO_TOKEN = "<|video_pad|>"
# Checked for every encoded image the server receives (strict loading); in-process inputs are only
# bounded by PIL's decompression-bomb guard.
MAX_IMAGE_BYTES = 8_000_000
MAX_IMAGE_SOURCE_PIXELS = 16_000_000
IMAGE_FORMATS = ("PNG", "JPEG", "WEBP")
DOWNLOAD_TIMEOUT_SECONDS = 30
MAX_DOWNLOAD_BYTES = 64 << 20
MERGE_TOKENS = 8192
# What the processor returns for an image batch; the fast path runs exactly these through the fused layers.
IMAGE_INPUTS = (
    "input_ids",
    "attention_mask",
    "mm_token_type_ids",
    "pixel_values",
    "image_grid_thw",
)
VIDEO_FPS = 2.0
VIDEO_MIN_FRAMES = 4
VIDEO_MAX_FRAMES = 32
# Per frame; one input token per 32 x 32 pixels for every two frames.
VIDEO_MAX_PIXELS = 200_704
VIDEO_MIN_PIXELS = 4_096
# All videos of one request.
VIDEO_MAX_TOKENS = 16_384
# Checked for every video the server receives (strict loading).
MAX_VIDEO_BYTES = 32_000_000
MAX_VIDEO_SOURCE_PIXELS = 8_294_400
MAX_VIDEO_SECONDS = 300
VIDEO_FORMATS = ("mp4", "webm", "quicktime", "x-matroska")
# A source frame rate outside this range is treated as unknown.
VIDEO_SOURCE_FPS = (0.1, 1000.0)
VIDEO_UNKNOWN_FPS = 24.0


class MaxLengthExceeded(ValueError):
    """A question prompt is longer than the checkpoint's input limit; nothing is truncated."""


# ---------------------------------------------------------------------------------------------
# Questions and prompts
# ---------------------------------------------------------------------------------------------


@dataclass
class Question:
    """One validated question: what the prompt renders and how the answer is reported."""

    kind: str  # choice | noul | score
    original: Mapping[str, Any]
    rendered: dict[str, Any]  # a choice or noul question in the d3_format contract
    keys: list[str]
    descriptions: list[Any]


def _json_value(value: Any) -> None:
    """Reject values that strict JSON cannot carry (NaN, infinities, non-JSON types)."""
    try:
        json.dumps(value, ensure_ascii=False, allow_nan=False)
    except (TypeError, ValueError, RecursionError) as exc:
        raise ValueError(f"not representable as JSON: {exc}") from exc


def normalize_question(question: Any) -> Question:
    """Validate one System One question; raises ValueError with the reason."""
    if not isinstance(question, Mapping):
        raise ValueError("a question is an object with a type")
    kind = question.get("type")
    instructions = question.get("instructions")
    if instructions is not None:
        _json_value(instructions)
    criteria = question.get("criteria")
    if kind == "choice":
        if not isinstance(criteria, Mapping) or not 1 <= len(criteria) <= MAX_OPTIONS:
            raise ValueError(
                f"choice criteria must map 1 to {MAX_OPTIONS} option keys to descriptions"
            )
        if any(not isinstance(k, str) or not k for k in criteria):
            raise ValueError("choice option keys must be nonempty strings")
        _json_value(dict(criteria))
        rendered = {
            "type": "choice",
            "instructions": instructions,
            "criteria": dict(criteria),
        }
        keys = list(criteria)
        return Question(kind, question, rendered, keys, [criteria[k] for k in keys])
    if kind == "noul":
        if criteria is not None:
            if not isinstance(criteria, Mapping) or set(criteria) - {"false", "true"}:
                raise ValueError('noul criteria may only describe "false" and "true"')
            _json_value(dict(criteria))
        rendered = {"type": "noul", "instructions": instructions}
        if criteria is not None:
            rendered["criteria"] = dict(criteria)
        keys, texts = options(rendered)
        return Question(kind, question, rendered, keys, texts)
    if kind == "score":
        low, high = SCORE_LEVELS
        if not isinstance(criteria, (list, tuple)) or not low <= len(criteria) <= high:
            raise ValueError(
                f"score criteria must be an ordered list of {low} to {high} levels"
            )
        _json_value(list(criteria))
        levels = [describe(level) for level in criteria]
        if any(not text for text in levels) or len(set(levels)) != len(levels):
            raise ValueError("score levels must be distinct and nonempty")
        # The ordered levels are the options, each shown by its text alone (code order = level order).
        rendered = {
            "type": "choice",
            "instructions": instructions,
            "criteria": {text: None for text in levels},
        }
        return Question(
            kind,
            question,
            rendered,
            [str(i) for i in range(len(levels))],
            list(criteria),
        )
    raise ValueError(f"unsupported question type {kind!r} (choice, noul or score)")


def render(
    tokenizer, prompt: str, state: Any, question: dict[str, Any], codes: Sequence[str]
) -> str:
    if prompt == "d3":
        return render_d3(tokenizer, state, question, codes)
    raise ValueError(f"unknown prompt family {prompt!r}")


def image_messages(
    prompt: str,
    state: Any,
    question: dict[str, Any],
    codes: Sequence[str],
    n_images: int,
) -> list[dict[str, Any]]:
    """The prompt family's messages with one image placeholder per image before the user text."""
    if n_images < 1:
        raise ValueError(f"an image prompt takes at least 1 image, got {n_images}")
    images: list[dict[str, Any]] = [{"type": "image"} for _ in range(n_images)]
    if prompt == "d3":
        text = {"type": "text", "text": user_prompt(state, question, codes)}
        return [
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user", "content": images + [text]},
        ]
    raise ValueError(f"unknown prompt family {prompt!r}")


def video_messages(
    prompt: str,
    state: Any,
    question: dict[str, Any],
    codes: Sequence[str],
    n_images: int,
    n_videos: int,
) -> list[dict[str, Any]]:
    """The prompt family's messages with the image placeholders, then the video placeholders, before the user text."""
    if n_videos < 1:
        raise ValueError(f"a video prompt takes at least 1 video, got {n_videos}")
    media: list[dict[str, Any]] = [{"type": "image"} for _ in range(n_images)]
    media += [{"type": "video"} for _ in range(n_videos)]
    if prompt == "d3":
        text = {"type": "text", "text": user_prompt(state, question, codes)}
        return [
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user", "content": media + [text]},
        ]
    raise ValueError(f"unknown prompt family {prompt!r}")


def reversed_question(question: Question) -> dict[str, Any] | None:
    """The rendered choice question with its options in reverse order (None when there is nothing to permute)."""
    if question.kind != "choice" or len(question.keys) < 2:
        return None
    criteria = question.rendered["criteria"]
    return dict(
        question.rendered, criteria={key: criteria[key] for key in reversed(criteria)}
    )


def average_orders(forward: Sequence[float], backward: Sequence[float]) -> list[float]:
    """Per-option mean of the original-order and reversed-order distributions, in original option order."""
    return [(a + b) / 2 for a, b in zip(forward, reversed(backward))]


def _canonical(value: Any) -> str:
    """Stable text for a score level: strings verbatim, anything else as compact key-sorted JSON."""
    if isinstance(value, str):
        return value
    return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))


def _confidence(values: Sequence[float]) -> float:
    """One minus the normalized Shannon entropy of a distribution, in [0, 1]."""
    n = len(values)
    if n < 2:
        return 1.0
    entropy = -sum(p * math.log(p) for p in values if p > 0)
    return max(0.0, min(1.0, 1 - entropy / math.log(n)))


def product_answer(
    question: Question, probabilities: Sequence[float]
) -> dict[str, Any]:
    """The public answer object of one question from its option probabilities (prompt order)."""
    if question.kind == "score":
        shares = normalized(probabilities, len(question.keys))
        return {
            "type": "score",
            "score": sum(level * p for level, p in enumerate(shares)),
            "probabilities": dict(zip(question.keys, shares)),
            "confidence": _confidence(shares),
            "legend": {
                key: _canonical(text)
                for key, text in zip(question.keys, question.descriptions)
            },
        }
    answer = to_answer(question.rendered, probabilities)
    if question.kind == "choice":
        answer["confidence"] = _confidence(list(answer["probabilities"].values()))
    return answer


# ---------------------------------------------------------------------------------------------
# Images
# ---------------------------------------------------------------------------------------------


# The subtypes the server takes in ``data:image/<subtype>;base64,`` URLs.
DATA_URL_IMAGE_TYPES = ("png", "jpeg", "jpg", "webp")


def _split_data_url(value: str, media: str) -> tuple[str, str]:
    """``data:<media>/<rest>;base64,<text>`` -> ``(<rest>, <text>)``.

    Everything before the first comma is matched lower-cased and stripped of surrounding whitespace; ``<rest>``
    is the subtype with any parameters, ``<text>`` comes back exactly as given.
    """
    head, comma, text = value.partition(",")
    head = head.strip().lower()
    opening, closing = f"data:{media}/", ";base64"
    if not comma or not head.startswith(opening) or not head.endswith(closing):
        raise ValueError(
            f"expected a base64 data URL: data:{media}/<type>;base64,<data>"
        )
    return head[len(opening) : -len(closing)], text


def _decode_base64(text: str, limit: int | None = None) -> bytes:
    """Base64 text decoded with validation. With ``limit`` the text is refused unread when it is longer than
    ``limit`` bytes can encode to, and the decoded bytes may not exceed ``limit``."""
    if limit is not None and len(text) > 4 * math.ceil(limit / 3):
        raise ValueError(f"the encoded data exceeds the {limit:,} byte limit")
    try:
        data = base64.b64decode(text, validate=True)
    except (binascii.Error, ValueError) as exc:  # bad alphabet or padding, non-ASCII
        raise ValueError("the data URL does not hold valid base64") from exc
    if limit is not None and len(data) > limit:
        raise ValueError(
            f"the decoded data has {len(data):,} bytes, over the {limit:,} byte limit"
        )
    return data


def _image_from_bytes(data: bytes):
    """The RGB pixels of an encoded image; ValueError when PIL cannot identify or decode it."""
    from PIL import Image

    try:
        with Image.open(io.BytesIO(data)) as image:
            return image.convert("RGB")
    except Exception as exc:  # noqa: BLE001 - PIL raises several classes
        raise ValueError(
            f"the image could not be decoded ({type(exc).__name__}: {exc})"
        ) from exc


def _checked_image(value: str):
    """The server's reading of an image data URL (see ``load_image``)."""
    from PIL import Image

    subtype, text = _split_data_url(value, "image")
    if subtype not in DATA_URL_IMAGE_TYPES:
        raise ValueError(
            f"image data URLs take the types {', '.join(DATA_URL_IMAGE_TYPES)}, got {subtype!r}"
        )
    data = _decode_base64(text, MAX_IMAGE_BYTES)
    try:
        probe = Image.open(io.BytesIO(data))
    except Exception as exc:  # noqa: BLE001 - unidentified, decompression bomb, ...
        raise ValueError(
            f"the image could not be identified ({type(exc).__name__}: {exc})"
        ) from exc
    with probe:
        if probe.format not in IMAGE_FORMATS:
            raise ValueError(
                f"the image is {probe.format or 'of an unknown format'}; send PNG, JPEG or WebP"
            )
        width, height = probe.size
        if width * height > MAX_IMAGE_SOURCE_PIXELS:
            raise ValueError(
                f"the image has {width * height:,} pixels; at most {MAX_IMAGE_SOURCE_PIXELS:,} are accepted"
            )
        try:
            probe.verify()
        except Exception as exc:  # noqa: BLE001 - PIL raises several classes
            raise ValueError(
                f"the image failed the integrity check ({type(exc).__name__}: {exc})"
            ) from exc
    return _image_from_bytes(data)


def _download(url: str, what: str = "image") -> bytes:
    import urllib.request

    request = urllib.request.Request(url, headers={"User-Agent": RUNTIME})
    with urllib.request.urlopen(request, timeout=DOWNLOAD_TIMEOUT_SECONDS) as response:
        payload = response.read(MAX_DOWNLOAD_BYTES + 1)
    if len(payload) > MAX_DOWNLOAD_BYTES:
        raise ValueError(
            f"the {what} at {url} is larger than {MAX_DOWNLOAD_BYTES:,} bytes"
        )
    return payload


def load_image(value: Any, *, strict: bool = False):
    """One image input as an RGB PIL image: pixels converted to RGB, no EXIF rotation and no resize.

    In process ``value`` is a PIL image, a local path (``~`` expanded), an http(s) URL or a
    ``data:image/<type>;base64,`` URL in any format PIL reads, bounded only by PIL's decompression-bomb guard.
    ``strict`` (the server) takes data URLs only: PNG, JPEG or WebP files of at most ``MAX_IMAGE_BYTES`` bytes and
    ``MAX_IMAGE_SOURCE_PIXELS`` pixels that pass PIL's integrity check. Anything else raises ValueError.
    """
    from PIL import Image

    if strict:
        if not isinstance(value, str) or not value.startswith("data:"):
            raise ValueError(
                "strict mode: images must be given as a data: URL string, not a file path, web address or PIL image"
            )
        return _checked_image(value)
    if isinstance(value, Image.Image):
        return value.convert("RGB")
    if isinstance(value, os.PathLike):
        value = os.fspath(value)
    if not isinstance(value, str) or not value:
        raise ValueError(
            "an image is a PIL image, a local path, an http(s) URL or a data:image/...;base64 URL"
        )
    if value.startswith("data:"):
        return _image_from_bytes(_decode_base64(_split_data_url(value, "image")[1]))
    try:
        if value.startswith(("http://", "https://")):
            data = _download(value)
        else:
            data = Path(value).expanduser().read_bytes()
    except OSError as exc:
        raise ValueError(f"cannot read the image {value} ({exc})") from exc
    return _image_from_bytes(data)


def load_processor(root: Path):
    """The checkpoint's ``AutoProcessor`` with left padding and the 1.6 MP image budget."""
    from transformers import AutoProcessor

    processor = AutoProcessor.from_pretrained(str(root))
    processor.tokenizer.padding_side = "left"
    processor.image_processor.size = {
        "shortest_edge": IMAGE_MIN_PIXELS,
        "longest_edge": IMAGE_MAX_PIXELS,
    }
    return processor


def visual_tokens(image_processor, width: int, height: int) -> int:
    """Input tokens of one image after the processor's resize (raises ValueError on aspect ratio > 200)."""
    patches = image_processor.get_number_of_image_patches(height, width, {})
    return patches // image_processor.merge_size**2


def _linear_patch_embed_forward(self, hidden_states):
    import torch.nn.functional as F

    weight = self.proj.weight
    flat = hidden_states.reshape(-1, weight[0].numel()).to(weight.dtype)
    out = F.linear(flat, weight.reshape(weight.shape[0], -1), self.proj.bias)
    return out.view(-1, self.embed_dim)


def linearize_patch_embed(model) -> int:
    """Run the vision patch embedding as the matrix product it equals; returns how many were patched.

    The patch embedding is a Conv3d whose kernel equals its stride over inputs already cut into single
    patches, i.e. a linear map of each flattened patch (same weights, same result up to summation order).
    On ROCm, MIOpen searches a Conv3d kernel for every new patch count, so each new image size would stall
    for minutes; the evaluation engine runs the same matrix product.
    """
    import types

    import torch

    count = 0
    for module in model.modules():
        proj = getattr(module, "proj", None)
        if (
            isinstance(proj, torch.nn.Conv3d)
            and tuple(proj.kernel_size) == tuple(proj.stride)
            and hasattr(module, "embed_dim")
        ):
            module.forward = types.MethodType(_linear_patch_embed_forward, module)
            count += 1
    return count


# ---------------------------------------------------------------------------------------------
# Videos
# ---------------------------------------------------------------------------------------------


@dataclass
class Video:
    """A decoded video: the sampled RGB frames and where they come from in the source.

    ``frames`` is a uint8 array ``[frames, height, width, 3]`` holding an even number of frames (the last
    one is repeated when needed); ``indices`` are their source frame numbers and ``fps`` the source frame
    rate, which give each frame its timestamp.
    """

    frames: Any
    fps: float
    indices: list[int]
    total_frames: int

    @property
    def size(self) -> tuple[int, int]:
        """(width, height) of the frames."""
        return int(self.frames.shape[2]), int(self.frames.shape[1])

    def metadata(self) -> dict[str, Any]:
        """The video processor's ``video_metadata`` for these frames."""
        width, height = self.size
        return {
            "total_num_frames": self.total_frames,
            "fps": self.fps,
            "width": width,
            "height": height,
            "duration": self.total_frames / self.fps,
            "frames_indices": list(self.indices),
        }


def video_sample_indices(total: int, fps: float) -> list[int]:
    """Source frames to read: 2 per second, at least 4 and at most 32, spread evenly over the whole video."""
    import numpy as np

    count = int(total / fps * VIDEO_FPS)
    count = min(max(count, VIDEO_MIN_FRAMES), VIDEO_MAX_FRAMES, total)
    return np.linspace(0, total - 1, count).round().astype(int).tolist()


def _video(frames: list[Any], indices: list[int], fps: float, total: int) -> Video:
    import numpy as np

    if len(frames) % 2:
        frames.append(frames[-1])
        indices.append(indices[-1])
    try:
        stacked = np.stack(frames)
    except ValueError as exc:
        raise ValueError("all frames of a video must have the same size") from exc
    return Video(np.ascontiguousarray(stacked), float(fps), list(indices), int(total))


def _video_data_url_payload(value: str, strict: bool) -> bytes:
    """The bytes of a ``data:video/<type>[;<parameters>];base64,`` URL; ``<type>`` ends at the first ``;``.

    ``strict`` (the server) takes only the ``VIDEO_FORMATS`` types and at most ``MAX_VIDEO_BYTES`` decoded bytes,
    refusing base64 text too long for that limit before decoding it.
    """
    rest, text = _split_data_url(value, "video")
    if not strict:
        return _decode_base64(text)
    subtype = rest.split(";", 1)[0]
    if subtype not in VIDEO_FORMATS:
        raise ValueError(
            f"video data URLs take the types {', '.join(VIDEO_FORMATS)}, got {subtype!r}"
        )
    return _decode_base64(text, MAX_VIDEO_BYTES)


def _decode_video(
    source: bytes | str, *, strict: bool, end_frame: int | None = None
) -> Video:
    """Decode a video file (path) or encoded video (bytes) with OpenCV and sample its frames."""
    try:
        import cv2
    except ImportError as exc:
        raise ValueError(
            "decoding video files needs OpenCV (pip install opencv-python-headless)"
        ) from exc
    import tempfile

    try:
        api = (
            cv2.CAP_FFMPEG
            if cv2.videoio_registry.hasBackend(cv2.CAP_FFMPEG)
            else cv2.CAP_ANY
        )
    except (AttributeError, cv2.error):
        api = cv2.CAP_ANY
    temporary = None
    if not isinstance(source, str):
        # Encoded bytes are read from a temporary file, so they decode exactly as the same file given by path.
        with tempfile.NamedTemporaryFile(prefix="d3-video-", delete=False) as handle:
            handle.write(source)
        source = temporary = handle.name

    def capture():
        return cv2.VideoCapture(source, api)

    frames, kept, reader = [], [], None
    try:
        reader = capture()
        if not reader.isOpened():
            raise ValueError("the video could not be opened")
        width = int(reader.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(reader.get(cv2.CAP_PROP_FRAME_HEIGHT))
        if width <= 0 or height <= 0:
            raise ValueError("the video has no frame size")
        if strict and width * height > MAX_VIDEO_SOURCE_PIXELS:
            raise ValueError(
                f"video frames must have at most {MAX_VIDEO_SOURCE_PIXELS:,} pixels"
            )
        fps = float(reader.get(cv2.CAP_PROP_FPS) or 0.0)
        if not (
            math.isfinite(fps) and VIDEO_SOURCE_FPS[0] <= fps <= VIDEO_SOURCE_FPS[1]
        ):
            fps = VIDEO_UNKNOWN_FPS
        total = int(reader.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
        if total <= 0:  # the container does not say: count the frames, then read again
            limit = int(MAX_VIDEO_SECONDS * fps) if strict else None
            while reader.grab():
                total += 1
                if limit is not None and total > limit:
                    break
            reader.release()
            reader = capture()
        if end_frame is not None:
            total = min(total, int(end_frame))
        if total <= 0:
            raise ValueError("the video has no frames")
        if strict and total / fps > MAX_VIDEO_SECONDS:
            raise ValueError(
                f"each video must be at most {MAX_VIDEO_SECONDS} seconds long"
            )
        indices = video_sample_indices(total, fps)
        wanted = set(indices)
        for number in range(indices[-1] + 1):
            if not reader.grab():
                continue
            if number in wanted:
                ok, frame = reader.retrieve()
                if ok and frame is not None and frame.size:
                    frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
                    kept.append(number)
    except cv2.error as exc:
        raise ValueError(f"invalid video data ({exc.__class__.__name__})") from exc
    finally:
        if reader is not None:
            reader.release()
        if temporary is not None:
            os.unlink(temporary)
    if not frames:
        raise ValueError("no frame of the video could be decoded")
    return _video(frames, kept, fps, total)


def _frames_video(value: Any, end_frame: int | None = None) -> Video:
    """A frame array as a video sampled at ``VIDEO_FPS``, thinned evenly to at most ``VIDEO_MAX_FRAMES``."""
    import numpy as np
    from PIL import Image

    if hasattr(value, "detach"):
        value = value.detach().cpu().numpy()
    if isinstance(value, (list, tuple)):
        if not value:
            raise ValueError("a video given as frames needs at least one frame")
        value = [
            (
                np.asarray(frame.convert("RGB"))
                if isinstance(frame, Image.Image)
                else np.asarray(frame)
            )
            for frame in value
        ]
        if len({frame.shape for frame in value}) != 1:
            raise ValueError("all frames of a video must have the same size")
        value = np.stack(value)
    array = np.asarray(value)
    if (
        array.ndim != 4
        or array.shape[-1] != 3
        or array.dtype != np.uint8
        or not len(array)
    ):
        raise ValueError(
            "a video given as frames is a uint8 [frames, height, width, 3] RGB array or a list of "
            "PIL images or [height, width, 3] arrays"
        )
    if end_frame is not None:
        array = array[: max(1, int(end_frame))]
    total = len(array)
    if total > VIDEO_MAX_FRAMES:
        indices = (
            np.linspace(0, total - 1, VIDEO_MAX_FRAMES).round().astype(int).tolist()
        )
    else:
        indices = list(range(total))
    return _video([array[i] for i in indices], indices, VIDEO_FPS, total)


def load_video(
    value: Any, *, strict: bool = False, end_frame: int | None = None
) -> Video:
    """One video input, decoded and sampled (see the module docstring).

    ``value`` is a local path, an http(s) URL, a ``data:video/<format>;base64,`` URL, a frame array (a uint8
    ``[frames, height, width, 3]`` RGB array or a list of PIL images or ``[height, width, 3]`` arrays, read
    as frames sampled at 2 per second) or a ``Video`` this function returned. ``end_frame`` reads only the
    frames before it. ``strict`` (the server) accepts only data URLs of MP4, WebM, QuickTime or Matroska
    videos of at most 32,000,000 bytes, 300 seconds and 8,294,400 pixels per frame.
    """
    if isinstance(value, Video) and not strict:
        return value
    if isinstance(value, os.PathLike):
        value = os.fspath(value)
    if not isinstance(value, str):
        if strict:
            raise ValueError(
                "cannot load this video with strict=True; pass the clip inline as a data:video/... URL"
            )
        return _frames_video(value, end_frame)
    if not value:
        raise ValueError(
            "a video is a path, an http(s) URL, a data:video/...;base64 URL or a frame array"
        )
    if strict and not value.startswith("data:"):
        raise ValueError(
            "cannot load this video with strict=True; pass the clip inline as a data:video/... URL"
        )
    try:
        if value.startswith("data:"):
            source: bytes | str = _video_data_url_payload(value, strict)
        elif value.startswith(("http://", "https://")):
            source = _download(value, "video")
        else:
            path = Path(value).expanduser()
            if not path.is_file():
                raise ValueError(f"no video file at {value}")
            source = str(path)
    except OSError as exc:
        raise ValueError(f"video unreadable ({exc})") from exc
    return _decode_video(source, strict=strict, end_frame=end_frame)


def configure_video_processor(processor) -> None:
    """Read already-sampled frames at up to ``VIDEO_MAX_PIXELS`` each with the checkpoint's video processor."""
    video = processor.video_processor
    factor = video.patch_size * video.merge_size
    video.size = {
        "shortest_edge": VIDEO_MIN_PIXELS,
        "longest_edge": VIDEO_MAX_PIXELS * VIDEO_MAX_FRAMES,
    }
    video.cap_pixels_per_frame = True
    video.max_video_tokens = VIDEO_MAX_PIXELS // factor**2
    video.do_sample_frames = False


# ---------------------------------------------------------------------------------------------
# Package files
# ---------------------------------------------------------------------------------------------


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with open(path, "rb") as stream:
        for block in iter(lambda: stream.read(16 << 20), b""):
            digest.update(block)
    return digest.hexdigest()


def load_file_module(path: Path):
    """Import a flat package file by its path, once per file (two checkpoints can share a process)."""
    import importlib.util
    import sys

    # Not resolved: in a Hub snapshot the .py name is a link to a blob without a suffix, which has no loader.
    path = Path(path).absolute()
    name = f"{path.stem}_{hashlib.sha256(str(path).encode()).hexdigest()[:16]}"
    if name not in sys.modules:
        spec = importlib.util.spec_from_file_location(name, str(path))
        if spec is None or spec.loader is None:
            raise ImportError(f"no loader for {path}")
        module = importlib.util.module_from_spec(spec)
        sys.modules[name] = module
        try:
            spec.loader.exec_module(module)
        except BaseException:
            del sys.modules[name]
            raise
    return sys.modules[name]


def fast_module(root: Path):
    """``d3_fast.py`` of the checkpoint directory when it is not importable as a module.

    ``AutoModel.from_pretrained(..., trust_remote_code=True)`` copies only the modeling files and their
    ``from .x import y`` imports into its module cache; the package's own copy (verified against
    ``MODEL_MANIFEST.json``) is next to the weights.
    """
    path = Path(root) / "d3_fast.py"
    return load_file_module(path) if path.is_file() else None


def resolve_dir(
    name_or_path: str | os.PathLike, revision: str | None = None, **hub: Any
) -> Path:
    """A local checkpoint directory, or a Hub snapshot (one commit) in the Hugging Face cache."""
    path = Path(os.fspath(name_or_path)).expanduser()
    if path.is_dir():
        return path
    from huggingface_hub import snapshot_download

    options = {k: v for k, v in hub.items() if v is not None and v is not False}
    return Path(snapshot_download(str(name_or_path), revision=revision, **options))


def verify_package(root: Path, mode: str = "fast") -> dict[str, Any] | None:
    """Check the files against ``MODEL_MANIFEST.json``; returns the manifest (None without one)."""
    if mode not in VERIFY_MODES:
        raise ValueError(f"verify must be one of {VERIFY_MODES}")
    path = root / MANIFEST
    if not path.is_file():
        return None
    manifest = json.loads(path.read_text(encoding="utf-8"))
    if manifest.get("schema") != MANIFEST_SCHEMA:
        raise ValueError(f"{MANIFEST} is not a {MANIFEST_SCHEMA} manifest")
    if mode == "none":
        return manifest
    sizes = manifest.get("files_bytes") or {}
    problems = []
    for name, digest in sorted((manifest.get("files_sha256") or {}).items()):
        target = root / name
        if not target.is_file():
            problems.append(f"missing {name}")
            continue
        size = target.stat().st_size
        if name in sizes and size != sizes[name]:
            problems.append(f"{name}: {size} bytes, manifest says {sizes[name]}")
            continue
        if mode == "full" or size <= FAST_HASH_BYTES:
            if sha256_file(target) != digest:
                problems.append(f"{name}: SHA-256 differs from {MANIFEST}")
    if problems:
        raise ValueError("package verification failed: " + "; ".join(problems[:8]))
    return manifest


def vision_weights_present(root: Path) -> bool:
    """True when the checkpoint stores the vision tower (``visual.*`` tensors)."""
    index = root / "model.safetensors.index.json"
    if index.is_file():
        names = list(json.loads(index.read_text(encoding="utf-8"))["weight_map"])
    else:
        from safetensors import safe_open

        names = []
        for path in sorted(root.glob("model*.safetensors")):
            with safe_open(str(path), framework="pt") as handle:
                names += list(handle.keys())
    return any(name.startswith(("visual.", "model.visual.")) for name in names)


# ---------------------------------------------------------------------------------------------
# Model
# ---------------------------------------------------------------------------------------------


def backbone_type(root: Path) -> str:
    """``model_type`` of the checkpoint's ``config.json``: ``qwen3_5`` or ``qwen3_vl``."""
    kind = json.loads((Path(root) / "config.json").read_text(encoding="utf-8")).get(
        "model_type"
    )
    if kind not in BACKBONES:
        raise ValueError(f"unsupported backbone model_type {kind!r}")
    return kind


def backbone_class(kind: str):
    """The transformers backbone class of a ``model_type``."""
    if kind == "qwen3_vl":
        from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLModel

        return Qwen3VLModel
    from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5Model

    return Qwen3_5Model


def sdpa_backends(device):
    """CUDA builds skip the cuDNN SDPA backend; ROCm keeps the defaults.

    ``D3_CUDNN_SDPA=1`` keeps PyTorch's default backend choice on CUDA too.
    """
    import contextlib

    import torch

    if (
        torch.device(device).type != "cuda"
        or not torch.version.cuda
        or getattr(torch.version, "hip", None)
        or os.environ.get("D3_CUDNN_SDPA") == "1"
    ):
        return contextlib.nullcontext()
    from torch.nn.attention import SDPBackend, sdpa_kernel

    return sdpa_kernel(
        [SDPBackend.FLASH_ATTENTION, SDPBackend.EFFICIENT_ATTENTION, SDPBackend.MATH]
    )


def kernel_report() -> dict[str, str]:
    """Which implementation transformers bound for the Gated DeltaNet ops (kernel package or PyTorch)."""
    from transformers.models.qwen3_5 import modeling_qwen3_5 as m

    def bound(fn) -> str:
        seen, stack = set(), [fn]
        while stack:
            f = stack.pop()
            if id(f) in seen or not callable(f):
                continue
            seen.add(id(f))
            module = getattr(f, "__module__", "") or ""
            if module.startswith(("fla", "causal_conv1d", "kernels")):
                return f"{module}.{getattr(f, '__name__', '?')}"
            for cell in getattr(f, "__closure__", None) or ():
                try:
                    stack.append(cell.cell_contents)
                except ValueError:
                    pass
        return "torch-reference"

    names = ("torch_chunk_gated_delta_rule", "causal_conv1d_fn")
    report = {name: bound(getattr(m, name)) for name in names if hasattr(m, name)}
    try:
        import fla

        report["fla"] = getattr(fla, "__version__", "?")
    except Exception as exc:  # noqa: BLE001
        report["fla"] = f"missing ({type(exc).__name__})"
    return report


class PromptEncoder:
    """The text layers of a Qwen3.5 backbone over a left-padded prompt batch, with no causal mask.

    The batch goes through the decoder layers in order as one forward pass without a cache. Every softmax
    attention layer gets the boolean key mask ``[B, 1, 1, L]`` (True on prompt tokens), so SDPA lets each query
    position read every prompt position of its own row, earlier or later, and none of the left padding. The
    Gated DeltaNet layers keep their causal scan and get transformers' recurrent padding mask for the batch
    (``create_recurrent_attention_mask``: the ``[B, L]`` mask, or None when no row is padded), as in the stock
    forward pass. RoPE positions are the stock ones: ``0..L-1`` on every row of a text batch, the multimodal
    positions of the backbone for image and video prompts. The result is the final-norm hidden states.
    """

    def __init__(self, language_model: Any):
        from transformers.masking_utils import create_recurrent_attention_mask

        config = language_model.config
        depth = config.num_hidden_layers
        self.language_model = language_model
        self.recurrent_mask = create_recurrent_attention_mask
        self.stack = [
            (layer, kind == "full_attention")
            for layer, kind in zip(
                language_model.layers[:depth], config.layer_types[:depth]
            )
        ]

    def __call__(self, embeds: Any, present: Any, mrope: Any | None = None) -> Any:
        """``embeds`` [B, L, D]; ``present`` [B, L] (1 on prompt tokens, 0 on left padding); ``mrope`` the
        [3, B, L] multimodal positions of an image or video batch, None for text. Returns [B, L, D].
        """
        import torch

        model = self.language_model
        rows, width = present.shape
        if mrope is None:
            ramp = torch.arange(width, device=embeds.device).view(1, 1, -1)
            ramp = ramp.expand(4, rows, -1)
            text_positions, mrope = ramp[0], ramp[1:]
        else:
            text_positions = None
        rope = model.rotary_emb(embeds, mrope)
        keys = present.bool()[:, None, None, :]
        scan = self.recurrent_mask(
            config=model.config,
            inputs_embeds=embeds,
            attention_mask=present,
            past_key_values=None,
            position_ids=text_positions,
        )
        hidden = embeds
        for layer, softmax in self.stack:
            hidden = layer(
                hidden,
                position_embeddings=rope,
                attention_mask=keys if softmax else scan,
                position_ids=text_positions,
                past_key_values=None,
                use_cache=False,
            )
        return model.norm(hidden)


@dataclass
class Prepared:
    """A tokenized request: runnable questions in request order plus the per-question errors.

    A text request holds token ``sequences``; an image request holds the decoded ``images`` (shared by
    every question), the rendered prompt ``texts`` and the planned input ``lengths`` (text tokens plus
    image tokens), and the processor tokenizes it when it runs. A video request also holds the decoded
    ``videos``, their processed pixels (``media``, one copy per image and video) and each video's
    placeholder expansion (``video_texts``).
    """

    keys: list[str]
    questions: dict[str, Question] = field(default_factory=dict)
    sequences: dict[str, list[int]] = field(default_factory=dict)
    errors: dict[str, dict[str, Any]] = field(default_factory=dict)
    images: list[Any] = field(default_factory=list)
    texts: dict[str, str] = field(default_factory=dict)
    lengths: dict[str, int] = field(default_factory=dict)
    videos: list[Video] = field(default_factory=list)
    media: dict[str, Any] = field(default_factory=dict)
    video_texts: list[str] = field(default_factory=list)
    # Reversed-option prompts of the choice questions (permutation_average only).
    reversed_sequences: dict[str, list[int]] = field(default_factory=dict)
    reversed_texts: dict[str, str] = field(default_factory=dict)
    reversed_lengths: dict[str, int] = field(default_factory=dict)

    @property
    def runnable(self) -> list[str]:
        return [k for k in self.keys if k in self.sequences or k in self.texts]


class D3:
    """A loaded code-readout checkpoint answering System One requests."""

    def __init__(
        self,
        root: Path,
        *,
        device: str | None = None,
        batch_size: int = DEFAULT_BATCH_SIZE,
        manifest: dict[str, Any] | None = None,
        max_length: int | None = None,
        readout_dtype: str | None = None,
        model_name: str | None = None,
        permutation_average: bool = False,
    ):
        import torch
        from safetensors.torch import load_file
        from transformers import AutoTokenizer

        self.torch = torch
        self.root = Path(root)
        self.manifest = manifest
        started = time.perf_counter()
        config_file = self.root / "decision_config.json"
        config = json.loads(config_file.read_text(encoding="utf-8"))
        if not isinstance(config, dict):
            raise ValueError(f"{config_file.name} must hold a JSON object")
        if config.get("format_version") != FORMAT_VERSION:
            raise ValueError(
                f"{config_file.name} has format_version {config.get('format_version')!r}; "
                f"this runtime reads version {FORMAT_VERSION}"
            )
        self.config = config
        self.backbone_type = backbone_type(self.root)
        picked = {}
        for key, default, allowed in (
            ("prompt", "d3", PROMPTS),
            ("attention_mode", "causal", ATTENTION_MODES),
            ("pooling", "last", ("last",)),
        ):
            picked[key] = config.get(key, default)
            if picked[key] not in allowed:
                raise ValueError(
                    f"{config_file.name}: {key} must be one of {', '.join(allowed)}, got {picked[key]!r}"
                )
        self.prompt = picked["prompt"]
        self.attention_mode = picked["attention_mode"]
        if (
            self.attention_mode == "noncausal_full_attention"
            and self.backbone_type != "qwen3_5"
        ):
            raise ValueError(
                f"{self.attention_mode} needs a qwen3_5 backbone; this checkpoint is {self.backbone_type}"
            )
        self.temperature = float(config.get("temperature", 1.0))
        if not (math.isfinite(self.temperature) and self.temperature > 0):
            raise ValueError(
                f"{config_file.name}: temperature must be finite and positive, got {self.temperature}"
            )
        self.readout_dtype = readout_dtype or config.get("readout_dtype", "float32")
        if self.readout_dtype not in ("float32", "bfloat16"):
            raise ValueError(
                f"readout_dtype must be float32 or bfloat16, got {self.readout_dtype!r}"
            )
        limit = config.get("max_length") if max_length is None else max_length
        self.max_length = int(limit) if limit else None
        self.batch_size = int(batch_size)
        if self.batch_size < 1:
            raise ValueError(f"batch_size must be at least 1, got {self.batch_size}")
        self.model_name = (
            model_name or (manifest or {}).get("model_name") or self.root.name
        )
        self.permutation_average = bool(permutation_average)
        self.tokenizer = AutoTokenizer.from_pretrained(str(self.root))
        self.tokenizer.padding_side = "left"
        codes, token_ids = answer_codes(self.tokenizer)
        if config.get("codes") != codes or config.get("token_ids") != token_ids:
            raise ValueError(
                f"the answer codes or token ids in {config_file.name} differ from the ones this tokenizer gives"
            )
        self.codes, self.token_ids = codes, token_ids
        probe = "State:\nA"
        if (
            self.tokenizer(probe)["input_ids"]
            != self.tokenizer(probe, add_special_tokens=False)["input_ids"]
        ):
            raise ValueError(
                "tokenizer adds special tokens; prompts are tokenized as rendered"
            )
        self.pad_id = self.tokenizer.pad_token_id
        if self.pad_id is None:
            raise ValueError("tokenizer has no pad token")

        if device is None:
            device = "cuda:0" if torch.cuda.is_available() else "cpu"
        self.device = torch.device(device)
        if self.device.type == "cuda":
            torch.cuda.set_device(self.device)
        self.kernels = kernel_report() if self.backbone_type == "qwen3_5" else {}
        if self.device.type == "cpu" and any(
            v.startswith(("fla", "causal_conv1d"))
            for k, v in self.kernels.items()
            if k != "fla"
        ):
            raise RuntimeError(
                "flash-linear-attention / causal-conv1d kernels are GPU-only; run on a GPU or "
                "use an environment without them for CPU inference"
            )
        torch.manual_seed(20260919)
        self.backbone = backbone_class(self.backbone_type).from_pretrained(
            str(self.root),
            dtype=torch.bfloat16,
            attn_implementation="sdpa",
            device_map={"": str(self.device)},
        )
        weight = load_file(str(self.root / "readout.safetensors"))["weight"]
        hidden = self.backbone.config.text_config.hidden_size
        if tuple(weight.shape) != (MAX_OPTIONS, hidden):
            raise ValueError(
                f"readout weight has shape {tuple(weight.shape)}, expected {(MAX_OPTIONS, hidden)}"
            )
        self.readout = weight.to(self.device, getattr(torch, self.readout_dtype))
        self.backbone.eval().requires_grad_(False)
        # The plain path's text layers for noncausal checkpoints; causal ones keep the stock forward.
        self.encoder = (
            PromptEncoder(self.backbone.language_model)
            if self.attention_mode == "noncausal_full_attention"
            else None
        )
        self.processor = None
        self.image_unavailable: str | None = None
        if not vision_weights_present(self.root):
            self.image_unavailable = (
                "the checkpoint has no vision tower (visual.* weights)"
            )
        elif not linearize_patch_embed(self.backbone):
            self.image_unavailable = "the vision patch embedding was not found"
        else:
            try:
                self.processor = load_processor(self.root)
            except (
                Exception
            ) as exc:  # noqa: BLE001 - text requests do not use the processor
                self.image_unavailable = (
                    f"the image processor failed to load ({type(exc).__name__}: {exc})"
                )
        self.video_unavailable: str | None = None
        if self.image_unavailable is not None:
            self.video_unavailable = self.image_unavailable
        elif self.backbone_type != "qwen3_5":
            self.video_unavailable = "video inputs need a Qwen3.5 backbone"
        elif getattr(self.processor, "video_processor", None) is None:
            self.video_unavailable = "the checkpoint has no video processor"
        else:
            try:
                configure_video_processor(self.processor)
            except (
                Exception
            ) as exc:  # noqa: BLE001 - text and image requests do not use it
                self.video_unavailable = (
                    f"the video processor failed to load ({type(exc).__name__}: {exc})"
                )
        fast, reason = d3_fast, "d3_fast.py is not present"
        if fast is None:
            try:
                fast = fast_module(self.root)
            except Exception as exc:  # noqa: BLE001 - the plain path always loads
                reason = f"d3_fast.py failed to import ({type(exc).__name__}: {exc})"
        self.fast_skipped = None if fast else reason
        self.fast = fast.install(self) if fast else None
        self.loaded_seconds = time.perf_counter() - started

    @classmethod
    def from_pretrained(
        cls,
        name_or_path: str | os.PathLike,
        *,
        revision: str | None = None,
        device: str | None = None,
        batch_size: int = DEFAULT_BATCH_SIZE,
        verify: str = "fast",
        token: str | bool | None = None,
        cache_dir: str | os.PathLike | None = None,
        local_files_only: bool = False,
        force_download: bool = False,
        max_length: int | None = None,
        readout_dtype: str | None = None,
        model_name: str | None = None,
        permutation_average: bool = False,
    ) -> D3:
        """Load a package directory or Hub repository.

        ``verify``: ``fast`` (default) hashes every file of ``MODEL_MANIFEST.json`` up to 64 MiB and checks
        the size of the weight shards; ``full`` hashes every file; ``none`` skips the check. A checkpoint
        without a manifest (a plain code-readout export) loads unverified. ``permutation_average``: see the
        module docstring.
        """
        root = resolve_dir(
            name_or_path,
            revision,
            token=token,
            cache_dir=cache_dir,
            local_files_only=local_files_only,
            force_download=force_download,
        )
        manifest = verify_package(root, verify)
        return cls(
            root,
            device=device,
            batch_size=batch_size,
            manifest=manifest,
            max_length=max_length,
            readout_dtype=readout_dtype,
            model_name=model_name,
            permutation_average=permutation_average,
        )

    # ------------------------------------------------------------------ requests

    def text(self, state: Any, question: dict[str, Any]) -> str:
        return render(self.tokenizer, self.prompt, state, question, self.codes)

    def image_text(self, state: Any, question: dict[str, Any], n_images: int) -> str:
        return self.processor.apply_chat_template(
            image_messages(self.prompt, state, question, self.codes, n_images),
            tokenize=False,
            add_generation_prompt=True,
            enable_thinking=False,
        )

    def video_text(
        self, state: Any, question: dict[str, Any], n_images: int, n_videos: int
    ) -> str:
        return self.processor.apply_chat_template(
            video_messages(
                self.prompt, state, question, self.codes, n_images, n_videos
            ),
            tokenize=False,
            add_generation_prompt=True,
            enable_thinking=False,
        )

    def load_videos(
        self, videos: Sequence[Any], *, strict: bool = False
    ) -> list[Video]:
        """Decode and sample a request's videos, any number; raises ValueError for a video it cannot read."""
        if not videos:
            return []
        if self.video_unavailable is not None:
            raise ValueError(
                f"video inputs are not available: {self.video_unavailable}"
            )
        decoded = []
        for number, value in enumerate(videos):
            try:
                decoded.append(load_video(value, strict=strict))
            except ValueError as exc:
                raise ValueError(f"videos[{number}]: {exc}") from exc
        return decoded

    def load_images(self, images: Sequence[Any], *, strict: bool = False) -> list[Any]:
        """Decode a request's images (RGB PIL), any number; raises ValueError for an image it cannot read."""
        if not images:
            return []
        if self.image_unavailable is not None:
            raise ValueError(
                f"image inputs are not available: {self.image_unavailable}"
            )
        decoded = []
        for number, value in enumerate(images):
            try:
                decoded.append(load_image(value, strict=strict))
            except ValueError as exc:
                raise ValueError(f"images[{number}]: {exc}") from exc
        return decoded

    def prepare(
        self,
        state: Any,
        questions: Mapping[str, Any],
        images: Sequence[Any] | None = None,
        videos: Sequence[Any] | None = None,
    ) -> Prepared:
        """Validate and tokenize one request; malformed ``state``, ``questions``, ``images`` or ``videos`` raise
        ValueError.

        ``images`` (a list of any number of images shared by every question) select the image path, ``videos``
        (a list of any number of videos, with or without images) the video path. Without them the request takes
        the text path unchanged.
        """
        if not isinstance(questions, Mapping) or not questions:
            raise ValueError(
                "questions must be a nonempty mapping of question IDs to questions"
            )
        if any(not isinstance(key, str) or not key for key in questions):
            raise ValueError("question IDs must be nonempty strings")
        _json_value(state)
        if images is not None and not isinstance(images, (list, tuple)):
            raise ValueError("images must be a list of images")
        if videos is not None and not isinstance(videos, (list, tuple)):
            raise ValueError("videos must be a list of videos")
        if videos:
            return self._prepare_videos(
                state,
                questions,
                self.load_images(images or []),
                self.load_videos(videos),
            )
        if images:
            return self._prepare_images(state, questions, self.load_images(images))
        prepared = Prepared(keys=list(questions))
        texts = []
        for key, question in questions.items():
            try:
                normalized = normalize_question(question)
                texts.append((key, self.text(state, normalized.rendered)))
            except ValueError as exc:
                kind = question.get("type") if isinstance(question, Mapping) else None
                prepared.errors[key] = {
                    "type": kind,
                    "error": "invalid_question",
                    "message": str(exc),
                }
                continue
            prepared.questions[key] = normalized
        if texts:
            ids = self.tokenizer([t for _, t in texts], add_special_tokens=False)[
                "input_ids"
            ]
            for (key, _), sequence in zip(texts, ids):
                if self.max_length is not None and len(sequence) > self.max_length:
                    prepared.errors[key] = {
                        "type": prepared.questions[key].kind,
                        "error": "max_length_exceeded",
                        "message": f"the question prompt has {len(sequence)} tokens, over the maximum context "
                        f"length of {self.max_length} tokens; nothing was truncated",
                    }
                    continue
                prepared.sequences[key] = sequence
        if self.permutation_average:
            self._prepare_reversed(state, prepared)
        return prepared

    def _prepare_reversed(
        self, state: Any, prepared: Prepared, visual: int = 0
    ) -> None:
        """Render and tokenize the reversed-option prompt of every runnable choice question.

        ``visual``: the image and video tokens of the request (image and video paths), counted like
        ``_prepare_images`` and ``_prepare_videos`` count them.
        """
        n_images, n_videos = len(prepared.images), len(prepared.videos)
        texts = []
        for key in prepared.runnable:
            flipped = reversed_question(prepared.questions[key])
            if flipped is not None:
                texts.append(
                    (
                        key,
                        (
                            self.video_text(state, flipped, n_images, n_videos)
                            if n_videos
                            else (
                                self.image_text(state, flipped, n_images)
                                if n_images
                                else self.text(state, flipped)
                            )
                        ),
                    )
                )
        if not texts:
            return
        tokenizer = self.processor.tokenizer if n_images or n_videos else self.tokenizer
        ids = tokenizer([t for _, t in texts], add_special_tokens=False)["input_ids"]
        for (key, text), sequence in zip(texts, ids):
            length = len(sequence) - n_images - n_videos + visual
            if self.max_length is not None and length > self.max_length:
                for planned in (prepared.sequences, prepared.texts, prepared.lengths):
                    planned.pop(key, None)
                prepared.errors[key] = {
                    "type": prepared.questions[key].kind,
                    "error": "max_length_exceeded",
                    "message": f"the reversed-option prompt has {length} tokens, over the maximum context "
                    f"length of {self.max_length} tokens; nothing was truncated",
                }
            elif n_images or n_videos:
                prepared.reversed_texts[key] = text
                prepared.reversed_lengths[key] = length
            else:
                prepared.reversed_sequences[key] = sequence

    def _prepare_images(
        self, state: Any, questions: Mapping[str, Any], images: list[Any]
    ) -> Prepared:
        """Render every question with the images in front and plan its input tokens (images not run yet)."""
        try:
            visual = [
                visual_tokens(self.processor.image_processor, *image.size)
                for image in images
            ]
        except ValueError as exc:
            raise ValueError(f"image rejected by the processor: {exc}") from exc
        prepared = Prepared(keys=list(questions), images=images)
        texts = []
        for key, question in questions.items():
            try:
                normalized = normalize_question(question)
                text = self.image_text(state, normalized.rendered, len(images))
            except ValueError as exc:
                kind = question.get("type") if isinstance(question, Mapping) else None
                prepared.errors[key] = {
                    "type": kind,
                    "error": "invalid_question",
                    "message": str(exc),
                }
                continue
            if text.count(IMAGE_TOKEN) != len(images) or VIDEO_TOKEN in text:
                prepared.errors[key] = {
                    "type": normalized.kind,
                    "error": "invalid_question",
                    "message": "the state or question contains a literal image or video placeholder token",
                }
                continue
            prepared.questions[key] = normalized
            texts.append((key, text))
        if texts:
            ids = self.processor.tokenizer(
                [t for _, t in texts], add_special_tokens=False
            )["input_ids"]
            for (key, text), sequence in zip(texts, ids):
                length = len(sequence) - len(images) + sum(visual)
                if self.max_length is not None and length > self.max_length:
                    prepared.errors[key] = {
                        "type": prepared.questions[key].kind,
                        "error": "max_length_exceeded",
                        "message": f"the question prompt has {length} tokens ({sum(visual)} for "
                        f"{len(images)} image(s)), over the maximum context length of {self.max_length} "
                        "tokens; nothing was truncated",
                    }
                    continue
                prepared.texts[key] = text
                prepared.lengths[key] = length
        if self.permutation_average:
            self._prepare_reversed(state, prepared, sum(visual))
        return prepared

    def _prepare_videos(
        self,
        state: Any,
        questions: Mapping[str, Any],
        images: list[Any],
        videos: list[Video],
    ) -> Prepared:
        """Process the images and videos once, render every question with them in front and plan its tokens."""
        processor = self.processor
        try:
            visual = [
                visual_tokens(processor.image_processor, *image.size)
                for image in images
            ]
        except ValueError as exc:
            raise ValueError(f"image rejected by the processor: {exc}") from exc
        merge = processor.video_processor.merge_size**2
        try:
            planned = (
                sum(
                    processor.video_processor.get_num_of_video_patches(
                        len(video.frames), video.size[1], video.size[0]
                    )
                    for video in videos
                )
                // merge
            )
        except ValueError as exc:
            raise ValueError(f"video rejected by the processor: {exc}") from exc
        if planned > VIDEO_MAX_TOKENS:
            raise ValueError(
                f"the videos take {planned:,} input tokens, over the {VIDEO_MAX_TOKENS:,} tokens a "
                "request may spend on videos; send fewer or shorter videos"
            )
        try:
            media = dict(
                processor.video_processor(
                    videos=[video.frames for video in videos],
                    video_metadata=[video.metadata() for video in videos],
                    return_metadata=True,
                    return_tensors="pt",
                )
            )
        except ValueError as exc:
            raise ValueError(f"video rejected by the processor: {exc}") from exc
        video_texts = [
            processor.replace_video_token(media, video_idx=i)
            for i in range(len(videos))
        ]
        if int(media["video_grid_thw"].prod(-1).sum()) // merge != planned:
            raise RuntimeError("the video processor and its token count disagree")
        expanded = sum(
            len(ids)
            for ids in processor.tokenizer(video_texts, add_special_tokens=False)[
                "input_ids"
            ]
        )
        media = {
            "pixel_values_videos": media["pixel_values_videos"],
            "video_grid_thw": media["video_grid_thw"],
        }
        if images:
            media.update(processor.image_processor(images=images, return_tensors="pt"))
        prepared = Prepared(
            keys=list(questions),
            images=images,
            videos=videos,
            media=media,
            video_texts=video_texts,
        )
        texts = []
        for key, question in questions.items():
            try:
                normalized = normalize_question(question)
                text = self.video_text(
                    state, normalized.rendered, len(images), len(videos)
                )
            except ValueError as exc:
                kind = question.get("type") if isinstance(question, Mapping) else None
                prepared.errors[key] = {
                    "type": kind,
                    "error": "invalid_question",
                    "message": str(exc),
                }
                continue
            if text.count(IMAGE_TOKEN) != len(images) or text.count(VIDEO_TOKEN) != len(
                videos
            ):
                prepared.errors[key] = {
                    "type": normalized.kind,
                    "error": "invalid_question",
                    "message": "the state or question contains a literal image or video placeholder token",
                }
                continue
            prepared.questions[key] = normalized
            texts.append((key, text))
        if texts:
            ids = processor.tokenizer([t for _, t in texts], add_special_tokens=False)[
                "input_ids"
            ]
            for (key, text), sequence in zip(texts, ids):
                length = (
                    len(sequence) - len(images) - len(videos) + sum(visual) + expanded
                )
                if self.max_length is not None and length > self.max_length:
                    prepared.errors[key] = {
                        "type": prepared.questions[key].kind,
                        "error": "max_length_exceeded",
                        "message": f"the question prompt has {length} tokens ({sum(visual) + expanded} for "
                        f"{len(images)} image(s) and {len(videos)} video(s)), over the maximum context length "
                        f"of {self.max_length} tokens; nothing was truncated",
                    }
                    continue
                prepared.texts[key] = text
                prepared.lengths[key] = length
        if self.permutation_average:
            self._prepare_reversed(state, prepared, sum(visual) + expanded)
        return prepared

    def logits(self, sequences: Sequence[Sequence[int]], counts: Sequence[int]):
        """Masked code logits (FP32, [B, 255]) for one left-padded batch."""
        torch = self.torch
        width = max(len(s) for s in sequences)
        ids = torch.full((len(sequences), width), self.pad_id, dtype=torch.long)
        mask = torch.zeros((len(sequences), width), dtype=torch.long)
        for i, sequence in enumerate(sequences):
            ids[i, width - len(sequence) :] = torch.as_tensor(
                sequence, dtype=torch.long
            )
            mask[i, width - len(sequence) :] = 1
        ids, mask = ids.to(self.device, non_blocking=True), mask.to(
            self.device, non_blocking=True
        )
        with torch.inference_mode(), sdpa_backends(self.device):
            if self.encoder is not None:
                embeds = self.backbone.get_input_embeddings()(ids)
                hidden = self.encoder(embeds, mask)[:, -1]
            else:
                hidden = self.backbone(
                    input_ids=ids, attention_mask=mask, use_cache=False
                ).last_hidden_state[:, -1]
            if self.readout_dtype == "float32":
                logits = hidden.float() @ self.readout.T
            else:
                logits = torch.nn.functional.linear(hidden, self.readout).float()
            limit = torch.as_tensor(list(counts), device=self.device)[:, None]
            invalid = torch.arange(MAX_OPTIONS, device=self.device)[None] >= limit
            return logits.masked_fill(invalid, float("-inf"))

    def probabilities(
        self, sequences: Sequence[Sequence[int]], counts: Sequence[int]
    ) -> list[list[float]]:
        """Option probabilities of one batch, each row cut to its question's option count."""
        if self.fast is not None:
            return self.fast.probabilities(sequences, counts)
        scaled = self.logits(sequences, counts) / self.temperature
        rows = self.torch.softmax(scaled, dim=-1).cpu().tolist()
        return [row[:count] for row, count in zip(rows, counts)]

    def image_embeddings(self, inputs: Mapping[str, Any]):
        """Input embeddings with the image features in place, and the multimodal RoPE positions, of an image
        batch: the steps the backbone's own forward takes before its text layers."""
        torch = self.torch
        unexpected = sorted(set(inputs) - set(IMAGE_INPUTS))
        if unexpected:
            raise RuntimeError(
                f"the processor returned inputs the runtime does not read: {unexpected}"
            )
        backbone = self.backbone
        ids = inputs["input_ids"]
        embeds = backbone.get_input_embeddings()(ids)
        pooled = backbone.get_image_features(
            inputs["pixel_values"], inputs["image_grid_thw"], return_dict=True
        ).pooler_output
        pooled = torch.cat(pooled, dim=0).to(embeds.device, embeds.dtype)
        slots, _ = backbone.get_placeholder_mask(
            ids, inputs_embeds=embeds, image_features=pooled
        )
        embeds = embeds.masked_scatter(slots, pooled)
        mrope = backbone.compute_3d_position_ids(
            input_ids=ids,
            image_grid_thw=inputs["image_grid_thw"],
            video_grid_thw=None,
            inputs_embeds=embeds,
            attention_mask=inputs["attention_mask"],
            past_key_values=None,
            mm_token_type_ids=inputs["mm_token_type_ids"],
        )
        return embeds, mrope

    def image_logits(
        self,
        texts: Sequence[str],
        images: Sequence[Any],
        counts: Sequence[int],
        width: int,
    ):
        """Masked code logits (FP32, [B, 255]) for one left-padded batch of prompts that share ``images``.

        The processor expands each prompt's image placeholders, resizes the images and pads the batch to
        ``width`` tokens (the longest planned prompt); the backbone gets every tensor it returns.
        """
        torch = self.torch
        encoded = self.processor(
            text=list(texts),
            images=[image for _ in texts for image in images],
            padding=True,
            return_tensors="pt",
        )
        if encoded["input_ids"].shape[1] != width:
            raise RuntimeError(
                f"planned {width} input tokens, the processor produced {encoded['input_ids'].shape[1]}"
            )
        inputs = {name: value.to(self.device) for name, value in encoded.items()}
        fused = self.fast is not None and self.fast.fused is not None
        with torch.inference_mode(), sdpa_backends(self.device):
            if fused and set(inputs) == set(IMAGE_INPUTS):
                padded = not bool(encoded["attention_mask"].all())
                hidden = self.fast.image_hidden(inputs, padded)[:, -1]
            elif self.encoder is not None:
                embeds, mrope = self.image_embeddings(inputs)
                hidden = self.encoder(embeds, inputs["attention_mask"], mrope)[:, -1]
            else:
                hidden = self.backbone(**inputs, use_cache=False).last_hidden_state[
                    :, -1
                ]
            if self.readout_dtype == "float32":
                logits = hidden.float() @ self.readout.T
            else:
                logits = torch.nn.functional.linear(hidden, self.readout).float()
            limit = torch.as_tensor(list(counts), device=self.device)[:, None]
            invalid = torch.arange(MAX_OPTIONS, device=self.device)[None] >= limit
            return logits.masked_fill(invalid, float("-inf"))

    def image_probabilities(
        self,
        texts: Sequence[str],
        images: Sequence[Any],
        counts: Sequence[int],
        width: int,
    ) -> list[list[float]]:
        probs = (
            (self.image_logits(texts, images, counts, width) / self.temperature)
            .softmax(-1)
            .cpu()
            .tolist()
        )
        return [p[:c] for p, c in zip(probs, counts)]

    def _run_images(self, prepared: Prepared) -> tuple[dict[str, list[float]], int]:
        keys = prepared.runnable
        out: dict[str, list[float]] = {}
        for start in range(0, len(keys), self.batch_size):
            chunk = keys[start : start + self.batch_size]
            extra = [k for k in chunk if k in prepared.reversed_texts]
            texts = [prepared.texts[k] for k in chunk] + [
                prepared.reversed_texts[k] for k in extra
            ]
            counts = [len(prepared.questions[k].keys) for k in chunk + extra]
            widths = [prepared.lengths[k] for k in chunk] + [
                prepared.reversed_lengths[k] for k in extra
            ]
            n = len(chunk)
            if extra and len(widths) * max(widths) > MERGE_TOKENS:
                probs = self.image_probabilities(
                    texts[:n], prepared.images, counts[:n], max(widths[:n])
                ) + self.image_probabilities(
                    texts[n:], prepared.images, counts[n:], max(widths[n:])
                )
            else:
                probs = self.image_probabilities(
                    texts, prepared.images, counts, max(widths)
                )
            out.update(zip(chunk, probs))
            for key, backward in zip(extra, probs[n:]):
                out[key] = average_orders(out[key], backward)
        return out, sum(prepared.lengths[k] for k in keys) + sum(
            prepared.reversed_lengths.values()
        )

    def media_features(self, media: Mapping[str, Any]) -> dict[str, list[Any]]:
        """Vision-tower features of each image and video of a video request, computed once per request."""
        torch = self.torch
        backbone = self.backbone
        features: dict[str, list[Any]] = {}
        with torch.inference_mode(), sdpa_backends(self.device):
            if "pixel_values" in media:
                features["image"] = list(
                    backbone.get_image_features(
                        media["pixel_values"].to(self.device),
                        media["image_grid_thw"].to(self.device),
                        return_dict=True,
                    ).pooler_output
                )
            features["video"] = list(
                backbone.get_video_features(
                    media["pixel_values_videos"].to(self.device),
                    media["video_grid_thw"].to(self.device),
                    return_dict=True,
                ).pooler_output
            )
        return features

    def video_logits(
        self,
        texts: Sequence[str],
        prepared: Prepared,
        features: Mapping[str, list[Any]],
        counts: Sequence[int],
        width: int,
    ):
        """Masked code logits (FP32, [B, 255]) for one left-padded batch of prompts that share the request's videos.

        Each prompt's placeholders are expanded with the request's images and videos and the batch is padded to
        ``width`` tokens (the longest planned prompt); the vision features of the request go into every prompt,
        then the text layers run as for an image batch (the fused layers when the fast path is active).
        """
        torch = self.torch
        processor = self.processor
        backbone = self.backbone
        media = prepared.media
        image_texts = [
            processor.replace_image_token(media, image_idx=i)
            for i in range(len(prepared.images))
        ]
        expanded = [
            processor.get_text_with_replacements(
                [text], image_texts, prepared.video_texts
            )[0][0]
            for text in texts
        ]
        encoded = processor.tokenizer(
            expanded, padding=True, return_token_type_ids=False, return_tensors="pt"
        )
        if encoded["input_ids"].shape[1] != width:
            raise RuntimeError(
                f"planned {width} input tokens, the processor produced {encoded['input_ids'].shape[1]}"
            )
        rows = len(texts)
        ids = encoded["input_ids"].to(self.device)
        mask = encoded["attention_mask"].to(self.device)
        types = torch.as_tensor(
            processor.create_mm_token_type_ids(encoded["input_ids"]), device=self.device
        )
        padded = not bool(encoded["attention_mask"].all())
        with torch.inference_mode(), sdpa_backends(self.device):
            embeds = backbone.get_input_embeddings()(ids)
            image_grid = None
            if "image" in features:
                values = torch.cat(features["image"] * rows).to(
                    embeds.device, embeds.dtype
                )
                image_mask, _ = backbone.get_placeholder_mask(
                    ids, inputs_embeds=embeds, image_features=values
                )
                embeds = embeds.masked_scatter(image_mask, values)
                image_grid = media["image_grid_thw"].repeat(rows, 1).to(self.device)
            values = torch.cat(features["video"] * rows).to(embeds.device, embeds.dtype)
            _, video_mask = backbone.get_placeholder_mask(
                ids, inputs_embeds=embeds, video_features=values
            )
            embeds = embeds.masked_scatter(video_mask, values)
            positions = backbone.compute_3d_position_ids(
                input_ids=ids,
                image_grid_thw=image_grid,
                video_grid_thw=media["video_grid_thw"].repeat(rows, 1).to(self.device),
                inputs_embeds=embeds,
                attention_mask=mask,
                past_key_values=None,
                mm_token_type_ids=types,
            )
            fused = self.fast.fused if self.fast is not None else None
            if fused is not None:
                rotary = self.fast.lm.rotary_emb(embeds, positions)
                hidden = fused.forward(
                    embeds,
                    mask.bool()[:, None, None, :],
                    mask.reshape(-1) if padded else None,
                    rotary,
                )[:, -1]
            elif self.encoder is not None:
                hidden = self.encoder(embeds, mask, positions)[:, -1]
            else:
                hidden = backbone.language_model(
                    input_ids=None,
                    position_ids=positions,
                    attention_mask=mask,
                    past_key_values=None,
                    inputs_embeds=embeds,
                    use_cache=False,
                ).last_hidden_state[:, -1]
            if self.readout_dtype == "float32":
                logits = hidden.float() @ self.readout.T
            else:
                logits = torch.nn.functional.linear(hidden, self.readout).float()
            limit = torch.as_tensor(list(counts), device=self.device)[:, None]
            invalid = torch.arange(MAX_OPTIONS, device=self.device)[None] >= limit
            return logits.masked_fill(invalid, float("-inf"))

    def _run_videos(self, prepared: Prepared) -> tuple[dict[str, list[float]], int]:
        keys = prepared.runnable
        out: dict[str, list[float]] = {}
        if not keys:
            return out, 0
        features = self.media_features(prepared.media)

        def probabilities(texts, counts, width):
            probs = (
                (
                    self.video_logits(texts, prepared, features, counts, width)
                    / self.temperature
                )
                .softmax(-1)
                .cpu()
                .tolist()
            )
            return [p[:c] for p, c in zip(probs, counts)]

        for start in range(0, len(keys), self.batch_size):
            chunk = keys[start : start + self.batch_size]
            extra = [k for k in chunk if k in prepared.reversed_texts]
            texts = [prepared.texts[k] for k in chunk] + [
                prepared.reversed_texts[k] for k in extra
            ]
            counts = [len(prepared.questions[k].keys) for k in chunk + extra]
            widths = [prepared.lengths[k] for k in chunk] + [
                prepared.reversed_lengths[k] for k in extra
            ]
            n = len(chunk)
            if extra and len(widths) * max(widths) > MERGE_TOKENS:
                probs = probabilities(
                    texts[:n], counts[:n], max(widths[:n])
                ) + probabilities(texts[n:], counts[n:], max(widths[n:]))
            else:
                probs = probabilities(texts, counts, max(widths))
            out.update(zip(chunk, probs))
            for key, backward in zip(extra, probs[n:]):
                out[key] = average_orders(out[key], backward)
        return out, sum(prepared.lengths[k] for k in keys) + sum(
            prepared.reversed_lengths.values()
        )

    def run(self, prepared: Prepared) -> tuple[dict[str, list[float]], int]:
        """Probabilities per runnable question (request order, ``batch_size`` per pass) and the input tokens."""
        if prepared.videos:
            return self._run_videos(prepared)
        if prepared.images:
            return self._run_images(prepared)
        keys = prepared.runnable
        out: dict[str, list[float]] = {}
        for start in range(0, len(keys), self.batch_size):
            chunk = keys[start : start + self.batch_size]
            extra = [k for k in chunk if k in prepared.reversed_sequences]
            sequences = [prepared.sequences[k] for k in chunk] + [
                prepared.reversed_sequences[k] for k in extra
            ]
            counts = [len(prepared.questions[k].keys) for k in chunk + extra]
            n = len(chunk)
            if extra and len(sequences) * max(map(len, sequences)) > MERGE_TOKENS:
                probs = self.probabilities(
                    sequences[:n], counts[:n]
                ) + self.probabilities(sequences[n:], counts[n:])
            else:
                probs = self.probabilities(sequences, counts)
            out.update(zip(chunk, probs))
            for key, backward in zip(extra, probs[n:]):
                out[key] = average_orders(out[key], backward)
        return out, sum(len(prepared.sequences[k]) for k in keys) + sum(
            len(s) for s in prepared.reversed_sequences.values()
        )

    def respond(
        self,
        prepared: Prepared,
        probabilities: Mapping[str, Sequence[float]],
        tokens: int,
    ) -> dict[str, Any]:
        answers: dict[str, Any] = {}
        for key in prepared.keys:
            if key in prepared.errors:
                answers[key] = prepared.errors[key]
                continue
            question = prepared.questions[key]
            try:
                answers[key] = product_answer(question, probabilities[key])
            except ValueError as exc:
                answers[key] = {
                    "type": question.kind,
                    "error": "invalid_model_output",
                    "message": str(exc),
                }
        return {
            "model": self.model_name,
            "answers": answers,
            "usage": {"input_tokens": tokens, "output_tokens": 0},
        }

    def system_one(
        self,
        *,
        state: Any,
        questions: Mapping[str, Any],
        images: Sequence[Any] | None = None,
        videos: Sequence[Any] | None = None,
    ) -> dict[str, Any]:
        """Typed Choice / Noul / Score answers about one state: ``{"model", "answers", "usage"}``.

        ``images``: any number of images (PIL images, paths, http(s) or data URLs) that every question sees.
        ``videos``: any number of videos (paths, http(s) or data URLs, frame arrays) that every question sees.
        A question that cannot be answered gets ``{"type", "error", "message"}`` with ``error`` one of
        ``invalid_question``, ``max_length_exceeded`` (never truncated) or ``invalid_model_output``; the
        other questions of the request are still answered.
        """
        prepared = self.prepare(state, questions, images, videos)
        probabilities, tokens = self.run(prepared)
        return self.respond(prepared, probabilities, tokens)

    def warmup(
        self,
        lengths: Sequence[int] = (37, 64, 320, 333, 1000, 1024),
        images: bool = True,
    ) -> float:
        """Compile and autotune the kernels for every batch size up to ``batch_size`` (twice that with
        ``permutation_average``) before serving.

        The Gated DeltaNet kernels take the batch size as a compile-time constant, and Triton specializes their
        length and chunk-count arguments on being 1 or a multiple of 16; these lengths cover every combination
        (chunks of 64 tokens). A fresh process otherwise pays several seconds on the first request of each new
        class. With ``images`` (and a vision tower) three image requests (one small image, one 1.6 MP image,
        four 1.6 MP images) also warm the vision tower. Answers are unchanged.
        """
        started = time.perf_counter()
        if self.fast is not None:
            self.fast.capture_all()
        widest = self.batch_size * (2 if self.permutation_average else 1)
        for size in range(1, widest + 1):
            for length in lengths:
                sequences = [
                    [
                        self.token_ids[(i + j) % len(self.token_ids)]
                        for j in range(max(8, length - 7 * i))
                    ]
                    for i in range(size)
                ]
                self.probabilities(sequences, [2] * size)
        if images and self.image_unavailable is None:
            from PIL import Image

            small = Image.new("RGB", (448, 336), (128, 128, 128))
            large = Image.new("RGB", (1280, 1280), (96, 160, 224))
            for batch in ([small], [large], [large] * 4):
                self.system_one(
                    state="warm-up", questions={"q": {"type": "noul"}}, images=batch
                )
        self.synchronize()
        return time.perf_counter() - started

    # ------------------------------------------------------------------ device and records

    def synchronize(self) -> None:
        if self.device.type == "cuda":
            self.torch.cuda.synchronize(self.device)

    def to(self, device: str) -> D3:
        target = self.torch.device(device)
        self.backbone.to(target)
        self.readout = self.readout.to(target)
        self.device = target
        if self.fast is not None:
            self.fast, self.fast_skipped = None, "moved after loading"
        return self

    def parameter_count(self) -> int:
        return sum(p.numel() for p in self.backbone.parameters()) + self.readout.numel()

    def provenance(self) -> dict[str, Any]:
        files = {
            name: sha256_file(self.root / name)
            for name in (
                "decision_config.json",
                "readout.safetensors",
                "config.json",
                MANIFEST,
            )
            if (self.root / name).is_file()
        }
        identity = (self.manifest or {}).get("identity", {})
        record = {
            "kind": "d3-code-readout",
            "runtime": RUNTIME,
            "model_name": self.model_name,
            "repo_id": (self.manifest or {}).get("repo_id"),
            "model_sha256": identity.get("model_sha256"),
            "format_id": self.config.get("format_id"),
            "prompt": self.prompt,
            "attention_mode": self.attention_mode,
            "pooling": "last",
            "temperature": self.temperature,
            "max_length": self.max_length,
            "readout_dtype": self.readout_dtype,
            "backbone_dtype": "bfloat16",
            "attn_implementation": "sdpa",
            "batch_size": self.batch_size,
            "files_sha256": files,
            "kernels": self.kernels,
            "policy": "One forward pass per question; options under single-token answer codes; last-token readout "
            "over the question's codes only; argmax choice; no truncation (over-limit questions are "
            "refused); no option filtering; one fixed prompt for every request.",
            "images": self.image_contract(),
            "videos": self.video_contract(),
        }
        if self.permutation_average:
            record["permutation_average"] = True
            record["policy"] += (
                " Choice questions with two or more options are also scored with their options in reversed "
                "order, in the same forward passes, and answered with the per-option mean of both distributions."
            )
        return record

    def image_contract(self) -> dict[str, Any]:
        """How image inputs are read (or why they are not available)."""
        contract: dict[str, Any] = {
            "supported": self.image_unavailable is None,
            "min_pixels": IMAGE_MIN_PIXELS,
            "max_pixels": IMAGE_MAX_PIXELS,
            "placement": "before the text of the user turn, one placeholder per image, request order",
            "patch_embedding": "matrix product (equal to the Conv3d with kernel = stride)",
        }
        if self.processor is not None:
            contract["image_processor"] = type(self.processor.image_processor).__name__
            try:
                import torchvision

                contract["torchvision"] = torchvision.__version__
            except Exception:  # noqa: BLE001
                contract["torchvision"] = None
        if self.image_unavailable is not None:
            contract["unavailable"] = self.image_unavailable
        return contract

    def video_contract(self) -> dict[str, Any]:
        """How video inputs are read (or why they are not available)."""
        contract: dict[str, Any] = {
            "supported": self.video_unavailable is None,
            "fps": VIDEO_FPS,
            "min_frames": VIDEO_MIN_FRAMES,
            "max_frames": VIDEO_MAX_FRAMES,
            "max_pixels_per_frame": VIDEO_MAX_PIXELS,
            "max_tokens_per_request": VIDEO_MAX_TOKENS,
            "placement": "after the images, before the text of the user turn, one placeholder per video, "
            "request order; every two frames are one token group after their timestamp",
            "sampling": "source frames spread evenly over the whole video (frame arrays: frames at 2 per second)",
        }
        if self.video_unavailable is None:
            contract["video_processor"] = type(self.processor.video_processor).__name__
            try:
                import cv2

                contract["opencv"] = cv2.__version__
            except Exception:  # noqa: BLE001
                contract["opencv"] = None
        else:
            contract["unavailable"] = self.video_unavailable
        return contract

    def runtime_info(self) -> dict[str, Any]:
        import transformers

        torch = self.torch
        info = {
            "torch": torch.__version__,
            "transformers": transformers.__version__,
            "device": str(self.device),
            "cuda": torch.version.cuda,
            "hip": getattr(torch.version, "hip", None),
            "loaded_seconds": round(self.loaded_seconds, 2),
        }
        if self.device.type == "cuda":
            info["gpu"] = torch.cuda.get_device_name(self.device)
        info["fast_path"] = self.fast_report()
        return info

    def fast_report(self) -> dict[str, Any]:
        """What the ROCm fast path (``d3_fast.py``) does in this process, or why it is off."""
        if self.fast is None:
            return {"active": False, "reason": self.fast_skipped}
        return {"active": True, **self.fast.report()}