Release isolated single-pass SPEV decision head

MANIFEST.json

@@ -1,164 +0,0 @@
-{
-  "model": "SpragAI/SPEV",
-  "package_format": "transformers-autoclasses",
-  "candidate": "clean57/joint/candidate",
-  "source_adapter_sha256": "1e1ffd17099c9e70ab061547286a43a6bdddd8ce8aa1d304803f65550fc8a7b5",
-  "source_head_sha256": "6f1c6891aec8cb6862493ce24c3895e33e2c5b95ce8781c0ab61dd27b0702f94",
-  "files": [
-    {
-      "path": "LICENSE",
-      "bytes": 11358,
-      "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"
-    },
-    {
-      "path": "README.md",
-      "bytes": 4119,
-      "sha256": "73d6815884bd212924e2cd7450f42897ac67bc4feeb9883eaa9d8d39bb287993"
-    },
-    {
-      "path": "THIRD_PARTY_NOTICES.md",
-      "bytes": 358,
-      "sha256": "4c8a4cecaf46c00fc31b21aaed23e9c15099b312f07e922ecbf49945df683b45"
-    },
-    {
-      "path": "__init__.py",
-      "bytes": 0,
-      "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-    },
-    {
-      "path": "chat_template.jinja",
-      "bytes": 6519,
-      "sha256": "45777d2e798aae1d1ce6844e69edcb23fc0c710009554711c78efab9044da2c8"
-    },
-    {
-      "path": "config.json",
-      "bytes": 6341,
-      "sha256": "98f5092ed524600d44113f192c45c2ef8cb078d4ae77885ab2376db88a818b4f"
-    },
-    {
-      "path": "configuration_spev.py",
-      "bytes": 1750,
-      "sha256": "8390f4ded6259839601d878e62c4f44559b44999f63c3ae141879a911003de03"
-    },
-    {
-      "path": "decision_utils.py",
-      "bytes": 3867,
-      "sha256": "bc38e41c1123880b2038696489272cb489a1c62da98f672f467e694078636208"
-    },
-    {
-      "path": "examples/turn_decision.py",
-      "bytes": 981,
-      "sha256": "e1ce54067b9cd8b6348b97309ee7920c6a0eadddf75880676f3335ae45c883d8"
-    },
-    {
-      "path": "examples/turn_policy.json",
-      "bytes": 730,
-      "sha256": "d07af2090edf412ab86af84887b11e7d96e09b22325799788b84921d130e3406"
-    },
-    {
-      "path": "model-00001-of-00013.safetensors",
-      "bytes": 4375707904,
-      "sha256": "27a8786a74a21747745003b524f343d7d39b5acfe3edf4ce8127cbfc7ccc7788"
-    },
-    {
-      "path": "model-00002-of-00013.safetensors",
-      "bytes": 4985172432,
-      "sha256": "ec6b95123dead3f88cc7b7ee87da4ab49cb50878c7ae7446d0b8d2b81990fe13"
-    },
-    {
-      "path": "model-00003-of-00013.safetensors",
-      "bytes": 4985172432,
-      "sha256": "14d8680d304fcedd57440d3722591ad3347530d2e1d3952713cdbffcbc4bce00"
-    },
-    {
-      "path": "model-00004-of-00013.safetensors",
-      "bytes": 4985173960,
-      "sha256": "82223f49e7ad8eb263f442642f6b260bb251df8a7703968e36289b04845cf228"
-    },
-    {
-      "path": "model-00005-of-00013.safetensors",
-      "bytes": 4985174008,
-      "sha256": "ff1cd35f862c04aa8bbb8cd2c7ce909c8833866964170185fe6ef69ee6f79328"
-    },
-    {
-      "path": "model-00006-of-00013.safetensors",
-      "bytes": 4985174008,
-      "sha256": "efb7cf9badc49260df295f1f3b9ee852195643579a22b17930ed847cee1b801f"
-    },
-    {
-      "path": "model-00007-of-00013.safetensors",
-      "bytes": 4985174008,
-      "sha256": "aeeed3d751bd7f4b8a4d49c56fd4f84ce5ca6352e989136ac0f7be9f7a1266ad"
-    },
-    {
-      "path": "model-00008-of-00013.safetensors",
-      "bytes": 4985174008,
-      "sha256": "1b1ec719c4854406189f4727cd57eaf6f29756eae7a5a02afaaa351c45fecde3"
-    },
-    {
-      "path": "model-00009-of-00013.safetensors",
-      "bytes": 4985174008,
-      "sha256": "b85cf0be2c40324cb3372de572b6e38e60e02264d9444eb0c2054aac98984d60"
-    },
-    {
-      "path": "model-00010-of-00013.safetensors",
-      "bytes": 4985174008,
-      "sha256": "4cae94eb7f02d03a1ae90d055b6119ae8b3fdc877827ee714ec41f33029007cf"
-    },
-    {
-      "path": "model-00011-of-00013.safetensors",
-      "bytes": 4985174008,
-      "sha256": "9b74e898729090f19b9e0ba76a9f4736bd6fcc09ebb1422d1717a9b95645ece7"
-    },
-    {
-      "path": "model-00012-of-00013.safetensors",
-      "bytes": 4985174008,
-      "sha256": "6d6a2d55237097a1086f1522fb41fef8877873f04a110a4393e4e218fd28726f"
-    },
-    {
-      "path": "model-00013-of-00013.safetensors",
-      "bytes": 3605546880,
-      "sha256": "089e485dcc0bb546286a7f0acc906d15f080502d7bd9ee4b2441b9e2bb33be41"
-    },
-    {
-      "path": "model-decision.safetensors",
-      "bytes": 55615444,
-      "sha256": "e711007c2b401a337442e7b9e1ae6d89c36a2a2dee67cda9a817730b766ca1a1"
-    },
-    {
-      "path": "model.safetensors.index.json",
-      "bytes": 1964764,
-      "sha256": "61c7968601e201e1713058cc0a32619ce0edd8958b574ce7f442a6c1fa4e44d4"
-    },
-    {
-      "path": "modeling_spev.py",
-      "bytes": 4750,
-      "sha256": "625f3c60581eed62a278c23f68cd9494c16023707e9d8ddb30e3ae47fcd6ba87"
-    },
-    {
-      "path": "processing_spev.py",
-      "bytes": 4330,
-      "sha256": "821524b54c2de7a7cc6800dc8fe4679318542a86cdbffc6646fe82ccc64e1a96"
-    },
-    {
-      "path": "processor_config.json",
-      "bytes": 2704,
-      "sha256": "5712ee707a83a1c8fbc230df116395e6c61be58770f648a61cbf8c504f9dc380"
-    },
-    {
-      "path": "requirements.txt",
-      "bytes": 205,
-      "sha256": "4b107e5fca893565799ebe5b2600c5708d28d618c0d581125bdfa3df52c15435"
-    },
-    {
-      "path": "tokenizer.json",
-      "bytes": 11424262,
-      "sha256": "a7d41145c408f3062e824f965be1c29854cd809e111cdf8170aa8b0bcd5d1fab"
-    },
-    {
-      "path": "tokenizer_config.json",
-      "bytes": 1067,
-      "sha256": "6d6cadf5fc644eef4fe90b62f053f759733e4543921f52f37adb22cc48492779"
-    }
-  ]
-}

README.md

@@ -29,7 +29,7 @@ returns an answer with a probability for each choice.
 ## Model
 
 - **Backbone:** Qwen3-Omni-30B-A3B Thinker, with a trained LoRA adapter.
-- **Decision head:** a shared scorer over contextual representations of the supplied choices.
+- **Decision head:** an evidence-attention head that scores each question independently.
 - **Output:** per-choice probabilities and the highest-probability answer for each question.
 
 ## Quickstart
@@ -104,23 +104,12 @@ do not supply a sample-rate argument. Supply a video's soundtrack separately as
 | `model-*.safetensors`, `model.safetensors.index.json` | Backbone, adapter and decision-head weights |
 | `config.json`, `configuration_spev.py` | Model architecture and scoring configuration |
 | `modeling_spev.py` | Transformers model implementation |
+| `decision_head.py`, `decision_layout.py`, `candidate_tree.py`, `branch_attention.py`, `token_operations.py` | Decision scoring and question isolation |
 | `processing_spev.py`, `decision_utils.py` | Media preparation, question rendering and answer formatting |
 | Tokenizer, processor and chat-template files | Text, audio, image and video processing |
-| `examples/turn_policy.json` | Evaluated turn-taking question and application threshold |
 | `requirements.txt` | Inference dependencies |
 | `LICENSE` | Apache-2.0 license |
 
-## Results
-
-| Input | SPEV p50 / p95 | OpenAI p50 / p95 |
-|---|---:|---:|
-| 5-second audio | 166 / 191 ms | 752 / 1331 ms |
-| 20-second audio | 216 / 270 ms | 997 / 1326 ms |
-| 512 × 512 image | 183 / 283 ms | 194 / 334 ms |
-| 1024 × 1024 image | 322 / 642 ms | 373 / 763 ms |
-
-Warm public-HTTPS latency including RTT, measured from the same client with concurrency one. SPEV ran on an RTX PRO6000 with native vLLM pooling. Each row has 30 measured pairs over six fixtures after three warmup pairs. OpenAI used `gpt-6-luna`; audio includes a fresh `gpt-4o-mini-transcribe` request before Decisions. These measurements cover latency, not accuracy.
-
 ## License
 
 Apache-2.0. See [LICENSE](LICENSE) and [third-party notices](THIRD_PARTY_NOTICES.md).

branch_attention.py

@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Shared-prefix branch attention with request-local immutable mask metadata."""
+
+from collections import defaultdict
+from contextlib import contextmanager
+from contextvars import ContextVar
+
+import torch
+from torch.nn import functional as F
+
+REQUEST = ContextVar("spev_attention_request", default=None)
+
+
+class RequestMaskCache:
+    def __init__(self):
+        self.tree = None
+        self.device = None
+        self.plan = None
+        self.builds = 0
+        self.hits = 0
+
+    def __call__(self, tree, device):
+        if tree is self.tree and device == self.device:
+            self.hits += 1
+            return self.plan
+        buckets = defaultdict(list)
+        start = tree.prefix_length
+        for length in tree.branch_lengths:
+            buckets[((length + 31) // 32) * 32].append((start, length))
+            start += length
+        assert start == tree.length
+        plan = []
+        p = tree.prefix_length
+        for width, branches in sorted(buckets.items()):
+            for offset in range(0, len(branches), 8):
+                block = branches[offset : offset + 8]
+                allowed = torch.zeros(8, 1, width, p + width, device=device, dtype=torch.bool)
+                allowed[:, :, :, 0] = True
+                for slot, (_, length) in enumerate(block):
+                    allowed[slot, 0, :length, :p] = True
+                    allowed[slot, 0, :length, p : p + length] = torch.ones(
+                        length, length, device=device, dtype=torch.bool
+                    ).tril()
+                plan.append((width, block, allowed))
+        self.tree = tree
+        self.device = device
+        self.plan = plan
+        self.builds += 1
+        return plan
+
+    def stats(self):
+        return dict(
+            builds=self.builds,
+            hits=self.hits,
+            blocks=len(self.plan or []),
+            branches=sum(len(block) for _, block, _ in self.plan or []),
+        )
+
+
+def branch_attention(
+    module, query, key, value, attention_mask=None, dropout=0.0, scaling=None, sliding_window=None, **kwargs
+):
+    request = REQUEST.get()
+    tree = request[0] if request else None
+    if tree is None or query.shape[0] != 1 or key.shape[-2] != tree.length or query.shape[-2] != tree.length:
+        raise ValueError("Branch attention requires an explicit single-request tree layout")
+    if sliding_window is not None:
+        raise ValueError("Sliding-window branch attention is not implemented")
+    if dropout:
+        raise ValueError("Branch attention reference requires zero attention dropout")
+    groups = query.shape[1] // key.shape[1]
+    if groups > 1:
+        key = key.repeat_interleave(groups, dim=1)
+        value = value.repeat_interleave(groups, dim=1)
+    p = tree.prefix_length
+    output = torch.zeros_like(query)
+    output[:, :, :p] = F.scaled_dot_product_attention(
+        query[:, :, :p], key[:, :, :p], value[:, :, :p], is_causal=True, scale=scaling
+    )
+    cache = request[1]
+    heads, dim = query.shape[1], query.shape[-1]
+    batch_slots = 8
+    for width, block, allowed in cache(tree, query.device):
+        q = query.new_zeros(batch_slots, heads, width, dim)
+        k = key.new_zeros(batch_slots, heads, p + width, dim)
+        v = value.new_zeros(batch_slots, heads, p + width, dim)
+        for slot, (start, length) in enumerate(block):
+            q[slot, :, :length] = query[0, :, start : start + length]
+            k[slot, :, :p] = key[0, :, :p]
+            v[slot, :, :p] = value[0, :, :p]
+            k[slot, :, p : p + length] = key[0, :, start : start + length]
+            v[slot, :, p : p + length] = value[0, :, start : start + length]
+        values = F.scaled_dot_product_attention(q, k, v, attn_mask=allowed, scale=scaling)
+        for slot, (start, length) in enumerate(block):
+            output[0, :, start : start + length] = values[slot, :, :length]
+    return output.transpose(1, 2).contiguous(), None
+
+
+def install(model):
+    from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+
+    implementation = f"{__name__}.isolated_branches"
+    ALL_ATTENTION_FUNCTIONS.register(implementation, branch_attention)
+    for layer in model.model.layers:
+        layer.self_attn.config._attn_implementation = implementation
+
+
+@contextmanager
+def request_context(tree):
+    token = REQUEST.set((tree, RequestMaskCache()))
+    try:
+        yield
+    finally:
+        REQUEST.reset(token)

candidate_tree.py

@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Shared evidence prefix and isolated candidate branch layout."""
+
+from dataclasses import dataclass
+
+import torch
+
+
+@dataclass(frozen=True)
+class CandidateTree:
+    prefix_length: int
+    branch_lengths: tuple[int, ...]
+
+    def __post_init__(self):
+        if self.prefix_length <= 0 or not self.branch_lengths or min(self.branch_lengths) <= 0:
+            raise ValueError("A nonempty prefix and nonempty branches are required")
+
+    @property
+    def length(self):
+        return self.prefix_length + sum(self.branch_lengths)
+
+    def allowed(self, *, device=None):
+        groups = [-1] * self.prefix_length
+        for index, length in enumerate(self.branch_lengths):
+            groups.extend([index] * length)
+        groups = torch.tensor(groups, device=device)
+        physical = torch.arange(self.length, device=device)
+        causal = physical[None, :] <= physical[:, None]
+        return causal & ((groups[None, :] == -1) | (groups[:, None] == groups[None, :]))
+
+    def additive_mask(self, *, dtype=torch.float32, device=None):
+        allowed = self.allowed(device=device)
+        return torch.zeros_like(allowed, dtype=dtype).masked_fill(~allowed, float("-inf"))[None, None]
+
+    def position_ids(self, prefix_positions):
+        if prefix_positions.ndim != 3 or prefix_positions.shape[-1] != self.prefix_length:
+            raise ValueError("Expected native prefix positions with shape (axes, batch, prefix)")
+        if prefix_positions.shape[1] != 1:
+            raise ValueError("This reference packs one request")
+        origin = int(prefix_positions.max()) + 1
+        parts = [prefix_positions]
+        for length in self.branch_lengths:
+            branch = torch.arange(origin, origin + length, device=prefix_positions.device)
+            parts.append(branch.view(1, 1, -1).expand(prefix_positions.shape[0], 1, -1))
+        return torch.cat(parts, dim=-1)
+
+    def attention_pairs(self):
+        prefix = self.prefix_length
+        return prefix * (prefix + 1) // 2 + sum(
+            prefix * length + length * (length + 1) // 2 for length in self.branch_lengths
+        )

config.json

@@ -9,10 +9,14 @@
     "AutoModel": "modeling_spev.SPEVModel"
   },
   "dtype": "bfloat16",
+  "head_architecture": "isolated_evidence_v1",
+  "head_depth": 2,
+  "head_ff": 1024,
+  "head_heads": 8,
   "head_width": 256,
   "max_tokens": 8192,
   "model_type": "spev",
-  "temperature": 1.3234945284875692,
+  "temperature": 1.0,
   "thinker_config": {
     "architectures": [
       "Qwen3OmniMoeThinkerForConditionalGeneration"

configuration_spev.py

@@ -16,6 +16,10 @@ class SPEVConfig(PretrainedConfig):
         adapter_rank=32,
         adapter_alpha=64,
         head_width=256,
+        head_heads=8,
+        head_ff=1024,
+        head_depth=2,
+        head_architecture="isolated_evidence_v1",
         temperature=1.0,
         max_tokens=8192,
         **kwargs,
@@ -26,6 +30,9 @@ class SPEVConfig(PretrainedConfig):
         for name, value in (
             ("adapter_rank", adapter_rank),
             ("head_width", head_width),
+            ("head_heads", head_heads),
+            ("head_ff", head_ff),
+            ("head_depth", head_depth),
             ("max_tokens", max_tokens),
         ):
             if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
@@ -37,6 +44,14 @@ class SPEVConfig(PretrainedConfig):
         self.adapter_rank = adapter_rank
         self.adapter_alpha = adapter_alpha
         self.head_width = head_width
+        if head_architecture != "isolated_evidence_v1":
+            raise ValueError("Unsupported SPEV decision head architecture")
+        if head_width % head_heads:
+            raise ValueError("head_width must be divisible by head_heads")
+        self.head_heads = head_heads
+        self.head_ff = head_ff
+        self.head_depth = head_depth
+        self.head_architecture = head_architecture
         self.temperature = temperature
         self.max_tokens = max_tokens
         kwargs.setdefault("tie_word_embeddings", False)

decision_head.py

@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Question-local evidence routing for SPEV decision checkpoints."""
+
+from dataclasses import dataclass
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+class EvidenceBlock(nn.Module):
+    def __init__(self, width: int, heads: int, ff: int):
+        super().__init__()
+        self.qnorm = nn.LayerNorm(width)
+        self.mnorm = nn.LayerNorm(width)
+        self.attn = nn.MultiheadAttention(width, heads, batch_first=True, dropout=0.0)
+        self.fnorm = nn.LayerNorm(width)
+        self.ff = nn.Sequential(nn.Linear(width, ff), nn.GELU(), nn.Linear(ff, width))
+
+    def forward(self, query, memory, memory_pad):
+        memory = self.mnorm(memory)
+        value, _ = self.attn(self.qnorm(query), memory, memory, key_padding_mask=memory_pad, need_weights=False)
+        query = query + value
+        return query + self.ff(self.fnorm(query))
+
+
+class IsolatedDecisionHead(nn.Module):
+    def __init__(self, hidden: int, width: int, heads: int, ff: int, depth: int):
+        super().__init__()
+        self.norm = nn.LayerNorm(hidden)
+        self.memory = nn.Linear(hidden, width, bias=False)
+        self.qproj = nn.Linear(hidden, width, bias=False)
+        self.oqproj = nn.Linear(hidden, width, bias=False)
+        self.globalproj = nn.Linear(hidden, width, bias=False)
+        self.context = nn.Linear(hidden, width, bias=False)
+        self.lexical = nn.Linear(hidden, width, bias=False)
+        self.types = nn.Embedding(3, width)
+        self.evidence = nn.ModuleList([EvidenceBlock(width, heads, ff) for _ in range(depth)])
+        self.summarynorm = nn.LayerNorm(width)
+        self.joint = nn.ModuleList(
+            [
+                nn.TransformerDecoderLayer(
+                    width, heads, ff, dropout=0.0, activation="gelu", batch_first=True, norm_first=True
+                )
+                for _ in range(depth)
+            ]
+        )
+        self.fnorm = nn.LayerNorm(width)
+        self.onorm = nn.LayerNorm(width)
+        self.scorer = nn.Sequential(nn.Linear(width * 4, width), nn.GELU(), nn.Linear(width, 1))
+        self.prior_scale = nn.Parameter(torch.zeros(()))
+        self.joint_scale = nn.Parameter(torch.zeros(()))
+        self.gate = nn.Parameter(torch.zeros(()))
+
+    def forward(self, batch: dict[str, torch.Tensor]) -> torch.Tensor:
+        memory = self.memory(self.norm(batch["memory"]))
+        question = self.norm(batch["questions"])
+        context = self.norm(batch["options"])
+        global_feature = self.norm(batch["global"])
+        options = self.context(context) + self.lexical(batch["lexical"]) + self.oqproj(question)
+        for block in self.evidence:
+            options = block(options, memory, batch["memory_pad"])
+        fields = self.qproj(question)
+        membership = ~batch["option_pad"][:, None, :]
+        weights = torch.einsum("bqd,bcd->bqc", fields, options) / memory.shape[-1] ** 0.5
+        weights = weights.masked_fill(~membership, -1e9).softmax(-1) * membership
+        summary = torch.einsum("bqc,bcd->bqd", weights, options)
+        fields = fields + self.summarynorm(summary) + self.globalproj(global_feature)[:, None, :]
+        fields = fields + self.types(batch["types"])
+        for block in self.joint:
+            fields = block(fields, memory, memory_key_padding_mask=batch["memory_pad"])
+        fields = self.fnorm(fields).expand(-1, options.shape[1], -1)
+        options = self.onorm(options)
+        prior = (
+            F.normalize(question + global_feature[:, None, :], dim=-1) * F.normalize(batch["lexical"], dim=-1)
+        ).sum(-1) * self.prior_scale.clamp(max=4.605170186).exp()
+        joint = F.cosine_similarity(fields, options, dim=-1) * self.joint_scale.clamp(max=4.605170186).exp()
+        residual = self.scorer(torch.cat([fields, options, fields * options, (fields - options).abs()], -1)).squeeze(-1)
+        return prior + self.gate.sigmoid() * (joint + residual)
+
+
+@dataclass(frozen=True)
+class QuestionFeatures:
+    memory: torch.Tensor
+    question: torch.Tensor
+    options: torch.Tensor
+    lexical: torch.Tensor
+    global_feature: torch.Tensor
+    question_type: int
+
+
+def score_questions(head: IsolatedDecisionHead, records: list[QuestionFeatures]) -> list[torch.Tensor]:
+    """Score all questions in a request batch with one head invocation."""
+    if not records:
+        raise ValueError("At least one question is required")
+    device = head.norm.weight.device
+    hidden = head.norm.normalized_shape[0]
+    memories = max(len(record.memory) for record in records)
+    candidates = max(len(record.options) for record in records)
+    batch = {
+        name: torch.zeros(shape, device=device, dtype=torch.float32)
+        for name, shape in {
+            "memory": (len(records), memories, hidden),
+            "questions": (len(records), 1, hidden),
+            "options": (len(records), candidates, hidden),
+            "lexical": (len(records), candidates, hidden),
+            "global": (len(records), hidden),
+        }.items()
+    }
+    batch["memory_pad"] = torch.ones(len(records), memories, device=device, dtype=torch.bool)
+    batch["option_pad"] = torch.ones(len(records), candidates, device=device, dtype=torch.bool)
+    batch["types"] = torch.tensor([[record.question_type] for record in records], device=device)
+    for index, record in enumerate(records):
+        if not len(record.memory) or not len(record.options) or record.question_type not in (0, 1, 2):
+            raise ValueError("Invalid question features")
+        if record.lexical.shape != record.options.shape:
+            raise ValueError("Criterion embeddings must match candidate features")
+        for name, value in (("memory", record.memory), ("options", record.options), ("lexical", record.lexical)):
+            batch[name][index, : len(value)] = value.to(device=device, dtype=torch.float32)
+        batch["questions"][index, 0] = record.question.to(device=device, dtype=torch.float32)
+        batch["global"][index] = record.global_feature.to(device=device, dtype=torch.float32)
+        batch["memory_pad"][index, : len(record.memory)] = False
+        batch["option_pad"][index, : len(record.options)] = False
+    values = head(batch)
+    return [values[index, : len(record.options)] for index, record in enumerate(records)]

decision_layout.py

@@ -0,0 +1,84 @@
+"""Token layout for an evidence prefix and independent decision candidates."""
+
+from dataclasses import dataclass
+
+from .candidate_tree import CandidateTree
+from .decision_utils import MARKER, literal, locate_markers
+
+
+@dataclass(frozen=True)
+class DecisionLayout:
+    tree: CandidateTree
+    anchors: tuple[int, ...]
+    question_positions: tuple[tuple[int, ...], ...]
+    lexical_ids: tuple[tuple[int, ...], ...]
+    candidate_questions: tuple[int, ...]
+    question_types: tuple[int, ...]
+
+    def to_dict(self) -> dict:
+        return {
+            "prefix_length": self.tree.prefix_length,
+            "branch_lengths": self.tree.branch_lengths,
+            "anchors": self.anchors,
+            "question_positions": self.question_positions,
+            "lexical_ids": self.lexical_ids,
+            "candidate_questions": self.candidate_questions,
+            "question_types": self.question_types,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(
+            CandidateTree(data["prefix_length"], tuple(data["branch_lengths"])),
+            tuple(data["anchors"]), tuple(tuple(row) for row in data["question_positions"]),
+            tuple(tuple(row) for row in data["lexical_ids"]),
+            tuple(data["candidate_questions"]), tuple(data["question_types"]),
+        )
+
+    def validate(self, length: int):
+        count = len(self.tree.branch_lengths)
+        if self.tree.length != length or not (len(self.anchors) == len(self.lexical_ids) == len(self.candidate_questions) == count):
+            raise ValueError("Decision layout does not match the packed sequence")
+        if not self.question_types or len(self.question_positions) != len(self.question_types):
+            raise ValueError("Decision question metadata is incomplete")
+        if any(kind not in (0, 1, 2) for kind in self.question_types):
+            raise ValueError("Invalid decision question type")
+        if set(self.candidate_questions) != set(range(len(self.question_types))):
+            raise ValueError("Every question requires candidate branches")
+        start = self.tree.prefix_length
+        for index, size in enumerate(self.tree.branch_lengths):
+            if not start <= self.anchors[index] < start + size or not self.lexical_ids[index]:
+                raise ValueError("Invalid decision candidate metadata")
+            start += size
+        for index, positions in enumerate(self.question_positions):
+            branch = self.candidate_questions.index(index)
+            start = self.tree.prefix_length + sum(self.tree.branch_lengths[:branch])
+            end = start + self.tree.branch_lengths[branch]
+            if not positions or any(position < start or position >= end for position in positions):
+                raise ValueError("Question positions must belong to its first candidate branch")
+
+
+def pack_candidates(tokenizer, prefix: list[int], questions: dict) -> tuple[list[int], DecisionLayout]:
+    ids = list(prefix)
+    lengths, anchors, positions, lexical, owners, kinds = [], [], [], [], [], []
+    marker = tokenizer.encode(MARKER.lstrip("\n"), add_special_tokens=False)
+    for question_index, question in enumerate(questions.values()):
+        kinds.append({"predicate": 0, "choice": 1, "score": 2}[question["type"]])
+        for candidate_index, description in enumerate(question["criteria"].values()):
+            text = literal({"question": question["instructions"], "criterion": description}) + MARKER
+            encoded = tokenizer(text, add_special_tokens=False, return_offsets_mapping=True)
+            start = len(ids)
+            branch = encoded["input_ids"]
+            ids.extend(branch)
+            lengths.append(len(branch))
+            anchors.append(start + locate_markers(branch, marker, 1)[0])
+            owners.append(question_index)
+            lexical.append(tuple(tokenizer.encode(literal(description), add_special_tokens=False)))
+            if candidate_index == 0:
+                phrase = literal(question["instructions"])
+                begin = len('{"question": ')
+                end = begin + len(phrase)
+                positions.append(tuple(start + i for i, (a, b) in enumerate(encoded["offset_mapping"]) if a < end and b > begin))
+    layout = DecisionLayout(CandidateTree(len(prefix), tuple(lengths)), tuple(anchors), tuple(positions), tuple(lexical), tuple(owners), tuple(kinds))
+    layout.validate(len(ids))
+    return ids, layout

decision_utils.py

@@ -13,7 +13,7 @@ MARKER = "\n<spev_score>\n"
 
 class ChoiceQuestion(BaseModel):
     model_config = ConfigDict(extra="forbid", strict=True)
-    type: Literal["choice"] = "choice"
+    type: Literal["choice", "predicate", "score"] = "choice"
     instructions: str
     criteria: dict[str, str] = Field(min_length=2, max_length=255)
 

examples/turn_decision.py

@@ -1,28 +0,0 @@
-"""Evaluate an audio file using the packaged turn-taking question."""
-
-import argparse
-import json
-from pathlib import Path
-
-import torch
-from transformers import AutoModel, AutoProcessor
-
-parser = argparse.ArgumentParser()
-parser.add_argument("audio")
-parser.add_argument("--model", default=str(Path(__file__).resolve().parents[1]))
-args = parser.parse_args()
-policy = json.loads((Path(__file__).with_name("turn_policy.json")).read_text())
-processor = AutoProcessor.from_pretrained(args.model, trust_remote_code=True)
-model = AutoModel.from_pretrained(
-    args.model,
-    trust_remote_code=True,
-    dtype=torch.bfloat16,
-    device_map="auto",
-    attn_implementation="sdpa",
-)
-inputs = processor(audio=args.audio, state={}, questions=policy["questions"])
-inputs = inputs.to(model.device, dtype=torch.bfloat16)
-with torch.inference_mode():
-    outputs = model(**inputs)
-result = processor.post_process(outputs, questions=policy["questions"])
-print(json.dumps(result, indent=2))

examples/turn_policy.json

@@ -1,17 +0,0 @@
-{
-  "questions": {
-    "decision": {
-      "instructions": "At the end of this conversation audio, has the most recent speaker yielded the floor or are they continuing their turn?",
-      "criteria": {
-        "keep_listening": "The most recent speaker is continuing their turn; keep listening.",
-        "respond": "The most recent speaker has yielded the floor; another speaker can respond."
-      }
-    }
-  },
-  "temperature": 1.3234945284875692,
-  "response_threshold": 0.3856703463365996,
-  "comparison": "strictly greater than",
-  "scope": "Fixed English turn-taking schema; not a generic Choice threshold",
-  "temperature_fitted_on": "AMI designated calibration split",
-  "threshold_selected_on": "AMI development split"
-}

media.py

@@ -0,0 +1,50 @@
+"""Bounded video sampling for SPEV evidence."""
+
+import io
+import math
+from tempfile import NamedTemporaryFile
+
+import av
+from transformers.video_utils import load_video
+
+
+def decode_video(data, video_processor, *, fps, max_seconds, max_frames, max_bytes):
+    if len(data) > max_bytes:
+        raise ValueError("Video exceeds the checkpoint byte limit")
+    try:
+        with av.open(io.BytesIO(data)) as container:
+            if not container.streams.video:
+                raise ValueError("Video has no visual stream")
+            stream = container.streams.video[0]
+            duration = float(stream.duration * stream.time_base) if stream.duration is not None else None
+            if duration is None and container.duration is not None:
+                duration = container.duration / av.time_base
+            if duration is None or not math.isfinite(duration) or not 0 < duration <= max_seconds:
+                raise ValueError("Video duration is missing or exceeds the checkpoint limit")
+            frame_rate = float(stream.average_rate) if stream.average_rate is not None else None
+            if frame_rate is None or not math.isfinite(frame_rate) or frame_rate <= 0:
+                raise ValueError("Video frame rate is missing or invalid")
+            frame_count = stream.frames or math.ceil(duration * frame_rate)
+            if not 0 < frame_count <= max_frames:
+                raise ValueError("Video frame count is missing or exceeds the checkpoint limit")
+    except av.FFmpegError as exc:
+        raise ValueError("Invalid or unsupported video") from exc
+
+    def sample_indices(metadata, **kwargs):
+        metadata.total_num_frames = frame_count
+        metadata.fps = frame_rate
+        metadata.duration = frame_count / frame_rate
+        return video_processor.sample_frames(metadata=metadata, fps=fps)
+
+    with NamedTemporaryFile() as source:
+        source.write(data)
+        source.flush()
+        try:
+            frames, metadata = load_video(source.name, backend="pyav", sample_indices_fn=sample_indices)
+        except (av.FFmpegError, OSError) as exc:
+            raise ValueError("Invalid or unsupported video") from exc
+    if not len(frames) or not metadata.fps or not metadata.total_num_frames:
+        raise ValueError("Video has no decodable frames")
+    if metadata.frames_indices is None or len(frames) != len(metadata.frames_indices):
+        raise ValueError("Video sampled-frame decode is incomplete")
+    return frames, metadata

model-decision.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e711007c2b401a337442e7b9e1ae6d89c36a2a2dee67cda9a817730b766ca1a1
-size 55615444
+oid sha256:ea005b67b23edcb83fe48a98978452dfd382db9937c789ba445236a6e9feb6a2
+size 81916464

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 62871149796
+    "total_size": 62897443056
   },
   "weight_map": {
     "backbone.audio_tower.conv2d1.bias": "model-00001-of-00013.safetensors",
@@ -19937,11 +19937,91 @@
     "backbone.model.layers.9.self_attn.q_proj.b": "model-decision.safetensors",
     "backbone.model.layers.9.self_attn.v_proj.a": "model-decision.safetensors",
     "backbone.model.layers.9.self_attn.v_proj.b": "model-decision.safetensors",
+    "head.context.weight": "model-decision.safetensors",
+    "head.evidence.0.attn.in_proj_bias": "model-decision.safetensors",
+    "head.evidence.0.attn.in_proj_weight": "model-decision.safetensors",
+    "head.evidence.0.attn.out_proj.bias": "model-decision.safetensors",
+    "head.evidence.0.attn.out_proj.weight": "model-decision.safetensors",
+    "head.evidence.0.ff.0.bias": "model-decision.safetensors",
+    "head.evidence.0.ff.0.weight": "model-decision.safetensors",
+    "head.evidence.0.ff.2.bias": "model-decision.safetensors",
+    "head.evidence.0.ff.2.weight": "model-decision.safetensors",
+    "head.evidence.0.fnorm.bias": "model-decision.safetensors",
+    "head.evidence.0.fnorm.weight": "model-decision.safetensors",
+    "head.evidence.0.mnorm.bias": "model-decision.safetensors",
+    "head.evidence.0.mnorm.weight": "model-decision.safetensors",
+    "head.evidence.0.qnorm.bias": "model-decision.safetensors",
+    "head.evidence.0.qnorm.weight": "model-decision.safetensors",
+    "head.evidence.1.attn.in_proj_bias": "model-decision.safetensors",
+    "head.evidence.1.attn.in_proj_weight": "model-decision.safetensors",
+    "head.evidence.1.attn.out_proj.bias": "model-decision.safetensors",
+    "head.evidence.1.attn.out_proj.weight": "model-decision.safetensors",
+    "head.evidence.1.ff.0.bias": "model-decision.safetensors",
+    "head.evidence.1.ff.0.weight": "model-decision.safetensors",
+    "head.evidence.1.ff.2.bias": "model-decision.safetensors",
+    "head.evidence.1.ff.2.weight": "model-decision.safetensors",
+    "head.evidence.1.fnorm.bias": "model-decision.safetensors",
+    "head.evidence.1.fnorm.weight": "model-decision.safetensors",
+    "head.evidence.1.mnorm.bias": "model-decision.safetensors",
+    "head.evidence.1.mnorm.weight": "model-decision.safetensors",
+    "head.evidence.1.qnorm.bias": "model-decision.safetensors",
+    "head.evidence.1.qnorm.weight": "model-decision.safetensors",
+    "head.fnorm.bias": "model-decision.safetensors",
+    "head.fnorm.weight": "model-decision.safetensors",
+    "head.gate": "model-decision.safetensors",
+    "head.globalproj.weight": "model-decision.safetensors",
+    "head.joint.0.linear1.bias": "model-decision.safetensors",
+    "head.joint.0.linear1.weight": "model-decision.safetensors",
+    "head.joint.0.linear2.bias": "model-decision.safetensors",
+    "head.joint.0.linear2.weight": "model-decision.safetensors",
+    "head.joint.0.multihead_attn.in_proj_bias": "model-decision.safetensors",
+    "head.joint.0.multihead_attn.in_proj_weight": "model-decision.safetensors",
+    "head.joint.0.multihead_attn.out_proj.bias": "model-decision.safetensors",
+    "head.joint.0.multihead_attn.out_proj.weight": "model-decision.safetensors",
+    "head.joint.0.norm1.bias": "model-decision.safetensors",
+    "head.joint.0.norm1.weight": "model-decision.safetensors",
+    "head.joint.0.norm2.bias": "model-decision.safetensors",
+    "head.joint.0.norm2.weight": "model-decision.safetensors",
+    "head.joint.0.norm3.bias": "model-decision.safetensors",
+    "head.joint.0.norm3.weight": "model-decision.safetensors",
+    "head.joint.0.self_attn.in_proj_bias": "model-decision.safetensors",
+    "head.joint.0.self_attn.in_proj_weight": "model-decision.safetensors",
+    "head.joint.0.self_attn.out_proj.bias": "model-decision.safetensors",
+    "head.joint.0.self_attn.out_proj.weight": "model-decision.safetensors",
+    "head.joint.1.linear1.bias": "model-decision.safetensors",
+    "head.joint.1.linear1.weight": "model-decision.safetensors",
+    "head.joint.1.linear2.bias": "model-decision.safetensors",
+    "head.joint.1.linear2.weight": "model-decision.safetensors",
+    "head.joint.1.multihead_attn.in_proj_bias": "model-decision.safetensors",
+    "head.joint.1.multihead_attn.in_proj_weight": "model-decision.safetensors",
+    "head.joint.1.multihead_attn.out_proj.bias": "model-decision.safetensors",
+    "head.joint.1.multihead_attn.out_proj.weight": "model-decision.safetensors",
+    "head.joint.1.norm1.bias": "model-decision.safetensors",
+    "head.joint.1.norm1.weight": "model-decision.safetensors",
+    "head.joint.1.norm2.bias": "model-decision.safetensors",
+    "head.joint.1.norm2.weight": "model-decision.safetensors",
+    "head.joint.1.norm3.bias": "model-decision.safetensors",
+    "head.joint.1.norm3.weight": "model-decision.safetensors",
+    "head.joint.1.self_attn.in_proj_bias": "model-decision.safetensors",
+    "head.joint.1.self_attn.in_proj_weight": "model-decision.safetensors",
+    "head.joint.1.self_attn.out_proj.bias": "model-decision.safetensors",
+    "head.joint.1.self_attn.out_proj.weight": "model-decision.safetensors",
+    "head.joint_scale": "model-decision.safetensors",
+    "head.lexical.weight": "model-decision.safetensors",
+    "head.memory.weight": "model-decision.safetensors",
+    "head.norm.bias": "model-decision.safetensors",
+    "head.norm.weight": "model-decision.safetensors",
+    "head.onorm.bias": "model-decision.safetensors",
+    "head.onorm.weight": "model-decision.safetensors",
+    "head.oqproj.weight": "model-decision.safetensors",
+    "head.prior_scale": "model-decision.safetensors",
+    "head.qproj.weight": "model-decision.safetensors",
     "head.scorer.0.bias": "model-decision.safetensors",
     "head.scorer.0.weight": "model-decision.safetensors",
-    "head.scorer.1.bias": "model-decision.safetensors",
-    "head.scorer.1.weight": "model-decision.safetensors",
-    "head.scorer.3.bias": "model-decision.safetensors",
-    "head.scorer.3.weight": "model-decision.safetensors"
+    "head.scorer.2.bias": "model-decision.safetensors",
+    "head.scorer.2.weight": "model-decision.safetensors",
+    "head.summarynorm.bias": "model-decision.safetensors",
+    "head.summarynorm.weight": "model-decision.safetensors",
+    "head.types.weight": "model-decision.safetensors"
   }
 }

modeling_spev.py

@@ -10,6 +10,10 @@ from transformers import PreTrainedModel, Qwen3OmniMoeThinkerForConditionalGener
 from transformers.utils import ModelOutput
 
 from .configuration_spev import SPEVConfig
+from .branch_attention import install as install_attention, request_context
+from .decision_head import IsolatedDecisionHead, QuestionFeatures, score_questions
+from .decision_layout import DecisionLayout
+from .token_operations import install as install_token_operations
 
 
 class ResidualAdapter(nn.Module):
@@ -26,20 +30,6 @@ class ResidualAdapter(nn.Module):
         return self.base(x) + (residual * self.scale).to(x.dtype)
 
 
-class OptionHead(nn.Module):
-    def __init__(self, hidden_size, width):
-        super().__init__()
-        self.scorer = nn.Sequential(
-            nn.LayerNorm(hidden_size),
-            nn.Linear(hidden_size, width),
-            nn.GELU(),
-            nn.Linear(width, 1),
-        )
-
-    def forward(self, features):
-        return self.scorer(features.float()).squeeze(-1)
-
-
 @dataclass
 class SPEVOutput(ModelOutput):
     logits: torch.Tensor | None = None
@@ -57,7 +47,7 @@ class SPEVModel(PreTrainedModel):
     _no_split_modules: ClassVar[list[str]] = [
         "Qwen3OmniMoeThinkerTextDecoderLayer",
         "ResidualAdapter",
-        "OptionHead",
+        "IsolatedDecisionHead",
     ]
 
     def __init__(self, config):
@@ -76,10 +66,13 @@ class SPEVModel(PreTrainedModel):
                     leaf,
                     ResidualAdapter(module, config.adapter_rank, config.adapter_alpha),
                 )
-        self.head = OptionHead(
-            config.thinker_config.text_config.hidden_size, config.head_width
+        self.head = IsolatedDecisionHead(
+            config.thinker_config.text_config.hidden_size, config.head_width,
+            config.head_heads, config.head_ff, config.head_depth,
         )
         self.post_init()
+        install_attention(self.backbone)
+        install_token_operations(self, linears=True, moe=False)
 
     def get_input_embeddings(self):
         return self.backbone.get_input_embeddings()
@@ -88,46 +81,70 @@ class SPEVModel(PreTrainedModel):
         self.backbone.set_input_embeddings(value)
 
     def forward(
-        self, input_ids=None, attention_mask=None, scoring_positions=None, **kwargs
+        self, input_ids=None, attention_mask=None, decision_layout=None, **kwargs
     ):
         if input_ids is None or input_ids.ndim != 2 or input_ids.shape[0] != 1:
             raise ValueError("SPEV accepts one evidence sequence per forward pass")
         if input_ids.shape[1] > self.config.max_tokens:
             raise ValueError("Input exceeds the configured context limit")
-        if (
-            scoring_positions is None
-            or scoring_positions.ndim != 2
-            or scoring_positions.shape[0] != 1
-        ):
-            raise ValueError("scoring_positions must have shape (1, candidates)")
-        if (
-            scoring_positions.numel() == 0
-            or torch.any(scoring_positions < 0)
-            or torch.any(scoring_positions >= input_ids.shape[1])
-        ):
-            raise ValueError("Scoring positions are outside the input sequence")
+        if not isinstance(decision_layout, dict):
+            raise ValueError("A processor-generated decision_layout is required")
+        decision_layout = DecisionLayout.from_dict(decision_layout)
+        decision_layout.validate(input_ids.shape[1])
+        tree = decision_layout.tree
+        if attention_mask is not None and not torch.all(attention_mask == 1):
+            raise ValueError("SPEV accepts unpadded evidence sequences")
         kwargs.pop("use_cache", None)
         kwargs.pop("return_dict", None)
-        hidden = self.backbone(
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            use_cache=False,
-            return_dict=True,
-            **kwargs,
-        ).logits
-        positions = scoring_positions.to(hidden.device)
-        features = hidden.gather(
-            1, positions.unsqueeze(-1).expand(-1, -1, hidden.shape[-1])
-        )
-        head_device = self.head.scorer[0].weight.device
-        logits = self.head(features.to(device=head_device, dtype=torch.float32))
-        count = (
-            attention_mask.sum(-1)
-            if attention_mask is not None
-            else input_ids.new_tensor([input_ids.shape[1]])
+        kwargs.pop("position_ids", None)
+        prefix = input_ids[:, :tree.prefix_length]
+        rope_kwargs = {key: kwargs[key] for key in (
+            "image_grid_thw", "video_grid_thw", "use_audio_in_video", "second_per_grids"
+        ) if key in kwargs}
+        if "video_second_per_grid" in kwargs:
+            rope_kwargs["second_per_grids"] = kwargs["video_second_per_grid"]
+        if "feature_attention_mask" in kwargs:
+            rope_kwargs["audio_seqlens"] = kwargs["feature_attention_mask"].sum(1)
+        positions, _ = self.backbone.get_rope_index(
+            prefix, attention_mask=torch.ones_like(prefix), **rope_kwargs
         )
+        position_ids = tree.position_ids(positions).to(input_ids.device)
+        with request_context(tree):
+            hidden = self.backbone(
+                input_ids=input_ids,
+                attention_mask=tree.additive_mask(dtype=self.backbone.dtype, device=input_ids.device),
+                position_ids=position_ids,
+                use_cache=False,
+                return_dict=True,
+                **kwargs,
+            ).logits[0]
+        lexical = []
+        for tokens in decision_layout.lexical_ids:
+            tokens = input_ids.new_tensor(tokens)
+            lexical.append(self.get_input_embeddings()(tokens).mean(0))
+        records = []
+        starts = [tree.prefix_length]
+        for length in tree.branch_lengths:
+            starts.append(starts[-1] + length)
+        for question, kind in enumerate(decision_layout.question_types):
+            indices = [i for i, owner in enumerate(decision_layout.candidate_questions) if owner == question]
+            memory = torch.cat([hidden[:tree.prefix_length]] + [hidden[starts[i]:starts[i + 1]] for i in indices])
+            records.append(QuestionFeatures(
+                memory=memory,
+                question=hidden[list(decision_layout.question_positions[question])].mean(0),
+                options=hidden[[decision_layout.anchors[i] for i in indices]],
+                lexical=torch.stack([lexical[i] for i in indices]),
+                global_feature=hidden[tree.prefix_length - 1],
+                question_type=kind,
+            ))
+        grouped = score_questions(self.head, records)
+        logits = torch.empty(len(lexical), device=grouped[0].device, dtype=grouped[0].dtype)
+        for question, values in enumerate(grouped):
+            indices = [i for i, owner in enumerate(decision_layout.candidate_questions) if owner == question]
+            logits[indices] = values
         return SPEVOutput(
-            logits=logits, input_tokens=count, temperature=self.config.temperature
+            logits=logits[None], input_tokens=input_ids.new_tensor([input_ids.shape[1]]),
+            temperature=self.config.temperature
         )
 
 

processing_spev.py

@@ -7,10 +7,10 @@ from .decision_utils import (
     MARKER,
     ChoiceQuestions,
     answers,
-    locate_markers,
     prepare_audio,
-    render,
+    literal,
 )
+from .decision_layout import pack_candidates
 
 
 class SPEVProcessor(Qwen3OmniMoeProcessor):
@@ -23,6 +23,9 @@ class SPEVProcessor(Qwen3OmniMoeProcessor):
         chat_template=None,
         max_tokens=8192,
         video_fps=1.0,
+        max_video_seconds=60.0,
+        max_video_frames=1800,
+        max_media_bytes=12 * 1024 * 1024,
     ):
         super().__init__(
             image_processor,
@@ -33,6 +36,9 @@ class SPEVProcessor(Qwen3OmniMoeProcessor):
         )
         self.max_tokens = max_tokens
         self.video_fps = video_fps
+        self.max_video_seconds = max_video_seconds
+        self.max_video_frames = max_video_frames
+        self.max_media_bytes = max_media_bytes
         self.marker = self.tokenizer.encode(
             MARKER.lstrip("\n"), add_special_tokens=False
         )
@@ -41,7 +47,8 @@ class SPEVProcessor(Qwen3OmniMoeProcessor):
         questions = ChoiceQuestions.model_validate(
             {"questions": questions}
         ).model_dump()["questions"]
-        text, mapping = render({} if state is None else state, questions)
+        text = "Use the audio and state as evidence for the supplied questions.\n"
+        text += "State: " + literal({} if state is None else state) + "\n"
         content, media = [], {}
         if audio is not None:
             import soundfile as sf
@@ -56,20 +63,17 @@ class SPEVProcessor(Qwen3OmniMoeProcessor):
                 media["images"] = [source.convert("RGB")]
             content.append({"type": "image", "image": "provided-image"})
         if video is not None:
-            from functools import partial
+            from pathlib import Path
+            from .media import decode_video
 
-            from transformers.video_utils import load_video
-
-            frames, metadata = load_video(
-                str(video),
-                backend="pyav",
-                sample_indices_fn=partial(
-                    self.video_processor.sample_frames,
-                    fps=self.video_fps,
-                ),
+            source = Path(video)
+            if source.stat().st_size > self.max_media_bytes:
+                raise ValueError("Video exceeds the checkpoint byte limit")
+            frames, metadata = decode_video(
+                source.read_bytes(), self.video_processor, fps=self.video_fps,
+                max_seconds=self.max_video_seconds, max_frames=self.max_video_frames,
+                max_bytes=self.max_media_bytes,
             )
-            if not len(frames) or not metadata.fps or not metadata.total_num_frames:
-                raise ValueError("Video has no decodable frames or timing information")
             media["videos"] = [frames]
             media["fps"] = len(frames) * metadata.fps / metadata.total_num_frames
             media["do_sample_frames"] = False
@@ -93,10 +97,15 @@ class SPEVProcessor(Qwen3OmniMoeProcessor):
             **media,
         )
         ids = inputs["input_ids"][0].tolist()
+        ending = self.tokenizer.encode("<|im_end|>\n", add_special_tokens=False)
+        if ids[-len(ending):] != ending:
+            raise ValueError("Evidence chat template has an unexpected ending")
+        ids, layout = pack_candidates(self.tokenizer, ids[:-len(ending)], questions)
         if len(ids) > self.max_tokens:
             raise ValueError("Input exceeds the configured context limit")
-        positions = locate_markers(ids, self.marker, len(mapping))
-        inputs["scoring_positions"] = torch.tensor([positions], dtype=torch.long)
+        inputs["input_ids"] = torch.tensor([ids], dtype=torch.long)
+        inputs["attention_mask"] = torch.ones(1, len(ids), dtype=torch.long)
+        inputs["decision_layout"] = layout.to_dict()
         return inputs
 
     def post_process(self, outputs, *, questions):

processor_config.json

@@ -74,7 +74,10 @@
     },
     "temporal_patch_size": 2
   },
+  "max_media_bytes": 12582912,
   "max_tokens": 8192,
+  "max_video_frames": 1800,
+  "max_video_seconds": 60.0,
   "processor_class": "SPEVProcessor",
   "video_fps": 1.0,
   "video_processor": {

token_operations.py

@@ -0,0 +1,78 @@
+"""Reuse request-local gather indices for fixed-shape token-local operations."""
+from collections import defaultdict
+from .branch_attention import REQUEST
+from contextvars import ContextVar
+import types
+import torch
+
+DEPTH=ContextVar('spev_vectorized_token_ops_depth',default=0)
+
+
+class PackingCache:
+    def __init__(self):
+        self.tree=None;self.device=None;self.plan=None
+
+    def __call__(self,tree,device):
+        if tree is self.tree and device==self.device:return self.plan
+        buckets=defaultdict(list);start=tree.prefix_length
+        for index,length in enumerate(tree.branch_lengths):
+            buckets[((length+31)//32)*32].append((index,start,length));start+=length
+        assert start==tree.length
+        blocks=[];restore=[None]*len(tree.branch_lengths);offset=0
+        for width,branches in sorted(buckets.items()):
+            for begin in range(0,len(branches),8):
+                block=branches[begin:begin+8];indices=[0]*(8*width);valid=[False]*(8*width)
+                for slot,(index,start,length) in enumerate(block):
+                    a=slot*width;indices[a:a+length]=range(start,start+length);valid[a:a+length]=[True]*length
+                    restore[index]=torch.arange(length,device=device)+offset+a
+                blocks.append((width,torch.tensor(indices,device=device,dtype=torch.long),torch.tensor(valid,device=device,dtype=torch.bool)))
+                offset+=8*width
+        order=torch.cat(restore) if restore else torch.empty(0,device=device,dtype=torch.long)
+        self.tree=tree;self.device=device;self.plan=(blocks,order)
+        return self.plan
+
+
+def grouped_forward(original,tree_getter,plan_getter=None):
+    cache=PackingCache() if plan_getter is None else plan_getter
+    def forward(module,value,*args,**kwargs):
+        if DEPTH.get():return original(value,*args,**kwargs)
+        tree=tree_getter()
+        if tree is None:raise ValueError('Grouped token operations require a tree layout')
+        if value.ndim not in (2,3) or value.shape[-2]!=tree.length:raise ValueError('Grouped token operations require the packed token sequence')
+        if value.ndim==3 and value.shape[0]!=1:raise ValueError('Grouped token operations support one packed request')
+        x=value[0] if value.ndim==3 else value
+        blocks,order=cache(tree,x.device);token=DEPTH.set(DEPTH.get()+1)
+        try:
+            prefix=original(x[:tree.prefix_length].contiguous(),*args,**kwargs);outputs=[]
+            for width,indices,valid in blocks:
+                packed=x.index_select(0,indices).masked_fill(~valid[:,None],0).reshape(8,width,x.shape[-1])
+                output=original(packed,*args,**kwargs)
+                assert isinstance(output,torch.Tensor) and output.shape[:2]==packed.shape[:2]
+                outputs.append(output.reshape(8*width,output.shape[-1]))
+            branch=(outputs[0] if len(outputs)==1 else torch.cat(outputs,dim=0)).index_select(0,order) if outputs else prefix[:0]
+            output=torch.cat([prefix,branch],dim=0)
+            return output[None] if value.ndim==3 else output
+        finally:DEPTH.reset(token)
+    return forward
+
+
+def install(model,*,linears,moe):
+    if moe:raise ValueError('MoE dispatch is not a token-local linear operation')
+    originals=[]
+    def cache(tree, device):
+        request=REQUEST.get()
+        attention_cache=request[1]
+        if not hasattr(attention_cache, "packing"):
+            attention_cache.packing=PackingCache()
+        return attention_cache.packing(tree,device)
+    getter=lambda:REQUEST.get()[0] if REQUEST.get() else None
+    if linears:
+        for layer in model.backbone.model.layers:
+            for module in list(layer.modules()):
+                is_lora=all(hasattr(module,name) for name in ['a','b','base','scale'])
+                if isinstance(module,torch.nn.Linear) or is_lora:
+                    original=module.forward;originals.append((module,original))
+                    module.forward=types.MethodType(grouped_forward(original,getter,cache),module)
+    def restore():
+        for module,forward in reversed(originals):module.forward=forward
+    return restore