"""Attention layouts for non-recurrent visual latent reasoning chains. The sequence is laid out as:: [multimodal prompt ; clean question ; LOOK_1 ; THINK_1 ; ... ; answer] All rows are processed once by the native VLM decoder. A block-sparse causal graph makes LOOK rows the only latent rows with access to the multimodal prefix, while THINK rows integrate one visual read without directly seeing that prefix. Answer rows can see the clean question and THINK rows, but never the multimodal prefix or LOOK rows. Consequently the image-dependent answer path is image -> LOOK_k -> THINK_k -> answer, without a tied recurrent cell, a visual-state update, or an aggregation module. The transition-conditioned variant tightens the graph without adding a module. The first LOOK/THINK pair bootstraps from the complete prompt and the clean question. Every later pair sees the causal latent prefix, while its LOOK row receives only visual placeholder rows from the multimodal prefix. Thus later pairs cannot independently re-solve the original image/question prompt; their task semantics must arrive through earlier latent states. The last THINK is the sole latent exposed to answer rows and therefore acts as the native-transformer aggregation state. """ from __future__ import annotations from dataclasses import dataclass import torch @dataclass(frozen=True) class PerceiveDeliberateLayout: """Physical row layout of one padded training/generation sequence.""" multimodal_length: int question_length: int num_pairs: int answer_input_length: int = 0 def __post_init__(self) -> None: if self.multimodal_length < 1: raise ValueError("multimodal_length must be positive") if self.question_length < 1: raise ValueError("question_length must be positive") if self.num_pairs < 1: raise ValueError("num_pairs must be positive") if self.answer_input_length < 0: raise ValueError("answer_input_length must be non-negative") @property def num_latent_tokens(self) -> int: return 2 * self.num_pairs @property def question_start(self) -> int: return self.multimodal_length @property def latent_start(self) -> int: return self.multimodal_length + self.question_length @property def answer_start(self) -> int: return self.latent_start + self.num_latent_tokens @property def sequence_length(self) -> int: return self.answer_start + self.answer_input_length @property def question_slice(self) -> slice: return slice(self.question_start, self.latent_start) @property def answer_slice(self) -> slice: return slice(self.answer_start, self.sequence_length) @property def look_indices(self) -> tuple[int, ...]: return tuple(self.latent_start + 2 * index for index in range(self.num_pairs)) @property def think_indices(self) -> tuple[int, ...]: return tuple(index + 1 for index in self.look_indices) def build_perceive_deliberate_mask( layout: PerceiveDeliberateLayout, multimodal_attention_mask: torch.Tensor, question_attention_mask: torch.Tensor, answer_input_attention_mask: torch.Tensor | None = None, *, multimodal_visual_mask: torch.Tensor | None = None, local_visual_chain: bool = False, transition_conditioned: bool = False, question_visible_through_pair: int = 1, disable_chain_links: bool = False, disable_late_visual: bool = False, final_pair_only: bool = False, ) -> torch.BoolTensor: """Build the strict LOOK/THINK/answer visibility graph. The returned boolean mask has shape ``[B, 1, L, L]`` and follows PyTorch SDPA semantics: ``True`` means that the query may attend to the key. In the original graph, LOOK_k sees the valid multimodal prompt, the clean question, and (for k>1) only THINK_{k-1}; THINK_k sees the clean question and LOOK_k. In the transition-conditioned graph, pair 1 keeps those bootstrap inputs, but every later latent sees its complete causal latent prefix, later LOOK rows see only visual placeholder keys, and later THINK rows normally receive no independent question shortcut. For a controlled curriculum/intervention, ``question_visible_through_pair`` may temporarily retain the clean-question edge through a later pair; ``1`` is the strict inference graph. Answer rows see only the final THINK in that variant. Every row also sees itself so padded query rows remain numerically defined; padded rows are never exposed as keys to semantic rows. """ if multimodal_attention_mask.ndim != 2: raise ValueError("multimodal_attention_mask must have shape [B, L_mm]") if question_attention_mask.ndim != 2: raise ValueError("question_attention_mask must have shape [B, L_q]") batch_size = multimodal_attention_mask.shape[0] expected_mm = (batch_size, layout.multimodal_length) expected_q = (batch_size, layout.question_length) if tuple(multimodal_attention_mask.shape) != expected_mm: raise ValueError( "multimodal mask/layout mismatch: " f"expected {expected_mm}, got {tuple(multimodal_attention_mask.shape)}" ) if tuple(question_attention_mask.shape) != expected_q: raise ValueError( "question mask/layout mismatch: " f"expected {expected_q}, got {tuple(question_attention_mask.shape)}" ) if layout.answer_input_length: if answer_input_attention_mask is None: raise ValueError("answer_input_attention_mask is required") expected_answer = (batch_size, layout.answer_input_length) if tuple(answer_input_attention_mask.shape) != expected_answer: raise ValueError( "answer mask/layout mismatch: " f"expected {expected_answer}, got " f"{tuple(answer_input_attention_mask.shape)}" ) elif answer_input_attention_mask is not None and answer_input_attention_mask.numel(): raise ValueError("received answer mask for an empty answer-input segment") device = multimodal_attention_mask.device length = layout.sequence_length visible = torch.zeros( batch_size, length, length, dtype=torch.bool, device=device ) mm_valid = multimodal_attention_mask.bool() q_valid = question_attention_mask.bool() visual_valid = None if transition_conditioned or local_visual_chain: if not 1 <= question_visible_through_pair <= layout.num_pairs: raise ValueError( "question_visible_through_pair must lie in " f"[1,{layout.num_pairs}]" ) if multimodal_visual_mask is None: raise ValueError( "multimodal_visual_mask is required for the " "transition-conditioned graph" ) if tuple(multimodal_visual_mask.shape) != expected_mm: raise ValueError( "visual mask/layout mismatch: " f"expected {expected_mm}, got " f"{tuple(multimodal_visual_mask.shape)}" ) visual_valid = multimodal_visual_mask.bool() if bool((visual_valid & ~mm_valid).any()): raise ValueError("visual keys must be a subset of valid multimodal keys") # The source multimodal prompt keeps the native causal graph. Padded query # rows are harmless and receive a self edge below. mm_causal = torch.ones( layout.multimodal_length, layout.multimodal_length, dtype=torch.bool, device=device, ).tril() visible[:, : layout.multimodal_length, : layout.multimodal_length] = ( mm_causal.unsqueeze(0) & mm_valid[:, None, :] ) # The duplicated question is deliberately text-only: it is causally # connected only to preceding valid rows of its own segment. q_causal = torch.ones( layout.question_length, layout.question_length, dtype=torch.bool, device=device, ).tril() visible[ :, layout.question_slice, layout.question_slice ] = q_causal.unsqueeze(0) & q_valid[:, None, :] if local_visual_chain: # Homogeneous one-pass latent chain. Visual memory is persistent and # available at every position, while task state has exactly one local # predecessor edge. No latent may consume the complete causal prefix. latent_indices = range(layout.latent_start, layout.answer_start) for latent_offset, latent in enumerate(latent_indices): visible[:, latent, : layout.multimodal_length] = visual_valid if latent_offset == 0: visible[:, latent, layout.question_slice] = q_valid else: visible[:, latent, latent - 1] = True if layout.answer_input_length: answer_valid = answer_input_attention_mask.bool() answer_causal = torch.ones( layout.answer_input_length, layout.answer_input_length, dtype=torch.bool, device=device, ).tril() visible[:, layout.answer_slice, layout.question_slice] = q_valid[:, None, :] visible[:, layout.answer_slice, layout.answer_start - 1] = True visible[:, layout.answer_slice, layout.answer_slice] = ( answer_causal.unsqueeze(0) & answer_valid[:, None, :] ) diagonal = torch.arange(length, device=device) visible[:, diagonal, diagonal] = True return visible.unsqueeze(1) for pair_index, (look, think) in enumerate( zip(layout.look_indices, layout.think_indices) ): if final_pair_only and pair_index != layout.num_pairs - 1: continue if transition_conditioned and not final_pair_only: # Pair 1 bootstraps the causal latent prefix. Later pairs cannot # recover the task from the original prompt: only image placeholder # rows remain visible outside the latent prefix. if pair_index == 0: visible[:, look, : layout.multimodal_length] = mm_valid visible[:, look, layout.question_slice] = q_valid visible[:, think, layout.question_slice] = q_valid else: if not disable_late_visual: visible[:, look, : layout.multimodal_length] = visual_valid if not disable_chain_links: visible[:, look, layout.latent_start:look] = True visible[:, think, layout.latent_start:look] = True if pair_index < question_visible_through_pair: visible[:, look, layout.question_slice] = q_valid visible[:, think, layout.question_slice] = q_valid # The local LOOK -> THINK edge is never an inter-pair chain-link # intervention and therefore remains present in every arm. visible[:, think, look] = True else: # Original graph, also used by the final-pair-only capacity # control: each active pair may independently consume prompt + Q. if not disable_late_visual or pair_index == 0 or final_pair_only: visible[:, look, : layout.multimodal_length] = mm_valid visible[:, look, layout.question_slice] = q_valid if pair_index and not disable_chain_links and not final_pair_only: visible[:, look, layout.think_indices[pair_index - 1]] = True # Deliberation: no original multimodal key can be consumed here. visible[:, think, layout.question_slice] = q_valid visible[:, think, look] = True if layout.answer_input_length: answer_valid = answer_input_attention_mask.bool() answer_causal = torch.ones( layout.answer_input_length, layout.answer_input_length, dtype=torch.bool, device=device, ).tril() visible[:, layout.answer_slice, layout.question_slice] = q_valid[:, None, :] answer_thinks = ( [layout.think_indices[-1]] if final_pair_only or transition_conditioned else list(layout.think_indices) ) visible[:, layout.answer_slice, answer_thinks] = True visible[:, layout.answer_slice, layout.answer_slice] = ( answer_causal.unsqueeze(0) & answer_valid[:, None, :] ) # Avoid all-masked softmax rows for physical padding. Semantic queries do # not receive padded keys because every segment assignment above uses its # validity mask. diagonal = torch.arange(length, device=device) visible[:, diagonal, diagonal] = True return visible.unsqueeze(1) def build_perceive_deliberate_positions( layout: PerceiveDeliberateLayout, multimodal_position_ids: torch.LongTensor, multimodal_attention_mask: torch.Tensor, question_attention_mask: torch.Tensor, answer_input_attention_mask: torch.Tensor | None = None, ) -> torch.LongTensor: """Extend native multimodal M-RoPE with logical text positions. The clean question keeps ordinary relative text positions but is shifted after the largest valid multimodal M-RoPE coordinate. Latent and answer rows then continue from each item's *logical* question length, independent of right padding. All three M-RoPE coordinates are equal for these non-spatial rows. """ if multimodal_position_ids.ndim != 3 or multimodal_position_ids.shape[0] != 3: raise ValueError("multimodal_position_ids must have shape [3, B, L_mm]") batch_size = multimodal_attention_mask.shape[0] if tuple(multimodal_position_ids.shape[1:]) != ( batch_size, layout.multimodal_length, ): raise ValueError("multimodal position/layout mismatch") mm_valid = multimodal_attention_mask.bool() masked_mm = multimodal_position_ids.masked_fill(~mm_valid.unsqueeze(0), -1) continuation = masked_mm.amax(dim=(0, 2)).clamp_min(-1) + 1 # [B] q_valid = question_attention_mask.bool() q_relative = q_valid.long().cumsum(dim=-1) - 1 q_relative = q_relative.masked_fill(~q_valid, 0) q_positions = continuation[:, None] + q_relative q_lengths = q_valid.sum(dim=-1) latent_relative = torch.arange( layout.num_latent_tokens, device=multimodal_position_ids.device, ) latent_positions = ( continuation[:, None] + q_lengths[:, None] + latent_relative[None, :] ) segments = [multimodal_position_ids, q_positions.unsqueeze(0).expand(3, -1, -1)] segments.append(latent_positions.unsqueeze(0).expand(3, -1, -1)) if layout.answer_input_length: if answer_input_attention_mask is None: raise ValueError("answer_input_attention_mask is required") answer_relative = torch.arange( layout.answer_input_length, device=multimodal_position_ids.device, ) answer_positions = ( continuation[:, None] + q_lengths[:, None] + layout.num_latent_tokens + answer_relative[None, :] ) segments.append(answer_positions.unsqueeze(0).expand(3, -1, -1)) return torch.cat(segments, dim=-1) def build_answer_decode_mask( layout: PerceiveDeliberateLayout, question_attention_mask: torch.Tensor, generated_attention_mask: torch.Tensor, *, local_visual_chain: bool = False, final_pair_only: bool = False, transition_conditioned: bool = False, ) -> torch.BoolTensor: """Visibility for one cached answer query after the latent prefill. ``generated_attention_mask`` includes the current answer token. Physical multimodal and LOOK cache rows remain present but are invisible. """ if generated_attention_mask.ndim != 2: raise ValueError("generated_attention_mask must have shape [B, N]") batch_size, generated_length = generated_attention_mask.shape if tuple(question_attention_mask.shape) != ( batch_size, layout.question_length, ): raise ValueError("question mask/layout mismatch") total_keys = layout.answer_start + generated_length visible = torch.zeros( batch_size, 1, 1, total_keys, dtype=torch.bool, device=question_attention_mask.device, ) visible[:, 0, 0, layout.question_slice] = question_attention_mask.bool() if local_visual_chain: visible[:, 0, 0, layout.answer_start - 1] = True else: answer_thinks = ( [layout.think_indices[-1]] if final_pair_only or transition_conditioned else list(layout.think_indices) ) visible[:, 0, 0, answer_thinks] = True visible[:, 0, 0, layout.answer_start:] = generated_attention_mask.bool() return visible __all__ = [ "PerceiveDeliberateLayout", "build_answer_decode_mask", "build_perceive_deliberate_mask", "build_perceive_deliberate_positions", ]