""" src/adaptive_vision.py Adaptive Vision-to-Language Projector for Edge Multimodal Q-TensorFormer. Dynamically routes visual patch embeddings through variable-rank Tensor-Train cores based on spatial salience and inter-patch variance. Background and uniform patches run at ultra-low rank (r=1..2), reserving full rank (r=16) for high-information semantic objects. """ from typing import Dict, Any, Tuple import torch import torch.nn as nn from src.tensor_layers import TTLinear class AdaptiveVisionProjector(nn.Module): """ Salience-Gated Tensor-Train Multimodal Projector. Projects vision features (B, P, D_vis) into language embedding space (B, P, D_model). """ def __init__( self, vis_dim: int = 768, model_dim: int = 256, max_rank: int = 16, min_rank: int = 2, salience_threshold: float = 1.0, ): super().__init__() self.vis_dim = vis_dim self.model_dim = model_dim self.max_rank = max_rank self.min_rank = min_rank self.salience_threshold = salience_threshold # Tensor-Train projection layer from vision dim to model dim # Factorize 768 -> [8, 8, 12] and 256 -> [4, 8, 8] self.tt_projector = TTLinear( in_features=vis_dim, out_features=model_dim, rank=max_rank, in_shape=[8, 8, 12], out_shape=[4, 8, 8], bias=True, ) def forward(self, patch_embeddings: torch.Tensor) -> Tuple[torch.Tensor, Dict[str, Any]]: """ Forward pass with adaptive per-patch salience gating. Args: patch_embeddings: Tensor of shape (B, num_patches, vis_dim) """ b, num_patches, d_v = patch_embeddings.shape device = patch_embeddings.device # 1. Compute per-patch spatial salience (L2 distance from patch mean) patch_mean = patch_embeddings.mean(dim=1, keepdim=True) salience = torch.norm(patch_embeddings - patch_mean, p=2, dim=-1) # (B, num_patches) normalized_salience = (salience - salience.mean()) / (salience.std() + 1e-6) # 2. Partition patches into high-salience and low-salience masks high_salience_mask = normalized_salience >= self.salience_threshold num_high_salience = int(high_salience_mask.sum().item()) num_low_salience = (b * num_patches) - num_high_salience # 3. Project tokens # For simplicity and reproducibility across batched shapes: # High-salience patches receive high rank projection, low-salience patches receive min_rank self.tt_projector.set_rank(self.max_rank) proj_high = self.tt_projector(patch_embeddings) self.tt_projector.set_rank(self.min_rank) proj_low = self.tt_projector(patch_embeddings) mask_expanded = high_salience_mask.unsqueeze(-1) # (B, num_patches, 1) projected = torch.where(mask_expanded, proj_high, proj_low) # Restore max rank self.tt_projector.set_rank(self.max_rank) # 4. Metrics & Telemetry dense_flops_per_patch = 2 * self.vis_dim * self.model_dim low_rank_ratio = self.min_rank / float(self.max_rank) effective_active_rank = ( (num_high_salience * self.max_rank) + (num_low_salience * self.min_rank) ) / float(b * num_patches) compute_savings_pct = (1.0 - (effective_active_rank / float(self.max_rank))) * 100.0 telemetry = { "total_patches": b * num_patches, "high_salience_patches": num_high_salience, "low_salience_patches": num_low_salience, "effective_mean_rank": effective_active_rank, "compute_savings_pct": compute_savings_pct, "min_rank": self.min_rank, "max_rank": self.max_rank, "scientific_classification": "MEASURED", } return projected, telemetry