5.0 structure fixes

model-00002-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b73a606d306a09519e3fbe7bfd29077d39db48fee47ce19521b6b5c398cdcc32
-size 4054187824

model-00003-of-00003.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6941d35ff1feae1603946f8746a71205bb86343b57968402df2e737faf9258a2
-size 1244659840

model-00001-of-00003.safetensors → model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f60b6bc3c8ed16d95c88b5b6d33101d0aa9464f5f3f33e204342859b12e371bb
-size 4969539560
+oid sha256:b84184ec79aa409631e68dc76c3133bc1bbae76d61842fdcd4fe553dd6a3b579
+size 10268388224

modular_isaac.py

@@ -96,7 +96,6 @@ from transformers.models.siglip2.modeling_siglip2 import (
     Siglip2Attention,
     Siglip2Encoder,
     Siglip2EncoderLayer,
-    Siglip2VisionTransformer
 )
 
 
@@ -973,6 +972,23 @@ class IsaacVisionTransformer(nn.Module):
         # Return the full sequence of embeddings
         return hidden_states
 
+class IsaacMultiModalProjector(nn.Module):
+    def __init__(self, config: IsaacConfig):
+        super().__init__()
+        self.vision_hidden_size = config.vision_config.hidden_size * (
+            config.vision_config.pixel_shuffle_scale_factor**2
+        )
+        self.backbone_hidden_size = config.hidden_size
+        self.linear_1 = nn.Linear(self.vision_hidden_size, 4 * self.vision_hidden_size, bias=False)
+        self.silu = nn.SiLU()
+        self.linear_2 = nn.Linear(4 * self.vision_hidden_size, self.backbone_hidden_size, bias=False)
+
+    def forward(self, image_features):
+        hidden_states = self.linear_1(image_features)
+        hidden_states = self.silu(hidden_states)
+        hidden_states = self.linear_2(hidden_states)
+        return hidden_states
+
 
 class IsaacVisionEmbedding(nn.Module):
     """Vision embedding wrapper exposing tower and projector."""
@@ -982,14 +998,8 @@ class IsaacVisionEmbedding(nn.Module):
     def __init__(self, config: IsaacConfig):
         super().__init__()
         vision_cfg = config.vision_config
-        hidden_dim = vision_cfg.hidden_size * (vision_cfg.pixel_shuffle_scale_factor**2)
-
         self.vision_tower = IsaacVisionTransformer(vision_cfg)
-        self.multimodal_projector = nn.Sequential(
-            nn.Linear(hidden_dim, 4 * hidden_dim, bias=False),
-            nn.SiLU(),
-            nn.Linear(4 * hidden_dim, config.hidden_size, bias=False),
-        )
+        self.multimodal_projector = IsaacMultiModalProjector(config)
 
     def forward(self, vision_tokens: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor:
         hidden_states = self.vision_tower(vision_tokens)
@@ -1569,8 +1579,9 @@ class IsaacModel(Qwen3PreTrainedModel):
             raise ValueError("IsaacConfig should always have vision_config")
 
 
-        self.vision_embedding = IsaacVisionEmbedding(config)
-        self.vision_embedding._supports_sdpa = True
+
+        self.vision_tower = IsaacVisionTransformer(config.vision_config)
+        self.multimodal_projector = IsaacMultiModalProjector(config)
 
         # Dispatch table for TensorStream balanced embedding (text + vision)
         self.embed_fns = {
@@ -1622,10 +1633,6 @@ class IsaacModel(Qwen3PreTrainedModel):
     def vision_model(self) -> nn.Module:
         return self.vision_embedding.vision_tower
 
-    @property
-    def vision_tower(self) -> nn.Module:
-        return self.vision_embedding.vision_tower
-
     def embed_text_tokens(self, token_ids: torch.Tensor) -> torch.Tensor:
         """Embed text tokens, squeezing singleton dimensions."""
         # Text events are shaped as (..., 1); squeeze the singleton index dim
@@ -1637,7 +1644,7 @@ class IsaacModel(Qwen3PreTrainedModel):
     def embed_vision(self, vision_tokens: tuple[torch.Tensor, torch.Tensor]) -> torch.Tensor:
         """Embed vision tokens using the vision encoder."""
         # vision tokens is (seq_patches, token_grids)
-        return self.vision_embedding(vision_tokens)
+        return self.multimodal_projector(self.vision_tower(vision_tokens))
 
     def embed_stream(self, tensor_stream: TensorStream) -> torch.Tensor:
         """
@@ -2096,7 +2103,8 @@ def _compute_residual_p_frames(frames: torch.Tensor, is_p_frame: list[bool]) ->
 __all__ = [
     "IsaacConfig",
     "IsaacModel",
+    "IsaacPreTrainedModel",  # noqa: F822
     "IsaacForConditionalGeneration",
     "IsaacImageProcessorFast",
     "IsaacProcessor",
-]
+]