Update model weights after training (epoch 7, loss 4.8695)

audio_decoder.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c225077ec0e29909d0f390011f666158ae658fa3385cf8032280f5203da09cae
+oid sha256:be283b159de0c4a206d250a8791de6c6dd88188cbf4bca57c4ff4f1b0b83ebf7
 size 1458410612

cross_attention.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c5dc29d69984df0e49cf508c56c03b7a18a7a49baf89a414fa3128513d753e7e
+oid sha256:879ba97c8a30d794181570c76bd10ecbb10fb84fabcb10047d7f5d7f944cc707
 size 174191400

llm.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5de86313a868d4108f814a3debd9d1ed31dc72281458ef9c7824b9a4398ce28f
+oid sha256:0b534cad0e5014cdd5984e8a4bd04771ffc7e701c12cea42b5467e4d051224d9
 size 1506832040

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 7309365134,
+    "total_size": 7309254542,
     "format": "components"
   },
   "weight_map": {

modeling_xoron.py

@@ -3824,6 +3824,42 @@ class VideoTiTokTokenizer(nn.Module):
         print(f"      Temporal tokens: {self.num_temporal_tokens}, Content tokens: {self.num_content_tokens}")
         print(f"      Layers: {num_layers}, Heads: {num_heads}")
 
+    def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs):
+        """Production-grade hook to handle dynamic frame counts and token counts when loading checkpoints."""
+        
+        # 1. Handle temporal_pos: [1, max_frames, 1, hidden_size]
+        t_pos_key = prefix + 'temporal_pos'
+        if t_pos_key in state_dict:
+            ckpt_pos = state_dict[t_pos_key]
+            if ckpt_pos.shape != self.temporal_pos.shape:
+                print(f"   ⚠️ VideoTiTokTokenizer: Interpolating {t_pos_key} from {ckpt_pos.shape[1]} to {self.max_frames} frames.")
+                # [1, T, 1, D] -> [1, D, T]
+                ckpt_pos = ckpt_pos.squeeze(2).transpose(1, 2)
+                resized = F.interpolate(ckpt_pos, size=self.max_frames, mode='linear', align_corners=False)
+                state_dict[t_pos_key] = resized.transpose(1, 2).unsqueeze(2)
+                
+        # 2. Handle temporal_queries: [1, num_temporal_tokens, hidden_size]
+        t_query_key = prefix + 'temporal_queries'
+        if t_query_key in state_dict:
+            ckpt_query = state_dict[t_query_key]
+            if ckpt_query.shape != self.temporal_queries.shape:
+                print(f"   ⚠️ VideoTiTokTokenizer: Interpolating {t_query_key} from {ckpt_query.shape[1]} to {self.num_temporal_tokens} tokens.")
+                ckpt_query = ckpt_query.transpose(1, 2)
+                resized = F.interpolate(ckpt_query, size=self.num_temporal_tokens, mode='linear', align_corners=False)
+                state_dict[t_query_key] = resized.transpose(1, 2)
+                
+        # 3. Handle content_queries: [1, num_content_tokens, hidden_size]
+        c_query_key = prefix + 'content_queries'
+        if c_query_key in state_dict:
+            ckpt_query = state_dict[c_query_key]
+            if ckpt_query.shape != self.content_queries.shape:
+                print(f"   ⚠️ VideoTiTokTokenizer: Interpolating {c_query_key} from {ckpt_query.shape[1]} to {self.num_content_tokens} tokens.")
+                ckpt_query = ckpt_query.transpose(1, 2)
+                resized = F.interpolate(ckpt_query, size=self.num_content_tokens, mode='linear', align_corners=False)
+                state_dict[c_query_key] = resized.transpose(1, 2)
+
+        super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs)
+
     def _add_3d_pos_encoding(self, x: torch.Tensor, num_frames: int, patches_per_frame: int) -> torch.Tensor:
         """Add 3D positional encoding (temporal + spatial)."""
         B, seq_len, D = x.shape
@@ -4023,6 +4059,23 @@ class VideoEncoder(nn.Module):
 
         print(f"   🎬 Video encoder: max {max_frames} frames (multi-scale enabled)")
 
+    def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs):
+        """Production-grade hook to handle dynamic frame counts when loading checkpoints.
+        Interpolates temporal embeddings if the checkpoint frames differ from max_frames.
+        """
+        # Handle frame_pos_embed
+        embed_key = prefix + 'frame_pos_embed'
+        if embed_key in state_dict:
+            ckpt_embed = state_dict[embed_key]
+            if ckpt_embed.shape != self.frame_pos_embed.shape:
+                print(f"   ⚠️ VideoEncoder: Interpolating {embed_key} from {ckpt_embed.shape[1]} to {self.max_frames} frames.")
+                # [1, T, D] -> [1, D, T] for interpolation
+                ckpt_embed = ckpt_embed.transpose(1, 2)
+                resized = F.interpolate(ckpt_embed, size=self.max_frames, mode='linear', align_corners=False)
+                state_dict[embed_key] = resized.transpose(1, 2) # Back to [1, T, D]
+                
+        super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs)
+
     def _extract_frame_features(self, frames: torch.Tensor) -> torch.Tensor:
         """Extract per-frame features using vision encoder."""
         batch_size, num_frames = frames.shape[:2]

streaming_state.json

@@ -1,7 +1,7 @@
 {
-  "epoch": 35,
-  "unique_samples": 400,
-  "total_yields": 800,
+  "epoch": 48,
+  "unique_samples": 50,
+  "total_yields": 100,
   "dataset_positions": {
     "WebSight": 386,
     "ScienceQA": 364,
@@ -30,7 +30,7 @@
     "NoRobots": 450,
     "Synth-LanguageSetup": 200,
     "Function-Calling-ChatML": 200,
-    "Synth-CoT": 200,
+    "Synth-CoT": 550,
     "Python-Code-18k": 200,
     "Code-Feedback": 200,
     "HumanEval-CPP": 164,
@@ -148,7 +148,7 @@
     "audio": {}
   },
   "modality_counts": {
-    "text": 400,
+    "text": 50,
     "image": 0,
     "video": 0,
     "audio": 0

trainer_state.json

@@ -1,15 +1,15 @@
 {
   "best_model_checkpoint": "/kaggle/working/xoron-final",
-  "best_metric": 6.958861378133297,
-  "epoch": 5,
-  "epochs_completed": 5,
-  "global_step": 250,
+  "best_metric": 4.869536457061767,
+  "epoch": 7,
+  "epochs_completed": 7,
+  "global_step": 42,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [],
   "logging_steps": 50,
-  "max_steps": 250,
-  "num_train_epochs": 5,
+  "max_steps": 42,
+  "num_train_epochs": 7,
   "total_flos": 0,
   "train_batch_size": 1,
   "effective_batch_size": 16,

training_state.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a751ecf22021470154d58846b700d04286522c14cda7393ece31f907eff5a2c7
+oid sha256:5b7335b590f20d3da7af0b586bde304e2566d2306489e64d38993d1cd20df627
 size 1514911851

video_encoder.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f70226e533706675adf13f72c46122854021d13fe388445bc4d6b7495fa64e3a
-size 1923089112
+oid sha256:f79301c42561645d3000aa5d61769c0dd5089c6120c49a3fdca5a7eb5af9d2c9
+size 1922978512