Update model weights after training (epoch 2, loss 3.9784)

audio_decoder.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be283b159de0c4a206d250a8791de6c6dd88188cbf4bca57c4ff4f1b0b83ebf7
+oid sha256:49e6f986a4bddecd8227e793979c365ea238167aa0d87886dc13ece8b990cd89
 size 1458410612

chat_template.jinja

@@ -6,181 +6,22 @@
 {%- set user_end = '<|/user|>' -%}
 {%- set assistant_start = '<|assistant|>' -%}
 {%- set assistant_end = '<|/assistant|>' -%}
-{%- set image_start = '<|image|>' -%}
-{%- set image_end = '<|/image|>' -%}
-{%- set video_start = '<|video|>' -%}
-{%- set video_end = '<|/video|>' -%}
-{%- set audio_start = '<|audio|>' -%}
-{%- set audio_end = '<|/audio|>' -%}
-{%- set doc_start = '<|doc|>' -%}
-{%- set doc_end = '<|/doc|>' -%}
-{%- set tool_call_start = '<|tool_call|>' -%}
-{%- set tool_call_end = '<|/tool_call|>' -%}
-{%- set tool_result_start = '<|tool_result|>' -%}
-{%- set tool_result_end = '<|/tool_result|>' -%}
-{%- set tools_start = '<|tools|>' -%}
-{%- set tools_end = '<|/tools|>' -%}
-{%- set available_tools_start = '<|available_tools|>' -%}
-{%- set available_tools_end = '<|/available_tools|>' -%}
-{%- set function_name_start = '<|function_name|>' -%}
-{%- set function_name_end = '<|/function_name|>' -%}
-{%- set function_args_start = '<|function_args|>' -%}
-{%- set function_args_end = '<|/function_args|>' -%}
-{%- set think_start = '<|think|>' -%}
-{%- set think_end = '<|/think|>' -%}
-{%- set plan_start = '<|plan|>' -%}
-{%- set plan_end = '<|/plan|>' -%}
-{%- set critique_start = '<|critique|>' -%}
-{%- set critique_end = '<|/critique|>' -%}
-{%- set analysis_start = '<|analysis|>' -%}
-{%- set analysis_end = '<|/analysis|>' -%}
-{%- set observation_start = '<|observation|>' -%}
-{%- set observation_end = '<|/observation|>' -%}
-{%- set reflection_start = '<|reflection|>' -%}
-{%- set reflection_end = '<|/reflection|>' -%}
-{%- set conclusion_start = '<|conclusion|>' -%}
-{%- set conclusion_end = '<|/conclusion|>' -%}
-{%- set code_start = '<|code|>' -%}
-{%- set code_end = '<|/code|>' -%}
-{%- set exec_start = '<|exec|>' -%}
-{%- set exec_end = '<|/exec|>' -%}
-{%- set exec_result = '<|exec_result|>' -%}
-{%- set exec_result_end = '<|/exec_result|>' -%}
-{%- set jupyter_code = '<|jupyter_code|>' -%}
-{%- set jupyter_code_end = '<|/jupyter_code|>' -%}
-{%- set jupyter_output = '<|jupyter_output|>' -%}
-{%- set jupyter_output_end = '<|/jupyter_output|>' -%}
-{%- set gen_image_start = '<|gen_image|>' -%}
-{%- set gen_image_end = '<|/gen_image|>' -%}
-{%- set gen_video_start = '<|gen_video|>' -%}
-{%- set gen_video_end = '<|/gen_video|>' -%}
-{%- set speak_start = '<|speak|>' -%}
-{%- set speak_end = '<|/speak|>' -%}
-{%- set listen_start = '<|listen|>' -%}
-{%- set listen_end = '<|/listen|>' -%}
-{%- set memory_start = '<|memory|>' -%}
-{%- set memory_end = '<|/memory|>' -%}
-{%- set context_start = '<|context|>' -%}
-{%- set context_end = '<|/context|>' -%}
-{%- set uncertain_start = '<|uncertain|>' -%}
-{%- set uncertain_end = '<|/uncertain|>' -%}
-{%- set cite_start = '<|cite|>' -%}
-{%- set cite_end = '<|/cite|>' -%}
-{%- set eod = '<|eod|>' -%}
 
 {{- bos -}}
-{%- if messages[0]['role'] == 'system' -%}
-    {{- system_start + messages[0]['content'] + system_end -}}
-    {%- set messages = messages[1:] -%}
-{%- endif -%}
-{%- if available_tools is defined and available_tools -%}
-    {{- available_tools_start + available_tools + available_tools_end -}}
-{%- elif tools is defined and tools -%}
-    {{- tools_start + tools + tools_end -}}
-{%- endif -%}
-{%- if memory is defined and memory -%}
-    {{- memory_start + memory + memory_end -}}
-{%- endif -%}
-{%- if context is defined and context -%}
-    {{- context_start + context + context_end -}}
-{%- endif -%}
 {%- for message in messages -%}
     {%- if message['role'] == 'system' -%}
         {{- system_start + message['content'] + system_end -}}
     {%- elif message['role'] == 'user' -%}
-        {{- user_start -}}
-        {%- if message.get('images') -%}
-            {%- for img in message['images'] -%}
-                {{- image_start + img + image_end -}}
-            {%- endfor -%}
-        {%- endif -%}
-        {%- if message.get('videos') -%}
-            {%- for vid in message['videos'] -%}
-                {{- video_start + vid + video_end -}}
-            {%- endfor -%}
-        {%- endif -%}
-        {%- if message.get('audio') -%}
-            {%- for aud in message['audio'] -%}
-                {{- audio_start + aud + audio_end -}}
-            {%- endfor -%}
-        {%- endif -%}
-        {%- if message.get('documents') -%}
-            {%- for doc in message['documents'] -%}
-                {{- doc_start + doc + doc_end -}}
-            {%- endfor -%}
-        {%- endif -%}
-        {{- message['content'] + user_end -}}
+        {{- user_start + message['content'] + user_end -}}
     {%- elif message['role'] == 'assistant' -%}
-        {{- assistant_start -}}
-        {%- if message.get('thinking') -%}
-            {{- think_start + message['thinking'] + think_end -}}
-        {%- endif -%}
-        {%- if message.get('planning') -%}
-            {{- plan_start + message['planning'] + plan_end -}}
-        {%- endif -%}
-        {%- if message.get('analysis') -%}
-            {{- analysis_start + message['analysis'] + analysis_end -}}
-        {%- endif -%}
-        {%- if message.get('observation') -%}
-            {{- observation_start + message['observation'] + observation_end -}}
-        {%- endif -%}
-        {%- if message.get('reflection') -%}
-            {{- reflection_start + message['reflection'] + reflection_end -}}
-        {%- endif -%}
-        {%- if message.get('critique') -%}
-            {{- critique_start + message['critique'] + critique_end -}}
-        {%- endif -%}
-        {%- if message.get('conclusion') -%}
-            {{- conclusion_start + message['conclusion'] + conclusion_end -}}
-        {%- endif -%}
-        {%- if message.get('tool_calls') -%}
-            {%- for tool in message['tool_calls'] -%}
-                {{- tool_call_start -}}
-                {%- if tool is mapping -%}
-                    {{- function_name_start + tool.get('name', '') + function_name_end -}}
-                    {{- function_args_start + (tool.get('arguments', '') | tojson if tool.get('arguments') is mapping else tool.get('arguments', '')) + function_args_end -}}
-                {%- else -%}
-                    {{- tool -}}
-                {%- endif -%}
-                {{- tool_call_end -}}
-            {%- endfor -%}
-        {%- endif -%}
-        {%- if message.get('code') -%}
-            {{- code_start + message['code'] + code_end -}}
-        {%- endif -%}
-        {%- if message.get('exec') -%}
-            {{- exec_start + message['exec'] + exec_end -}}
-        {%- endif -%}
-        {%- if message.get('gen_image') -%}
-            {{- gen_image_start + message['gen_image'] + gen_image_end -}}
-        {%- endif -%}
-        {%- if message.get('gen_video') -%}
-            {{- gen_video_start + message['gen_video'] + gen_video_end -}}
-        {%- endif -%}
-        {%- if message.get('speak') -%}
-            {{- speak_start + message['speak'] + speak_end -}}
-        {%- endif -%}
-        {%- if message.get('uncertain') -%}
-            {{- uncertain_start + message['uncertain'] + uncertain_end -}}
-        {%- endif -%}
-        {%- if message.get('citation') -%}
-            {{- cite_start + message['citation'] + cite_end -}}
-        {%- endif -%}
-        {{- message['content'] -}}
+        {{- assistant_start + message['content'] -}}
         {%- if not loop.last or add_generation_prompt is not defined or not add_generation_prompt -%}
             {{- assistant_end -}}
         {%- endif -%}
-    {%- elif message['role'] == 'tool' -%}
-        {{- tool_result_start + message['content'] + tool_result_end -}}
-    {%- elif message['role'] == 'exec_result' -%}
-        {{- exec_result + message['content'] + exec_result_end -}}
-    {%- elif message['role'] == 'jupyter' -%}
-        {{- jupyter_output + message['content'] + jupyter_output_end -}}
+    {%- elif message['role'] == 'tool' or message['role'] == 'exec_result' or message['role'] == 'jupyter' -%}
+        {{- user_start + message['content'] + user_end -}}
     {%- endif -%}
 {%- endfor -%}
 {%- if add_generation_prompt is defined and add_generation_prompt -%}
     {{- assistant_start -}}
-    {%- if enable_thinking is defined and enable_thinking -%}
-        {{- think_start -}}
-    {%- endif -%}
 {%- endif -%}

cross_attention.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:879ba97c8a30d794181570c76bd10ecbb10fb84fabcb10047d7f5d7f944cc707
+oid sha256:4018c034a9aebf411e6668da372c89e821fa86e0ee23ccae8c2d5950c7be81cc
 size 174191400

llm.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b534cad0e5014cdd5984e8a4bd04771ffc7e701c12cea42b5467e4d051224d9
+oid sha256:8fcf68c777631c00259dbee807f3270c93a3a296b2bc123a7fa33ee83c531ca2
 size 1506832040

streaming_state.json

@@ -1,7 +1,7 @@
 {
-  "epoch": 48,
-  "unique_samples": 50,
-  "total_yields": 100,
+  "epoch": 51,
+  "unique_samples": 250,
+  "total_yields": 500,
   "dataset_positions": {
     "WebSight": 386,
     "ScienceQA": 364,
@@ -76,15 +76,15 @@
     "Tool-Calls-SingleTurn": 200,
     "Tool-Calls-Multiturn": 200,
     "OpenAssistant": 450,
-    "T2V-Sora-Preferences-2": 200,
-    "T2V-Human-Preferences": 200,
+    "T2V-Sora-Preferences-2": 300,
+    "T2V-Human-Preferences": 300,
     "Sora-Alignment-Likert": 198,
     "Sora-Style-Likert": 198,
     "I2V-Preference-Seedance": 198,
-    "WebVid-10M": 200,
+    "WebVid-10M": 300,
     "Sora-Physics-Likert": 198,
-    "TIP-I2V": 200,
-    "Pexels-I2V-350k": 200,
+    "TIP-I2V": 300,
+    "Pexels-I2V-350k": 300,
     "SmolTalk-OpenHermes": 250,
     "SmolTalk-All": 250
   },
@@ -135,22 +135,22 @@
       "MagicBrush": 386
     },
     "video": {
-      "T2V-Sora-Preferences-2": 200,
-      "T2V-Human-Preferences": 200,
+      "T2V-Sora-Preferences-2": 300,
+      "T2V-Human-Preferences": 300,
       "Sora-Alignment-Likert": 198,
       "Sora-Style-Likert": 198,
       "I2V-Preference-Seedance": 198,
-      "WebVid-10M": 200,
+      "WebVid-10M": 300,
       "Sora-Physics-Likert": 198,
-      "TIP-I2V": 200,
-      "Pexels-I2V-350k": 200
+      "TIP-I2V": 300,
+      "Pexels-I2V-350k": 300
     },
     "audio": {}
   },
   "modality_counts": {
-    "text": 50,
+    "text": 0,
     "image": 0,
-    "video": 0,
+    "video": 250,
     "audio": 0
   },
   "last_modality": null

trainer_state.json

@@ -1,32 +1,32 @@
 {
   "best_model_checkpoint": "/kaggle/working/xoron-final",
-  "best_metric": 4.869536457061767,
-  "epoch": 7,
-  "epochs_completed": 7,
-  "global_step": 42,
+  "best_metric": 3.9784093894741965,
+  "epoch": 2,
+  "epochs_completed": 2,
+  "global_step": 62,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [],
   "logging_steps": 50,
-  "max_steps": 42,
-  "num_train_epochs": 7,
+  "max_steps": 62,
+  "num_train_epochs": 2,
   "total_flos": 0,
   "train_batch_size": 1,
   "effective_batch_size": 16,
   "learning_rate": 0.0001,
   "max_grad_norm": 1.0,
   "trainable_components": [
+    "vision",
+    "video",
     "llm",
     "cross_attention",
+    "video_generation",
     "modality_markers"
   ],
   "frozen_components": [
-    "vision",
-    "video",
     "audio",
     "speech",
-    "image_generation",
-    "video_generation"
+    "image_generation"
   ],
   "trial_name": null,
   "trial_params": null

training_state.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b7335b590f20d3da7af0b586bde304e2566d2306489e64d38993d1cd20df627
-size 1514911851
+oid sha256:996b90fe9af05150cee2a37b6f085ac2f23791df3a528e52813fe7de22153097
+size 3426643671

video_generator.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4b113c1d2cf10b7fad0c03661c1093738604762583c5c8f0fb0c8c84bcdc6f4
+oid sha256:a368345b4cc4a315a5258ee40047027e7c0b837907c3c15200877187899ab8be
 size 61574134