Upload Order_Ablations without checkpoints

Order_Ablations/E114/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Order_Ablations/E114/chat_template.json

@@ -0,0 +1,3 @@
+{
+    "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+}

Order_Ablations/E114/config.json

@@ -0,0 +1,84 @@
+{
+  "architectures": [
+    "Qwen2_5_VLForConditionalGenerationWithVGGT"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "feature_fusion_method": "deepstack_language_add",
+  "fusion_num_layers": 1,
+  "geometry_encoder_layers": [
+    23,
+    17,
+    11
+  ],
+  "geometry_encoder_type": "vggt",
+  "geometry_fusion_layers": [
+    0,
+    1,
+    2
+  ],
+  "geometry_merger_type": "mlp",
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_token_id": 151655,
+  "include_camera_token": false,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 128000,
+  "max_window_layers": 70,
+  "model_type": "qwen2_5_vl",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "pos_encoding_type": "none",
+  "reference_frame": "first",
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "mrope_section": [
+      16,
+      24,
+      24
+    ],
+    "rope_type": "default",
+    "type": "default"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.50.0",
+  "use_cache": true,
+  "use_geometry_encoder": true,
+  "use_sliding_window": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "depth": 32,
+    "fullatt_block_indexes": [
+      7,
+      15,
+      23,
+      31
+    ],
+    "hidden_act": "silu",
+    "hidden_size": 1280,
+    "in_channels": 3,
+    "in_chans": 3,
+    "intermediate_size": 3420,
+    "model_type": "qwen2_5_vl",
+    "num_heads": 16,
+    "out_hidden_size": 2048,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "temporal_patch_size": 2,
+    "tokens_per_second": 2,
+    "torch_dtype": "bfloat16",
+    "window_size": 112
+  },
+  "vision_end_token_id": 151653,
+  "vision_language_fusion_layers": null,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
+  "vocab_size": 151936
+}

Order_Ablations/E114/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 1e-06,
+  "transformers_version": "4.50.0"
+}

Order_Ablations/E114/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa7e2f5f87c9d9ecf56ea5d6e2bd627f03f8d6d1867d9f65072147719a7188ea
+size 4962140952

Order_Ablations/E114/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80984c1d55eb294d309ddf2e82d00189995ec1a86aee5e78f7f7b75bcef12e02
+size 4869006472

Order_Ablations/E114/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00a6124968156780fab691e7154601acd20b9f093c9e6be0ea2c070cba932766
+size 622329984

Order_Ablations/E114/preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 451584,
+  "merge_size": 2,
+  "min_pixels": 12544,
+  "patch_size": 14,
+  "processor_class": "Qwen2_5_VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 451584,
+    "shortest_edge": 12544
+  },
+  "temporal_patch_size": 2
+}

Order_Ablations/E114/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Order_Ablations/E114/tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 12800,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

Order_Ablations/E115/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

Order_Ablations/E115/chat_template.json

@@ -0,0 +1,3 @@
+{
+    "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+}

Order_Ablations/E115/config.json

@@ -0,0 +1,84 @@
+{
+  "architectures": [
+    "Qwen2_5_VLForConditionalGenerationWithVGGT"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "feature_fusion_method": "deepstack_language_add",
+  "fusion_num_layers": 1,
+  "geometry_encoder_layers": [
+    11,
+    17,
+    23
+  ],
+  "geometry_encoder_type": "vggt",
+  "geometry_fusion_layers": [
+    0,
+    1,
+    2
+  ],
+  "geometry_merger_type": "mlp",
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_token_id": 151655,
+  "include_camera_token": false,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 128000,
+  "max_window_layers": 70,
+  "model_type": "qwen2_5_vl",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "pos_encoding_type": "none",
+  "reference_frame": "first",
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "mrope_section": [
+      16,
+      24,
+      24
+    ],
+    "rope_type": "default",
+    "type": "default"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.50.0",
+  "use_cache": true,
+  "use_geometry_encoder": true,
+  "use_sliding_window": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "depth": 32,
+    "fullatt_block_indexes": [
+      7,
+      15,
+      23,
+      31
+    ],
+    "hidden_act": "silu",
+    "hidden_size": 1280,
+    "in_channels": 3,
+    "in_chans": 3,
+    "intermediate_size": 3420,
+    "model_type": "qwen2_5_vl",
+    "num_heads": 16,
+    "out_hidden_size": 2048,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "temporal_patch_size": 2,
+    "tokens_per_second": 2,
+    "torch_dtype": "bfloat16",
+    "window_size": 112
+  },
+  "vision_end_token_id": 151653,
+  "vision_language_fusion_layers": null,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
+  "vocab_size": 151936
+}

Order_Ablations/E115/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 1e-06,
+  "transformers_version": "4.50.0"
+}

Order_Ablations/E115/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51061e4731c298354360396b931fe155dc55562cf3fa939a6cbb95605fffcc7c
+size 4962140952

Order_Ablations/E115/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:644ea7aa2c29d0b07fc0cd71dd1140c7ddfc8696e809aa6e619b2ed6efdd8226
+size 4869006472

Order_Ablations/E115/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad05072d5ca31590022ab2a32f0fc1b3b245526d95cad06536e213b280fb134d
+size 622329984

Order_Ablations/E115/preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 451584,
+  "merge_size": 2,
+  "min_pixels": 12544,
+  "patch_size": 14,
+  "processor_class": "Qwen2_5_VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 451584,
+    "shortest_edge": 12544
+  },
+  "temporal_patch_size": 2
+}

Order_Ablations/E115/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Order_Ablations/E115/tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 12800,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

Order_Ablations/E115/trainer_state.json

@@ -0,0 +1,2353 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9996222709073053,
+  "eval_steps": 500,
+  "global_step": 3308,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0030218327415577548,
+      "grad_norm": 166.1145477294922,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 13.8334,
+      "step": 10
+    },
+    {
+      "epoch": 0.0060436654831155096,
+      "grad_norm": 81.33020782470703,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 13.1022,
+      "step": 20
+    },
+    {
+      "epoch": 0.009065498224673264,
+      "grad_norm": 51.09122085571289,
+      "learning_rate": 3e-06,
+      "loss": 12.0899,
+      "step": 30
+    },
+    {
+      "epoch": 0.012087330966231019,
+      "grad_norm": 49.01457214355469,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 10.2942,
+      "step": 40
+    },
+    {
+      "epoch": 0.015109163707788774,
+      "grad_norm": 46.00205612182617,
+      "learning_rate": 5e-06,
+      "loss": 9.538,
+      "step": 50
+    },
+    {
+      "epoch": 0.01813099644934653,
+      "grad_norm": 41.61635971069336,
+      "learning_rate": 6e-06,
+      "loss": 8.3357,
+      "step": 60
+    },
+    {
+      "epoch": 0.021152829190904283,
+      "grad_norm": 31.584325790405273,
+      "learning_rate": 7e-06,
+      "loss": 6.9115,
+      "step": 70
+    },
+    {
+      "epoch": 0.024174661932462038,
+      "grad_norm": 51.664695739746094,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 6.7596,
+      "step": 80
+    },
+    {
+      "epoch": 0.027196494674019793,
+      "grad_norm": 31.014076232910156,
+      "learning_rate": 9e-06,
+      "loss": 7.0298,
+      "step": 90
+    },
+    {
+      "epoch": 0.030218327415577548,
+      "grad_norm": 37.792179107666016,
+      "learning_rate": 1e-05,
+      "loss": 6.7021,
+      "step": 100
+    },
+    {
+      "epoch": 0.0332401601571353,
+      "grad_norm": 37.68498611450195,
+      "learning_rate": 9.999760394462267e-06,
+      "loss": 7.3545,
+      "step": 110
+    },
+    {
+      "epoch": 0.03626199289869306,
+      "grad_norm": 38.4805793762207,
+      "learning_rate": 9.999041600813393e-06,
+      "loss": 7.0073,
+      "step": 120
+    },
+    {
+      "epoch": 0.03928382564025081,
+      "grad_norm": 32.300174713134766,
+      "learning_rate": 9.997843687944153e-06,
+      "loss": 6.2416,
+      "step": 130
+    },
+    {
+      "epoch": 0.04230565838180857,
+      "grad_norm": 29.263317108154297,
+      "learning_rate": 9.996166770665168e-06,
+      "loss": 5.5583,
+      "step": 140
+    },
+    {
+      "epoch": 0.04532749112336632,
+      "grad_norm": 33.3656005859375,
+      "learning_rate": 9.994011009695908e-06,
+      "loss": 5.6737,
+      "step": 150
+    },
+    {
+      "epoch": 0.048349323864924076,
+      "grad_norm": 32.699825286865234,
+      "learning_rate": 9.991376611649278e-06,
+      "loss": 6.0879,
+      "step": 160
+    },
+    {
+      "epoch": 0.05137115660648183,
+      "grad_norm": 27.45968246459961,
+      "learning_rate": 9.988263829011821e-06,
+      "loss": 5.4056,
+      "step": 170
+    },
+    {
+      "epoch": 0.054392989348039586,
+      "grad_norm": 25.30878448486328,
+      "learning_rate": 9.984672960119523e-06,
+      "loss": 5.3618,
+      "step": 180
+    },
+    {
+      "epoch": 0.05741482208959734,
+      "grad_norm": 40.055721282958984,
+      "learning_rate": 9.980604349129212e-06,
+      "loss": 5.7602,
+      "step": 190
+    },
+    {
+      "epoch": 0.060436654831155096,
+      "grad_norm": 26.245195388793945,
+      "learning_rate": 9.976058385985575e-06,
+      "loss": 5.186,
+      "step": 200
+    },
+    {
+      "epoch": 0.06345848757271286,
+      "grad_norm": 34.81965637207031,
+      "learning_rate": 9.971035506383791e-06,
+      "loss": 5.5341,
+      "step": 210
+    },
+    {
+      "epoch": 0.0664803203142706,
+      "grad_norm": 29.513893127441406,
+      "learning_rate": 9.96553619172777e-06,
+      "loss": 5.0542,
+      "step": 220
+    },
+    {
+      "epoch": 0.06950215305582837,
+      "grad_norm": 32.30284118652344,
+      "learning_rate": 9.959560969084004e-06,
+      "loss": 5.3365,
+      "step": 230
+    },
+    {
+      "epoch": 0.07252398579738611,
+      "grad_norm": 27.652576446533203,
+      "learning_rate": 9.953110411131073e-06,
+      "loss": 4.7513,
+      "step": 240
+    },
+    {
+      "epoch": 0.07554581853894388,
+      "grad_norm": 28.387413024902344,
+      "learning_rate": 9.946185136104736e-06,
+      "loss": 5.4127,
+      "step": 250
+    },
+    {
+      "epoch": 0.07856765128050162,
+      "grad_norm": 29.694316864013672,
+      "learning_rate": 9.938785807738692e-06,
+      "loss": 4.813,
+      "step": 260
+    },
+    {
+      "epoch": 0.08158948402205939,
+      "grad_norm": 31.964120864868164,
+      "learning_rate": 9.930913135200964e-06,
+      "loss": 5.4212,
+      "step": 270
+    },
+    {
+      "epoch": 0.08461131676361713,
+      "grad_norm": 23.594715118408203,
+      "learning_rate": 9.922567873025924e-06,
+      "loss": 5.2445,
+      "step": 280
+    },
+    {
+      "epoch": 0.0876331495051749,
+      "grad_norm": 23.0896053314209,
+      "learning_rate": 9.913750821041988e-06,
+      "loss": 4.5194,
+      "step": 290
+    },
+    {
+      "epoch": 0.09065498224673264,
+      "grad_norm": 25.44329833984375,
+      "learning_rate": 9.904462824294945e-06,
+      "loss": 4.6093,
+      "step": 300
+    },
+    {
+      "epoch": 0.0936768149882904,
+      "grad_norm": 27.408288955688477,
+      "learning_rate": 9.894704772966978e-06,
+      "loss": 4.512,
+      "step": 310
+    },
+    {
+      "epoch": 0.09669864772984815,
+      "grad_norm": 24.26542091369629,
+      "learning_rate": 9.884477602291343e-06,
+      "loss": 4.5071,
+      "step": 320
+    },
+    {
+      "epoch": 0.09972048047140591,
+      "grad_norm": 35.8819694519043,
+      "learning_rate": 9.873782292462727e-06,
+      "loss": 4.3557,
+      "step": 330
+    },
+    {
+      "epoch": 0.10274231321296366,
+      "grad_norm": 29.487594604492188,
+      "learning_rate": 9.862619868543323e-06,
+      "loss": 8.2236,
+      "step": 340
+    },
+    {
+      "epoch": 0.10576414595452142,
+      "grad_norm": 38.13749694824219,
+      "learning_rate": 9.850991400364557e-06,
+      "loss": 5.1538,
+      "step": 350
+    },
+    {
+      "epoch": 0.10878597869607917,
+      "grad_norm": 25.492799758911133,
+      "learning_rate": 9.838898002424586e-06,
+      "loss": 6.0666,
+      "step": 360
+    },
+    {
+      "epoch": 0.11180781143763693,
+      "grad_norm": 31.119089126586914,
+      "learning_rate": 9.826340833781448e-06,
+      "loss": 5.8633,
+      "step": 370
+    },
+    {
+      "epoch": 0.11482964417919468,
+      "grad_norm": 21.065149307250977,
+      "learning_rate": 9.813321097942005e-06,
+      "loss": 5.1017,
+      "step": 380
+    },
+    {
+      "epoch": 0.11785147692075244,
+      "grad_norm": 29.40814971923828,
+      "learning_rate": 9.79984004274658e-06,
+      "loss": 4.9132,
+      "step": 390
+    },
+    {
+      "epoch": 0.12087330966231019,
+      "grad_norm": 22.45477294921875,
+      "learning_rate": 9.785898960249365e-06,
+      "loss": 4.2496,
+      "step": 400
+    },
+    {
+      "epoch": 0.12389514240386795,
+      "grad_norm": 19.05487060546875,
+      "learning_rate": 9.771499186594586e-06,
+      "loss": 5.0767,
+      "step": 410
+    },
+    {
+      "epoch": 0.12691697514542571,
+      "grad_norm": 31.310686111450195,
+      "learning_rate": 9.756642101888449e-06,
+      "loss": 5.192,
+      "step": 420
+    },
+    {
+      "epoch": 0.12993880788698345,
+      "grad_norm": 25.689640045166016,
+      "learning_rate": 9.74132913006686e-06,
+      "loss": 3.445,
+      "step": 430
+    },
+    {
+      "epoch": 0.1329606406285412,
+      "grad_norm": 21.052574157714844,
+      "learning_rate": 9.725561738758956e-06,
+      "loss": 3.3354,
+      "step": 440
+    },
+    {
+      "epoch": 0.13598247337009897,
+      "grad_norm": 24.987884521484375,
+      "learning_rate": 9.709341439146452e-06,
+      "loss": 5.0777,
+      "step": 450
+    },
+    {
+      "epoch": 0.13900430611165673,
+      "grad_norm": 26.842397689819336,
+      "learning_rate": 9.692669785818787e-06,
+      "loss": 6.4292,
+      "step": 460
+    },
+    {
+      "epoch": 0.14202613885321447,
+      "grad_norm": 35.66836166381836,
+      "learning_rate": 9.675548376624149e-06,
+      "loss": 5.7348,
+      "step": 470
+    },
+    {
+      "epoch": 0.14504797159477223,
+      "grad_norm": 29.318471908569336,
+      "learning_rate": 9.657978852516318e-06,
+      "loss": 5.6924,
+      "step": 480
+    },
+    {
+      "epoch": 0.14806980433633,
+      "grad_norm": 23.544092178344727,
+      "learning_rate": 9.639962897397405e-06,
+      "loss": 4.183,
+      "step": 490
+    },
+    {
+      "epoch": 0.15109163707788775,
+      "grad_norm": 22.90180206298828,
+      "learning_rate": 9.621502237956452e-06,
+      "loss": 5.085,
+      "step": 500
+    },
+    {
+      "epoch": 0.1541134698194455,
+      "grad_norm": 23.748275756835938,
+      "learning_rate": 9.602598643503957e-06,
+      "loss": 3.2694,
+      "step": 510
+    },
+    {
+      "epoch": 0.15713530256100325,
+      "grad_norm": 29.096708297729492,
+      "learning_rate": 9.583253925802283e-06,
+      "loss": 4.2373,
+      "step": 520
+    },
+    {
+      "epoch": 0.160157135302561,
+      "grad_norm": 24.87314796447754,
+      "learning_rate": 9.563469938892023e-06,
+      "loss": 4.8482,
+      "step": 530
+    },
+    {
+      "epoch": 0.16317896804411877,
+      "grad_norm": 24.310091018676758,
+      "learning_rate": 9.543248578914309e-06,
+      "loss": 3.2299,
+      "step": 540
+    },
+    {
+      "epoch": 0.1662008007856765,
+      "grad_norm": 24.80878257751465,
+      "learning_rate": 9.522591783929069e-06,
+      "loss": 4.8424,
+      "step": 550
+    },
+    {
+      "epoch": 0.16922263352723427,
+      "grad_norm": 22.142215728759766,
+      "learning_rate": 9.501501533729297e-06,
+      "loss": 4.1786,
+      "step": 560
+    },
+    {
+      "epoch": 0.17224446626879203,
+      "grad_norm": 33.77587890625,
+      "learning_rate": 9.479979849651287e-06,
+      "loss": 5.7505,
+      "step": 570
+    },
+    {
+      "epoch": 0.1752662990103498,
+      "grad_norm": 25.414831161499023,
+      "learning_rate": 9.45802879438091e-06,
+      "loss": 6.3392,
+      "step": 580
+    },
+    {
+      "epoch": 0.17828813175190752,
+      "grad_norm": 25.716073989868164,
+      "learning_rate": 9.43565047175593e-06,
+      "loss": 4.1603,
+      "step": 590
+    },
+    {
+      "epoch": 0.1813099644934653,
+      "grad_norm": 25.389522552490234,
+      "learning_rate": 9.412847026564359e-06,
+      "loss": 3.9676,
+      "step": 600
+    },
+    {
+      "epoch": 0.18433179723502305,
+      "grad_norm": 22.911640167236328,
+      "learning_rate": 9.389620644338893e-06,
+      "loss": 4.1508,
+      "step": 610
+    },
+    {
+      "epoch": 0.1873536299765808,
+      "grad_norm": 36.27210998535156,
+      "learning_rate": 9.365973551147453e-06,
+      "loss": 4.691,
+      "step": 620
+    },
+    {
+      "epoch": 0.19037546271813854,
+      "grad_norm": 23.555246353149414,
+      "learning_rate": 9.341908013379832e-06,
+      "loss": 4.7148,
+      "step": 630
+    },
+    {
+      "epoch": 0.1933972954596963,
+      "grad_norm": 25.42097282409668,
+      "learning_rate": 9.317426337530477e-06,
+      "loss": 4.0105,
+      "step": 640
+    },
+    {
+      "epoch": 0.19641912820125407,
+      "grad_norm": 24.92901611328125,
+      "learning_rate": 9.292530869977432e-06,
+      "loss": 5.5589,
+      "step": 650
+    },
+    {
+      "epoch": 0.19944096094281183,
+      "grad_norm": 26.411352157592773,
+      "learning_rate": 9.26722399675745e-06,
+      "loss": 3.1881,
+      "step": 660
+    },
+    {
+      "epoch": 0.20246279368436956,
+      "grad_norm": 22.39121437072754,
+      "learning_rate": 9.24150814333732e-06,
+      "loss": 3.9177,
+      "step": 670
+    },
+    {
+      "epoch": 0.20548462642592732,
+      "grad_norm": 21.436046600341797,
+      "learning_rate": 9.215385774381395e-06,
+      "loss": 6.2124,
+      "step": 680
+    },
+    {
+      "epoch": 0.2085064591674851,
+      "grad_norm": 42.19996643066406,
+      "learning_rate": 9.188859393515382e-06,
+      "loss": 4.863,
+      "step": 690
+    },
+    {
+      "epoch": 0.21152829190904285,
+      "grad_norm": 24.43948745727539,
+      "learning_rate": 9.16193154308638e-06,
+      "loss": 6.0562,
+      "step": 700
+    },
+    {
+      "epoch": 0.21455012465060058,
+      "grad_norm": 36.5896110534668,
+      "learning_rate": 9.13460480391922e-06,
+      "loss": 6.1878,
+      "step": 710
+    },
+    {
+      "epoch": 0.21757195739215834,
+      "grad_norm": 39.19657897949219,
+      "learning_rate": 9.106881795069116e-06,
+      "loss": 6.4964,
+      "step": 720
+    },
+    {
+      "epoch": 0.2205937901337161,
+      "grad_norm": 19.438859939575195,
+      "learning_rate": 9.078765173570649e-06,
+      "loss": 3.1914,
+      "step": 730
+    },
+    {
+      "epoch": 0.22361562287527387,
+      "grad_norm": 26.316898345947266,
+      "learning_rate": 9.0502576341831e-06,
+      "loss": 4.0543,
+      "step": 740
+    },
+    {
+      "epoch": 0.2266374556168316,
+      "grad_norm": 21.5406436920166,
+      "learning_rate": 9.02136190913219e-06,
+      "loss": 5.4649,
+      "step": 750
+    },
+    {
+      "epoch": 0.22965928835838936,
+      "grad_norm": 38.014617919921875,
+      "learning_rate": 8.99208076784822e-06,
+      "loss": 4.6499,
+      "step": 760
+    },
+    {
+      "epoch": 0.23268112109994712,
+      "grad_norm": 16.046876907348633,
+      "learning_rate": 8.962417016700624e-06,
+      "loss": 3.0368,
+      "step": 770
+    },
+    {
+      "epoch": 0.2357029538415049,
+      "grad_norm": 25.170169830322266,
+      "learning_rate": 8.932373498729026e-06,
+      "loss": 4.6374,
+      "step": 780
+    },
+    {
+      "epoch": 0.23872478658306262,
+      "grad_norm": 28.294591903686523,
+      "learning_rate": 8.901953093370734e-06,
+      "loss": 4.0344,
+      "step": 790
+    },
+    {
+      "epoch": 0.24174661932462038,
+      "grad_norm": 25.618423461914062,
+      "learning_rate": 8.871158716184784e-06,
+      "loss": 3.9153,
+      "step": 800
+    },
+    {
+      "epoch": 0.24476845206617814,
+      "grad_norm": 33.044132232666016,
+      "learning_rate": 8.839993318572497e-06,
+      "loss": 4.852,
+      "step": 810
+    },
+    {
+      "epoch": 0.2477902848077359,
+      "grad_norm": 19.522127151489258,
+      "learning_rate": 8.808459887494617e-06,
+      "loss": 3.0679,
+      "step": 820
+    },
+    {
+      "epoch": 0.25081211754929367,
+      "grad_norm": 17.915157318115234,
+      "learning_rate": 8.77656144518502e-06,
+      "loss": 3.832,
+      "step": 830
+    },
+    {
+      "epoch": 0.25383395029085143,
+      "grad_norm": 18.468053817749023,
+      "learning_rate": 8.744301048861083e-06,
+      "loss": 2.9134,
+      "step": 840
+    },
+    {
+      "epoch": 0.25685578303240914,
+      "grad_norm": 25.19109535217285,
+      "learning_rate": 8.711681790430646e-06,
+      "loss": 2.9987,
+      "step": 850
+    },
+    {
+      "epoch": 0.2598776157739669,
+      "grad_norm": 27.227184295654297,
+      "learning_rate": 8.678706796195694e-06,
+      "loss": 4.7592,
+      "step": 860
+    },
+    {
+      "epoch": 0.26289944851552466,
+      "grad_norm": 28.04375457763672,
+      "learning_rate": 8.645379226552712e-06,
+      "loss": 3.7402,
+      "step": 870
+    },
+    {
+      "epoch": 0.2659212812570824,
+      "grad_norm": 21.457616806030273,
+      "learning_rate": 8.611702275689805e-06,
+      "loss": 4.6756,
+      "step": 880
+    },
+    {
+      "epoch": 0.2689431139986402,
+      "grad_norm": 35.01508331298828,
+      "learning_rate": 8.577679171280538e-06,
+      "loss": 4.5315,
+      "step": 890
+    },
+    {
+      "epoch": 0.27196494674019794,
+      "grad_norm": 20.160045623779297,
+      "learning_rate": 8.543313174174601e-06,
+      "loss": 5.2698,
+      "step": 900
+    },
+    {
+      "epoch": 0.2749867794817557,
+      "grad_norm": 22.52850341796875,
+      "learning_rate": 8.508607578085281e-06,
+      "loss": 3.849,
+      "step": 910
+    },
+    {
+      "epoch": 0.27800861222331347,
+      "grad_norm": 21.895462036132812,
+      "learning_rate": 8.473565709273786e-06,
+      "loss": 3.8616,
+      "step": 920
+    },
+    {
+      "epoch": 0.2810304449648712,
+      "grad_norm": 16.077316284179688,
+      "learning_rate": 8.438190926230439e-06,
+      "loss": 3.8386,
+      "step": 930
+    },
+    {
+      "epoch": 0.28405227770642894,
+      "grad_norm": 33.1984977722168,
+      "learning_rate": 8.40248661935281e-06,
+      "loss": 4.3994,
+      "step": 940
+    },
+    {
+      "epoch": 0.2870741104479867,
+      "grad_norm": 27.1571102142334,
+      "learning_rate": 8.366456210620756e-06,
+      "loss": 3.1029,
+      "step": 950
+    },
+    {
+      "epoch": 0.29009594318954446,
+      "grad_norm": 31.706750869750977,
+      "learning_rate": 8.330103153268464e-06,
+      "loss": 3.7567,
+      "step": 960
+    },
+    {
+      "epoch": 0.2931177759311022,
+      "grad_norm": 24.30504608154297,
+      "learning_rate": 8.29343093145347e-06,
+      "loss": 3.6988,
+      "step": 970
+    },
+    {
+      "epoch": 0.29613960867266,
+      "grad_norm": 24.231523513793945,
+      "learning_rate": 8.25644305992275e-06,
+      "loss": 3.6097,
+      "step": 980
+    },
+    {
+      "epoch": 0.29916144141421774,
+      "grad_norm": 19.621383666992188,
+      "learning_rate": 8.21914308367584e-06,
+      "loss": 4.5566,
+      "step": 990
+    },
+    {
+      "epoch": 0.3021832741557755,
+      "grad_norm": 21.627859115600586,
+      "learning_rate": 8.181534577625088e-06,
+      "loss": 3.7714,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3052051068973332,
+      "grad_norm": 14.206421852111816,
+      "learning_rate": 8.143621146253022e-06,
+      "loss": 4.6373,
+      "step": 1010
+    },
+    {
+      "epoch": 0.308226939638891,
+      "grad_norm": 27.084983825683594,
+      "learning_rate": 8.105406423266884e-06,
+      "loss": 4.6538,
+      "step": 1020
+    },
+    {
+      "epoch": 0.31124877238044873,
+      "grad_norm": 20.950910568237305,
+      "learning_rate": 8.066894071250374e-06,
+      "loss": 4.4614,
+      "step": 1030
+    },
+    {
+      "epoch": 0.3142706051220065,
+      "grad_norm": 20.357742309570312,
+      "learning_rate": 8.02808778131262e-06,
+      "loss": 3.7694,
+      "step": 1040
+    },
+    {
+      "epoch": 0.31729243786356426,
+      "grad_norm": 18.685476303100586,
+      "learning_rate": 7.988991272734407e-06,
+      "loss": 4.4575,
+      "step": 1050
+    },
+    {
+      "epoch": 0.320314270605122,
+      "grad_norm": 24.249338150024414,
+      "learning_rate": 7.94960829261172e-06,
+      "loss": 4.4394,
+      "step": 1060
+    },
+    {
+      "epoch": 0.3233361033466798,
+      "grad_norm": 22.846027374267578,
+      "learning_rate": 7.909942615496613e-06,
+      "loss": 4.7241,
+      "step": 1070
+    },
+    {
+      "epoch": 0.32635793608823754,
+      "grad_norm": 30.40308952331543,
+      "learning_rate": 7.869998043035442e-06,
+      "loss": 5.3999,
+      "step": 1080
+    },
+    {
+      "epoch": 0.32937976882979525,
+      "grad_norm": 17.647789001464844,
+      "learning_rate": 7.829778403604512e-06,
+      "loss": 5.0469,
+      "step": 1090
+    },
+    {
+      "epoch": 0.332401601571353,
+      "grad_norm": 33.98617935180664,
+      "learning_rate": 7.789287551943158e-06,
+      "loss": 6.0896,
+      "step": 1100
+    },
+    {
+      "epoch": 0.3354234343129108,
+      "grad_norm": 21.646024703979492,
+      "learning_rate": 7.748529368784293e-06,
+      "loss": 4.5196,
+      "step": 1110
+    },
+    {
+      "epoch": 0.33844526705446853,
+      "grad_norm": 18.94881820678711,
+      "learning_rate": 7.707507760482473e-06,
+      "loss": 6.1607,
+      "step": 1120
+    },
+    {
+      "epoch": 0.3414670997960263,
+      "grad_norm": 18.058412551879883,
+      "learning_rate": 7.666226658639507e-06,
+      "loss": 3.7909,
+      "step": 1130
+    },
+    {
+      "epoch": 0.34448893253758406,
+      "grad_norm": 22.541349411010742,
+      "learning_rate": 7.624690019727636e-06,
+      "loss": 3.638,
+      "step": 1140
+    },
+    {
+      "epoch": 0.3475107652791418,
+      "grad_norm": 23.882991790771484,
+      "learning_rate": 7.58290182471034e-06,
+      "loss": 4.53,
+      "step": 1150
+    },
+    {
+      "epoch": 0.3505325980206996,
+      "grad_norm": 19.6879940032959,
+      "learning_rate": 7.5408660786607976e-06,
+      "loss": 3.6987,
+      "step": 1160
+    },
+    {
+      "epoch": 0.3535544307622573,
+      "grad_norm": 20.6401309967041,
+      "learning_rate": 7.498586810378019e-06,
+      "loss": 2.9513,
+      "step": 1170
+    },
+    {
+      "epoch": 0.35657626350381505,
+      "grad_norm": 22.658132553100586,
+      "learning_rate": 7.456068072000731e-06,
+      "loss": 2.8103,
+      "step": 1180
+    },
+    {
+      "epoch": 0.3595980962453728,
+      "grad_norm": 23.935726165771484,
+      "learning_rate": 7.4133139386190026e-06,
+      "loss": 4.5498,
+      "step": 1190
+    },
+    {
+      "epoch": 0.3626199289869306,
+      "grad_norm": 18.697385787963867,
+      "learning_rate": 7.3703285078836796e-06,
+      "loss": 5.2042,
+      "step": 1200
+    },
+    {
+      "epoch": 0.36564176172848833,
+      "grad_norm": 17.5216064453125,
+      "learning_rate": 7.3271158996136625e-06,
+      "loss": 3.7229,
+      "step": 1210
+    },
+    {
+      "epoch": 0.3686635944700461,
+      "grad_norm": 18.313034057617188,
+      "learning_rate": 7.283680255401049e-06,
+      "loss": 4.403,
+      "step": 1220
+    },
+    {
+      "epoch": 0.37168542721160386,
+      "grad_norm": 19.784748077392578,
+      "learning_rate": 7.240025738214193e-06,
+      "loss": 6.1978,
+      "step": 1230
+    },
+    {
+      "epoch": 0.3747072599531616,
+      "grad_norm": 33.28024673461914,
+      "learning_rate": 7.196156531998718e-06,
+      "loss": 4.4892,
+      "step": 1240
+    },
+    {
+      "epoch": 0.3777290926947193,
+      "grad_norm": 20.449913024902344,
+      "learning_rate": 7.152076841276527e-06,
+      "loss": 3.6566,
+      "step": 1250
+    },
+    {
+      "epoch": 0.3807509254362771,
+      "grad_norm": 19.441957473754883,
+      "learning_rate": 7.1077908907428154e-06,
+      "loss": 3.7812,
+      "step": 1260
+    },
+    {
+      "epoch": 0.38377275817783485,
+      "grad_norm": 32.515724182128906,
+      "learning_rate": 7.063302924861182e-06,
+      "loss": 3.8969,
+      "step": 1270
+    },
+    {
+      "epoch": 0.3867945909193926,
+      "grad_norm": 22.129140853881836,
+      "learning_rate": 7.018617207456821e-06,
+      "loss": 3.5997,
+      "step": 1280
+    },
+    {
+      "epoch": 0.3898164236609504,
+      "grad_norm": 19.576011657714844,
+      "learning_rate": 6.973738021307872e-06,
+      "loss": 3.6646,
+      "step": 1290
+    },
+    {
+      "epoch": 0.39283825640250813,
+      "grad_norm": 17.848796844482422,
+      "learning_rate": 6.9286696677349455e-06,
+      "loss": 5.9623,
+      "step": 1300
+    },
+    {
+      "epoch": 0.3958600891440659,
+      "grad_norm": 15.815289497375488,
+      "learning_rate": 6.883416466188881e-06,
+      "loss": 3.6821,
+      "step": 1310
+    },
+    {
+      "epoch": 0.39888192188562366,
+      "grad_norm": 17.62392807006836,
+      "learning_rate": 6.837982753836755e-06,
+      "loss": 2.8778,
+      "step": 1320
+    },
+    {
+      "epoch": 0.40190375462718136,
+      "grad_norm": 34.39213180541992,
+      "learning_rate": 6.7923728851461955e-06,
+      "loss": 6.0046,
+      "step": 1330
+    },
+    {
+      "epoch": 0.4049255873687391,
+      "grad_norm": 22.834793090820312,
+      "learning_rate": 6.74659123146805e-06,
+      "loss": 3.6498,
+      "step": 1340
+    },
+    {
+      "epoch": 0.4079474201102969,
+      "grad_norm": 18.146869659423828,
+      "learning_rate": 6.70064218061742e-06,
+      "loss": 2.8181,
+      "step": 1350
+    },
+    {
+      "epoch": 0.41096925285185465,
+      "grad_norm": 18.262357711791992,
+      "learning_rate": 6.654530136453119e-06,
+      "loss": 4.3635,
+      "step": 1360
+    },
+    {
+      "epoch": 0.4139910855934124,
+      "grad_norm": 18.1636905670166,
+      "learning_rate": 6.608259518455599e-06,
+      "loss": 5.2127,
+      "step": 1370
+    },
+    {
+      "epoch": 0.4170129183349702,
+      "grad_norm": 17.246234893798828,
+      "learning_rate": 6.5618347613033875e-06,
+      "loss": 5.1173,
+      "step": 1380
+    },
+    {
+      "epoch": 0.42003475107652793,
+      "grad_norm": 19.54306983947754,
+      "learning_rate": 6.5152603144480406e-06,
+      "loss": 5.9817,
+      "step": 1390
+    },
+    {
+      "epoch": 0.4230565838180857,
+      "grad_norm": 31.445457458496094,
+      "learning_rate": 6.468540641687716e-06,
+      "loss": 4.5568,
+      "step": 1400
+    },
+    {
+      "epoch": 0.4260784165596434,
+      "grad_norm": 19.258493423461914,
+      "learning_rate": 6.421680220739337e-06,
+      "loss": 3.9311,
+      "step": 1410
+    },
+    {
+      "epoch": 0.42910024930120116,
+      "grad_norm": 33.21185302734375,
+      "learning_rate": 6.374683542809447e-06,
+      "loss": 7.8417,
+      "step": 1420
+    },
+    {
+      "epoch": 0.4321220820427589,
+      "grad_norm": 19.956239700317383,
+      "learning_rate": 6.327555112163761e-06,
+      "loss": 4.3582,
+      "step": 1430
+    },
+    {
+      "epoch": 0.4351439147843167,
+      "grad_norm": 19.256486892700195,
+      "learning_rate": 6.280299445695469e-06,
+      "loss": 5.2,
+      "step": 1440
+    },
+    {
+      "epoch": 0.43816574752587445,
+      "grad_norm": 20.045286178588867,
+      "learning_rate": 6.232921072492319e-06,
+      "loss": 4.3409,
+      "step": 1450
+    },
+    {
+      "epoch": 0.4411875802674322,
+      "grad_norm": 24.16641616821289,
+      "learning_rate": 6.185424533402543e-06,
+      "loss": 4.3162,
+      "step": 1460
+    },
+    {
+      "epoch": 0.44420941300899,
+      "grad_norm": 23.316164016723633,
+      "learning_rate": 6.13781438059966e-06,
+      "loss": 3.5112,
+      "step": 1470
+    },
+    {
+      "epoch": 0.44723124575054773,
+      "grad_norm": 34.204627990722656,
+      "learning_rate": 6.090095177146178e-06,
+      "loss": 5.1696,
+      "step": 1480
+    },
+    {
+      "epoch": 0.45025307849210544,
+      "grad_norm": 17.53434181213379,
+      "learning_rate": 6.042271496556255e-06,
+      "loss": 2.7874,
+      "step": 1490
+    },
+    {
+      "epoch": 0.4532749112336632,
+      "grad_norm": 21.362934112548828,
+      "learning_rate": 5.994347922357372e-06,
+      "loss": 3.8133,
+      "step": 1500
+    },
+    {
+      "epoch": 0.45629674397522096,
+      "grad_norm": 19.935638427734375,
+      "learning_rate": 5.946329047651037e-06,
+      "loss": 3.592,
+      "step": 1510
+    },
+    {
+      "epoch": 0.4593185767167787,
+      "grad_norm": 17.95412826538086,
+      "learning_rate": 5.8982194746725686e-06,
+      "loss": 2.7345,
+      "step": 1520
+    },
+    {
+      "epoch": 0.4623404094583365,
+      "grad_norm": 24.026193618774414,
+      "learning_rate": 5.850023814350007e-06,
+      "loss": 4.2519,
+      "step": 1530
+    },
+    {
+      "epoch": 0.46536224219989425,
+      "grad_norm": 12.00658893585205,
+      "learning_rate": 5.801746685862197e-06,
+      "loss": 6.0717,
+      "step": 1540
+    },
+    {
+      "epoch": 0.468384074941452,
+      "grad_norm": 14.519695281982422,
+      "learning_rate": 5.753392716196069e-06,
+      "loss": 2.8474,
+      "step": 1550
+    },
+    {
+      "epoch": 0.4714059076830098,
+      "grad_norm": 15.277630805969238,
+      "learning_rate": 5.704966539703185e-06,
+      "loss": 3.6301,
+      "step": 1560
+    },
+    {
+      "epoch": 0.4744277404245675,
+      "grad_norm": 17.934938430786133,
+      "learning_rate": 5.656472797655571e-06,
+      "loss": 4.4189,
+      "step": 1570
+    },
+    {
+      "epoch": 0.47744957316612524,
+      "grad_norm": 17.185529708862305,
+      "learning_rate": 5.60791613780088e-06,
+      "loss": 2.7758,
+      "step": 1580
+    },
+    {
+      "epoch": 0.480471405907683,
+      "grad_norm": 25.111557006835938,
+      "learning_rate": 5.5593012139169525e-06,
+      "loss": 4.296,
+      "step": 1590
+    },
+    {
+      "epoch": 0.48349323864924076,
+      "grad_norm": 23.77570343017578,
+      "learning_rate": 5.510632685365777e-06,
+      "loss": 4.4462,
+      "step": 1600
+    },
+    {
+      "epoch": 0.4865150713907985,
+      "grad_norm": 17.37128448486328,
+      "learning_rate": 5.461915216646938e-06,
+      "loss": 2.7426,
+      "step": 1610
+    },
+    {
+      "epoch": 0.4895369041323563,
+      "grad_norm": 23.484580993652344,
+      "learning_rate": 5.41315347695055e-06,
+      "loss": 4.2378,
+      "step": 1620
+    },
+    {
+      "epoch": 0.49255873687391405,
+      "grad_norm": 23.495826721191406,
+      "learning_rate": 5.364352139709758e-06,
+      "loss": 4.8879,
+      "step": 1630
+    },
+    {
+      "epoch": 0.4955805696154718,
+      "grad_norm": 16.23356819152832,
+      "learning_rate": 5.315515882152822e-06,
+      "loss": 3.5359,
+      "step": 1640
+    },
+    {
+      "epoch": 0.4986024023570295,
+      "grad_norm": 16.77799415588379,
+      "learning_rate": 5.266649384854842e-06,
+      "loss": 4.2516,
+      "step": 1650
+    },
+    {
+      "epoch": 0.5016242350985873,
+      "grad_norm": 21.264799118041992,
+      "learning_rate": 5.217757331289165e-06,
+      "loss": 3.6844,
+      "step": 1660
+    },
+    {
+      "epoch": 0.5046460678401451,
+      "grad_norm": 18.198184967041016,
+      "learning_rate": 5.168844407378506e-06,
+      "loss": 4.8485,
+      "step": 1670
+    },
+    {
+      "epoch": 0.5076679005817029,
+      "grad_norm": 13.497072219848633,
+      "learning_rate": 5.119915301045836e-06,
+      "loss": 2.8835,
+      "step": 1680
+    },
+    {
+      "epoch": 0.5106897333232605,
+      "grad_norm": 24.342716217041016,
+      "learning_rate": 5.070974701765089e-06,
+      "loss": 5.1527,
+      "step": 1690
+    },
+    {
+      "epoch": 0.5137115660648183,
+      "grad_norm": 25.917234420776367,
+      "learning_rate": 5.022027300111712e-06,
+      "loss": 4.3981,
+      "step": 1700
+    },
+    {
+      "epoch": 0.516733398806376,
+      "grad_norm": 15.280237197875977,
+      "learning_rate": 4.973077787313099e-06,
+      "loss": 4.4554,
+      "step": 1710
+    },
+    {
+      "epoch": 0.5197552315479338,
+      "grad_norm": 17.290264129638672,
+      "learning_rate": 4.924130854798983e-06,
+      "loss": 5.1108,
+      "step": 1720
+    },
+    {
+      "epoch": 0.5227770642894916,
+      "grad_norm": 15.63051700592041,
+      "learning_rate": 4.875191193751803e-06,
+      "loss": 2.8006,
+      "step": 1730
+    },
+    {
+      "epoch": 0.5257988970310493,
+      "grad_norm": 15.663633346557617,
+      "learning_rate": 4.826263494657077e-06,
+      "loss": 3.4979,
+      "step": 1740
+    },
+    {
+      "epoch": 0.5288207297726071,
+      "grad_norm": 35.42136001586914,
+      "learning_rate": 4.777352446853863e-06,
+      "loss": 4.9996,
+      "step": 1750
+    },
+    {
+      "epoch": 0.5318425625141648,
+      "grad_norm": 23.063594818115234,
+      "learning_rate": 4.72846273808533e-06,
+      "loss": 3.509,
+      "step": 1760
+    },
+    {
+      "epoch": 0.5348643952557226,
+      "grad_norm": 21.706233978271484,
+      "learning_rate": 4.679599054049458e-06,
+      "loss": 3.3899,
+      "step": 1770
+    },
+    {
+      "epoch": 0.5378862279972804,
+      "grad_norm": 20.82579231262207,
+      "learning_rate": 4.630766077949965e-06,
+      "loss": 5.9861,
+      "step": 1780
+    },
+    {
+      "epoch": 0.5409080607388381,
+      "grad_norm": 32.06898880004883,
+      "learning_rate": 4.5819684900474484e-06,
+      "loss": 4.3172,
+      "step": 1790
+    },
+    {
+      "epoch": 0.5439298934803959,
+      "grad_norm": 16.330984115600586,
+      "learning_rate": 4.5332109672108245e-06,
+      "loss": 4.4365,
+      "step": 1800
+    },
+    {
+      "epoch": 0.5469517262219536,
+      "grad_norm": 17.189834594726562,
+      "learning_rate": 4.484498182469085e-06,
+      "loss": 3.6319,
+      "step": 1810
+    },
+    {
+      "epoch": 0.5499735589635114,
+      "grad_norm": 19.211336135864258,
+      "learning_rate": 4.435834804563422e-06,
+      "loss": 5.8999,
+      "step": 1820
+    },
+    {
+      "epoch": 0.5529953917050692,
+      "grad_norm": 26.310638427734375,
+      "learning_rate": 4.387225497499767e-06,
+      "loss": 3.5792,
+      "step": 1830
+    },
+    {
+      "epoch": 0.5560172244466269,
+      "grad_norm": 20.680715560913086,
+      "learning_rate": 4.3386749201017856e-06,
+      "loss": 3.4555,
+      "step": 1840
+    },
+    {
+      "epoch": 0.5590390571881846,
+      "grad_norm": 15.533769607543945,
+      "learning_rate": 4.290187725564356e-06,
+      "loss": 6.0278,
+      "step": 1850
+    },
+    {
+      "epoch": 0.5620608899297423,
+      "grad_norm": 13.684257507324219,
+      "learning_rate": 4.2417685610076135e-06,
+      "loss": 3.4758,
+      "step": 1860
+    },
+    {
+      "epoch": 0.5650827226713001,
+      "grad_norm": 15.711587905883789,
+      "learning_rate": 4.193422067031535e-06,
+      "loss": 4.3166,
+      "step": 1870
+    },
+    {
+      "epoch": 0.5681045554128579,
+      "grad_norm": 18.764991760253906,
+      "learning_rate": 4.145152877271196e-06,
+      "loss": 4.1625,
+      "step": 1880
+    },
+    {
+      "epoch": 0.5711263881544156,
+      "grad_norm": 19.19873809814453,
+      "learning_rate": 4.096965617952667e-06,
+      "loss": 4.4233,
+      "step": 1890
+    },
+    {
+      "epoch": 0.5741482208959734,
+      "grad_norm": 20.817365646362305,
+      "learning_rate": 4.048864907449619e-06,
+      "loss": 3.5268,
+      "step": 1900
+    },
+    {
+      "epoch": 0.5771700536375312,
+      "grad_norm": 18.440645217895508,
+      "learning_rate": 4.000855355840695e-06,
+      "loss": 3.5747,
+      "step": 1910
+    },
+    {
+      "epoch": 0.5801918863790889,
+      "grad_norm": 15.997143745422363,
+      "learning_rate": 3.952941564467665e-06,
+      "loss": 4.2257,
+      "step": 1920
+    },
+    {
+      "epoch": 0.5832137191206467,
+      "grad_norm": 20.629562377929688,
+      "learning_rate": 3.905128125494427e-06,
+      "loss": 4.3136,
+      "step": 1930
+    },
+    {
+      "epoch": 0.5862355518622044,
+      "grad_norm": 33.730995178222656,
+      "learning_rate": 3.8574196214668876e-06,
+      "loss": 4.509,
+      "step": 1940
+    },
+    {
+      "epoch": 0.5892573846037622,
+      "grad_norm": 30.045576095581055,
+      "learning_rate": 3.8098206248737486e-06,
+      "loss": 5.139,
+      "step": 1950
+    },
+    {
+      "epoch": 0.59227921734532,
+      "grad_norm": 23.693470001220703,
+      "learning_rate": 3.7623356977082794e-06,
+      "loss": 2.5913,
+      "step": 1960
+    },
+    {
+      "epoch": 0.5953010500868777,
+      "grad_norm": 18.655092239379883,
+      "learning_rate": 3.714969391031084e-06,
+      "loss": 4.3328,
+      "step": 1970
+    },
+    {
+      "epoch": 0.5983228828284355,
+      "grad_norm": 15.45345687866211,
+      "learning_rate": 3.6677262445339136e-06,
+      "loss": 3.5691,
+      "step": 1980
+    },
+    {
+      "epoch": 0.6013447155699932,
+      "grad_norm": 21.302995681762695,
+      "learning_rate": 3.6206107861045803e-06,
+      "loss": 2.5934,
+      "step": 1990
+    },
+    {
+      "epoch": 0.604366548311551,
+      "grad_norm": 13.75935173034668,
+      "learning_rate": 3.5736275313929826e-06,
+      "loss": 4.3405,
+      "step": 2000
+    },
+    {
+      "epoch": 0.6073883810531087,
+      "grad_norm": 17.593429565429688,
+      "learning_rate": 3.5267809833783213e-06,
+      "loss": 4.8443,
+      "step": 2010
+    },
+    {
+      "epoch": 0.6104102137946664,
+      "grad_norm": 23.467853546142578,
+      "learning_rate": 3.4800756319375326e-06,
+      "loss": 3.4879,
+      "step": 2020
+    },
+    {
+      "epoch": 0.6134320465362242,
+      "grad_norm": 25.12725830078125,
+      "learning_rate": 3.433515953414953e-06,
+      "loss": 2.7966,
+      "step": 2030
+    },
+    {
+      "epoch": 0.616453879277782,
+      "grad_norm": 33.0245475769043,
+      "learning_rate": 3.387106410193308e-06,
+      "loss": 5.8078,
+      "step": 2040
+    },
+    {
+      "epoch": 0.6194757120193397,
+      "grad_norm": 18.8001651763916,
+      "learning_rate": 3.3408514502660195e-06,
+      "loss": 5.2049,
+      "step": 2050
+    },
+    {
+      "epoch": 0.6224975447608975,
+      "grad_norm": 16.787553787231445,
+      "learning_rate": 3.2947555068109057e-06,
+      "loss": 3.3988,
+      "step": 2060
+    },
+    {
+      "epoch": 0.6255193775024552,
+      "grad_norm": 21.532262802124023,
+      "learning_rate": 3.248822997765295e-06,
+      "loss": 2.815,
+      "step": 2070
+    },
+    {
+      "epoch": 0.628541210244013,
+      "grad_norm": 24.630603790283203,
+      "learning_rate": 3.203058325402599e-06,
+      "loss": 4.3332,
+      "step": 2080
+    },
+    {
+      "epoch": 0.6315630429855708,
+      "grad_norm": 16.667922973632812,
+      "learning_rate": 3.1574658759103904e-06,
+      "loss": 4.3038,
+      "step": 2090
+    },
+    {
+      "epoch": 0.6345848757271285,
+      "grad_norm": 20.671772003173828,
+      "learning_rate": 3.1120500189700204e-06,
+      "loss": 3.4132,
+      "step": 2100
+    },
+    {
+      "epoch": 0.6376067084686863,
+      "grad_norm": 21.932987213134766,
+      "learning_rate": 3.066815107337815e-06,
+      "loss": 4.1988,
+      "step": 2110
+    },
+    {
+      "epoch": 0.640628541210244,
+      "grad_norm": 17.348411560058594,
+      "learning_rate": 3.0217654764279114e-06,
+      "loss": 3.5937,
+      "step": 2120
+    },
+    {
+      "epoch": 0.6436503739518018,
+      "grad_norm": 25.625871658325195,
+      "learning_rate": 2.9769054438967192e-06,
+      "loss": 5.9817,
+      "step": 2130
+    },
+    {
+      "epoch": 0.6466722066933596,
+      "grad_norm": 31.0660457611084,
+      "learning_rate": 2.9322393092291256e-06,
+      "loss": 5.6772,
+      "step": 2140
+    },
+    {
+      "epoch": 0.6496940394349173,
+      "grad_norm": 20.511960983276367,
+      "learning_rate": 2.887771353326422e-06,
+      "loss": 4.2915,
+      "step": 2150
+    },
+    {
+      "epoch": 0.6527158721764751,
+      "grad_norm": 17.798234939575195,
+      "learning_rate": 2.8435058380959957e-06,
+      "loss": 2.642,
+      "step": 2160
+    },
+    {
+      "epoch": 0.6557377049180327,
+      "grad_norm": 18.133886337280273,
+      "learning_rate": 2.7994470060428835e-06,
+      "loss": 4.1208,
+      "step": 2170
+    },
+    {
+      "epoch": 0.6587595376595905,
+      "grad_norm": 18.74016571044922,
+      "learning_rate": 2.7555990798631436e-06,
+      "loss": 4.8817,
+      "step": 2180
+    },
+    {
+      "epoch": 0.6617813704011483,
+      "grad_norm": 15.885804176330566,
+      "learning_rate": 2.711966262039145e-06,
+      "loss": 3.3242,
+      "step": 2190
+    },
+    {
+      "epoch": 0.664803203142706,
+      "grad_norm": 24.100414276123047,
+      "learning_rate": 2.668552734436802e-06,
+      "loss": 4.3377,
+      "step": 2200
+    },
+    {
+      "epoch": 0.6678250358842638,
+      "grad_norm": 17.113306045532227,
+      "learning_rate": 2.6253626579047653e-06,
+      "loss": 5.7855,
+      "step": 2210
+    },
+    {
+      "epoch": 0.6708468686258215,
+      "grad_norm": 33.268699645996094,
+      "learning_rate": 2.582400171875638e-06,
+      "loss": 3.4326,
+      "step": 2220
+    },
+    {
+      "epoch": 0.6738687013673793,
+      "grad_norm": 29.673768997192383,
+      "learning_rate": 2.5396693939692474e-06,
+      "loss": 4.8596,
+      "step": 2230
+    },
+    {
+      "epoch": 0.6768905341089371,
+      "grad_norm": 14.550185203552246,
+      "learning_rate": 2.4971744195979985e-06,
+      "loss": 5.1031,
+      "step": 2240
+    },
+    {
+      "epoch": 0.6799123668504948,
+      "grad_norm": 32.16508102416992,
+      "learning_rate": 2.4549193215743706e-06,
+      "loss": 5.833,
+      "step": 2250
+    },
+    {
+      "epoch": 0.6829341995920526,
+      "grad_norm": 18.873088836669922,
+      "learning_rate": 2.4129081497205536e-06,
+      "loss": 3.3544,
+      "step": 2260
+    },
+    {
+      "epoch": 0.6859560323336104,
+      "grad_norm": 31.875137329101562,
+      "learning_rate": 2.3711449304803174e-06,
+      "loss": 4.0864,
+      "step": 2270
+    },
+    {
+      "epoch": 0.6889778650751681,
+      "grad_norm": 27.996572494506836,
+      "learning_rate": 2.329633666533103e-06,
+      "loss": 4.0582,
+      "step": 2280
+    },
+    {
+      "epoch": 0.6919996978167259,
+      "grad_norm": 19.299062728881836,
+      "learning_rate": 2.288378336410398e-06,
+      "loss": 4.2188,
+      "step": 2290
+    },
+    {
+      "epoch": 0.6950215305582836,
+      "grad_norm": 21.146148681640625,
+      "learning_rate": 2.2473828941144277e-06,
+      "loss": 4.8756,
+      "step": 2300
+    },
+    {
+      "epoch": 0.6980433632998414,
+      "grad_norm": 28.3226261138916,
+      "learning_rate": 2.20665126873919e-06,
+      "loss": 3.3593,
+      "step": 2310
+    },
+    {
+      "epoch": 0.7010651960413992,
+      "grad_norm": 16.02470588684082,
+      "learning_rate": 2.1661873640938818e-06,
+      "loss": 4.1255,
+      "step": 2320
+    },
+    {
+      "epoch": 0.7040870287829568,
+      "grad_norm": 21.263837814331055,
+      "learning_rate": 2.1259950583287633e-06,
+      "loss": 4.145,
+      "step": 2330
+    },
+    {
+      "epoch": 0.7071088615245146,
+      "grad_norm": 22.879661560058594,
+      "learning_rate": 2.086078203563439e-06,
+      "loss": 4.7453,
+      "step": 2340
+    },
+    {
+      "epoch": 0.7101306942660723,
+      "grad_norm": 15.726652145385742,
+      "learning_rate": 2.0464406255176967e-06,
+      "loss": 4.019,
+      "step": 2350
+    },
+    {
+      "epoch": 0.7131525270076301,
+      "grad_norm": 30.606904983520508,
+      "learning_rate": 2.0070861231448142e-06,
+      "loss": 4.9014,
+      "step": 2360
+    },
+    {
+      "epoch": 0.7161743597491879,
+      "grad_norm": 17.185054779052734,
+      "learning_rate": 1.968018468267472e-06,
+      "loss": 4.1918,
+      "step": 2370
+    },
+    {
+      "epoch": 0.7191961924907456,
+      "grad_norm": 15.510167121887207,
+      "learning_rate": 1.929241405216254e-06,
+      "loss": 4.0934,
+      "step": 2380
+    },
+    {
+      "epoch": 0.7222180252323034,
+      "grad_norm": 20.12055206298828,
+      "learning_rate": 1.8907586504707776e-06,
+      "loss": 4.701,
+      "step": 2390
+    },
+    {
+      "epoch": 0.7252398579738611,
+      "grad_norm": 19.135282516479492,
+      "learning_rate": 1.8525738923035002e-06,
+      "loss": 2.5439,
+      "step": 2400
+    },
+    {
+      "epoch": 0.7282616907154189,
+      "grad_norm": 19.167003631591797,
+      "learning_rate": 1.8146907904262268e-06,
+      "loss": 4.2791,
+      "step": 2410
+    },
+    {
+      "epoch": 0.7312835234569767,
+      "grad_norm": 24.79986572265625,
+      "learning_rate": 1.7771129756393545e-06,
+      "loss": 3.4256,
+      "step": 2420
+    },
+    {
+      "epoch": 0.7343053561985344,
+      "grad_norm": 20.59393310546875,
+      "learning_rate": 1.7398440494838947e-06,
+      "loss": 3.5206,
+      "step": 2430
+    },
+    {
+      "epoch": 0.7373271889400922,
+      "grad_norm": 25.903627395629883,
+      "learning_rate": 1.7028875838962822e-06,
+      "loss": 4.1281,
+      "step": 2440
+    },
+    {
+      "epoch": 0.74034902168165,
+      "grad_norm": 35.45489501953125,
+      "learning_rate": 1.6662471208660392e-06,
+      "loss": 4.0468,
+      "step": 2450
+    },
+    {
+      "epoch": 0.7433708544232077,
+      "grad_norm": 20.3117618560791,
+      "learning_rate": 1.6299261720963095e-06,
+      "loss": 4.1749,
+      "step": 2460
+    },
+    {
+      "epoch": 0.7463926871647655,
+      "grad_norm": 15.878867149353027,
+      "learning_rate": 1.5939282186672705e-06,
+      "loss": 4.8916,
+      "step": 2470
+    },
+    {
+      "epoch": 0.7494145199063232,
+      "grad_norm": 19.15277099609375,
+      "learning_rate": 1.5582567107025237e-06,
+      "loss": 4.8288,
+      "step": 2480
+    },
+    {
+      "epoch": 0.7524363526478809,
+      "grad_norm": 29.44374656677246,
+      "learning_rate": 1.5229150670384057e-06,
+      "loss": 3.3806,
+      "step": 2490
+    },
+    {
+      "epoch": 0.7554581853894387,
+      "grad_norm": 23.206140518188477,
+      "learning_rate": 1.4879066748963295e-06,
+      "loss": 2.5563,
+      "step": 2500
+    },
+    {
+      "epoch": 0.7584800181309964,
+      "grad_norm": 27.133193969726562,
+      "learning_rate": 1.4532348895581466e-06,
+      "loss": 3.4434,
+      "step": 2510
+    },
+    {
+      "epoch": 0.7615018508725542,
+      "grad_norm": 29.599319458007812,
+      "learning_rate": 1.4189030340445648e-06,
+      "loss": 6.7087,
+      "step": 2520
+    },
+    {
+      "epoch": 0.7645236836141119,
+      "grad_norm": 17.123348236083984,
+      "learning_rate": 1.3849143987966646e-06,
+      "loss": 4.9595,
+      "step": 2530
+    },
+    {
+      "epoch": 0.7675455163556697,
+      "grad_norm": 16.49233627319336,
+      "learning_rate": 1.3512722413605356e-06,
+      "loss": 4.0857,
+      "step": 2540
+    },
+    {
+      "epoch": 0.7705673490972275,
+      "grad_norm": 16.6666316986084,
+      "learning_rate": 1.3179797860750654e-06,
+      "loss": 4.8943,
+      "step": 2550
+    },
+    {
+      "epoch": 0.7735891818387852,
+      "grad_norm": 19.440494537353516,
+      "learning_rate": 1.2850402237629184e-06,
+      "loss": 4.1448,
+      "step": 2560
+    },
+    {
+      "epoch": 0.776611014580343,
+      "grad_norm": 14.674943923950195,
+      "learning_rate": 1.2524567114247083e-06,
+      "loss": 3.3491,
+      "step": 2570
+    },
+    {
+      "epoch": 0.7796328473219007,
+      "grad_norm": 16.349637985229492,
+      "learning_rate": 1.2202323719364324e-06,
+      "loss": 3.2897,
+      "step": 2580
+    },
+    {
+      "epoch": 0.7826546800634585,
+      "grad_norm": 19.67890739440918,
+      "learning_rate": 1.1883702937501708e-06,
+      "loss": 4.0901,
+      "step": 2590
+    },
+    {
+      "epoch": 0.7856765128050163,
+      "grad_norm": 21.339618682861328,
+      "learning_rate": 1.1568735305980694e-06,
+      "loss": 4.1003,
+      "step": 2600
+    },
+    {
+      "epoch": 0.788698345546574,
+      "grad_norm": 21.269119262695312,
+      "learning_rate": 1.1257451011996807e-06,
+      "loss": 3.4165,
+      "step": 2610
+    },
+    {
+      "epoch": 0.7917201782881318,
+      "grad_norm": 33.041419982910156,
+      "learning_rate": 1.0949879889726295e-06,
+      "loss": 3.4622,
+      "step": 2620
+    },
+    {
+      "epoch": 0.7947420110296896,
+      "grad_norm": 28.960115432739258,
+      "learning_rate": 1.0646051417466801e-06,
+      "loss": 3.4136,
+      "step": 2630
+    },
+    {
+      "epoch": 0.7977638437712473,
+      "grad_norm": 24.76239013671875,
+      "learning_rate": 1.0345994714812135e-06,
+      "loss": 4.1335,
+      "step": 2640
+    },
+    {
+      "epoch": 0.800785676512805,
+      "grad_norm": 15.773963928222656,
+      "learning_rate": 1.0049738539861332e-06,
+      "loss": 3.2818,
+      "step": 2650
+    },
+    {
+      "epoch": 0.8038075092543627,
+      "grad_norm": 21.248395919799805,
+      "learning_rate": 9.757311286462428e-07,
+      "loss": 4.1348,
+      "step": 2660
+    },
+    {
+      "epoch": 0.8068293419959205,
+      "grad_norm": 23.75290298461914,
+      "learning_rate": 9.468740981491143e-07,
+      "loss": 4.1947,
+      "step": 2670
+    },
+    {
+      "epoch": 0.8098511747374783,
+      "grad_norm": 16.7280330657959,
+      "learning_rate": 9.1840552821647e-07,
+      "loss": 4.0364,
+      "step": 2680
+    },
+    {
+      "epoch": 0.812873007479036,
+      "grad_norm": 17.696247100830078,
+      "learning_rate": 8.903281473391152e-07,
+      "loss": 3.3641,
+      "step": 2690
+    },
+    {
+      "epoch": 0.8158948402205938,
+      "grad_norm": 16.840299606323242,
+      "learning_rate": 8.62644646515427e-07,
+      "loss": 5.7446,
+      "step": 2700
+    },
+    {
+      "epoch": 0.8189166729621515,
+      "grad_norm": 13.25534725189209,
+      "learning_rate": 8.353576789934436e-07,
+      "loss": 3.3763,
+      "step": 2710
+    },
+    {
+      "epoch": 0.8219385057037093,
+      "grad_norm": 19.88932991027832,
+      "learning_rate": 8.084698600165797e-07,
+      "loss": 3.5133,
+      "step": 2720
+    },
+    {
+      "epoch": 0.8249603384452671,
+      "grad_norm": 17.921199798583984,
+      "learning_rate": 7.819837665729596e-07,
+      "loss": 4.1018,
+      "step": 2730
+    },
+    {
+      "epoch": 0.8279821711868248,
+      "grad_norm": 29.57664680480957,
+      "learning_rate": 7.559019371484521e-07,
+      "loss": 3.3378,
+      "step": 2740
+    },
+    {
+      "epoch": 0.8310040039283826,
+      "grad_norm": 17.720863342285156,
+      "learning_rate": 7.302268714833622e-07,
+      "loss": 4.1487,
+      "step": 2750
+    },
+    {
+      "epoch": 0.8340258366699403,
+      "grad_norm": 17.34684944152832,
+      "learning_rate": 7.049610303328541e-07,
+      "loss": 3.5199,
+      "step": 2760
+    },
+    {
+      "epoch": 0.8370476694114981,
+      "grad_norm": 16.739910125732422,
+      "learning_rate": 6.80106835231113e-07,
+      "loss": 4.2899,
+      "step": 2770
+    },
+    {
+      "epoch": 0.8400695021530559,
+      "grad_norm": 17.1294002532959,
+      "learning_rate": 6.556666682592494e-07,
+      "loss": 3.3016,
+      "step": 2780
+    },
+    {
+      "epoch": 0.8430913348946136,
+      "grad_norm": 14.801079750061035,
+      "learning_rate": 6.316428718170037e-07,
+      "loss": 2.4169,
+      "step": 2790
+    },
+    {
+      "epoch": 0.8461131676361714,
+      "grad_norm": 19.354856491088867,
+      "learning_rate": 6.080377483982425e-07,
+      "loss": 3.2883,
+      "step": 2800
+    },
+    {
+      "epoch": 0.849135000377729,
+      "grad_norm": 17.925838470458984,
+      "learning_rate": 5.848535603702798e-07,
+      "loss": 3.3497,
+      "step": 2810
+    },
+    {
+      "epoch": 0.8521568331192868,
+      "grad_norm": 20.340959548950195,
+      "learning_rate": 5.62092529757054e-07,
+      "loss": 6.4132,
+      "step": 2820
+    },
+    {
+      "epoch": 0.8551786658608446,
+      "grad_norm": 21.507797241210938,
+      "learning_rate": 5.397568380261559e-07,
+      "loss": 2.3404,
+      "step": 2830
+    },
+    {
+      "epoch": 0.8582004986024023,
+      "grad_norm": 16.9514102935791,
+      "learning_rate": 5.178486258797555e-07,
+      "loss": 4.0876,
+      "step": 2840
+    },
+    {
+      "epoch": 0.8612223313439601,
+      "grad_norm": 14.505171775817871,
+      "learning_rate": 4.963699930494365e-07,
+      "loss": 3.3715,
+      "step": 2850
+    },
+    {
+      "epoch": 0.8642441640855179,
+      "grad_norm": 22.551313400268555,
+      "learning_rate": 4.75322998094942e-07,
+      "loss": 4.2347,
+      "step": 2860
+    },
+    {
+      "epoch": 0.8672659968270756,
+      "grad_norm": 20.145078659057617,
+      "learning_rate": 4.5470965820689384e-07,
+      "loss": 2.5903,
+      "step": 2870
+    },
+    {
+      "epoch": 0.8702878295686334,
+      "grad_norm": 17.447914123535156,
+      "learning_rate": 4.345319490134453e-07,
+      "loss": 3.0177,
+      "step": 2880
+    },
+    {
+      "epoch": 0.8733096623101911,
+      "grad_norm": 16.10365104675293,
+      "learning_rate": 4.147918043909405e-07,
+      "loss": 4.764,
+      "step": 2890
+    },
+    {
+      "epoch": 0.8763314950517489,
+      "grad_norm": 19.066129684448242,
+      "learning_rate": 3.9549111627856794e-07,
+      "loss": 4.7699,
+      "step": 2900
+    },
+    {
+      "epoch": 0.8793533277933067,
+      "grad_norm": 19.604887008666992,
+      "learning_rate": 3.766317344970288e-07,
+      "loss": 4.1165,
+      "step": 2910
+    },
+    {
+      "epoch": 0.8823751605348644,
+      "grad_norm": 17.465734481811523,
+      "learning_rate": 3.582154665712473e-07,
+      "loss": 2.4443,
+      "step": 2920
+    },
+    {
+      "epoch": 0.8853969932764222,
+      "grad_norm": 22.400236129760742,
+      "learning_rate": 3.402440775571364e-07,
+      "loss": 4.0664,
+      "step": 2930
+    },
+    {
+      "epoch": 0.88841882601798,
+      "grad_norm": 21.420312881469727,
+      "learning_rate": 3.227192898724252e-07,
+      "loss": 5.7203,
+      "step": 2940
+    },
+    {
+      "epoch": 0.8914406587595377,
+      "grad_norm": 23.331478118896484,
+      "learning_rate": 3.056427831315878e-07,
+      "loss": 3.367,
+      "step": 2950
+    },
+    {
+      "epoch": 0.8944624915010955,
+      "grad_norm": 21.29648208618164,
+      "learning_rate": 2.890161939848535e-07,
+      "loss": 4.1604,
+      "step": 2960
+    },
+    {
+      "epoch": 0.8974843242426531,
+      "grad_norm": 15.172201156616211,
+      "learning_rate": 2.72841115961357e-07,
+      "loss": 4.2335,
+      "step": 2970
+    },
+    {
+      "epoch": 0.9005061569842109,
+      "grad_norm": 16.736038208007812,
+      "learning_rate": 2.5711909931640633e-07,
+      "loss": 3.9793,
+      "step": 2980
+    },
+    {
+      "epoch": 0.9035279897257686,
+      "grad_norm": 22.6779727935791,
+      "learning_rate": 2.418516508829e-07,
+      "loss": 2.4922,
+      "step": 2990
+    },
+    {
+      "epoch": 0.9065498224673264,
+      "grad_norm": 32.2912712097168,
+      "learning_rate": 2.270402339269162e-07,
+      "loss": 5.6454,
+      "step": 3000
+    },
+    {
+      "epoch": 0.9095716552088842,
+      "grad_norm": 18.107574462890625,
+      "learning_rate": 2.126862680074643e-07,
+      "loss": 5.0056,
+      "step": 3010
+    },
+    {
+      "epoch": 0.9125934879504419,
+      "grad_norm": 32.63033676147461,
+      "learning_rate": 1.9879112884043317e-07,
+      "loss": 2.5369,
+      "step": 3020
+    },
+    {
+      "epoch": 0.9156153206919997,
+      "grad_norm": 18.089956283569336,
+      "learning_rate": 1.853561481667404e-07,
+      "loss": 2.4556,
+      "step": 3030
+    },
+    {
+      "epoch": 0.9186371534335575,
+      "grad_norm": 13.772138595581055,
+      "learning_rate": 1.7238261362469256e-07,
+      "loss": 3.0884,
+      "step": 3040
+    },
+    {
+      "epoch": 0.9216589861751152,
+      "grad_norm": 22.537776947021484,
+      "learning_rate": 1.5987176862657883e-07,
+      "loss": 3.2805,
+      "step": 3050
+    },
+    {
+      "epoch": 0.924680818916673,
+      "grad_norm": 30.13243865966797,
+      "learning_rate": 1.4782481223949597e-07,
+      "loss": 3.2507,
+      "step": 3060
+    },
+    {
+      "epoch": 0.9277026516582307,
+      "grad_norm": 20.858510971069336,
+      "learning_rate": 1.3624289907042787e-07,
+      "loss": 4.1981,
+      "step": 3070
+    },
+    {
+      "epoch": 0.9307244843997885,
+      "grad_norm": 30.669658660888672,
+      "learning_rate": 1.2512713915559027e-07,
+      "loss": 4.9341,
+      "step": 3080
+    },
+    {
+      "epoch": 0.9337463171413463,
+      "grad_norm": 32.03891372680664,
+      "learning_rate": 1.1447859785403359e-07,
+      "loss": 4.8266,
+      "step": 3090
+    },
+    {
+      "epoch": 0.936768149882904,
+      "grad_norm": 18.382429122924805,
+      "learning_rate": 1.0429829574554573e-07,
+      "loss": 3.4044,
+      "step": 3100
+    },
+    {
+      "epoch": 0.9397899826244618,
+      "grad_norm": 16.341550827026367,
+      "learning_rate": 9.458720853282977e-08,
+      "loss": 4.1438,
+      "step": 3110
+    },
+    {
+      "epoch": 0.9428118153660195,
+      "grad_norm": 32.575286865234375,
+      "learning_rate": 8.534626694799485e-08,
+      "loss": 5.6917,
+      "step": 3120
+    },
+    {
+      "epoch": 0.9458336481075772,
+      "grad_norm": 19.515989303588867,
+      "learning_rate": 7.657635666335317e-08,
+      "loss": 2.5437,
+      "step": 3130
+    },
+    {
+      "epoch": 0.948855480849135,
+      "grad_norm": 18.81734275817871,
+      "learning_rate": 6.827831820653163e-08,
+      "loss": 2.5297,
+      "step": 3140
+    },
+    {
+      "epoch": 0.9518773135906927,
+      "grad_norm": 20.44892120361328,
+      "learning_rate": 6.045294687991643e-08,
+      "loss": 5.3046,
+      "step": 3150
+    },
+    {
+      "epoch": 0.9548991463322505,
+      "grad_norm": 30.222261428833008,
+      "learning_rate": 5.310099268443114e-08,
+      "loss": 7.1585,
+      "step": 3160
+    },
+    {
+      "epoch": 0.9579209790738082,
+      "grad_norm": 22.93487548828125,
+      "learning_rate": 4.622316024765039e-08,
+      "loss": 3.9296,
+      "step": 3170
+    },
+    {
+      "epoch": 0.960942811815366,
+      "grad_norm": 20.129398345947266,
+      "learning_rate": 3.982010875626885e-08,
+      "loss": 3.2971,
+      "step": 3180
+    },
+    {
+      "epoch": 0.9639646445569238,
+      "grad_norm": 20.64815330505371,
+      "learning_rate": 3.389245189292622e-08,
+      "loss": 4.1501,
+      "step": 3190
+    },
+    {
+      "epoch": 0.9669864772984815,
+      "grad_norm": 19.435129165649414,
+      "learning_rate": 2.8440757777385976e-08,
+      "loss": 4.9552,
+      "step": 3200
+    },
+    {
+      "epoch": 0.9700083100400393,
+      "grad_norm": 17.719867706298828,
+      "learning_rate": 2.3465548912088298e-08,
+      "loss": 2.6329,
+      "step": 3210
+    },
+    {
+      "epoch": 0.973030142781597,
+      "grad_norm": 21.178937911987305,
+      "learning_rate": 1.896730213207132e-08,
+      "loss": 4.0836,
+      "step": 3220
+    },
+    {
+      "epoch": 0.9760519755231548,
+      "grad_norm": 16.906330108642578,
+      "learning_rate": 1.4946448559270964e-08,
+      "loss": 2.397,
+      "step": 3230
+    },
+    {
+      "epoch": 0.9790738082647126,
+      "grad_norm": 23.301292419433594,
+      "learning_rate": 1.1403373561199583e-08,
+      "loss": 4.2365,
+      "step": 3240
+    },
+    {
+      "epoch": 0.9820956410062703,
+      "grad_norm": 20.07245635986328,
+      "learning_rate": 8.338416714013254e-09,
+      "loss": 3.444,
+      "step": 3250
+    },
+    {
+      "epoch": 0.9851174737478281,
+      "grad_norm": 16.27911949157715,
+      "learning_rate": 5.751871769965056e-09,
+      "loss": 5.5038,
+      "step": 3260
+    },
+    {
+      "epoch": 0.9881393064893859,
+      "grad_norm": 21.404827117919922,
+      "learning_rate": 3.643986629253138e-09,
+      "loss": 4.1734,
+      "step": 3270
+    },
+    {
+      "epoch": 0.9911611392309436,
+      "grad_norm": 32.63972473144531,
+      "learning_rate": 2.014963316257501e-09,
+      "loss": 4.9837,
+      "step": 3280
+    },
+    {
+      "epoch": 0.9941829719725013,
+      "grad_norm": 19.831165313720703,
+      "learning_rate": 8.649579601810454e-10,
+      "loss": 4.9867,
+      "step": 3290
+    },
+    {
+      "epoch": 0.997204804714059,
+      "grad_norm": 21.53673553466797,
+      "learning_rate": 1.9408078008431587e-10,
+      "loss": 3.3738,
+      "step": 3300
+    },
+    {
+      "epoch": 0.9996222709073053,
+      "step": 3308,
+      "total_flos": 1.175877708593234e+19,
+      "train_loss": 4.4771003486744005,
+      "train_runtime": 52781.7624,
+      "train_samples_per_second": 4.013,
+      "train_steps_per_second": 0.063
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3309,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.175877708593234e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

Order_Ablations/E115/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5862fe46b077140f25daedc4df5e3d677f94e6124396e1490481d467231aacc3
+size 7697

Order_Ablations/E115_smoke100/smoke100.log

@@ -0,0 +1,214 @@
+/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
+  import pynvml  # type: ignore[import]
+/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
+  import pynvml  # type: ignore[import]
+Trainer._get_train_sampler replaced with custom implementation.
+[2026-01-27 16:42:03,814] [INFO] [real_accelerator.py:222:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2026-01-27 16:42:05,037] [INFO] [comm.py:658:init_distributed] cdb=None
+[2026-01-27 16:42:05,037] [INFO] [comm.py:689:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
+Warning: FlashAttention 3 is not available, falling back to PyTorch's scaled_dot_product_attention
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+
+Some weights of Qwen2_5_VLForConditionalGenerationWithVGGT were not initialized from the model checkpoint at Qwen/Qwen2.5-VL-3B-Instruct and are newly initialized: ['geometry_encoder.vggt.aggregator.camera_token', 'geometry_encoder.vggt.aggregator.frame_blocks.0.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.0.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.0.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.0.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.0.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.0.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.0.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.0.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.0.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.0.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.0.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.0.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.0.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.0.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.0.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.0.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.0.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.0.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.1.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.1.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.1.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.1.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.1.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.1.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.1.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.1.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.1.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.1.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.1.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.1.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.1.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.1.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.1.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.1.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.1.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.1.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.10.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.10.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.10.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.10.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.10.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.10.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.10.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.10.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.10.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.10.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.10.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.10.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.10.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.10.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.10.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.10.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.10.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.10.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.11.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.11.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.11.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.11.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.11.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.11.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.11.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.11.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.11.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.11.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.11.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.11.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.11.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.11.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.11.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.11.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.11.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.11.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.12.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.12.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.12.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.12.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.12.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.12.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.12.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.12.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.12.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.12.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.12.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.12.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.12.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.12.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.12.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.12.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.12.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.12.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.13.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.13.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.13.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.13.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.13.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.13.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.13.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.13.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.13.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.13.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.13.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.13.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.13.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.13.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.13.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.13.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.13.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.13.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.14.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.14.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.14.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.14.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.14.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.14.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.14.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.14.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.14.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.14.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.14.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.14.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.14.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.14.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.14.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.14.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.14.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.14.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.15.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.15.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.15.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.15.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.15.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.15.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.15.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.15.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.15.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.15.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.15.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.15.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.15.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.15.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.15.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.15.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.15.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.15.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.16.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.16.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.16.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.16.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.16.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.16.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.16.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.16.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.16.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.16.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.16.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.16.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.16.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.16.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.16.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.16.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.16.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.16.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.17.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.17.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.17.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.17.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.17.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.17.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.17.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.17.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.17.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.17.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.17.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.17.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.17.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.17.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.17.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.17.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.17.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.17.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.18.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.18.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.18.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.18.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.18.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.18.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.18.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.18.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.18.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.18.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.18.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.18.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.18.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.18.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.18.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.18.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.18.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.18.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.19.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.19.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.19.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.19.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.19.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.19.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.19.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.19.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.19.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.19.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.19.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.19.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.19.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.19.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.19.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.19.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.19.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.19.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.2.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.2.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.2.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.2.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.2.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.2.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.2.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.2.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.2.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.2.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.2.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.2.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.2.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.2.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.2.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.2.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.2.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.2.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.20.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.20.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.20.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.20.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.20.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.20.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.20.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.20.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.20.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.20.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.20.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.20.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.20.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.20.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.20.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.20.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.20.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.20.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.21.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.21.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.21.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.21.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.21.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.21.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.21.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.21.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.21.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.21.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.21.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.21.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.21.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.21.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.21.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.21.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.21.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.21.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.22.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.22.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.22.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.22.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.22.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.22.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.22.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.22.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.22.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.22.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.22.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.22.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.22.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.22.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.22.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.22.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.22.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.22.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.23.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.23.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.23.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.23.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.23.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.23.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.23.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.23.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.23.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.23.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.23.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.23.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.23.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.23.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.23.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.23.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.23.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.23.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.3.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.3.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.3.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.3.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.3.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.3.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.3.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.3.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.3.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.3.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.3.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.3.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.3.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.3.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.3.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.3.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.3.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.3.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.4.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.4.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.4.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.4.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.4.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.4.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.4.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.4.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.4.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.4.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.4.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.4.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.4.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.4.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.4.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.4.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.4.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.4.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.5.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.5.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.5.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.5.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.5.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.5.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.5.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.5.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.5.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.5.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.5.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.5.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.5.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.5.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.5.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.5.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.5.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.5.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.6.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.6.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.6.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.6.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.6.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.6.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.6.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.6.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.6.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.6.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.6.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.6.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.6.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.6.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.6.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.6.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.6.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.6.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.7.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.7.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.7.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.7.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.7.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.7.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.7.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.7.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.7.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.7.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.7.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.7.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.7.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.7.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.7.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.7.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.7.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.7.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.8.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.8.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.8.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.8.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.8.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.8.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.8.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.8.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.8.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.8.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.8.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.8.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.8.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.8.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.8.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.8.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.8.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.8.norm2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.9.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.9.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.9.attn.proj.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.9.attn.proj.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.9.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.9.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.9.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.9.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.9.ls1.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.9.ls2.gamma', 'geometry_encoder.vggt.aggregator.frame_blocks.9.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.9.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.9.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.9.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.9.norm1.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.9.norm1.weight', 'geometry_encoder.vggt.aggregator.frame_blocks.9.norm2.bias', 'geometry_encoder.vggt.aggregator.frame_blocks.9.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.0.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.0.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.0.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.0.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.0.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.0.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.0.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.0.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.0.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.0.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.0.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.0.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.0.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.0.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.0.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.0.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.0.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.0.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.1.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.1.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.1.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.1.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.1.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.1.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.1.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.1.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.1.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.1.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.1.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.1.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.1.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.1.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.1.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.1.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.1.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.1.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.10.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.10.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.10.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.10.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.10.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.10.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.10.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.10.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.10.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.10.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.10.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.10.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.10.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.10.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.10.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.10.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.10.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.10.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.11.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.11.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.11.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.11.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.11.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.11.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.11.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.11.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.11.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.11.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.11.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.11.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.11.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.11.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.11.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.11.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.11.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.11.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.12.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.12.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.12.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.12.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.12.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.12.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.12.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.12.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.12.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.12.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.12.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.12.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.12.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.12.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.12.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.12.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.12.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.12.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.13.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.13.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.13.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.13.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.13.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.13.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.13.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.13.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.13.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.13.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.13.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.13.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.13.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.13.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.13.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.13.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.13.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.13.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.14.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.14.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.14.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.14.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.14.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.14.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.14.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.14.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.14.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.14.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.14.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.14.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.14.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.14.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.14.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.14.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.14.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.14.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.15.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.15.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.15.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.15.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.15.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.15.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.15.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.15.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.15.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.15.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.15.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.15.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.15.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.15.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.15.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.15.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.15.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.15.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.16.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.16.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.16.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.16.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.16.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.16.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.16.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.16.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.16.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.16.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.16.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.16.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.16.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.16.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.16.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.16.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.16.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.16.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.17.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.17.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.17.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.17.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.17.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.17.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.17.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.17.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.17.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.17.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.17.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.17.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.17.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.17.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.17.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.17.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.17.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.17.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.18.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.18.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.18.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.18.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.18.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.18.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.18.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.18.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.18.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.18.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.18.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.18.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.18.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.18.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.18.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.18.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.18.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.18.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.19.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.19.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.19.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.19.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.19.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.19.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.19.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.19.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.19.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.19.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.19.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.19.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.19.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.19.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.19.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.19.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.19.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.19.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.2.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.2.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.2.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.2.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.2.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.2.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.2.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.2.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.2.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.2.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.2.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.2.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.2.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.2.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.2.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.2.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.2.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.2.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.20.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.20.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.20.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.20.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.20.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.20.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.20.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.20.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.20.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.20.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.20.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.20.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.20.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.20.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.20.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.20.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.20.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.20.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.21.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.21.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.21.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.21.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.21.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.21.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.21.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.21.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.21.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.21.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.21.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.21.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.21.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.21.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.21.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.21.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.21.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.21.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.22.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.22.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.22.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.22.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.22.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.22.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.22.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.22.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.22.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.22.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.22.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.22.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.22.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.22.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.22.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.22.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.22.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.22.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.23.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.23.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.23.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.23.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.23.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.23.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.23.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.23.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.23.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.23.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.23.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.23.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.23.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.23.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.23.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.23.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.23.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.23.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.3.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.3.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.3.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.3.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.3.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.3.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.3.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.3.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.3.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.3.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.3.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.3.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.3.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.3.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.3.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.3.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.3.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.3.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.4.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.4.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.4.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.4.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.4.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.4.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.4.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.4.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.4.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.4.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.4.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.4.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.4.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.4.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.4.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.4.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.4.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.4.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.5.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.5.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.5.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.5.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.5.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.5.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.5.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.5.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.5.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.5.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.5.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.5.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.5.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.5.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.5.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.5.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.5.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.5.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.6.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.6.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.6.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.6.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.6.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.6.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.6.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.6.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.6.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.6.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.6.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.6.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.6.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.6.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.6.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.6.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.6.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.6.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.7.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.7.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.7.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.7.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.7.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.7.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.7.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.7.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.7.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.7.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.7.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.7.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.7.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.7.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.7.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.7.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.7.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.7.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.8.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.8.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.8.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.8.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.8.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.8.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.8.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.8.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.8.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.8.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.8.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.8.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.8.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.8.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.8.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.8.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.8.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.8.norm2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.9.attn.k_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.9.attn.k_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.9.attn.proj.bias', 'geometry_encoder.vggt.aggregator.global_blocks.9.attn.proj.weight', 'geometry_encoder.vggt.aggregator.global_blocks.9.attn.q_norm.bias', 'geometry_encoder.vggt.aggregator.global_blocks.9.attn.q_norm.weight', 'geometry_encoder.vggt.aggregator.global_blocks.9.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.global_blocks.9.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.global_blocks.9.ls1.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.9.ls2.gamma', 'geometry_encoder.vggt.aggregator.global_blocks.9.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.9.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.9.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.9.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.global_blocks.9.norm1.bias', 'geometry_encoder.vggt.aggregator.global_blocks.9.norm1.weight', 'geometry_encoder.vggt.aggregator.global_blocks.9.norm2.bias', 'geometry_encoder.vggt.aggregator.global_blocks.9.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.0.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.1.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.10.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.11.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.12.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.13.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.14.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.15.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.16.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.17.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.18.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.19.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.2.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.20.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.21.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.22.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.23.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.3.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.4.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.5.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.6.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.7.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.8.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.attn.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.attn.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.attn.qkv.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.attn.qkv.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.ls1.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.ls2.gamma', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.mlp.fc1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.mlp.fc1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.mlp.fc2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.mlp.fc2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.norm1.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.norm1.weight', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.norm2.bias', 'geometry_encoder.vggt.aggregator.patch_embed.blocks.9.norm2.weight', 'geometry_encoder.vggt.aggregator.patch_embed.cls_token', 'geometry_encoder.vggt.aggregator.patch_embed.mask_token', 'geometry_encoder.vggt.aggregator.patch_embed.norm.bias', 'geometry_encoder.vggt.aggregator.patch_embed.norm.weight', 'geometry_encoder.vggt.aggregator.patch_embed.patch_embed.proj.bias', 'geometry_encoder.vggt.aggregator.patch_embed.patch_embed.proj.weight', 'geometry_encoder.vggt.aggregator.patch_embed.pos_embed', 'geometry_encoder.vggt.aggregator.patch_embed.register_tokens', 'geometry_encoder.vggt.aggregator.register_token', 'language_feature_fusion.fusion_layers.0.0.geo_ln.weight', 'language_feature_fusion.fusion_layers.0.0.geo_mlp.0.bias', 'language_feature_fusion.fusion_layers.0.0.geo_mlp.0.weight', 'language_feature_fusion.fusion_layers.0.0.geo_mlp.2.bias', 'language_feature_fusion.fusion_layers.0.0.geo_mlp.2.weight', 'language_feature_fusion.fusion_layers.1.0.geo_ln.weight', 'language_feature_fusion.fusion_layers.1.0.geo_mlp.0.bias', 'language_feature_fusion.fusion_layers.1.0.geo_mlp.0.weight', 'language_feature_fusion.fusion_layers.1.0.geo_mlp.2.bias', 'language_feature_fusion.fusion_layers.1.0.geo_mlp.2.weight', 'language_feature_fusion.fusion_layers.2.0.geo_ln.weight', 'language_feature_fusion.fusion_layers.2.0.geo_mlp.0.bias', 'language_feature_fusion.fusion_layers.2.0.geo_mlp.0.weight', 'language_feature_fusion.fusion_layers.2.0.geo_mlp.2.bias', 'language_feature_fusion.fusion_layers.2.0.geo_mlp.2.weight', 'multi_layer_feature_fusion.fusion_layers.0.0.geo_ln.weight', 'multi_layer_feature_fusion.fusion_layers.0.0.geo_mlp.0.bias', 'multi_layer_feature_fusion.fusion_layers.0.0.geo_mlp.0.weight', 'multi_layer_feature_fusion.fusion_layers.0.0.geo_mlp.2.bias', 'multi_layer_feature_fusion.fusion_layers.0.0.geo_mlp.2.weight', 'multi_layer_feature_fusion.fusion_layers.1.0.geo_ln.weight', 'multi_layer_feature_fusion.fusion_layers.1.0.geo_mlp.0.bias', 'multi_layer_feature_fusion.fusion_layers.1.0.geo_mlp.0.weight', 'multi_layer_feature_fusion.fusion_layers.1.0.geo_mlp.2.bias', 'multi_layer_feature_fusion.fusion_layers.1.0.geo_mlp.2.weight', 'multi_layer_feature_fusion.fusion_layers.2.0.geo_ln.weight', 'multi_layer_feature_fusion.fusion_layers.2.0.geo_mlp.0.bias', 'multi_layer_feature_fusion.fusion_layers.2.0.geo_mlp.0.weight', 'multi_layer_feature_fusion.fusion_layers.2.0.geo_mlp.2.bias', 'multi_layer_feature_fusion.fusion_layers.2.0.geo_mlp.2.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.50, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.
+Vision Module - Attention Blocks:
+Trainable Block Indices: None
+Non-Trainable Block Indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
+Merger Module Trainable: False
+LLM Module - Embed Tokens Trainable: True
+LLM Module - Trainable Layer Indices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35]
+LLM Module - Non-Trainable Layer Indices: None
+Qwen2_5_VLConfig {
+  "_attn_implementation_autoset": true,
+  "architectures": [
+    "Qwen2_5_VLForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "feature_fusion_method": "deepstack_language_add",
+  "fusion_num_layers": 1,
+  "geometry_encoder_layers": [
+    11,
+    17,
+    23
+  ],
+  "geometry_encoder_type": "vggt",
+  "geometry_fusion_layers": [
+    0,
+    1,
+    2
+  ],
+  "geometry_merger_type": "mlp",
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_token_id": 151655,
+  "include_camera_token": false,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 128000,
+  "max_window_layers": 70,
+  "model_type": "qwen2_5_vl",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "pos_encoding_type": "none",
+  "reference_frame": "first",
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "mrope_section": [
+      16,
+      24,
+      24
+    ],
+    "rope_type": "default",
+    "type": "default"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.50.0",
+  "use_cache": false,
+  "use_geometry_encoder": true,
+  "use_sliding_window": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "depth": 32,
+    "fullatt_block_indexes": [
+      7,
+      15,
+      23,
+      31
+    ],
+    "hidden_act": "silu",
+    "hidden_size": 1280,
+    "in_channels": 3,
+    "in_chans": 3,
+    "intermediate_size": 3420,
+    "model_type": "qwen2_5_vl",
+    "num_heads": 16,
+    "out_hidden_size": 2048,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "temporal_patch_size": 2,
+    "tokens_per_second": 2,
+    "torch_dtype": "bfloat16",
+    "window_size": 112
+  },
+  "vision_end_token_id": 151653,
+  "vision_language_fusion_layers": null,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
+  "vocab_size": 151936
+}
+
+Loading datasets: [{'annotation_path': 'data/train/spar_234k.json', 'data_path': 'data/media', 'tag': '3d', 'sampling_rate': 0.6, 'dataset_name': 'spar_234k'}, {'annotation_path': 'data/train/llava_hound_64k.json', 'data_path': 'data/media', 'tag': '2d', 'sampling_rate': 0.6, 'dataset_name': 'llava_hound_64k'}, {'annotation_path': 'data/vlm3r/annotations/vsibench_train/merged_qa_scannet_train.json', 'data_path': 'data/vlm3r/media', 'tag': '3d', 'sampling_rate': 0.6, 'dataset_name': 'vlm3r_scannet'}, {'annotation_path': 'data/vsi_590k/annotations/vsi_appearance_order_vsibench_scannet.json', 'data_path': 'data/vsi_590k/media', 'tag': '3d', 'sampling_rate': 0.5, 'dataset_name': 'vsi_appr_order'}]
+sampling 140566 examples from dataset {'annotation_path': 'data/train/spar_234k.json', 'data_path': 'data/media', 'tag': '3d', 'sampling_rate': 0.6, 'dataset_name': 'spar_234k'}
+sampling 38250 examples from dataset {'annotation_path': 'data/train/llava_hound_64k.json', 'data_path': 'data/media', 'tag': '2d', 'sampling_rate': 0.6, 'dataset_name': 'llava_hound_64k'}
+sampling 31067 examples from dataset {'annotation_path': 'data/vlm3r/annotations/vsibench_train/merged_qa_scannet_train.json', 'data_path': 'data/vlm3r/media', 'tag': '3d', 'sampling_rate': 0.6, 'dataset_name': 'vlm3r_scannet'}
+sampling 1909 examples from dataset {'annotation_path': 'data/vsi_590k/annotations/vsi_appearance_order_vsibench_scannet.json', 'data_path': 'data/vsi_590k/media', 'tag': '3d', 'sampling_rate': 0.5, 'dataset_name': 'vsi_appr_order'}
+Total training samples: 211792
+Formatting inputs...Skip in lazy mode
+
  0%|          | 0/100 [00:00<?, ?it/s]/usr/local/lib/python3.12/dist-packages/torch/utils/checkpoint.py:85: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn(
+/usr/local/lib/python3.12/dist-packages/torch/utils/checkpoint.py:85: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn(
+/workspace/src/qwen_vl/model/geometry_encoders/vggt_encoder.py:68: FutureWarning: `torch.cuda.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cuda', args...)` instead.
+  with torch.cuda.amp.autocast(dtype=dtype):
+
  1%|          | 1/100 [00:08<13:56,  8.45s/it]
  2%|▏         | 2/100 [00:09<06:21,  3.89s/it]
  3%|▎         | 3/100 [00:09<03:49,  2.36s/it]
  4%|▍         | 4/100 [00:10<02:37,  1.64s/it][Try #0] Failed to fetch sample 84896. Exception: [Errno 2] No such file or directory: 'data/media/spar/scannetpp/images/eb4bc76767/image_color/1250.jpg'
+
  5%|▌         | 5/100 [00:10<01:59,  1.26s/it][Try #1] Failed to fetch sample 84896. Exception: [Errno 2] No such file or directory: 'data/media/spar/scannetpp/images/eb4bc76767/image_color/1250.jpg'
+
  6%|▌         | 6/100 [00:11<01:34,  1.01s/it]
  7%|▋         | 7/100 [00:11<01:19,  1.17it/s][Try #2] Failed to fetch sample 84896. Exception: [Errno 2] No such file or directory: 'data/media/spar/scannetpp/images/eb4bc76767/image_color/1250.jpg'
+
  8%|▊         | 8/100 [00:12<01:09,  1.32it/s][Try #0] Failed to fetch sample 153781. Exception: [Errno 2] No such file or directory: 'data/media/spar/scannetpp/images/216b9e55e8/image_color/11130.jpg'
+
  9%|▉         | 9/100 [00:14<01:47,  1.19s/it]File not exist: data/media/llava_hound/frames/qgvsc6MuTxU
+[Try #0] Failed to fetch sample 78603. Exception: 
+[Try #1] Failed to fetch sample 153781. Exception: [Errno 2] No such file or directory: 'data/media/spar/scannetpp/images/216b9e55e8/image_color/11130.jpg'
+
 10%|█         | 10/100 [00:15<01:28,  1.02it/s]
                                                
+
 10%|█         | 10/100 [00:15<01:28,  1.02it/s]File not exist: data/media/llava_hound/frames/qgvsc6MuTxU
+[Try #1] Failed to fetch sample 78603. Exception: 
+[Try #2] Failed to fetch sample 153781. Exception: [Errno 2] No such file or directory: 'data/media/spar/scannetpp/images/216b9e55e8/image_color/11130.jpg'
+File not exist: data/media/llava_hound/frames/qgvsc6MuTxU
+[Try #2] Failed to fetch sample 78603. Exception: 
+
 11%|█         | 11/100 [00:17<02:11,  1.47s/it]
 12%|█▏        | 12/100 [00:18<01:43,  1.18s/it]
 13%|█▎        | 13/100 [00:18<01:25,  1.01it/s][Try #0] Failed to fetch sample 207772. Exception: [Errno 2] No such file or directory: 'data/media/spar/scannetpp/images/e9e16b6043/video_color/frame0_0.jpg'
+
 14%|█▍        | 14/100 [00:19<01:20,  1.07it/s][Try #0] Failed to fetch sample 192806. Exception: [Errno 2] No such file or directory: 'data/media/spar/scannetpp/images/54b6127146/image_color/2110.jpg'
+[Try #1] Failed to fetch sample 207772. Exception: [Errno 2] No such file or directory: 'data/media/spar/scannetpp/images/e9e16b6043/video_color/frame0_0.jpg'
+
 15%|█▌        | 15/100 [00:20<01:08,  1.24it/s][Try #1] Failed to fetch sample 192806. Exception: [Errno 2] No such file or directory: 'data/media/spar/scannetpp/images/54b6127146/image_color/2110.jpg'
+
 16%|█▌        | 16/100 [00:20<01:01,  1.38it/s][Try #2] Failed to fetch sample 207772. Exception: [Errno 2] No such file or directory: 'data/media/spar/scannetpp/images/e9e16b6043/video_color/frame0_0.jpg'
+
 17%|█▋        | 17/100 [00:21<00:55,  1.49it/s][Try #2] Failed to fetch sample 192806. Exception: [Errno 2] No such file or directory: 'data/media/spar/scannetpp/images/54b6127146/image_color/2110.jpg'
+
 18%|█▊        | 18/100 [00:22<01:23,  1.02s/it]
 19%|█▉        | 19/100 [00:23<01:10,  1.15it/s]Token indices sequence length is longer than the specified maximum sequence length for this model (8364 > 4096). Running this sequence through the model will result in indexing errors
+
 20%|██        | 20/100 [00:25<01:39,  1.24s/it]
                                                
+
 20%|██        | 20/100 [00:25<01:39,  1.24s/it][rank0]: Traceback (most recent call last):
+[rank0]:   File "/workspace/src/qwen_vl/train/train_qwen.py", line 249, in <module>
+[rank0]:     train(attn_implementation="flash_attention_2")
+[rank0]:   File "/workspace/src/qwen_vl/train/train_qwen.py", line 217, in train
+[rank0]:     trainer.train()
+[rank0]:   File "/usr/local/lib/python3.12/dist-packages/transformers/trainer.py", line 2245, in train
+[rank0]:     return inner_training_loop(
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.12/dist-packages/transformers/trainer.py", line 2556, in _inner_training_loop
+[rank0]:     tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
+[rank0]:                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.12/dist-packages/transformers/trainer.py", line 3718, in training_step
+[rank0]:     loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.12/dist-packages/transformers/trainer.py", line 3783, in compute_loss
+[rank0]:     outputs = model(**inputs)
+[rank0]:               ^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.12/dist-packages/deepspeed/utils/nvtx.py", line 18, in wrapped_fn
+[rank0]:     ret_val = func(*args, **kwargs)
+[rank0]:               ^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.12/dist-packages/deepspeed/runtime/engine.py", line 1987, in forward
+[rank0]:     loss = self.module(*inputs, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/workspace/src/qwen_vl/model/modeling_qwen2_5_vl.py", line 2292, in forward
+[rank0]:     raise ValueError(
+[rank0]: ValueError: Image features and image tokens do not match: tokens: 3965, features 8064
+
 20%|██        | 20/100 [00:27<01:49,  1.37s/it]
+[rank0]:[W127 16:43:20.601871192 ProcessGroupNCCL.cpp:1534] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+E0127 16:43:22.817000 174415 torch/distributed/elastic/multiprocessing/api.py:880] failed (exitcode: 1) local_rank: 0 (pid: 174483) of binary: /usr/bin/python
+Traceback (most recent call last):
+  File "/usr/local/bin/torchrun", line 7, in <module>
+    sys.exit(main())
+             ^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 936, in main
+    run(args)
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 927, in run
+    elastic_launch(
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 151, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 288, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+src/qwen_vl/train/train_qwen.py FAILED
+------------------------------------------------------------
+Failures:
+  <NO_OTHER_FAILURES>
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2026-01-27_16:43:22
+  host      : 1f486d0aa8fd
+  rank      : 0 (local_rank: 0)
+  exitcode  : 1 (pid: 174483)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================

Order_Ablations/E116/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

Order_Ablations/E116/chat_template.json

@@ -0,0 +1,3 @@
+{
+    "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+}

Order_Ablations/E116/config.json

@@ -0,0 +1,78 @@
+{
+  "architectures": [
+    "Qwen2_5_VLForConditionalGenerationWithVGGT"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "feature_fusion_method": "add",
+  "fusion_num_layers": 1,
+  "geometry_encoder_layers": [
+    21
+  ],
+  "geometry_encoder_type": "vggt",
+  "geometry_fusion_layers": null,
+  "geometry_merger_type": "mlp",
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_token_id": 151655,
+  "include_camera_token": false,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 128000,
+  "max_window_layers": 70,
+  "model_type": "qwen2_5_vl",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "pos_encoding_type": "none",
+  "reference_frame": "first",
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "mrope_section": [
+      16,
+      24,
+      24
+    ],
+    "rope_type": "default",
+    "type": "default"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.50.0",
+  "use_cache": true,
+  "use_geometry_encoder": true,
+  "use_sliding_window": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "depth": 32,
+    "fullatt_block_indexes": [
+      7,
+      15,
+      23,
+      31
+    ],
+    "hidden_act": "silu",
+    "hidden_size": 1280,
+    "in_channels": 3,
+    "in_chans": 3,
+    "intermediate_size": 3420,
+    "model_type": "qwen2_5_vl",
+    "num_heads": 16,
+    "out_hidden_size": 2048,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "temporal_patch_size": 2,
+    "tokens_per_second": 2,
+    "torch_dtype": "bfloat16",
+    "window_size": 112
+  },
+  "vision_end_token_id": 151653,
+  "vision_language_fusion_layers": null,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
+  "vocab_size": 151936
+}

Order_Ablations/E116/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 1e-06,
+  "transformers_version": "4.50.0"
+}

Order_Ablations/E116/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b0f1c41e389acd9dea4fd5fce691a73295b4a71ecbb60e16a1004906ba7345d
+size 4960002296

Order_Ablations/E116/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6d332fb671fee9d6cc0c6f9abcedf2994148a80ab5de7fe27f2ff17d1580dad
+size 4451629296

Order_Ablations/E116/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d110a4b5d2ebebd722c61565d28562a69698cdb36769e18fad6cf713674b6057
+size 622329984

Order_Ablations/E116/preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 451584,
+  "merge_size": 2,
+  "min_pixels": 12544,
+  "patch_size": 14,
+  "processor_class": "Qwen2_5_VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 451584,
+    "shortest_edge": 12544
+  },
+  "temporal_patch_size": 2
+}

Order_Ablations/E116/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Order_Ablations/E116/tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 12800,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

Order_Ablations/E116/trainer_state.json

@@ -0,0 +1,2507 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9997450352701209,
+  "eval_steps": 500,
+  "global_step": 3529,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002832941443100371,
+      "grad_norm": 174.56146240234375,
+      "learning_rate": 9.433962264150944e-07,
+      "loss": 17.5632,
+      "step": 10
+    },
+    {
+      "epoch": 0.005665882886200742,
+      "grad_norm": 99.76522064208984,
+      "learning_rate": 1.8867924528301889e-06,
+      "loss": 16.5212,
+      "step": 20
+    },
+    {
+      "epoch": 0.008498824329301113,
+      "grad_norm": 61.843570709228516,
+      "learning_rate": 2.830188679245283e-06,
+      "loss": 14.66,
+      "step": 30
+    },
+    {
+      "epoch": 0.011331765772401484,
+      "grad_norm": 58.26314926147461,
+      "learning_rate": 3.7735849056603777e-06,
+      "loss": 12.8877,
+      "step": 40
+    },
+    {
+      "epoch": 0.014164707215501856,
+      "grad_norm": 64.67306518554688,
+      "learning_rate": 4.716981132075472e-06,
+      "loss": 12.6102,
+      "step": 50
+    },
+    {
+      "epoch": 0.016997648658602225,
+      "grad_norm": 40.95338439941406,
+      "learning_rate": 5.660377358490566e-06,
+      "loss": 10.3169,
+      "step": 60
+    },
+    {
+      "epoch": 0.019830590101702596,
+      "grad_norm": 59.99547576904297,
+      "learning_rate": 6.60377358490566e-06,
+      "loss": 10.4793,
+      "step": 70
+    },
+    {
+      "epoch": 0.022663531544802967,
+      "grad_norm": 69.4089584350586,
+      "learning_rate": 7.5471698113207555e-06,
+      "loss": 10.2298,
+      "step": 80
+    },
+    {
+      "epoch": 0.02549647298790334,
+      "grad_norm": 38.2027702331543,
+      "learning_rate": 8.49056603773585e-06,
+      "loss": 8.5679,
+      "step": 90
+    },
+    {
+      "epoch": 0.028329414431003713,
+      "grad_norm": 41.05867004394531,
+      "learning_rate": 9.433962264150944e-06,
+      "loss": 8.6834,
+      "step": 100
+    },
+    {
+      "epoch": 0.031162355874104083,
+      "grad_norm": 45.03547286987305,
+      "learning_rate": 9.999966306552455e-06,
+      "loss": 7.8071,
+      "step": 110
+    },
+    {
+      "epoch": 0.03399529731720445,
+      "grad_norm": 43.03623580932617,
+      "learning_rate": 9.999587260482597e-06,
+      "loss": 8.2945,
+      "step": 120
+    },
+    {
+      "epoch": 0.036828238760304825,
+      "grad_norm": 37.273048400878906,
+      "learning_rate": 9.998787083568112e-06,
+      "loss": 8.0273,
+      "step": 130
+    },
+    {
+      "epoch": 0.03966118020340519,
+      "grad_norm": 55.94173049926758,
+      "learning_rate": 9.997565843210401e-06,
+      "loss": 7.1597,
+      "step": 140
+    },
+    {
+      "epoch": 0.04249412164650557,
+      "grad_norm": 50.30799102783203,
+      "learning_rate": 9.995923642278351e-06,
+      "loss": 7.2908,
+      "step": 150
+    },
+    {
+      "epoch": 0.045327063089605935,
+      "grad_norm": 53.113243103027344,
+      "learning_rate": 9.993860619099673e-06,
+      "loss": 5.9006,
+      "step": 160
+    },
+    {
+      "epoch": 0.04816000453270631,
+      "grad_norm": 51.57769012451172,
+      "learning_rate": 9.991376947449254e-06,
+      "loss": 8.8304,
+      "step": 170
+    },
+    {
+      "epoch": 0.05099294597580668,
+      "grad_norm": 38.17790985107422,
+      "learning_rate": 9.988472836534509e-06,
+      "loss": 8.6644,
+      "step": 180
+    },
+    {
+      "epoch": 0.05382588741890705,
+      "grad_norm": 34.28203582763672,
+      "learning_rate": 9.985148530977767e-06,
+      "loss": 6.6272,
+      "step": 190
+    },
+    {
+      "epoch": 0.056658828862007425,
+      "grad_norm": 34.18405532836914,
+      "learning_rate": 9.981404310795667e-06,
+      "loss": 8.2372,
+      "step": 200
+    },
+    {
+      "epoch": 0.05949177030510779,
+      "grad_norm": 40.83757400512695,
+      "learning_rate": 9.97724049137556e-06,
+      "loss": 5.3928,
+      "step": 210
+    },
+    {
+      "epoch": 0.06232471174820817,
+      "grad_norm": 36.30077362060547,
+      "learning_rate": 9.972657423448961e-06,
+      "loss": 6.5196,
+      "step": 220
+    },
+    {
+      "epoch": 0.06515765319130853,
+      "grad_norm": 38.03015899658203,
+      "learning_rate": 9.96765549306199e-06,
+      "loss": 7.1923,
+      "step": 230
+    },
+    {
+      "epoch": 0.0679905946344089,
+      "grad_norm": 42.84525680541992,
+      "learning_rate": 9.962235121542858e-06,
+      "loss": 6.9849,
+      "step": 240
+    },
+    {
+      "epoch": 0.07082353607750928,
+      "grad_norm": 34.071800231933594,
+      "learning_rate": 9.956396765466382e-06,
+      "loss": 7.0978,
+      "step": 250
+    },
+    {
+      "epoch": 0.07365647752060965,
+      "grad_norm": 37.807029724121094,
+      "learning_rate": 9.950140916615526e-06,
+      "loss": 7.5077,
+      "step": 260
+    },
+    {
+      "epoch": 0.07648941896371002,
+      "grad_norm": 36.55296325683594,
+      "learning_rate": 9.943468101939968e-06,
+      "loss": 6.6867,
+      "step": 270
+    },
+    {
+      "epoch": 0.07932236040681039,
+      "grad_norm": 31.735977172851562,
+      "learning_rate": 9.936378883511722e-06,
+      "loss": 8.5626,
+      "step": 280
+    },
+    {
+      "epoch": 0.08215530184991077,
+      "grad_norm": 51.97509002685547,
+      "learning_rate": 9.92887385847779e-06,
+      "loss": 7.1104,
+      "step": 290
+    },
+    {
+      "epoch": 0.08498824329301113,
+      "grad_norm": 56.030025482177734,
+      "learning_rate": 9.920953659009863e-06,
+      "loss": 6.6099,
+      "step": 300
+    },
+    {
+      "epoch": 0.0878211847361115,
+      "grad_norm": 37.08720016479492,
+      "learning_rate": 9.912618952251071e-06,
+      "loss": 4.9933,
+      "step": 310
+    },
+    {
+      "epoch": 0.09065412617921187,
+      "grad_norm": 34.61451721191406,
+      "learning_rate": 9.903870440259787e-06,
+      "loss": 5.7727,
+      "step": 320
+    },
+    {
+      "epoch": 0.09348706762231225,
+      "grad_norm": 35.92675018310547,
+      "learning_rate": 9.89470885995049e-06,
+      "loss": 4.8536,
+      "step": 330
+    },
+    {
+      "epoch": 0.09632000906541262,
+      "grad_norm": 31.899490356445312,
+      "learning_rate": 9.885134983031694e-06,
+      "loss": 5.6988,
+      "step": 340
+    },
+    {
+      "epoch": 0.09915295050851299,
+      "grad_norm": 37.71702194213867,
+      "learning_rate": 9.875149615940943e-06,
+      "loss": 8.0547,
+      "step": 350
+    },
+    {
+      "epoch": 0.10198589195161337,
+      "grad_norm": 32.81459426879883,
+      "learning_rate": 9.864753599776883e-06,
+      "loss": 5.7466,
+      "step": 360
+    },
+    {
+      "epoch": 0.10481883339471373,
+      "grad_norm": 34.12638854980469,
+      "learning_rate": 9.853947810228416e-06,
+      "loss": 6.3535,
+      "step": 370
+    },
+    {
+      "epoch": 0.1076517748378141,
+      "grad_norm": 34.04792022705078,
+      "learning_rate": 9.842733157500932e-06,
+      "loss": 5.7424,
+      "step": 380
+    },
+    {
+      "epoch": 0.11048471628091447,
+      "grad_norm": 33.2330322265625,
+      "learning_rate": 9.831110586239643e-06,
+      "loss": 5.4935,
+      "step": 390
+    },
+    {
+      "epoch": 0.11331765772401485,
+      "grad_norm": 33.370567321777344,
+      "learning_rate": 9.819081075450014e-06,
+      "loss": 5.5257,
+      "step": 400
+    },
+    {
+      "epoch": 0.11615059916711522,
+      "grad_norm": 34.091304779052734,
+      "learning_rate": 9.806645638415302e-06,
+      "loss": 6.1631,
+      "step": 410
+    },
+    {
+      "epoch": 0.11898354061021559,
+      "grad_norm": 28.293777465820312,
+      "learning_rate": 9.79380532261119e-06,
+      "loss": 6.2594,
+      "step": 420
+    },
+    {
+      "epoch": 0.12181648205331595,
+      "grad_norm": 31.169191360473633,
+      "learning_rate": 9.780561209617569e-06,
+      "loss": 5.428,
+      "step": 430
+    },
+    {
+      "epoch": 0.12464942349641633,
+      "grad_norm": 31.67852783203125,
+      "learning_rate": 9.766914415027426e-06,
+      "loss": 6.3704,
+      "step": 440
+    },
+    {
+      "epoch": 0.1274823649395167,
+      "grad_norm": 24.92995262145996,
+      "learning_rate": 9.752866088352882e-06,
+      "loss": 6.3413,
+      "step": 450
+    },
+    {
+      "epoch": 0.13031530638261707,
+      "grad_norm": 30.267122268676758,
+      "learning_rate": 9.738417412928348e-06,
+      "loss": 6.1918,
+      "step": 460
+    },
+    {
+      "epoch": 0.13314824782571744,
+      "grad_norm": 34.71146011352539,
+      "learning_rate": 9.72356960581087e-06,
+      "loss": 5.2388,
+      "step": 470
+    },
+    {
+      "epoch": 0.1359811892688178,
+      "grad_norm": 32.87137985229492,
+      "learning_rate": 9.7083239176776e-06,
+      "loss": 4.2622,
+      "step": 480
+    },
+    {
+      "epoch": 0.13881413071191817,
+      "grad_norm": 37.83769226074219,
+      "learning_rate": 9.692681632720448e-06,
+      "loss": 4.1838,
+      "step": 490
+    },
+    {
+      "epoch": 0.14164707215501857,
+      "grad_norm": 29.84713363647461,
+      "learning_rate": 9.676644068537915e-06,
+      "loss": 6.1015,
+      "step": 500
+    },
+    {
+      "epoch": 0.14448001359811893,
+      "grad_norm": 37.30936050415039,
+      "learning_rate": 9.660212576024102e-06,
+      "loss": 6.1121,
+      "step": 510
+    },
+    {
+      "epoch": 0.1473129550412193,
+      "grad_norm": 43.746490478515625,
+      "learning_rate": 9.64338853925493e-06,
+      "loss": 6.0385,
+      "step": 520
+    },
+    {
+      "epoch": 0.15014589648431967,
+      "grad_norm": 32.99515151977539,
+      "learning_rate": 9.62617337537154e-06,
+      "loss": 6.1911,
+      "step": 530
+    },
+    {
+      "epoch": 0.15297883792742004,
+      "grad_norm": 46.957340240478516,
+      "learning_rate": 9.608568534460938e-06,
+      "loss": 5.822,
+      "step": 540
+    },
+    {
+      "epoch": 0.1558117793705204,
+      "grad_norm": 33.011844635009766,
+      "learning_rate": 9.590575499433837e-06,
+      "loss": 7.1735,
+      "step": 550
+    },
+    {
+      "epoch": 0.15864472081362077,
+      "grad_norm": 31.042083740234375,
+      "learning_rate": 9.572195785899756e-06,
+      "loss": 6.8695,
+      "step": 560
+    },
+    {
+      "epoch": 0.16147766225672117,
+      "grad_norm": 26.289737701416016,
+      "learning_rate": 9.553430942039352e-06,
+      "loss": 6.9135,
+      "step": 570
+    },
+    {
+      "epoch": 0.16431060369982153,
+      "grad_norm": 29.071701049804688,
+      "learning_rate": 9.534282548474008e-06,
+      "loss": 5.075,
+      "step": 580
+    },
+    {
+      "epoch": 0.1671435451429219,
+      "grad_norm": 46.65534973144531,
+      "learning_rate": 9.514752218132703e-06,
+      "loss": 7.0842,
+      "step": 590
+    },
+    {
+      "epoch": 0.16997648658602227,
+      "grad_norm": 31.195234298706055,
+      "learning_rate": 9.494841596116138e-06,
+      "loss": 4.9555,
+      "step": 600
+    },
+    {
+      "epoch": 0.17280942802912264,
+      "grad_norm": 26.25963020324707,
+      "learning_rate": 9.474552359558167e-06,
+      "loss": 8.0829,
+      "step": 610
+    },
+    {
+      "epoch": 0.175642369472223,
+      "grad_norm": 25.132131576538086,
+      "learning_rate": 9.453886217484536e-06,
+      "loss": 4.0549,
+      "step": 620
+    },
+    {
+      "epoch": 0.17847531091532337,
+      "grad_norm": 47.06711196899414,
+      "learning_rate": 9.432844910668914e-06,
+      "loss": 6.1125,
+      "step": 630
+    },
+    {
+      "epoch": 0.18130825235842374,
+      "grad_norm": 41.20506286621094,
+      "learning_rate": 9.41143021148627e-06,
+      "loss": 6.7009,
+      "step": 640
+    },
+    {
+      "epoch": 0.18414119380152413,
+      "grad_norm": 27.037729263305664,
+      "learning_rate": 9.389643923763573e-06,
+      "loss": 6.8328,
+      "step": 650
+    },
+    {
+      "epoch": 0.1869741352446245,
+      "grad_norm": 44.91098403930664,
+      "learning_rate": 9.367487882627866e-06,
+      "loss": 5.0284,
+      "step": 660
+    },
+    {
+      "epoch": 0.18980707668772487,
+      "grad_norm": 32.71237564086914,
+      "learning_rate": 9.344963954351662e-06,
+      "loss": 6.0377,
+      "step": 670
+    },
+    {
+      "epoch": 0.19264001813082524,
+      "grad_norm": 24.608020782470703,
+      "learning_rate": 9.32207403619577e-06,
+      "loss": 3.9539,
+      "step": 680
+    },
+    {
+      "epoch": 0.1954729595739256,
+      "grad_norm": 45.37845230102539,
+      "learning_rate": 9.298820056249459e-06,
+      "loss": 7.6906,
+      "step": 690
+    },
+    {
+      "epoch": 0.19830590101702597,
+      "grad_norm": 26.40629768371582,
+      "learning_rate": 9.275203973268064e-06,
+      "loss": 5.7302,
+      "step": 700
+    },
+    {
+      "epoch": 0.20113884246012634,
+      "grad_norm": 25.433490753173828,
+      "learning_rate": 9.251227776507989e-06,
+      "loss": 5.6252,
+      "step": 710
+    },
+    {
+      "epoch": 0.20397178390322673,
+      "grad_norm": 25.276575088500977,
+      "learning_rate": 9.226893485559146e-06,
+      "loss": 5.8884,
+      "step": 720
+    },
+    {
+      "epoch": 0.2068047253463271,
+      "grad_norm": 45.13107681274414,
+      "learning_rate": 9.202203150174836e-06,
+      "loss": 9.215,
+      "step": 730
+    },
+    {
+      "epoch": 0.20963766678942747,
+      "grad_norm": 26.52821922302246,
+      "learning_rate": 9.177158850099099e-06,
+      "loss": 5.7232,
+      "step": 740
+    },
+    {
+      "epoch": 0.21247060823252784,
+      "grad_norm": 43.29339599609375,
+      "learning_rate": 9.151762694891522e-06,
+      "loss": 6.7846,
+      "step": 750
+    },
+    {
+      "epoch": 0.2153035496756282,
+      "grad_norm": 29.308732986450195,
+      "learning_rate": 9.12601682374955e-06,
+      "loss": 5.8371,
+      "step": 760
+    },
+    {
+      "epoch": 0.21813649111872857,
+      "grad_norm": 25.373172760009766,
+      "learning_rate": 9.099923405328293e-06,
+      "loss": 3.9846,
+      "step": 770
+    },
+    {
+      "epoch": 0.22096943256182894,
+      "grad_norm": 33.12062454223633,
+      "learning_rate": 9.073484637557852e-06,
+      "loss": 4.8174,
+      "step": 780
+    },
+    {
+      "epoch": 0.2238023740049293,
+      "grad_norm": 24.834850311279297,
+      "learning_rate": 9.046702747458186e-06,
+      "loss": 5.8073,
+      "step": 790
+    },
+    {
+      "epoch": 0.2266353154480297,
+      "grad_norm": 23.760942459106445,
+      "learning_rate": 9.019579990951514e-06,
+      "loss": 5.6668,
+      "step": 800
+    },
+    {
+      "epoch": 0.22946825689113007,
+      "grad_norm": 25.230995178222656,
+      "learning_rate": 8.992118652672302e-06,
+      "loss": 5.6386,
+      "step": 810
+    },
+    {
+      "epoch": 0.23230119833423044,
+      "grad_norm": 27.033655166625977,
+      "learning_rate": 8.964321045774808e-06,
+      "loss": 5.1316,
+      "step": 820
+    },
+    {
+      "epoch": 0.2351341397773308,
+      "grad_norm": 36.77193832397461,
+      "learning_rate": 8.936189511738254e-06,
+      "loss": 5.0568,
+      "step": 830
+    },
+    {
+      "epoch": 0.23796708122043117,
+      "grad_norm": 21.841785430908203,
+      "learning_rate": 8.907726420169583e-06,
+      "loss": 5.5521,
+      "step": 840
+    },
+    {
+      "epoch": 0.24080002266353154,
+      "grad_norm": 46.69823455810547,
+      "learning_rate": 8.878934168603865e-06,
+      "loss": 6.7058,
+      "step": 850
+    },
+    {
+      "epoch": 0.2436329641066319,
+      "grad_norm": 23.228717803955078,
+      "learning_rate": 8.849815182302345e-06,
+      "loss": 7.8944,
+      "step": 860
+    },
+    {
+      "epoch": 0.2464659055497323,
+      "grad_norm": 33.86655807495117,
+      "learning_rate": 8.820371914048153e-06,
+      "loss": 4.8468,
+      "step": 870
+    },
+    {
+      "epoch": 0.24929884699283267,
+      "grad_norm": 25.241182327270508,
+      "learning_rate": 8.790606843939705e-06,
+      "loss": 3.709,
+      "step": 880
+    },
+    {
+      "epoch": 0.25213178843593304,
+      "grad_norm": 25.6811580657959,
+      "learning_rate": 8.760522479181784e-06,
+      "loss": 4.844,
+      "step": 890
+    },
+    {
+      "epoch": 0.2549647298790334,
+      "grad_norm": 43.929115295410156,
+      "learning_rate": 8.730121353874365e-06,
+      "loss": 6.7687,
+      "step": 900
+    },
+    {
+      "epoch": 0.25779767132213377,
+      "grad_norm": 25.351106643676758,
+      "learning_rate": 8.69940602879915e-06,
+      "loss": 3.7733,
+      "step": 910
+    },
+    {
+      "epoch": 0.26063061276523414,
+      "grad_norm": 45.543373107910156,
+      "learning_rate": 8.66837909120387e-06,
+      "loss": 6.5226,
+      "step": 920
+    },
+    {
+      "epoch": 0.2634635542083345,
+      "grad_norm": 35.3692626953125,
+      "learning_rate": 8.637043154584351e-06,
+      "loss": 7.782,
+      "step": 930
+    },
+    {
+      "epoch": 0.2662964956514349,
+      "grad_norm": 23.175140380859375,
+      "learning_rate": 8.60540085846437e-06,
+      "loss": 3.7581,
+      "step": 940
+    },
+    {
+      "epoch": 0.26912943709453524,
+      "grad_norm": 22.461284637451172,
+      "learning_rate": 8.573454868173325e-06,
+      "loss": 3.8114,
+      "step": 950
+    },
+    {
+      "epoch": 0.2719623785376356,
+      "grad_norm": 30.49061393737793,
+      "learning_rate": 8.541207874621718e-06,
+      "loss": 5.6752,
+      "step": 960
+    },
+    {
+      "epoch": 0.274795319980736,
+      "grad_norm": 43.03390121459961,
+      "learning_rate": 8.508662594074496e-06,
+      "loss": 5.8459,
+      "step": 970
+    },
+    {
+      "epoch": 0.27762826142383634,
+      "grad_norm": 144.74916076660156,
+      "learning_rate": 8.475821767922254e-06,
+      "loss": 6.1604,
+      "step": 980
+    },
+    {
+      "epoch": 0.28046120286693677,
+      "grad_norm": 26.303081512451172,
+      "learning_rate": 8.442688162450315e-06,
+      "loss": 4.7632,
+      "step": 990
+    },
+    {
+      "epoch": 0.28329414431003713,
+      "grad_norm": 28.066007614135742,
+      "learning_rate": 8.409264568605714e-06,
+      "loss": 8.0062,
+      "step": 1000
+    },
+    {
+      "epoch": 0.2861270857531375,
+      "grad_norm": 23.32230567932129,
+      "learning_rate": 8.375553801762119e-06,
+      "loss": 3.9505,
+      "step": 1010
+    },
+    {
+      "epoch": 0.28896002719623787,
+      "grad_norm": 53.46368408203125,
+      "learning_rate": 8.34155870148267e-06,
+      "loss": 4.7727,
+      "step": 1020
+    },
+    {
+      "epoch": 0.29179296863933823,
+      "grad_norm": 28.557096481323242,
+      "learning_rate": 8.307282131280805e-06,
+      "loss": 3.876,
+      "step": 1030
+    },
+    {
+      "epoch": 0.2946259100824386,
+      "grad_norm": 26.72674560546875,
+      "learning_rate": 8.272726978379049e-06,
+      "loss": 3.6362,
+      "step": 1040
+    },
+    {
+      "epoch": 0.29745885152553897,
+      "grad_norm": 33.64091110229492,
+      "learning_rate": 8.23789615346582e-06,
+      "loss": 4.8435,
+      "step": 1050
+    },
+    {
+      "epoch": 0.30029179296863934,
+      "grad_norm": 25.513519287109375,
+      "learning_rate": 8.202792590450246e-06,
+      "loss": 4.8615,
+      "step": 1060
+    },
+    {
+      "epoch": 0.3031247344117397,
+      "grad_norm": 26.183082580566406,
+      "learning_rate": 8.167419246215042e-06,
+      "loss": 3.7897,
+      "step": 1070
+    },
+    {
+      "epoch": 0.30595767585484007,
+      "grad_norm": 37.91279983520508,
+      "learning_rate": 8.131779100367438e-06,
+      "loss": 3.8092,
+      "step": 1080
+    },
+    {
+      "epoch": 0.30879061729794044,
+      "grad_norm": 20.643037796020508,
+      "learning_rate": 8.09587515498819e-06,
+      "loss": 5.8217,
+      "step": 1090
+    },
+    {
+      "epoch": 0.3116235587410408,
+      "grad_norm": 24.009424209594727,
+      "learning_rate": 8.059710434378717e-06,
+      "loss": 4.6594,
+      "step": 1100
+    },
+    {
+      "epoch": 0.3144565001841412,
+      "grad_norm": 26.472389221191406,
+      "learning_rate": 8.02328798480635e-06,
+      "loss": 6.5989,
+      "step": 1110
+    },
+    {
+      "epoch": 0.31728944162724154,
+      "grad_norm": 42.69245529174805,
+      "learning_rate": 7.986610874247736e-06,
+      "loss": 6.7969,
+      "step": 1120
+    },
+    {
+      "epoch": 0.3201223830703419,
+      "grad_norm": 20.50579833984375,
+      "learning_rate": 7.949682192130407e-06,
+      "loss": 7.532,
+      "step": 1130
+    },
+    {
+      "epoch": 0.32295532451344233,
+      "grad_norm": 33.428890228271484,
+      "learning_rate": 7.912505049072559e-06,
+      "loss": 5.5098,
+      "step": 1140
+    },
+    {
+      "epoch": 0.3257882659565427,
+      "grad_norm": 33.04521560668945,
+      "learning_rate": 7.875082576621024e-06,
+      "loss": 5.7852,
+      "step": 1150
+    },
+    {
+      "epoch": 0.32862120739964307,
+      "grad_norm": 29.72992706298828,
+      "learning_rate": 7.837417926987496e-06,
+      "loss": 3.8586,
+      "step": 1160
+    },
+    {
+      "epoch": 0.33145414884274343,
+      "grad_norm": 22.467132568359375,
+      "learning_rate": 7.799514272783014e-06,
+      "loss": 5.6287,
+      "step": 1170
+    },
+    {
+      "epoch": 0.3342870902858438,
+      "grad_norm": 25.866819381713867,
+      "learning_rate": 7.761374806750712e-06,
+      "loss": 3.7462,
+      "step": 1180
+    },
+    {
+      "epoch": 0.33712003172894417,
+      "grad_norm": 21.801698684692383,
+      "learning_rate": 7.723002741496892e-06,
+      "loss": 5.6068,
+      "step": 1190
+    },
+    {
+      "epoch": 0.33995297317204454,
+      "grad_norm": 41.93526840209961,
+      "learning_rate": 7.684401309220416e-06,
+      "loss": 5.8573,
+      "step": 1200
+    },
+    {
+      "epoch": 0.3427859146151449,
+      "grad_norm": 23.245235443115234,
+      "learning_rate": 7.645573761440444e-06,
+      "loss": 4.5851,
+      "step": 1210
+    },
+    {
+      "epoch": 0.34561885605824527,
+      "grad_norm": 24.502330780029297,
+      "learning_rate": 7.606523368722554e-06,
+      "loss": 6.4644,
+      "step": 1220
+    },
+    {
+      "epoch": 0.34845179750134564,
+      "grad_norm": 16.042354583740234,
+      "learning_rate": 7.567253420403249e-06,
+      "loss": 5.6877,
+      "step": 1230
+    },
+    {
+      "epoch": 0.351284738944446,
+      "grad_norm": 26.405628204345703,
+      "learning_rate": 7.527767224312883e-06,
+      "loss": 4.764,
+      "step": 1240
+    },
+    {
+      "epoch": 0.3541176803875464,
+      "grad_norm": 40.40938186645508,
+      "learning_rate": 7.488068106497035e-06,
+      "loss": 5.8002,
+      "step": 1250
+    },
+    {
+      "epoch": 0.35695062183064674,
+      "grad_norm": 25.338321685791016,
+      "learning_rate": 7.448159410936348e-06,
+      "loss": 5.5113,
+      "step": 1260
+    },
+    {
+      "epoch": 0.3597835632737471,
+      "grad_norm": 39.411128997802734,
+      "learning_rate": 7.4080444992648534e-06,
+      "loss": 5.5444,
+      "step": 1270
+    },
+    {
+      "epoch": 0.3626165047168475,
+      "grad_norm": 22.218137741088867,
+      "learning_rate": 7.3677267504868055e-06,
+      "loss": 4.4882,
+      "step": 1280
+    },
+    {
+      "epoch": 0.3654494461599479,
+      "grad_norm": 43.15862274169922,
+      "learning_rate": 7.327209560692063e-06,
+      "loss": 6.6107,
+      "step": 1290
+    },
+    {
+      "epoch": 0.36828238760304827,
+      "grad_norm": 20.51604652404785,
+      "learning_rate": 7.2864963427700284e-06,
+      "loss": 5.6351,
+      "step": 1300
+    },
+    {
+      "epoch": 0.37111532904614863,
+      "grad_norm": 21.392065048217773,
+      "learning_rate": 7.2455905261221585e-06,
+      "loss": 5.7755,
+      "step": 1310
+    },
+    {
+      "epoch": 0.373948270489249,
+      "grad_norm": 28.160072326660156,
+      "learning_rate": 7.204495556373106e-06,
+      "loss": 6.5779,
+      "step": 1320
+    },
+    {
+      "epoch": 0.37678121193234937,
+      "grad_norm": 41.15205764770508,
+      "learning_rate": 7.163214895080479e-06,
+      "loss": 6.4435,
+      "step": 1330
+    },
+    {
+      "epoch": 0.37961415337544974,
+      "grad_norm": 26.100757598876953,
+      "learning_rate": 7.121752019443266e-06,
+      "loss": 6.5864,
+      "step": 1340
+    },
+    {
+      "epoch": 0.3824470948185501,
+      "grad_norm": 41.462791442871094,
+      "learning_rate": 7.080110422008937e-06,
+      "loss": 5.6488,
+      "step": 1350
+    },
+    {
+      "epoch": 0.38528003626165047,
+      "grad_norm": 22.369388580322266,
+      "learning_rate": 7.038293610379255e-06,
+      "loss": 4.4922,
+      "step": 1360
+    },
+    {
+      "epoch": 0.38811297770475084,
+      "grad_norm": 19.927444458007812,
+      "learning_rate": 6.996305106914824e-06,
+      "loss": 4.5791,
+      "step": 1370
+    },
+    {
+      "epoch": 0.3909459191478512,
+      "grad_norm": 25.871030807495117,
+      "learning_rate": 6.954148448438389e-06,
+      "loss": 4.5578,
+      "step": 1380
+    },
+    {
+      "epoch": 0.3937788605909516,
+      "grad_norm": 18.420751571655273,
+      "learning_rate": 6.911827185936914e-06,
+      "loss": 4.6252,
+      "step": 1390
+    },
+    {
+      "epoch": 0.39661180203405194,
+      "grad_norm": 27.263010025024414,
+      "learning_rate": 6.869344884262473e-06,
+      "loss": 5.5235,
+      "step": 1400
+    },
+    {
+      "epoch": 0.3994447434771523,
+      "grad_norm": 24.479764938354492,
+      "learning_rate": 6.8267051218319766e-06,
+      "loss": 5.6514,
+      "step": 1410
+    },
+    {
+      "epoch": 0.4022776849202527,
+      "grad_norm": 23.21695899963379,
+      "learning_rate": 6.7839114903257404e-06,
+      "loss": 7.5326,
+      "step": 1420
+    },
+    {
+      "epoch": 0.40511062636335304,
+      "grad_norm": 21.287368774414062,
+      "learning_rate": 6.74096759438496e-06,
+      "loss": 3.895,
+      "step": 1430
+    },
+    {
+      "epoch": 0.40794356780645347,
+      "grad_norm": 25.839454650878906,
+      "learning_rate": 6.697877051308067e-06,
+      "loss": 6.3928,
+      "step": 1440
+    },
+    {
+      "epoch": 0.41077650924955383,
+      "grad_norm": 22.896682739257812,
+      "learning_rate": 6.654643490746042e-06,
+      "loss": 4.5232,
+      "step": 1450
+    },
+    {
+      "epoch": 0.4136094506926542,
+      "grad_norm": 25.252422332763672,
+      "learning_rate": 6.611270554396676e-06,
+      "loss": 6.5998,
+      "step": 1460
+    },
+    {
+      "epoch": 0.41644239213575457,
+      "grad_norm": 24.610836029052734,
+      "learning_rate": 6.567761895697816e-06,
+      "loss": 4.6121,
+      "step": 1470
+    },
+    {
+      "epoch": 0.41927533357885494,
+      "grad_norm": 19.092580795288086,
+      "learning_rate": 6.524121179519625e-06,
+      "loss": 3.6029,
+      "step": 1480
+    },
+    {
+      "epoch": 0.4221082750219553,
+      "grad_norm": 22.915136337280273,
+      "learning_rate": 6.480352081855884e-06,
+      "loss": 3.6352,
+      "step": 1490
+    },
+    {
+      "epoch": 0.42494121646505567,
+      "grad_norm": 29.044233322143555,
+      "learning_rate": 6.436458289514342e-06,
+      "loss": 4.6979,
+      "step": 1500
+    },
+    {
+      "epoch": 0.42777415790815604,
+      "grad_norm": 39.80937194824219,
+      "learning_rate": 6.392443499806175e-06,
+      "loss": 4.6673,
+      "step": 1510
+    },
+    {
+      "epoch": 0.4306070993512564,
+      "grad_norm": 22.760765075683594,
+      "learning_rate": 6.348311420234542e-06,
+      "loss": 4.6801,
+      "step": 1520
+    },
+    {
+      "epoch": 0.4334400407943568,
+      "grad_norm": 21.216337203979492,
+      "learning_rate": 6.304065768182295e-06,
+      "loss": 5.7451,
+      "step": 1530
+    },
+    {
+      "epoch": 0.43627298223745714,
+      "grad_norm": 20.622943878173828,
+      "learning_rate": 6.259710270598848e-06,
+      "loss": 5.6216,
+      "step": 1540
+    },
+    {
+      "epoch": 0.4391059236805575,
+      "grad_norm": 40.299949645996094,
+      "learning_rate": 6.215248663686251e-06,
+      "loss": 6.5508,
+      "step": 1550
+    },
+    {
+      "epoch": 0.4419388651236579,
+      "grad_norm": 28.81671905517578,
+      "learning_rate": 6.170684692584469e-06,
+      "loss": 3.5039,
+      "step": 1560
+    },
+    {
+      "epoch": 0.44477180656675824,
+      "grad_norm": 43.010169982910156,
+      "learning_rate": 6.126022111055929e-06,
+      "loss": 6.4925,
+      "step": 1570
+    },
+    {
+      "epoch": 0.4476047480098586,
+      "grad_norm": 23.351240158081055,
+      "learning_rate": 6.081264681169317e-06,
+      "loss": 3.4456,
+      "step": 1580
+    },
+    {
+      "epoch": 0.45043768945295903,
+      "grad_norm": 40.19292449951172,
+      "learning_rate": 6.0364161729826905e-06,
+      "loss": 4.4953,
+      "step": 1590
+    },
+    {
+      "epoch": 0.4532706308960594,
+      "grad_norm": 25.595369338989258,
+      "learning_rate": 5.991480364225924e-06,
+      "loss": 6.2619,
+      "step": 1600
+    },
+    {
+      "epoch": 0.45610357233915977,
+      "grad_norm": 32.5233268737793,
+      "learning_rate": 5.946461039982485e-06,
+      "loss": 5.5702,
+      "step": 1610
+    },
+    {
+      "epoch": 0.45893651378226014,
+      "grad_norm": 25.565658569335938,
+      "learning_rate": 5.901361992370614e-06,
+      "loss": 3.5389,
+      "step": 1620
+    },
+    {
+      "epoch": 0.4617694552253605,
+      "grad_norm": 21.443763732910156,
+      "learning_rate": 5.856187020223901e-06,
+      "loss": 4.6532,
+      "step": 1630
+    },
+    {
+      "epoch": 0.46460239666846087,
+      "grad_norm": 26.775903701782227,
+      "learning_rate": 5.8109399287712935e-06,
+      "loss": 5.7745,
+      "step": 1640
+    },
+    {
+      "epoch": 0.46743533811156124,
+      "grad_norm": 20.02845001220703,
+      "learning_rate": 5.765624529316573e-06,
+      "loss": 5.506,
+      "step": 1650
+    },
+    {
+      "epoch": 0.4702682795546616,
+      "grad_norm": 22.177770614624023,
+      "learning_rate": 5.7202446389173225e-06,
+      "loss": 3.5255,
+      "step": 1660
+    },
+    {
+      "epoch": 0.473101220997762,
+      "grad_norm": 27.885957717895508,
+      "learning_rate": 5.674804080063392e-06,
+      "loss": 3.5088,
+      "step": 1670
+    },
+    {
+      "epoch": 0.47593416244086234,
+      "grad_norm": 33.34544372558594,
+      "learning_rate": 5.62930668035493e-06,
+      "loss": 4.4746,
+      "step": 1680
+    },
+    {
+      "epoch": 0.4787671038839627,
+      "grad_norm": 24.865848541259766,
+      "learning_rate": 5.5837562721799644e-06,
+      "loss": 6.4182,
+      "step": 1690
+    },
+    {
+      "epoch": 0.4816000453270631,
+      "grad_norm": 20.06027603149414,
+      "learning_rate": 5.538156692391592e-06,
+      "loss": 3.499,
+      "step": 1700
+    },
+    {
+      "epoch": 0.48443298677016344,
+      "grad_norm": 28.240829467773438,
+      "learning_rate": 5.4925117819847925e-06,
+      "loss": 5.4651,
+      "step": 1710
+    },
+    {
+      "epoch": 0.4872659282132638,
+      "grad_norm": 39.07200241088867,
+      "learning_rate": 5.44682538577288e-06,
+      "loss": 4.7134,
+      "step": 1720
+    },
+    {
+      "epoch": 0.4900988696563642,
+      "grad_norm": 31.383825302124023,
+      "learning_rate": 5.4011013520636466e-06,
+      "loss": 4.4705,
+      "step": 1730
+    },
+    {
+      "epoch": 0.4929318110994646,
+      "grad_norm": 40.832984924316406,
+      "learning_rate": 5.355343532335215e-06,
+      "loss": 7.2469,
+      "step": 1740
+    },
+    {
+      "epoch": 0.49576475254256497,
+      "grad_norm": 20.33405303955078,
+      "learning_rate": 5.309555780911604e-06,
+      "loss": 5.4482,
+      "step": 1750
+    },
+    {
+      "epoch": 0.49859769398566534,
+      "grad_norm": 22.8585262298584,
+      "learning_rate": 5.263741954638072e-06,
+      "loss": 4.4573,
+      "step": 1760
+    },
+    {
+      "epoch": 0.5014306354287656,
+      "grad_norm": 42.46244430541992,
+      "learning_rate": 5.217905912556248e-06,
+      "loss": 5.5277,
+      "step": 1770
+    },
+    {
+      "epoch": 0.5042635768718661,
+      "grad_norm": 21.30562973022461,
+      "learning_rate": 5.172051515579065e-06,
+      "loss": 5.4764,
+      "step": 1780
+    },
+    {
+      "epoch": 0.5070965183149664,
+      "grad_norm": 18.9359130859375,
+      "learning_rate": 5.126182626165547e-06,
+      "loss": 6.4232,
+      "step": 1790
+    },
+    {
+      "epoch": 0.5099294597580668,
+      "grad_norm": 33.8026123046875,
+      "learning_rate": 5.080303107995461e-06,
+      "loss": 6.6042,
+      "step": 1800
+    },
+    {
+      "epoch": 0.5127624012011671,
+      "grad_norm": 40.52323913574219,
+      "learning_rate": 5.034416825643868e-06,
+      "loss": 5.5848,
+      "step": 1810
+    },
+    {
+      "epoch": 0.5155953426442675,
+      "grad_norm": 27.342744827270508,
+      "learning_rate": 4.988527644255591e-06,
+      "loss": 5.2504,
+      "step": 1820
+    },
+    {
+      "epoch": 0.518428284087368,
+      "grad_norm": 19.118297576904297,
+      "learning_rate": 4.942639429219661e-06,
+      "loss": 4.5668,
+      "step": 1830
+    },
+    {
+      "epoch": 0.5212612255304683,
+      "grad_norm": 41.146236419677734,
+      "learning_rate": 4.896756045843698e-06,
+      "loss": 6.0831,
+      "step": 1840
+    },
+    {
+      "epoch": 0.5240941669735687,
+      "grad_norm": 19.685937881469727,
+      "learning_rate": 4.85088135902834e-06,
+      "loss": 5.5025,
+      "step": 1850
+    },
+    {
+      "epoch": 0.526927108416669,
+      "grad_norm": 22.97096061706543,
+      "learning_rate": 4.805019232941689e-06,
+      "loss": 4.4157,
+      "step": 1860
+    },
+    {
+      "epoch": 0.5297600498597694,
+      "grad_norm": 27.627784729003906,
+      "learning_rate": 4.7591735306938144e-06,
+      "loss": 4.3861,
+      "step": 1870
+    },
+    {
+      "epoch": 0.5325929913028697,
+      "grad_norm": 25.308032989501953,
+      "learning_rate": 4.713348114011357e-06,
+      "loss": 7.2963,
+      "step": 1880
+    },
+    {
+      "epoch": 0.5354259327459702,
+      "grad_norm": 19.11351203918457,
+      "learning_rate": 4.667546842912239e-06,
+      "loss": 4.2907,
+      "step": 1890
+    },
+    {
+      "epoch": 0.5382588741890705,
+      "grad_norm": 28.81739044189453,
+      "learning_rate": 4.6217735753805235e-06,
+      "loss": 4.5385,
+      "step": 1900
+    },
+    {
+      "epoch": 0.5410918156321709,
+      "grad_norm": 20.510547637939453,
+      "learning_rate": 4.576032167041452e-06,
+      "loss": 7.2043,
+      "step": 1910
+    },
+    {
+      "epoch": 0.5439247570752712,
+      "grad_norm": 26.19765281677246,
+      "learning_rate": 4.530326470836659e-06,
+      "loss": 4.3494,
+      "step": 1920
+    },
+    {
+      "epoch": 0.5467576985183716,
+      "grad_norm": 25.779802322387695,
+      "learning_rate": 4.484660336699638e-06,
+      "loss": 5.3226,
+      "step": 1930
+    },
+    {
+      "epoch": 0.549590639961472,
+      "grad_norm": 26.97022247314453,
+      "learning_rate": 4.439037611231448e-06,
+      "loss": 6.5069,
+      "step": 1940
+    },
+    {
+      "epoch": 0.5524235814045724,
+      "grad_norm": 26.32407569885254,
+      "learning_rate": 4.393462137376696e-06,
+      "loss": 3.545,
+      "step": 1950
+    },
+    {
+      "epoch": 0.5552565228476727,
+      "grad_norm": 30.962535858154297,
+      "learning_rate": 4.347937754099841e-06,
+      "loss": 4.4292,
+      "step": 1960
+    },
+    {
+      "epoch": 0.5580894642907731,
+      "grad_norm": 38.1851921081543,
+      "learning_rate": 4.302468296061823e-06,
+      "loss": 4.3079,
+      "step": 1970
+    },
+    {
+      "epoch": 0.5609224057338735,
+      "grad_norm": 21.038278579711914,
+      "learning_rate": 4.257057593297055e-06,
+      "loss": 4.5294,
+      "step": 1980
+    },
+    {
+      "epoch": 0.5637553471769738,
+      "grad_norm": 20.618942260742188,
+      "learning_rate": 4.211709470890815e-06,
+      "loss": 7.2449,
+      "step": 1990
+    },
+    {
+      "epoch": 0.5665882886200743,
+      "grad_norm": 21.230995178222656,
+      "learning_rate": 4.166427748657034e-06,
+      "loss": 4.3681,
+      "step": 2000
+    },
+    {
+      "epoch": 0.5694212300631746,
+      "grad_norm": 20.577428817749023,
+      "learning_rate": 4.121216240816559e-06,
+      "loss": 5.3925,
+      "step": 2010
+    },
+    {
+      "epoch": 0.572254171506275,
+      "grad_norm": 21.1496524810791,
+      "learning_rate": 4.076078755675852e-06,
+      "loss": 5.0495,
+      "step": 2020
+    },
+    {
+      "epoch": 0.5750871129493753,
+      "grad_norm": 26.215744018554688,
+      "learning_rate": 4.0310190953062155e-06,
+      "loss": 5.5832,
+      "step": 2030
+    },
+    {
+      "epoch": 0.5779200543924757,
+      "grad_norm": 33.668174743652344,
+      "learning_rate": 3.986041055223526e-06,
+      "loss": 5.1639,
+      "step": 2040
+    },
+    {
+      "epoch": 0.580752995835576,
+      "grad_norm": 28.786453247070312,
+      "learning_rate": 3.9411484240685315e-06,
+      "loss": 3.3797,
+      "step": 2050
+    },
+    {
+      "epoch": 0.5835859372786765,
+      "grad_norm": 24.81963348388672,
+      "learning_rate": 3.8963449832877164e-06,
+      "loss": 6.3189,
+      "step": 2060
+    },
+    {
+      "epoch": 0.5864188787217768,
+      "grad_norm": 25.143753051757812,
+      "learning_rate": 3.851634506814782e-06,
+      "loss": 6.463,
+      "step": 2070
+    },
+    {
+      "epoch": 0.5892518201648772,
+      "grad_norm": 39.29959487915039,
+      "learning_rate": 3.8070207607527587e-06,
+      "loss": 7.5255,
+      "step": 2080
+    },
+    {
+      "epoch": 0.5920847616079775,
+      "grad_norm": 19.030284881591797,
+      "learning_rate": 3.7625075030567683e-06,
+      "loss": 4.2513,
+      "step": 2090
+    },
+    {
+      "epoch": 0.5949177030510779,
+      "grad_norm": 24.105989456176758,
+      "learning_rate": 3.718098483217484e-06,
+      "loss": 3.3586,
+      "step": 2100
+    },
+    {
+      "epoch": 0.5977506444941783,
+      "grad_norm": 38.95778274536133,
+      "learning_rate": 3.673797441945304e-06,
+      "loss": 4.2773,
+      "step": 2110
+    },
+    {
+      "epoch": 0.6005835859372787,
+      "grad_norm": 42.26526641845703,
+      "learning_rate": 3.629608110855248e-06,
+      "loss": 5.2586,
+      "step": 2120
+    },
+    {
+      "epoch": 0.6034165273803791,
+      "grad_norm": 21.60348892211914,
+      "learning_rate": 3.585534212152643e-06,
+      "loss": 4.4408,
+      "step": 2130
+    },
+    {
+      "epoch": 0.6062494688234794,
+      "grad_norm": 39.41062545776367,
+      "learning_rate": 3.5415794583195846e-06,
+      "loss": 4.5132,
+      "step": 2140
+    },
+    {
+      "epoch": 0.6090824102665798,
+      "grad_norm": 32.25893783569336,
+      "learning_rate": 3.497747551802221e-06,
+      "loss": 8.4284,
+      "step": 2150
+    },
+    {
+      "epoch": 0.6119153517096801,
+      "grad_norm": 34.556373596191406,
+      "learning_rate": 3.4540421846988916e-06,
+      "loss": 6.3801,
+      "step": 2160
+    },
+    {
+      "epoch": 0.6147482931527806,
+      "grad_norm": 27.99374771118164,
+      "learning_rate": 3.4104670384491234e-06,
+      "loss": 5.5573,
+      "step": 2170
+    },
+    {
+      "epoch": 0.6175812345958809,
+      "grad_norm": 23.997901916503906,
+      "learning_rate": 3.367025783523534e-06,
+      "loss": 4.2779,
+      "step": 2180
+    },
+    {
+      "epoch": 0.6204141760389813,
+      "grad_norm": 41.76970291137695,
+      "learning_rate": 3.3237220791146597e-06,
+      "loss": 5.241,
+      "step": 2190
+    },
+    {
+      "epoch": 0.6232471174820816,
+      "grad_norm": 27.922670364379883,
+      "learning_rate": 3.2805595728287255e-06,
+      "loss": 4.2649,
+      "step": 2200
+    },
+    {
+      "epoch": 0.626080058925182,
+      "grad_norm": 33.54890060424805,
+      "learning_rate": 3.2375419003783957e-06,
+      "loss": 6.0635,
+      "step": 2210
+    },
+    {
+      "epoch": 0.6289130003682823,
+      "grad_norm": 21.987178802490234,
+      "learning_rate": 3.1946726852765325e-06,
+      "loss": 5.1542,
+      "step": 2220
+    },
+    {
+      "epoch": 0.6317459418113828,
+      "grad_norm": 35.2348518371582,
+      "learning_rate": 3.1519555385309685e-06,
+      "loss": 4.2332,
+      "step": 2230
+    },
+    {
+      "epoch": 0.6345788832544831,
+      "grad_norm": 39.060691833496094,
+      "learning_rate": 3.1093940583403447e-06,
+      "loss": 8.0693,
+      "step": 2240
+    },
+    {
+      "epoch": 0.6374118246975835,
+      "grad_norm": 20.76451873779297,
+      "learning_rate": 3.066991829791024e-06,
+      "loss": 5.3108,
+      "step": 2250
+    },
+    {
+      "epoch": 0.6402447661406838,
+      "grad_norm": 40.92884826660156,
+      "learning_rate": 3.024752424555105e-06,
+      "loss": 4.2548,
+      "step": 2260
+    },
+    {
+      "epoch": 0.6430777075837842,
+      "grad_norm": 24.043121337890625,
+      "learning_rate": 2.982679400589569e-06,
+      "loss": 5.3648,
+      "step": 2270
+    },
+    {
+      "epoch": 0.6459106490268847,
+      "grad_norm": 22.929412841796875,
+      "learning_rate": 2.9407763018365854e-06,
+      "loss": 4.2817,
+      "step": 2280
+    },
+    {
+      "epoch": 0.648743590469985,
+      "grad_norm": 36.0571174621582,
+      "learning_rate": 2.899046657924992e-06,
+      "loss": 7.9167,
+      "step": 2290
+    },
+    {
+      "epoch": 0.6515765319130854,
+      "grad_norm": 23.849647521972656,
+      "learning_rate": 2.8574939838729844e-06,
+      "loss": 4.44,
+      "step": 2300
+    },
+    {
+      "epoch": 0.6544094733561857,
+      "grad_norm": 42.65750503540039,
+      "learning_rate": 2.8161217797920304e-06,
+      "loss": 5.6655,
+      "step": 2310
+    },
+    {
+      "epoch": 0.6572424147992861,
+      "grad_norm": 23.45660400390625,
+      "learning_rate": 2.774933530592054e-06,
+      "loss": 5.4841,
+      "step": 2320
+    },
+    {
+      "epoch": 0.6600753562423864,
+      "grad_norm": 21.22451400756836,
+      "learning_rate": 2.733932705687883e-06,
+      "loss": 3.3468,
+      "step": 2330
+    },
+    {
+      "epoch": 0.6629082976854869,
+      "grad_norm": 37.178993225097656,
+      "learning_rate": 2.693122758707013e-06,
+      "loss": 5.1606,
+      "step": 2340
+    },
+    {
+      "epoch": 0.6657412391285872,
+      "grad_norm": 24.34912109375,
+      "learning_rate": 2.652507127198689e-06,
+      "loss": 7.2961,
+      "step": 2350
+    },
+    {
+      "epoch": 0.6685741805716876,
+      "grad_norm": 40.61592483520508,
+      "learning_rate": 2.612089232344371e-06,
+      "loss": 6.3695,
+      "step": 2360
+    },
+    {
+      "epoch": 0.6714071220147879,
+      "grad_norm": 20.37811279296875,
+      "learning_rate": 2.571872478669528e-06,
+      "loss": 3.3039,
+      "step": 2370
+    },
+    {
+      "epoch": 0.6742400634578883,
+      "grad_norm": 25.745912551879883,
+      "learning_rate": 2.5318602537568904e-06,
+      "loss": 4.2973,
+      "step": 2380
+    },
+    {
+      "epoch": 0.6770730049009887,
+      "grad_norm": 22.395126342773438,
+      "learning_rate": 2.4920559279610886e-06,
+      "loss": 4.1162,
+      "step": 2390
+    },
+    {
+      "epoch": 0.6799059463440891,
+      "grad_norm": 34.32621383666992,
+      "learning_rate": 2.452462854124758e-06,
+      "loss": 4.1658,
+      "step": 2400
+    },
+    {
+      "epoch": 0.6827388877871894,
+      "grad_norm": 39.03499984741211,
+      "learning_rate": 2.413084367296127e-06,
+      "loss": 6.3083,
+      "step": 2410
+    },
+    {
+      "epoch": 0.6855718292302898,
+      "grad_norm": 47.788394927978516,
+      "learning_rate": 2.373923784448089e-06,
+      "loss": 4.2861,
+      "step": 2420
+    },
+    {
+      "epoch": 0.6884047706733902,
+      "grad_norm": 26.90192413330078,
+      "learning_rate": 2.3349844041988044e-06,
+      "loss": 4.3008,
+      "step": 2430
+    },
+    {
+      "epoch": 0.6912377121164905,
+      "grad_norm": 22.178869247436523,
+      "learning_rate": 2.296269506533846e-06,
+      "loss": 5.2767,
+      "step": 2440
+    },
+    {
+      "epoch": 0.694070653559591,
+      "grad_norm": 21.529335021972656,
+      "learning_rate": 2.2577823525299205e-06,
+      "loss": 7.1097,
+      "step": 2450
+    },
+    {
+      "epoch": 0.6969035950026913,
+      "grad_norm": 20.215675354003906,
+      "learning_rate": 2.2195261840801757e-06,
+      "loss": 7.1815,
+      "step": 2460
+    },
+    {
+      "epoch": 0.6997365364457917,
+      "grad_norm": 21.300861358642578,
+      "learning_rate": 2.18150422362112e-06,
+      "loss": 6.9142,
+      "step": 2470
+    },
+    {
+      "epoch": 0.702569477888892,
+      "grad_norm": 30.098453521728516,
+      "learning_rate": 2.1437196738611958e-06,
+      "loss": 4.4774,
+      "step": 2480
+    },
+    {
+      "epoch": 0.7054024193319924,
+      "grad_norm": 25.317970275878906,
+      "learning_rate": 2.1061757175110024e-06,
+      "loss": 4.4772,
+      "step": 2490
+    },
+    {
+      "epoch": 0.7082353607750927,
+      "grad_norm": 30.881681442260742,
+      "learning_rate": 2.0688755170152e-06,
+      "loss": 4.2296,
+      "step": 2500
+    },
+    {
+      "epoch": 0.7110683022181932,
+      "grad_norm": 23.95901107788086,
+      "learning_rate": 2.031822214286134e-06,
+      "loss": 5.0405,
+      "step": 2510
+    },
+    {
+      "epoch": 0.7139012436612935,
+      "grad_norm": 41.624210357666016,
+      "learning_rate": 1.9950189304391855e-06,
+      "loss": 6.3358,
+      "step": 2520
+    },
+    {
+      "epoch": 0.7167341851043939,
+      "grad_norm": 34.76797866821289,
+      "learning_rate": 1.958468765529853e-06,
+      "loss": 5.061,
+      "step": 2530
+    },
+    {
+      "epoch": 0.7195671265474942,
+      "grad_norm": 20.406444549560547,
+      "learning_rate": 1.9221747982926493e-06,
+      "loss": 5.1701,
+      "step": 2540
+    },
+    {
+      "epoch": 0.7224000679905946,
+      "grad_norm": 24.22311782836914,
+      "learning_rate": 1.8861400858817508e-06,
+      "loss": 4.2621,
+      "step": 2550
+    },
+    {
+      "epoch": 0.725233009433695,
+      "grad_norm": 37.65345001220703,
+      "learning_rate": 1.8503676636134882e-06,
+      "loss": 6.1661,
+      "step": 2560
+    },
+    {
+      "epoch": 0.7280659508767954,
+      "grad_norm": 20.813777923583984,
+      "learning_rate": 1.81486054471068e-06,
+      "loss": 5.3045,
+      "step": 2570
+    },
+    {
+      "epoch": 0.7308988923198958,
+      "grad_norm": 39.82976150512695,
+      "learning_rate": 1.7796217200488114e-06,
+      "loss": 6.4348,
+      "step": 2580
+    },
+    {
+      "epoch": 0.7337318337629961,
+      "grad_norm": 25.495925903320312,
+      "learning_rate": 1.7446541579041048e-06,
+      "loss": 4.2349,
+      "step": 2590
+    },
+    {
+      "epoch": 0.7365647752060965,
+      "grad_norm": 38.05914306640625,
+      "learning_rate": 1.7099608037034953e-06,
+      "loss": 5.2485,
+      "step": 2600
+    },
+    {
+      "epoch": 0.7393977166491968,
+      "grad_norm": 22.876413345336914,
+      "learning_rate": 1.6755445797765286e-06,
+      "loss": 4.263,
+      "step": 2610
+    },
+    {
+      "epoch": 0.7422306580922973,
+      "grad_norm": 22.402753829956055,
+      "learning_rate": 1.6414083851091973e-06,
+      "loss": 4.3153,
+      "step": 2620
+    },
+    {
+      "epoch": 0.7450635995353976,
+      "grad_norm": 20.86781883239746,
+      "learning_rate": 1.6075550950997592e-06,
+      "loss": 4.4095,
+      "step": 2630
+    },
+    {
+      "epoch": 0.747896540978498,
+      "grad_norm": 39.51744842529297,
+      "learning_rate": 1.5739875613165283e-06,
+      "loss": 6.2356,
+      "step": 2640
+    },
+    {
+      "epoch": 0.7507294824215983,
+      "grad_norm": 26.651187896728516,
+      "learning_rate": 1.5407086112576813e-06,
+      "loss": 4.1033,
+      "step": 2650
+    },
+    {
+      "epoch": 0.7535624238646987,
+      "grad_norm": 46.947757720947266,
+      "learning_rate": 1.5077210481130815e-06,
+      "loss": 8.1815,
+      "step": 2660
+    },
+    {
+      "epoch": 0.756395365307799,
+      "grad_norm": 41.29295349121094,
+      "learning_rate": 1.475027650528168e-06,
+      "loss": 6.1637,
+      "step": 2670
+    },
+    {
+      "epoch": 0.7592283067508995,
+      "grad_norm": 39.40729522705078,
+      "learning_rate": 1.442631172369896e-06,
+      "loss": 7.1273,
+      "step": 2680
+    },
+    {
+      "epoch": 0.7620612481939998,
+      "grad_norm": 39.58256912231445,
+      "learning_rate": 1.4105343424947654e-06,
+      "loss": 5.187,
+      "step": 2690
+    },
+    {
+      "epoch": 0.7648941896371002,
+      "grad_norm": 39.836185455322266,
+      "learning_rate": 1.378739864518971e-06,
+      "loss": 3.8889,
+      "step": 2700
+    },
+    {
+      "epoch": 0.7677271310802005,
+      "grad_norm": 40.20053482055664,
+      "learning_rate": 1.3472504165906614e-06,
+      "loss": 5.3001,
+      "step": 2710
+    },
+    {
+      "epoch": 0.7705600725233009,
+      "grad_norm": 23.571002960205078,
+      "learning_rate": 1.3160686511643505e-06,
+      "loss": 4.0238,
+      "step": 2720
+    },
+    {
+      "epoch": 0.7733930139664014,
+      "grad_norm": 23.623443603515625,
+      "learning_rate": 1.2851971947774987e-06,
+      "loss": 5.1091,
+      "step": 2730
+    },
+    {
+      "epoch": 0.7762259554095017,
+      "grad_norm": 31.367658615112305,
+      "learning_rate": 1.2546386478292604e-06,
+      "loss": 4.1048,
+      "step": 2740
+    },
+    {
+      "epoch": 0.7790588968526021,
+      "grad_norm": 39.296226501464844,
+      "learning_rate": 1.2243955843614558e-06,
+      "loss": 4.271,
+      "step": 2750
+    },
+    {
+      "epoch": 0.7818918382957024,
+      "grad_norm": 28.69118881225586,
+      "learning_rate": 1.1944705518417466e-06,
+      "loss": 4.0739,
+      "step": 2760
+    },
+    {
+      "epoch": 0.7847247797388028,
+      "grad_norm": 32.27414321899414,
+      "learning_rate": 1.1648660709490538e-06,
+      "loss": 5.1998,
+      "step": 2770
+    },
+    {
+      "epoch": 0.7875577211819031,
+      "grad_norm": 24.473217010498047,
+      "learning_rate": 1.135584635361232e-06,
+      "loss": 4.9601,
+      "step": 2780
+    },
+    {
+      "epoch": 0.7903906626250036,
+      "grad_norm": 27.856367111206055,
+      "learning_rate": 1.1066287115450242e-06,
+      "loss": 4.9381,
+      "step": 2790
+    },
+    {
+      "epoch": 0.7932236040681039,
+      "grad_norm": 17.640838623046875,
+      "learning_rate": 1.0780007385483005e-06,
+      "loss": 4.2145,
+      "step": 2800
+    },
+    {
+      "epoch": 0.7960565455112043,
+      "grad_norm": 34.375091552734375,
+      "learning_rate": 1.0497031277946062e-06,
+      "loss": 8.3028,
+      "step": 2810
+    },
+    {
+      "epoch": 0.7988894869543046,
+      "grad_norm": 23.346403121948242,
+      "learning_rate": 1.0217382628800465e-06,
+      "loss": 6.9337,
+      "step": 2820
+    },
+    {
+      "epoch": 0.801722428397405,
+      "grad_norm": 25.259016036987305,
+      "learning_rate": 9.94108499372507e-07,
+      "loss": 3.1855,
+      "step": 2830
+    },
+    {
+      "epoch": 0.8045553698405054,
+      "grad_norm": 33.022727966308594,
+      "learning_rate": 9.668161646132296e-07,
+      "loss": 5.2408,
+      "step": 2840
+    },
+    {
+      "epoch": 0.8073883112836058,
+      "grad_norm": 30.2951717376709,
+      "learning_rate": 9.398635575207854e-07,
+      "loss": 3.1828,
+      "step": 2850
+    },
+    {
+      "epoch": 0.8102212527267061,
+      "grad_norm": 51.273616790771484,
+      "learning_rate": 9.132529483974217e-07,
+      "loss": 5.0485,
+      "step": 2860
+    },
+    {
+      "epoch": 0.8130541941698065,
+      "grad_norm": 39.878597259521484,
+      "learning_rate": 8.869865787378262e-07,
+      "loss": 6.3068,
+      "step": 2870
+    },
+    {
+      "epoch": 0.8158871356129069,
+      "grad_norm": 21.064966201782227,
+      "learning_rate": 8.61066661040324e-07,
+      "loss": 3.3587,
+      "step": 2880
+    },
+    {
+      "epoch": 0.8187200770560072,
+      "grad_norm": 22.18380355834961,
+      "learning_rate": 8.354953786205133e-07,
+      "loss": 4.242,
+      "step": 2890
+    },
+    {
+      "epoch": 0.8215530184991077,
+      "grad_norm": 41.24013137817383,
+      "learning_rate": 8.102748854273468e-07,
+      "loss": 4.1017,
+      "step": 2900
+    },
+    {
+      "epoch": 0.824385959942208,
+      "grad_norm": 23.30076789855957,
+      "learning_rate": 7.854073058617112e-07,
+      "loss": 5.3308,
+      "step": 2910
+    },
+    {
+      "epoch": 0.8272189013853084,
+      "grad_norm": 21.42025375366211,
+      "learning_rate": 7.60894734597476e-07,
+      "loss": 8.113,
+      "step": 2920
+    },
+    {
+      "epoch": 0.8300518428284087,
+      "grad_norm": 18.741073608398438,
+      "learning_rate": 7.367392364050485e-07,
+      "loss": 5.1848,
+      "step": 2930
+    },
+    {
+      "epoch": 0.8328847842715091,
+      "grad_norm": 23.857194900512695,
+      "learning_rate": 7.129428459774618e-07,
+      "loss": 7.1581,
+      "step": 2940
+    },
+    {
+      "epoch": 0.8357177257146094,
+      "grad_norm": 28.530094146728516,
+      "learning_rate": 6.895075677589791e-07,
+      "loss": 6.2661,
+      "step": 2950
+    },
+    {
+      "epoch": 0.8385506671577099,
+      "grad_norm": 40.354949951171875,
+      "learning_rate": 6.664353757762515e-07,
+      "loss": 4.2647,
+      "step": 2960
+    },
+    {
+      "epoch": 0.8413836086008102,
+      "grad_norm": 21.305288314819336,
+      "learning_rate": 6.437282134720479e-07,
+      "loss": 4.9122,
+      "step": 2970
+    },
+    {
+      "epoch": 0.8442165500439106,
+      "grad_norm": 40.32603454589844,
+      "learning_rate": 6.21387993541544e-07,
+      "loss": 6.2095,
+      "step": 2980
+    },
+    {
+      "epoch": 0.8470494914870109,
+      "grad_norm": 39.293067932128906,
+      "learning_rate": 5.994165977712175e-07,
+      "loss": 4.1365,
+      "step": 2990
+    },
+    {
+      "epoch": 0.8498824329301113,
+      "grad_norm": 25.006118774414062,
+      "learning_rate": 5.778158768803294e-07,
+      "loss": 3.4504,
+      "step": 3000
+    },
+    {
+      "epoch": 0.8527153743732117,
+      "grad_norm": 38.37477111816406,
+      "learning_rate": 5.565876503650442e-07,
+      "loss": 4.2214,
+      "step": 3010
+    },
+    {
+      "epoch": 0.8555483158163121,
+      "grad_norm": 40.921207427978516,
+      "learning_rate": 5.357337063451601e-07,
+      "loss": 5.1103,
+      "step": 3020
+    },
+    {
+      "epoch": 0.8583812572594125,
+      "grad_norm": 26.225017547607422,
+      "learning_rate": 5.152558014134906e-07,
+      "loss": 5.9913,
+      "step": 3030
+    },
+    {
+      "epoch": 0.8612141987025128,
+      "grad_norm": 22.678930282592773,
+      "learning_rate": 4.951556604879049e-07,
+      "loss": 4.3731,
+      "step": 3040
+    },
+    {
+      "epoch": 0.8640471401456132,
+      "grad_norm": 29.937393188476562,
+      "learning_rate": 4.754349766660299e-07,
+      "loss": 4.2301,
+      "step": 3050
+    },
+    {
+      "epoch": 0.8668800815887135,
+      "grad_norm": 26.465091705322266,
+      "learning_rate": 4.5609541108263377e-07,
+      "loss": 6.0091,
+      "step": 3060
+    },
+    {
+      "epoch": 0.869713023031814,
+      "grad_norm": 25.58681297302246,
+      "learning_rate": 4.3713859276971026e-07,
+      "loss": 6.979,
+      "step": 3070
+    },
+    {
+      "epoch": 0.8725459644749143,
+      "grad_norm": 22.564706802368164,
+      "learning_rate": 4.1856611851925245e-07,
+      "loss": 5.0316,
+      "step": 3080
+    },
+    {
+      "epoch": 0.8753789059180147,
+      "grad_norm": 40.97758102416992,
+      "learning_rate": 4.003795527487508e-07,
+      "loss": 8.964,
+      "step": 3090
+    },
+    {
+      "epoch": 0.878211847361115,
+      "grad_norm": 32.80113220214844,
+      "learning_rate": 3.8258042736942446e-07,
+      "loss": 3.1517,
+      "step": 3100
+    },
+    {
+      "epoch": 0.8810447888042154,
+      "grad_norm": 30.950176239013672,
+      "learning_rate": 3.651702416571762e-07,
+      "loss": 4.345,
+      "step": 3110
+    },
+    {
+      "epoch": 0.8838777302473158,
+      "grad_norm": 21.242090225219727,
+      "learning_rate": 3.481504621263049e-07,
+      "loss": 6.1642,
+      "step": 3120
+    },
+    {
+      "epoch": 0.8867106716904162,
+      "grad_norm": 21.0790958404541,
+      "learning_rate": 3.315225224059809e-07,
+      "loss": 5.1734,
+      "step": 3130
+    },
+    {
+      "epoch": 0.8895436131335165,
+      "grad_norm": 41.8050537109375,
+      "learning_rate": 3.1528782311948226e-07,
+      "loss": 5.0608,
+      "step": 3140
+    },
+    {
+      "epoch": 0.8923765545766169,
+      "grad_norm": 23.527942657470703,
+      "learning_rate": 2.9944773176621756e-07,
+      "loss": 5.9961,
+      "step": 3150
+    },
+    {
+      "epoch": 0.8952094960197172,
+      "grad_norm": 28.754201889038086,
+      "learning_rate": 2.840035826065368e-07,
+      "loss": 3.8781,
+      "step": 3160
+    },
+    {
+      "epoch": 0.8980424374628176,
+      "grad_norm": 26.580829620361328,
+      "learning_rate": 2.689566765493451e-07,
+      "loss": 4.1426,
+      "step": 3170
+    },
+    {
+      "epoch": 0.9008753789059181,
+      "grad_norm": 18.550945281982422,
+      "learning_rate": 2.5430828104251684e-07,
+      "loss": 4.9139,
+      "step": 3180
+    },
+    {
+      "epoch": 0.9037083203490184,
+      "grad_norm": 20.301895141601562,
+      "learning_rate": 2.4005962996614174e-07,
+      "loss": 3.1654,
+      "step": 3190
+    },
+    {
+      "epoch": 0.9065412617921188,
+      "grad_norm": 37.804969787597656,
+      "learning_rate": 2.2621192352858702e-07,
+      "loss": 5.0736,
+      "step": 3200
+    },
+    {
+      "epoch": 0.9093742032352191,
+      "grad_norm": 29.193897247314453,
+      "learning_rate": 2.1276632816540077e-07,
+      "loss": 5.2175,
+      "step": 3210
+    },
+    {
+      "epoch": 0.9122071446783195,
+      "grad_norm": 23.378320693969727,
+      "learning_rate": 1.9972397644106023e-07,
+      "loss": 5.2508,
+      "step": 3220
+    },
+    {
+      "epoch": 0.9150400861214198,
+      "grad_norm": 18.92923355102539,
+      "learning_rate": 1.870859669535724e-07,
+      "loss": 5.2554,
+      "step": 3230
+    },
+    {
+      "epoch": 0.9178730275645203,
+      "grad_norm": 18.03963279724121,
+      "learning_rate": 1.7485336424193366e-07,
+      "loss": 5.1253,
+      "step": 3240
+    },
+    {
+      "epoch": 0.9207059690076206,
+      "grad_norm": 38.961456298828125,
+      "learning_rate": 1.6302719869646432e-07,
+      "loss": 5.0659,
+      "step": 3250
+    },
+    {
+      "epoch": 0.923538910450721,
+      "grad_norm": 20.624431610107422,
+      "learning_rate": 1.5160846647201132e-07,
+      "loss": 4.1776,
+      "step": 3260
+    },
+    {
+      "epoch": 0.9263718518938213,
+      "grad_norm": 21.755279541015625,
+      "learning_rate": 1.4059812940404093e-07,
+      "loss": 3.142,
+      "step": 3270
+    },
+    {
+      "epoch": 0.9292047933369217,
+      "grad_norm": 20.076051712036133,
+      "learning_rate": 1.2999711492762079e-07,
+      "loss": 5.2161,
+      "step": 3280
+    },
+    {
+      "epoch": 0.932037734780022,
+      "grad_norm": 25.841142654418945,
+      "learning_rate": 1.198063159992996e-07,
+      "loss": 5.3184,
+      "step": 3290
+    },
+    {
+      "epoch": 0.9348706762231225,
+      "grad_norm": 35.23577117919922,
+      "learning_rate": 1.1002659102188784e-07,
+      "loss": 3.3098,
+      "step": 3300
+    },
+    {
+      "epoch": 0.9377036176662228,
+      "grad_norm": 20.789785385131836,
+      "learning_rate": 1.006587637721551e-07,
+      "loss": 3.1742,
+      "step": 3310
+    },
+    {
+      "epoch": 0.9405365591093232,
+      "grad_norm": 40.675296783447266,
+      "learning_rate": 9.170362333143778e-08,
+      "loss": 7.3385,
+      "step": 3320
+    },
+    {
+      "epoch": 0.9433695005524236,
+      "grad_norm": 24.71589469909668,
+      "learning_rate": 8.316192401917667e-08,
+      "loss": 5.3478,
+      "step": 3330
+    },
+    {
+      "epoch": 0.946202441995524,
+      "grad_norm": 38.48093032836914,
+      "learning_rate": 7.503438532937169e-08,
+      "loss": 6.069,
+      "step": 3340
+    },
+    {
+      "epoch": 0.9490353834386244,
+      "grad_norm": 26.636127471923828,
+      "learning_rate": 6.732169186998372e-08,
+      "loss": 4.1179,
+      "step": 3350
+    },
+    {
+      "epoch": 0.9518683248817247,
+      "grad_norm": 42.95631790161133,
+      "learning_rate": 6.002449330526294e-08,
+      "loss": 6.9268,
+      "step": 3360
+    },
+    {
+      "epoch": 0.9547012663248251,
+      "grad_norm": 20.64594268798828,
+      "learning_rate": 5.31434043010276e-08,
+      "loss": 3.1192,
+      "step": 3370
+    },
+    {
+      "epoch": 0.9575342077679254,
+      "grad_norm": 44.144744873046875,
+      "learning_rate": 4.667900447288931e-08,
+      "loss": 6.0163,
+      "step": 3380
+    },
+    {
+      "epoch": 0.9603671492110258,
+      "grad_norm": 41.0361442565918,
+      "learning_rate": 4.0631838337427675e-08,
+      "loss": 5.265,
+      "step": 3390
+    },
+    {
+      "epoch": 0.9632000906541262,
+      "grad_norm": 28.538305282592773,
+      "learning_rate": 3.500241526632753e-08,
+      "loss": 5.07,
+      "step": 3400
+    },
+    {
+      "epoch": 0.9660330320972266,
+      "grad_norm": 24.23358154296875,
+      "learning_rate": 2.979120944346936e-08,
+      "loss": 4.3623,
+      "step": 3410
+    },
+    {
+      "epoch": 0.9688659735403269,
+      "grad_norm": 26.74643898010254,
+      "learning_rate": 2.499865982499128e-08,
+      "loss": 3.3373,
+      "step": 3420
+    },
+    {
+      "epoch": 0.9716989149834273,
+      "grad_norm": 22.635358810424805,
+      "learning_rate": 2.0625170102309687e-08,
+      "loss": 4.2529,
+      "step": 3430
+    },
+    {
+      "epoch": 0.9745318564265276,
+      "grad_norm": 25.663415908813477,
+      "learning_rate": 1.6671108668119828e-08,
+      "loss": 4.1368,
+      "step": 3440
+    },
+    {
+      "epoch": 0.977364797869628,
+      "grad_norm": 28.40155029296875,
+      "learning_rate": 1.3136808585361149e-08,
+      "loss": 5.9535,
+      "step": 3450
+    },
+    {
+      "epoch": 0.9801977393127284,
+      "grad_norm": 21.457969665527344,
+      "learning_rate": 1.0022567559164198e-08,
+      "loss": 6.1661,
+      "step": 3460
+    },
+    {
+      "epoch": 0.9830306807558288,
+      "grad_norm": 26.77224349975586,
+      "learning_rate": 7.328647911774567e-09,
+      "loss": 4.1479,
+      "step": 3470
+    },
+    {
+      "epoch": 0.9858636221989292,
+      "grad_norm": 34.7308464050293,
+      "learning_rate": 5.055276560454459e-09,
+      "loss": 3.216,
+      "step": 3480
+    },
+    {
+      "epoch": 0.9886965636420295,
+      "grad_norm": 27.362573623657227,
+      "learning_rate": 3.202644998370752e-09,
+      "loss": 5.132,
+      "step": 3490
+    },
+    {
+      "epoch": 0.9915295050851299,
+      "grad_norm": 33.32588195800781,
+      "learning_rate": 1.770909278464017e-09,
+      "loss": 6.1225,
+      "step": 3500
+    },
+    {
+      "epoch": 0.9943624465282302,
+      "grad_norm": 19.929155349731445,
+      "learning_rate": 7.601900003051388e-10,
+      "loss": 3.1458,
+      "step": 3510
+    },
+    {
+      "epoch": 0.9971953879713307,
+      "grad_norm": 26.834556579589844,
+      "learning_rate": 1.7057229993344693e-10,
+      "loss": 7.2351,
+      "step": 3520
+    },
+    {
+      "epoch": 0.9997450352701209,
+      "step": 3529,
+      "total_flos": 1.1540349697243742e+19,
+      "train_loss": 5.616373471816494,
+      "train_runtime": 84659.9545,
+      "train_samples_per_second": 2.502,
+      "train_steps_per_second": 0.042
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3529,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1540349697243742e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

Order_Ablations/E116/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d82c01b8b47f3dcb417bcd4d7e8a34808b14725c2e7b32fe00671fd3ca24372
+size 7697