Upload folder using huggingface_hub

spa3_vlm/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

spa3_vlm/chat_template.json

@@ -0,0 +1,3 @@
+{
+    "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+}

spa3_vlm/config.json

@@ -0,0 +1,71 @@
+{
+  "architectures": [
+    "SPA3_VLMForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "feature_fusion_method": "add",
+  "fusion_num_layers": 1,
+  "geometry_encoder_type": "spa3r",
+  "geometry_merger_type": "mlp",
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_token_id": 151655,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 128000,
+  "max_window_layers": 70,
+  "model_type": "qwen2_5_vl",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 36,
+  "num_key_value_heads": 2,
+  "reference_frame": "first",
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "mrope_section": [
+      16,
+      24,
+      24
+    ],
+    "rope_type": "default",
+    "type": "default"
+  },
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.50.0",
+  "use_cache": true,
+  "use_geometry_encoder": true,
+  "use_sliding_window": false,
+  "video_token_id": 151656,
+  "vision_config": {
+    "depth": 32,
+    "fullatt_block_indexes": [
+      7,
+      15,
+      23,
+      31
+    ],
+    "hidden_act": "silu",
+    "hidden_size": 1280,
+    "in_channels": 3,
+    "in_chans": 3,
+    "intermediate_size": 3420,
+    "model_type": "qwen2_5_vl",
+    "num_heads": 16,
+    "out_hidden_size": 2048,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "temporal_patch_size": 2,
+    "tokens_per_second": 2,
+    "torch_dtype": "bfloat16",
+    "window_size": 112
+  },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
+  "vocab_size": 151936
+}

spa3_vlm/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 1e-06,
+  "transformers_version": "4.50.0"
+}

spa3_vlm/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:906f0fe47a9b5d64136ef20c5040f074e1cfaf91ab13fabd15a1b9832c8c741a
+size 4997750760

spa3_vlm/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba0ceb83af36b1684ed11e139e4e8e4faeebe542f2acd04d3322f7070767261e
+size 4999471242

spa3_vlm/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1bd1a8ff22e7a378dee8c0db855aecd0835830b86c5f20ddb1d1471db531487
+size 1386799840

spa3_vlm/preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 451584,
+  "merge_size": 2,
+  "min_pixels": 12544,
+  "patch_size": 14,
+  "processor_class": "Qwen2_5_VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 451584,
+    "shortest_edge": 12544
+  },
+  "temporal_patch_size": 2
+}

spa3_vlm/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

spa3_vlm/tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 16384,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

spa3_vlm/trainer_state.json

@@ -0,0 +1,1331 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9999729122084676,
+  "eval_steps": 500,
+  "global_step": 9229,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005417558306471273,
+      "grad_norm": 7.781421661376953,
+      "learning_rate": 1.805054151624549e-06,
+      "loss": 0.7515,
+      "step": 50
+    },
+    {
+      "epoch": 0.010835116612942546,
+      "grad_norm": 7.788329124450684,
+      "learning_rate": 3.610108303249098e-06,
+      "loss": 0.5659,
+      "step": 100
+    },
+    {
+      "epoch": 0.01625267491941382,
+      "grad_norm": 6.9556708335876465,
+      "learning_rate": 5.415162454873647e-06,
+      "loss": 0.5196,
+      "step": 150
+    },
+    {
+      "epoch": 0.021670233225885092,
+      "grad_norm": 6.938107967376709,
+      "learning_rate": 7.220216606498196e-06,
+      "loss": 0.4934,
+      "step": 200
+    },
+    {
+      "epoch": 0.027087791532356367,
+      "grad_norm": 4.235167503356934,
+      "learning_rate": 9.025270758122744e-06,
+      "loss": 0.4951,
+      "step": 250
+    },
+    {
+      "epoch": 0.03250534983882764,
+      "grad_norm": 5.155239105224609,
+      "learning_rate": 9.999837125565374e-06,
+      "loss": 0.4739,
+      "step": 300
+    },
+    {
+      "epoch": 0.03792290814529892,
+      "grad_norm": 5.2537946701049805,
+      "learning_rate": 9.99835932872431e-06,
+      "loss": 0.4799,
+      "step": 350
+    },
+    {
+      "epoch": 0.043340466451770185,
+      "grad_norm": 5.433236598968506,
+      "learning_rate": 9.995342612258154e-06,
+      "loss": 0.4695,
+      "step": 400
+    },
+    {
+      "epoch": 0.04875802475824146,
+      "grad_norm": 7.882500171661377,
+      "learning_rate": 9.990787904968514e-06,
+      "loss": 0.4731,
+      "step": 450
+    },
+    {
+      "epoch": 0.054175583064712735,
+      "grad_norm": 4.700081825256348,
+      "learning_rate": 9.984696609181225e-06,
+      "loss": 0.4741,
+      "step": 500
+    },
+    {
+      "epoch": 0.05959314137118401,
+      "grad_norm": 4.382456302642822,
+      "learning_rate": 9.977070600314606e-06,
+      "loss": 0.4751,
+      "step": 550
+    },
+    {
+      "epoch": 0.06501069967765528,
+      "grad_norm": 5.6364426612854,
+      "learning_rate": 9.967912226302032e-06,
+      "loss": 0.4524,
+      "step": 600
+    },
+    {
+      "epoch": 0.07042825798412655,
+      "grad_norm": 3.0505552291870117,
+      "learning_rate": 9.957224306869053e-06,
+      "loss": 0.4493,
+      "step": 650
+    },
+    {
+      "epoch": 0.07584581629059783,
+      "grad_norm": 7.565575122833252,
+      "learning_rate": 9.945010132665239e-06,
+      "loss": 0.4549,
+      "step": 700
+    },
+    {
+      "epoch": 0.0812633745970691,
+      "grad_norm": 3.4038021564483643,
+      "learning_rate": 9.93127346425103e-06,
+      "loss": 0.4487,
+      "step": 750
+    },
+    {
+      "epoch": 0.08668093290354037,
+      "grad_norm": 3.1840455532073975,
+      "learning_rate": 9.916018530939939e-06,
+      "loss": 0.4416,
+      "step": 800
+    },
+    {
+      "epoch": 0.09209849121001165,
+      "grad_norm": 4.019189357757568,
+      "learning_rate": 9.89925002949638e-06,
+      "loss": 0.4562,
+      "step": 850
+    },
+    {
+      "epoch": 0.09751604951648292,
+      "grad_norm": 3.7514753341674805,
+      "learning_rate": 9.880973122689623e-06,
+      "loss": 0.4524,
+      "step": 900
+    },
+    {
+      "epoch": 0.1029336078229542,
+      "grad_norm": 4.089905738830566,
+      "learning_rate": 9.861193437704259e-06,
+      "loss": 0.429,
+      "step": 950
+    },
+    {
+      "epoch": 0.10835116612942547,
+      "grad_norm": 3.3279268741607666,
+      "learning_rate": 9.839917064407662e-06,
+      "loss": 0.4354,
+      "step": 1000
+    },
+    {
+      "epoch": 0.11376872443589674,
+      "grad_norm": 3.866124391555786,
+      "learning_rate": 9.817150553475016e-06,
+      "loss": 0.4403,
+      "step": 1050
+    },
+    {
+      "epoch": 0.11918628274236802,
+      "grad_norm": 5.939216136932373,
+      "learning_rate": 9.792900914372459e-06,
+      "loss": 0.4425,
+      "step": 1100
+    },
+    {
+      "epoch": 0.12460384104883929,
+      "grad_norm": 6.584618091583252,
+      "learning_rate": 9.767175613198979e-06,
+      "loss": 0.4396,
+      "step": 1150
+    },
+    {
+      "epoch": 0.13002139935531057,
+      "grad_norm": 3.3282132148742676,
+      "learning_rate": 9.739982570387706e-06,
+      "loss": 0.4353,
+      "step": 1200
+    },
+    {
+      "epoch": 0.13543895766178182,
+      "grad_norm": 2.8459200859069824,
+      "learning_rate": 9.71133015826734e-06,
+      "loss": 0.4222,
+      "step": 1250
+    },
+    {
+      "epoch": 0.1408565159682531,
+      "grad_norm": 2.846902370452881,
+      "learning_rate": 9.681227198484431e-06,
+      "loss": 0.4303,
+      "step": 1300
+    },
+    {
+      "epoch": 0.14627407427472439,
+      "grad_norm": 5.5621418952941895,
+      "learning_rate": 9.649682959287321e-06,
+      "loss": 0.4412,
+      "step": 1350
+    },
+    {
+      "epoch": 0.15169163258119567,
+      "grad_norm": 3.3358514308929443,
+      "learning_rate": 9.616707152672603e-06,
+      "loss": 0.4292,
+      "step": 1400
+    },
+    {
+      "epoch": 0.15710919088766692,
+      "grad_norm": 3.2699906826019287,
+      "learning_rate": 9.582309931394934e-06,
+      "loss": 0.4154,
+      "step": 1450
+    },
+    {
+      "epoch": 0.1625267491941382,
+      "grad_norm": 4.3978962898254395,
+      "learning_rate": 9.546501885841158e-06,
+      "loss": 0.4144,
+      "step": 1500
+    },
+    {
+      "epoch": 0.16794430750060949,
+      "grad_norm": 3.8703715801239014,
+      "learning_rate": 9.50929404076968e-06,
+      "loss": 0.4262,
+      "step": 1550
+    },
+    {
+      "epoch": 0.17336186580708074,
+      "grad_norm": 3.6931490898132324,
+      "learning_rate": 9.47069785191612e-06,
+      "loss": 0.4126,
+      "step": 1600
+    },
+    {
+      "epoch": 0.17877942411355202,
+      "grad_norm": 4.529512882232666,
+      "learning_rate": 9.430725202466256e-06,
+      "loss": 0.4037,
+      "step": 1650
+    },
+    {
+      "epoch": 0.1841969824200233,
+      "grad_norm": 3.3340742588043213,
+      "learning_rate": 9.389388399397365e-06,
+      "loss": 0.418,
+      "step": 1700
+    },
+    {
+      "epoch": 0.18961454072649456,
+      "grad_norm": 2.689044952392578,
+      "learning_rate": 9.346700169689107e-06,
+      "loss": 0.4047,
+      "step": 1750
+    },
+    {
+      "epoch": 0.19503209903296584,
+      "grad_norm": 3.985704183578491,
+      "learning_rate": 9.302673656405056e-06,
+      "loss": 0.4229,
+      "step": 1800
+    },
+    {
+      "epoch": 0.20044965733943712,
+      "grad_norm": 3.384211540222168,
+      "learning_rate": 9.257322414646177e-06,
+      "loss": 0.4166,
+      "step": 1850
+    },
+    {
+      "epoch": 0.2058672156459084,
+      "grad_norm": 2.8771655559539795,
+      "learning_rate": 9.210660407377383e-06,
+      "loss": 0.4097,
+      "step": 1900
+    },
+    {
+      "epoch": 0.21128477395237966,
+      "grad_norm": 4.303565502166748,
+      "learning_rate": 9.162702001128573e-06,
+      "loss": 0.3976,
+      "step": 1950
+    },
+    {
+      "epoch": 0.21670233225885094,
+      "grad_norm": 3.7666449546813965,
+      "learning_rate": 9.113461961571385e-06,
+      "loss": 0.4019,
+      "step": 2000
+    },
+    {
+      "epoch": 0.22211989056532222,
+      "grad_norm": 6.130232810974121,
+      "learning_rate": 9.062955448973063e-06,
+      "loss": 0.4046,
+      "step": 2050
+    },
+    {
+      "epoch": 0.22753744887179347,
+      "grad_norm": 3.652613878250122,
+      "learning_rate": 9.011198013528845e-06,
+      "loss": 0.3948,
+      "step": 2100
+    },
+    {
+      "epoch": 0.23295500717826476,
+      "grad_norm": 4.481716632843018,
+      "learning_rate": 8.958205590574289e-06,
+      "loss": 0.3966,
+      "step": 2150
+    },
+    {
+      "epoch": 0.23837256548473604,
+      "grad_norm": 4.057042598724365,
+      "learning_rate": 8.903994495679029e-06,
+      "loss": 0.3934,
+      "step": 2200
+    },
+    {
+      "epoch": 0.2437901237912073,
+      "grad_norm": 3.507650852203369,
+      "learning_rate": 8.848581419623446e-06,
+      "loss": 0.3931,
+      "step": 2250
+    },
+    {
+      "epoch": 0.24920768209767857,
+      "grad_norm": 3.1253199577331543,
+      "learning_rate": 8.791983423259836e-06,
+      "loss": 0.3922,
+      "step": 2300
+    },
+    {
+      "epoch": 0.25462524040414986,
+      "grad_norm": 3.4113478660583496,
+      "learning_rate": 8.734217932259627e-06,
+      "loss": 0.3983,
+      "step": 2350
+    },
+    {
+      "epoch": 0.26004279871062114,
+      "grad_norm": 4.213040828704834,
+      "learning_rate": 8.67530273174828e-06,
+      "loss": 0.3909,
+      "step": 2400
+    },
+    {
+      "epoch": 0.2654603570170924,
+      "grad_norm": 2.6129212379455566,
+      "learning_rate": 8.615255960829514e-06,
+      "loss": 0.3879,
+      "step": 2450
+    },
+    {
+      "epoch": 0.27087791532356364,
+      "grad_norm": 3.5744173526763916,
+      "learning_rate": 8.554096107000545e-06,
+      "loss": 0.3911,
+      "step": 2500
+    },
+    {
+      "epoch": 0.2762954736300349,
+      "grad_norm": 2.9202399253845215,
+      "learning_rate": 8.491842000460072e-06,
+      "loss": 0.3786,
+      "step": 2550
+    },
+    {
+      "epoch": 0.2817130319365062,
+      "grad_norm": 2.073516845703125,
+      "learning_rate": 8.428512808310738e-06,
+      "loss": 0.3852,
+      "step": 2600
+    },
+    {
+      "epoch": 0.2871305902429775,
+      "grad_norm": 2.9791362285614014,
+      "learning_rate": 8.364128028657869e-06,
+      "loss": 0.3858,
+      "step": 2650
+    },
+    {
+      "epoch": 0.29254814854944877,
+      "grad_norm": 3.5832362174987793,
+      "learning_rate": 8.298707484606294e-06,
+      "loss": 0.3902,
+      "step": 2700
+    },
+    {
+      "epoch": 0.29796570685592005,
+      "grad_norm": 3.3187594413757324,
+      "learning_rate": 8.232271318157122e-06,
+      "loss": 0.3912,
+      "step": 2750
+    },
+    {
+      "epoch": 0.30338326516239134,
+      "grad_norm": 3.5363144874572754,
+      "learning_rate": 8.164839984006312e-06,
+      "loss": 0.3815,
+      "step": 2800
+    },
+    {
+      "epoch": 0.30880082346886256,
+      "grad_norm": 2.7027199268341064,
+      "learning_rate": 8.09643424324698e-06,
+      "loss": 0.3766,
+      "step": 2850
+    },
+    {
+      "epoch": 0.31421838177533384,
+      "grad_norm": 2.912078380584717,
+      "learning_rate": 8.027075156977384e-06,
+      "loss": 0.3673,
+      "step": 2900
+    },
+    {
+      "epoch": 0.3196359400818051,
+      "grad_norm": 2.636035919189453,
+      "learning_rate": 7.95678407981651e-06,
+      "loss": 0.3765,
+      "step": 2950
+    },
+    {
+      "epoch": 0.3250534983882764,
+      "grad_norm": 5.268231391906738,
+      "learning_rate": 7.885582653329323e-06,
+      "loss": 0.3747,
+      "step": 3000
+    },
+    {
+      "epoch": 0.3304710566947477,
+      "grad_norm": 3.0212998390197754,
+      "learning_rate": 7.813492799363638e-06,
+      "loss": 0.3744,
+      "step": 3050
+    },
+    {
+      "epoch": 0.33588861500121897,
+      "grad_norm": 3.505629777908325,
+      "learning_rate": 7.740536713300731e-06,
+      "loss": 0.386,
+      "step": 3100
+    },
+    {
+      "epoch": 0.3413061733076902,
+      "grad_norm": 3.1320040225982666,
+      "learning_rate": 7.666736857221702e-06,
+      "loss": 0.3696,
+      "step": 3150
+    },
+    {
+      "epoch": 0.3467237316141615,
+      "grad_norm": 4.146528244018555,
+      "learning_rate": 7.592115952991741e-06,
+      "loss": 0.3629,
+      "step": 3200
+    },
+    {
+      "epoch": 0.35214128992063276,
+      "grad_norm": 4.457525253295898,
+      "learning_rate": 7.516696975264415e-06,
+      "loss": 0.3639,
+      "step": 3250
+    },
+    {
+      "epoch": 0.35755884822710404,
+      "grad_norm": 3.180326461791992,
+      "learning_rate": 7.440503144408105e-06,
+      "loss": 0.3587,
+      "step": 3300
+    },
+    {
+      "epoch": 0.3629764065335753,
+      "grad_norm": 3.1236040592193604,
+      "learning_rate": 7.3635579193568165e-06,
+      "loss": 0.3602,
+      "step": 3350
+    },
+    {
+      "epoch": 0.3683939648400466,
+      "grad_norm": 2.8138082027435303,
+      "learning_rate": 7.285884990387516e-06,
+      "loss": 0.3687,
+      "step": 3400
+    },
+    {
+      "epoch": 0.3738115231465179,
+      "grad_norm": 2.3730499744415283,
+      "learning_rate": 7.20750827182626e-06,
+      "loss": 0.3528,
+      "step": 3450
+    },
+    {
+      "epoch": 0.3792290814529891,
+      "grad_norm": 3.144355535507202,
+      "learning_rate": 7.128451894685329e-06,
+      "loss": 0.3553,
+      "step": 3500
+    },
+    {
+      "epoch": 0.3846466397594604,
+      "grad_norm": 3.7990708351135254,
+      "learning_rate": 7.048740199233655e-06,
+      "loss": 0.3593,
+      "step": 3550
+    },
+    {
+      "epoch": 0.3900641980659317,
+      "grad_norm": 3.613049030303955,
+      "learning_rate": 6.968397727502812e-06,
+      "loss": 0.3574,
+      "step": 3600
+    },
+    {
+      "epoch": 0.39548175637240296,
+      "grad_norm": 2.9559617042541504,
+      "learning_rate": 6.8874492157309045e-06,
+      "loss": 0.3485,
+      "step": 3650
+    },
+    {
+      "epoch": 0.40089931467887424,
+      "grad_norm": 4.09456205368042,
+      "learning_rate": 6.8059195867466435e-06,
+      "loss": 0.355,
+      "step": 3700
+    },
+    {
+      "epoch": 0.4063168729853455,
+      "grad_norm": 3.0889694690704346,
+      "learning_rate": 6.723833942295983e-06,
+      "loss": 0.3527,
+      "step": 3750
+    },
+    {
+      "epoch": 0.4117344312918168,
+      "grad_norm": 4.265697479248047,
+      "learning_rate": 6.64121755531368e-06,
+      "loss": 0.3562,
+      "step": 3800
+    },
+    {
+      "epoch": 0.41715198959828803,
+      "grad_norm": 2.60492205619812,
+      "learning_rate": 6.558095862142122e-06,
+      "loss": 0.3412,
+      "step": 3850
+    },
+    {
+      "epoch": 0.4225695479047593,
+      "grad_norm": 2.914607286453247,
+      "learning_rate": 6.4744944546998745e-06,
+      "loss": 0.3613,
+      "step": 3900
+    },
+    {
+      "epoch": 0.4279871062112306,
+      "grad_norm": 2.1643524169921875,
+      "learning_rate": 6.390439072602308e-06,
+      "loss": 0.3335,
+      "step": 3950
+    },
+    {
+      "epoch": 0.4334046645177019,
+      "grad_norm": 3.182020425796509,
+      "learning_rate": 6.305955595236753e-06,
+      "loss": 0.3454,
+      "step": 4000
+    },
+    {
+      "epoch": 0.43882222282417316,
+      "grad_norm": 3.495281219482422,
+      "learning_rate": 6.221070033794639e-06,
+      "loss": 0.3532,
+      "step": 4050
+    },
+    {
+      "epoch": 0.44423978113064444,
+      "grad_norm": 3.2466609477996826,
+      "learning_rate": 6.135808523263026e-06,
+      "loss": 0.3421,
+      "step": 4100
+    },
+    {
+      "epoch": 0.44965733943711567,
+      "grad_norm": 3.0939319133758545,
+      "learning_rate": 6.0501973143780455e-06,
+      "loss": 0.3437,
+      "step": 4150
+    },
+    {
+      "epoch": 0.45507489774358695,
+      "grad_norm": 3.692669153213501,
+      "learning_rate": 5.964262765542689e-06,
+      "loss": 0.3529,
+      "step": 4200
+    },
+    {
+      "epoch": 0.46049245605005823,
+      "grad_norm": 2.590897560119629,
+      "learning_rate": 5.878031334711452e-06,
+      "loss": 0.3468,
+      "step": 4250
+    },
+    {
+      "epoch": 0.4659100143565295,
+      "grad_norm": 3.975038528442383,
+      "learning_rate": 5.7915295712443285e-06,
+      "loss": 0.3302,
+      "step": 4300
+    },
+    {
+      "epoch": 0.4713275726630008,
+      "grad_norm": 3.6712021827697754,
+      "learning_rate": 5.704784107732669e-06,
+      "loss": 0.3322,
+      "step": 4350
+    },
+    {
+      "epoch": 0.4767451309694721,
+      "grad_norm": 3.436875820159912,
+      "learning_rate": 5.617821651799393e-06,
+      "loss": 0.338,
+      "step": 4400
+    },
+    {
+      "epoch": 0.48216268927594336,
+      "grad_norm": 2.936464309692383,
+      "learning_rate": 5.530668977876128e-06,
+      "loss": 0.3299,
+      "step": 4450
+    },
+    {
+      "epoch": 0.4875802475824146,
+      "grad_norm": 2.5533535480499268,
+      "learning_rate": 5.443352918959753e-06,
+      "loss": 0.3315,
+      "step": 4500
+    },
+    {
+      "epoch": 0.49299780588888586,
+      "grad_norm": 3.4961626529693604,
+      "learning_rate": 5.3559003583509305e-06,
+      "loss": 0.33,
+      "step": 4550
+    },
+    {
+      "epoch": 0.49841536419535715,
+      "grad_norm": 3.5909876823425293,
+      "learning_rate": 5.26833822137714e-06,
+      "loss": 0.3163,
+      "step": 4600
+    },
+    {
+      "epoch": 0.5038329225018284,
+      "grad_norm": 3.2068519592285156,
+      "learning_rate": 5.180693467102777e-06,
+      "loss": 0.3276,
+      "step": 4650
+    },
+    {
+      "epoch": 0.5092504808082997,
+      "grad_norm": 2.6540346145629883,
+      "learning_rate": 5.0929930800288555e-06,
+      "loss": 0.3196,
+      "step": 4700
+    },
+    {
+      "epoch": 0.514668039114771,
+      "grad_norm": 3.2414097785949707,
+      "learning_rate": 5.005264061784896e-06,
+      "loss": 0.319,
+      "step": 4750
+    },
+    {
+      "epoch": 0.5200855974212423,
+      "grad_norm": 2.983604669570923,
+      "learning_rate": 4.9175334228155236e-06,
+      "loss": 0.3191,
+      "step": 4800
+    },
+    {
+      "epoch": 0.5255031557277136,
+      "grad_norm": 2.9019827842712402,
+      "learning_rate": 4.829828174064359e-06,
+      "loss": 0.313,
+      "step": 4850
+    },
+    {
+      "epoch": 0.5309207140341848,
+      "grad_norm": 3.0724728107452393,
+      "learning_rate": 4.74217531865776e-06,
+      "loss": 0.3039,
+      "step": 4900
+    },
+    {
+      "epoch": 0.5363382723406561,
+      "grad_norm": 3.6231110095977783,
+      "learning_rate": 4.65460184359096e-06,
+      "loss": 0.3158,
+      "step": 4950
+    },
+    {
+      "epoch": 0.5417558306471273,
+      "grad_norm": 2.427344560623169,
+      "learning_rate": 4.567134711419181e-06,
+      "loss": 0.3224,
+      "step": 5000
+    },
+    {
+      "epoch": 0.5471733889535986,
+      "grad_norm": 3.4552643299102783,
+      "learning_rate": 4.479800851956265e-06,
+      "loss": 0.3125,
+      "step": 5050
+    },
+    {
+      "epoch": 0.5525909472600699,
+      "grad_norm": 3.9358913898468018,
+      "learning_rate": 4.3926271539833974e-06,
+      "loss": 0.3211,
+      "step": 5100
+    },
+    {
+      "epoch": 0.5580085055665411,
+      "grad_norm": 3.328477621078491,
+      "learning_rate": 4.305640456970448e-06,
+      "loss": 0.3048,
+      "step": 5150
+    },
+    {
+      "epoch": 0.5634260638730124,
+      "grad_norm": 2.9380927085876465,
+      "learning_rate": 4.2188675428125025e-06,
+      "loss": 0.305,
+      "step": 5200
+    },
+    {
+      "epoch": 0.5688436221794837,
+      "grad_norm": 2.578197479248047,
+      "learning_rate": 4.132335127584126e-06,
+      "loss": 0.3149,
+      "step": 5250
+    },
+    {
+      "epoch": 0.574261180485955,
+      "grad_norm": 3.2814207077026367,
+      "learning_rate": 4.046069853313882e-06,
+      "loss": 0.2958,
+      "step": 5300
+    },
+    {
+      "epoch": 0.5796787387924263,
+      "grad_norm": 3.4869089126586914,
+      "learning_rate": 3.960098279781651e-06,
+      "loss": 0.3057,
+      "step": 5350
+    },
+    {
+      "epoch": 0.5850962970988975,
+      "grad_norm": 3.0821373462677,
+      "learning_rate": 3.874446876341279e-06,
+      "loss": 0.3026,
+      "step": 5400
+    },
+    {
+      "epoch": 0.5905138554053688,
+      "grad_norm": 2.943082809448242,
+      "learning_rate": 3.7891420137710517e-06,
+      "loss": 0.3064,
+      "step": 5450
+    },
+    {
+      "epoch": 0.5959314137118401,
+      "grad_norm": 4.134742259979248,
+      "learning_rate": 3.7042099561545413e-06,
+      "loss": 0.294,
+      "step": 5500
+    },
+    {
+      "epoch": 0.6013489720183114,
+      "grad_norm": 2.8058090209960938,
+      "learning_rate": 3.6196768527942837e-06,
+      "loss": 0.3017,
+      "step": 5550
+    },
+    {
+      "epoch": 0.6067665303247827,
+      "grad_norm": 3.481924295425415,
+      "learning_rate": 3.535568730160799e-06,
+      "loss": 0.3,
+      "step": 5600
+    },
+    {
+      "epoch": 0.6121840886312538,
+      "grad_norm": 3.3531112670898438,
+      "learning_rate": 3.451911483879443e-06,
+      "loss": 0.2988,
+      "step": 5650
+    },
+    {
+      "epoch": 0.6176016469377251,
+      "grad_norm": 2.790590286254883,
+      "learning_rate": 3.3687308707575206e-06,
+      "loss": 0.3035,
+      "step": 5700
+    },
+    {
+      "epoch": 0.6230192052441964,
+      "grad_norm": 3.1013131141662598,
+      "learning_rate": 3.2860525008541622e-06,
+      "loss": 0.3004,
+      "step": 5750
+    },
+    {
+      "epoch": 0.6284367635506677,
+      "grad_norm": 3.09080171585083,
+      "learning_rate": 3.203901829595368e-06,
+      "loss": 0.2895,
+      "step": 5800
+    },
+    {
+      "epoch": 0.633854321857139,
+      "grad_norm": 3.0901217460632324,
+      "learning_rate": 3.122304149936666e-06,
+      "loss": 0.2943,
+      "step": 5850
+    },
+    {
+      "epoch": 0.6392718801636103,
+      "grad_norm": 3.16723370552063,
+      "learning_rate": 3.041284584575791e-06,
+      "loss": 0.2888,
+      "step": 5900
+    },
+    {
+      "epoch": 0.6446894384700815,
+      "grad_norm": 3.6559319496154785,
+      "learning_rate": 2.960868078217787e-06,
+      "loss": 0.2832,
+      "step": 5950
+    },
+    {
+      "epoch": 0.6501069967765528,
+      "grad_norm": 3.0292601585388184,
+      "learning_rate": 2.881079389894899e-06,
+      "loss": 0.292,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6555245550830241,
+      "grad_norm": 2.4390199184417725,
+      "learning_rate": 2.801943085343657e-06,
+      "loss": 0.2974,
+      "step": 6050
+    },
+    {
+      "epoch": 0.6609421133894954,
+      "grad_norm": 2.870307207107544,
+      "learning_rate": 2.723483529441435e-06,
+      "loss": 0.273,
+      "step": 6100
+    },
+    {
+      "epoch": 0.6663596716959667,
+      "grad_norm": 3.480215072631836,
+      "learning_rate": 2.6457248787048874e-06,
+      "loss": 0.2749,
+      "step": 6150
+    },
+    {
+      "epoch": 0.6717772300024379,
+      "grad_norm": 4.0242228507995605,
+      "learning_rate": 2.5686910738525062e-06,
+      "loss": 0.2874,
+      "step": 6200
+    },
+    {
+      "epoch": 0.6771947883089092,
+      "grad_norm": 3.3167226314544678,
+      "learning_rate": 2.492405832433652e-06,
+      "loss": 0.2785,
+      "step": 6250
+    },
+    {
+      "epoch": 0.6826123466153804,
+      "grad_norm": 3.381011724472046,
+      "learning_rate": 2.416892641526264e-06,
+      "loss": 0.2891,
+      "step": 6300
+    },
+    {
+      "epoch": 0.6880299049218517,
+      "grad_norm": 3.1961159706115723,
+      "learning_rate": 2.3421747505055443e-06,
+      "loss": 0.2831,
+      "step": 6350
+    },
+    {
+      "epoch": 0.693447463228323,
+      "grad_norm": 2.952812433242798,
+      "learning_rate": 2.2682751638858357e-06,
+      "loss": 0.2739,
+      "step": 6400
+    },
+    {
+      "epoch": 0.6988650215347942,
+      "grad_norm": 3.127049446105957,
+      "learning_rate": 2.1952166342378576e-06,
+      "loss": 0.2689,
+      "step": 6450
+    },
+    {
+      "epoch": 0.7042825798412655,
+      "grad_norm": 3.175241231918335,
+      "learning_rate": 2.1230216551835505e-06,
+      "loss": 0.262,
+      "step": 6500
+    },
+    {
+      "epoch": 0.7097001381477368,
+      "grad_norm": 2.993858814239502,
+      "learning_rate": 2.0517124544706202e-06,
+      "loss": 0.265,
+      "step": 6550
+    },
+    {
+      "epoch": 0.7151176964542081,
+      "grad_norm": 2.3723642826080322,
+      "learning_rate": 1.9813109871289486e-06,
+      "loss": 0.2721,
+      "step": 6600
+    },
+    {
+      "epoch": 0.7205352547606794,
+      "grad_norm": 3.4100069999694824,
+      "learning_rate": 1.9118389287109734e-06,
+      "loss": 0.2629,
+      "step": 6650
+    },
+    {
+      "epoch": 0.7259528130671506,
+      "grad_norm": 3.2423717975616455,
+      "learning_rate": 1.8433176686181104e-06,
+      "loss": 0.2759,
+      "step": 6700
+    },
+    {
+      "epoch": 0.7313703713736219,
+      "grad_norm": 3.0641541481018066,
+      "learning_rate": 1.7757683035152706e-06,
+      "loss": 0.2582,
+      "step": 6750
+    },
+    {
+      "epoch": 0.7367879296800932,
+      "grad_norm": 3.797774314880371,
+      "learning_rate": 1.709211630835534e-06,
+      "loss": 0.2607,
+      "step": 6800
+    },
+    {
+      "epoch": 0.7422054879865645,
+      "grad_norm": 3.2128851413726807,
+      "learning_rate": 1.6436681423769134e-06,
+      "loss": 0.2745,
+      "step": 6850
+    },
+    {
+      "epoch": 0.7476230462930358,
+      "grad_norm": 3.2048397064208984,
+      "learning_rate": 1.579158017993247e-06,
+      "loss": 0.2665,
+      "step": 6900
+    },
+    {
+      "epoch": 0.7530406045995071,
+      "grad_norm": 3.453686237335205,
+      "learning_rate": 1.5157011193811255e-06,
+      "loss": 0.2801,
+      "step": 6950
+    },
+    {
+      "epoch": 0.7584581629059782,
+      "grad_norm": 2.968740940093994,
+      "learning_rate": 1.4533169839647682e-06,
+      "loss": 0.2643,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7638757212124495,
+      "grad_norm": 2.3109071254730225,
+      "learning_rate": 1.392024818880754e-06,
+      "loss": 0.2621,
+      "step": 7050
+    },
+    {
+      "epoch": 0.7692932795189208,
+      "grad_norm": 3.5081787109375,
+      "learning_rate": 1.3318434950644232e-06,
+      "loss": 0.2597,
+      "step": 7100
+    },
+    {
+      "epoch": 0.7747108378253921,
+      "grad_norm": 2.864156484603882,
+      "learning_rate": 1.2727915414398113e-06,
+      "loss": 0.2636,
+      "step": 7150
+    },
+    {
+      "epoch": 0.7801283961318634,
+      "grad_norm": 2.492056131362915,
+      "learning_rate": 1.2148871392148614e-06,
+      "loss": 0.2689,
+      "step": 7200
+    },
+    {
+      "epoch": 0.7855459544383346,
+      "grad_norm": 2.6361782550811768,
+      "learning_rate": 1.1581481162837065e-06,
+      "loss": 0.249,
+      "step": 7250
+    },
+    {
+      "epoch": 0.7909635127448059,
+      "grad_norm": 3.332240104675293,
+      "learning_rate": 1.1025919417377345e-06,
+      "loss": 0.2595,
+      "step": 7300
+    },
+    {
+      "epoch": 0.7963810710512772,
+      "grad_norm": 4.205018520355225,
+      "learning_rate": 1.0482357204871136e-06,
+      "loss": 0.2608,
+      "step": 7350
+    },
+    {
+      "epoch": 0.8017986293577485,
+      "grad_norm": 3.955824375152588,
+      "learning_rate": 9.95096187994447e-07,
+      "loss": 0.2486,
+      "step": 7400
+    },
+    {
+      "epoch": 0.8072161876642198,
+      "grad_norm": 3.6599531173706055,
+      "learning_rate": 9.431897051221795e-07,
+      "loss": 0.2465,
+      "step": 7450
+    },
+    {
+      "epoch": 0.812633745970691,
+      "grad_norm": 3.454878091812134,
+      "learning_rate": 8.925322530953301e-07,
+      "loss": 0.2505,
+      "step": 7500
+    },
+    {
+      "epoch": 0.8180513042771623,
+      "grad_norm": 3.9138760566711426,
+      "learning_rate": 8.431394285811156e-07,
+      "loss": 0.2586,
+      "step": 7550
+    },
+    {
+      "epoch": 0.8234688625836336,
+      "grad_norm": 2.8272411823272705,
+      "learning_rate": 7.950264388869721e-07,
+      "loss": 0.2598,
+      "step": 7600
+    },
+    {
+      "epoch": 0.8288864208901048,
+      "grad_norm": 3.8992950916290283,
+      "learning_rate": 7.482080972784511e-07,
+      "loss": 0.2571,
+      "step": 7650
+    },
+    {
+      "epoch": 0.8343039791965761,
+      "grad_norm": 4.094478130340576,
+      "learning_rate": 7.026988184184407e-07,
+      "loss": 0.2463,
+      "step": 7700
+    },
+    {
+      "epoch": 0.8397215375030473,
+      "grad_norm": 3.7604846954345703,
+      "learning_rate": 6.585126139291076e-07,
+      "loss": 0.2591,
+      "step": 7750
+    },
+    {
+      "epoch": 0.8451390958095186,
+      "grad_norm": 2.4784250259399414,
+      "learning_rate": 6.156630880779252e-07,
+      "loss": 0.2527,
+      "step": 7800
+    },
+    {
+      "epoch": 0.8505566541159899,
+      "grad_norm": 4.052146911621094,
+      "learning_rate": 5.741634335891294e-07,
+      "loss": 0.255,
+      "step": 7850
+    },
+    {
+      "epoch": 0.8559742124224612,
+      "grad_norm": 3.8481266498565674,
+      "learning_rate": 5.340264275818785e-07,
+      "loss": 0.2607,
+      "step": 7900
+    },
+    {
+      "epoch": 0.8613917707289325,
+      "grad_norm": 1.80850088596344,
+      "learning_rate": 4.952644276363644e-07,
+      "loss": 0.2564,
+      "step": 7950
+    },
+    {
+      "epoch": 0.8668093290354038,
+      "grad_norm": 5.037997245788574,
+      "learning_rate": 4.57889367989105e-07,
+      "loss": 0.2482,
+      "step": 8000
+    },
+    {
+      "epoch": 0.872226887341875,
+      "grad_norm": 3.722865343093872,
+      "learning_rate": 4.219127558585662e-07,
+      "loss": 0.2397,
+      "step": 8050
+    },
+    {
+      "epoch": 0.8776444456483463,
+      "grad_norm": 3.6952178478240967,
+      "learning_rate": 3.8734566790227023e-07,
+      "loss": 0.25,
+      "step": 8100
+    },
+    {
+      "epoch": 0.8830620039548176,
+      "grad_norm": 3.290282726287842,
+      "learning_rate": 3.541987468064556e-07,
+      "loss": 0.251,
+      "step": 8150
+    },
+    {
+      "epoch": 0.8884795622612889,
+      "grad_norm": 3.523390769958496,
+      "learning_rate": 3.224821980093523e-07,
+      "loss": 0.2487,
+      "step": 8200
+    },
+    {
+      "epoch": 0.8938971205677602,
+      "grad_norm": 3.55690336227417,
+      "learning_rate": 2.9220578655908384e-07,
+      "loss": 0.2479,
+      "step": 8250
+    },
+    {
+      "epoch": 0.8993146788742313,
+      "grad_norm": 2.6040093898773193,
+      "learning_rate": 2.6337883410715583e-07,
+      "loss": 0.2569,
+      "step": 8300
+    },
+    {
+      "epoch": 0.9047322371807026,
+      "grad_norm": 2.797853946685791,
+      "learning_rate": 2.3601021603845754e-07,
+      "loss": 0.2487,
+      "step": 8350
+    },
+    {
+      "epoch": 0.9101497954871739,
+      "grad_norm": 2.88952374458313,
+      "learning_rate": 2.1010835873866654e-07,
+      "loss": 0.2508,
+      "step": 8400
+    },
+    {
+      "epoch": 0.9155673537936452,
+      "grad_norm": 3.0155351161956787,
+      "learning_rate": 1.8568123699989004e-07,
+      "loss": 0.2504,
+      "step": 8450
+    },
+    {
+      "epoch": 0.9209849121001165,
+      "grad_norm": 3.56722354888916,
+      "learning_rate": 1.6273637156534738e-07,
+      "loss": 0.2451,
+      "step": 8500
+    },
+    {
+      "epoch": 0.9264024704065877,
+      "grad_norm": 2.918964147567749,
+      "learning_rate": 1.4128082681384602e-07,
+      "loss": 0.2462,
+      "step": 8550
+    },
+    {
+      "epoch": 0.931820028713059,
+      "grad_norm": 3.4673612117767334,
+      "learning_rate": 1.213212085847637e-07,
+      "loss": 0.2453,
+      "step": 8600
+    },
+    {
+      "epoch": 0.9372375870195303,
+      "grad_norm": 4.0660271644592285,
+      "learning_rate": 1.0286366214421472e-07,
+      "loss": 0.2495,
+      "step": 8650
+    },
+    {
+      "epoch": 0.9426551453260016,
+      "grad_norm": 3.0246922969818115,
+      "learning_rate": 8.591387029301013e-08,
+      "loss": 0.2461,
+      "step": 8700
+    },
+    {
+      "epoch": 0.9480727036324729,
+      "grad_norm": 4.276732444763184,
+      "learning_rate": 7.047705161701002e-08,
+      "loss": 0.2456,
+      "step": 8750
+    },
+    {
+      "epoch": 0.9534902619389442,
+      "grad_norm": 3.485605001449585,
+      "learning_rate": 5.655795888040272e-08,
+      "loss": 0.2462,
+      "step": 8800
+    },
+    {
+      "epoch": 0.9589078202454154,
+      "grad_norm": 3.392996072769165,
+      "learning_rate": 4.4160877562393625e-08,
+      "loss": 0.2611,
+      "step": 8850
+    },
+    {
+      "epoch": 0.9643253785518867,
+      "grad_norm": 3.8503029346466064,
+      "learning_rate": 3.328962453777895e-08,
+      "loss": 0.2593,
+      "step": 8900
+    },
+    {
+      "epoch": 0.9697429368583579,
+      "grad_norm": 2.5636746883392334,
+      "learning_rate": 2.3947546901779696e-08,
+      "loss": 0.2497,
+      "step": 8950
+    },
+    {
+      "epoch": 0.9751604951648292,
+      "grad_norm": 3.013471841812134,
+      "learning_rate": 1.613752093952703e-08,
+      "loss": 0.243,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9805780534713004,
+      "grad_norm": 4.137032508850098,
+      "learning_rate": 9.86195124049405e-09,
+      "loss": 0.2564,
+      "step": 9050
+    },
+    {
+      "epoch": 0.9859956117777717,
+      "grad_norm": 3.717381715774536,
+      "learning_rate": 5.12276995816019e-09,
+      "loss": 0.2444,
+      "step": 9100
+    },
+    {
+      "epoch": 0.991413170084243,
+      "grad_norm": 4.272940158843994,
+      "learning_rate": 1.921436215129302e-09,
+      "loss": 0.2332,
+      "step": 9150
+    },
+    {
+      "epoch": 0.9968307283907143,
+      "grad_norm": 2.4432733058929443,
+      "learning_rate": 2.589356538906795e-10,
+      "loss": 0.2537,
+      "step": 9200
+    },
+    {
+      "epoch": 0.9999729122084676,
+      "step": 9229,
+      "total_flos": 3.697200260631128e+19,
+      "train_loss": 0.3376897029027299,
+      "train_runtime": 181851.6542,
+      "train_samples_per_second": 3.248,
+      "train_steps_per_second": 0.051
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 9229,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.697200260631128e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

spa3_vlm/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b8d1b8b81bdfe6e6c9a8f094641e023641c72322563d0de0aabe5e506d1cd51
+size 7313

spa3r.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce8b388558747c8735f62c6d1a8c9a876f8c274593ee10ccb8d06b6cfe07da78
+size 7216681563