Dispatch SDF 4x post_dolmino: 20260810T113248Z-corefix

.gitattributes

@@ -50,3 +50,4 @@ sft_4epoch/coin/checkpoint-48/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 sft_4epoch/charter/checkpoint-4/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 sft_4epoch/charter/checkpoint-48/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 sdf/1x/shared/post_dolmino/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+sdf/4x/shared/post_dolmino/tokenizer.json filter=lfs diff=lfs merge=lfs -text

sdf/4x/shared/post_dolmino/config.json

@@ -0,0 +1,125 @@
+{
+  "architectures": [
+    "Gemma3ForConditionalGeneration"
+  ],
+  "boi_token_index": 255999,
+  "bos_token_id": 2,
+  "dtype": "bfloat16",
+  "eoi_token_index": 256000,
+  "eos_token_id": 1,
+  "image_token_index": 262144,
+  "initializer_range": 0.02,
+  "mm_tokens_per_image": 256,
+  "model_type": "gemma3",
+  "pad_token_id": 0,
+  "text_config": {
+    "_sliding_window_pattern": 6,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_logit_softcapping": null,
+    "bos_token_id": 2,
+    "cache_implementation": "hybrid",
+    "dtype": "bfloat16",
+    "eos_token_id": 1,
+    "final_logit_softcapping": null,
+    "head_dim": 256,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 3840,
+    "initializer_range": 0.02,
+    "intermediate_size": 15360,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 131072,
+    "model_type": "gemma3_text",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 48,
+    "num_key_value_heads": 8,
+    "pad_token_id": 0,
+    "query_pre_attn_scalar": 256,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "full_attention": {
+        "factor": 8.0,
+        "rope_theta": 1000000.0,
+        "rope_type": "linear"
+      },
+      "sliding_attention": {
+        "rope_theta": 10000.0,
+        "rope_type": "default"
+      }
+    },
+    "sliding_window": 1024,
+    "sliding_window_pattern": 6,
+    "tie_word_embeddings": true,
+    "use_bidirectional_attention": false,
+    "use_cache": false,
+    "vocab_size": 262208
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.9.0",
+  "unsloth_fixed": true,
+  "use_cache": false,
+  "vision_config": {
+    "attention_dropout": 0.0,
+    "dtype": "bfloat16",
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "image_size": 896,
+    "intermediate_size": 4304,
+    "layer_norm_eps": 1e-06,
+    "model_type": "siglip_vision_model",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 27,
+    "patch_size": 14,
+    "vision_use_head": false
+  }
+}

sdf/4x/shared/post_dolmino/generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "pad_token_id": 0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "5.9.0"
+}

sdf/4x/shared/post_dolmino/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7bb3d1e4080b96d19447dc9b9188898eb99cedf9cfe073f7aa6d37117a329a8
+size 26388552360

sdf/4x/shared/post_dolmino/preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_pan_and_scan": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Gemma3ImageProcessor",
+  "image_seq_length": 256,
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "pan_and_scan_max_num_crops": null,
+  "pan_and_scan_min_crop_size": null,
+  "pan_and_scan_min_ratio_to_activate": null,
+  "processor_class": "Gemma3Processor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 896,
+    "width": 896
+  }
+}

sdf/4x/shared/post_dolmino/processor_config.json

@@ -0,0 +1,4 @@
+{
+  "image_seq_length": 256,
+  "processor_class": "Gemma3Processor"
+}

sdf/4x/shared/post_dolmino/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daab2354f8a74e70d70b4d1f804939b68a8c9624dd06cb7858e52dd8970e9726
+size 33384567

sdf/4x/shared/post_dolmino/tokenizer_config.json

@@ -0,0 +1,26 @@
+{
+  "backend": "tokenizers",
+  "boi_token": "<start_of_image>",
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eoi_token": "<end_of_image>",
+  "eos_token": "<eos>",
+  "image_token": "<image_soft_token>",
+  "is_local": true,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "model_max_length": 131072,
+  "model_specific_special_tokens": {
+    "boi_token": "<start_of_image>",
+    "eoi_token": "<end_of_image>",
+    "image_token": "<image_soft_token>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "processor_class": "Gemma3Processor",
+  "sp_model_kwargs": null,
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

sdf/4x/shared/post_dolmino/tokens_state.json

@@ -0,0 +1 @@
+{"total": 15990784, "trainable": 15955136}

sdf/4x/shared/post_dolmino/trainer_state.json

@@ -0,0 +1,930 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.0,
+  "eval_steps": 500,
+  "global_step": 64,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06557377049180328,
+      "grad_norm": 8.9375,
+      "learning_rate": 0.0,
+      "loss": 1.400146484375,
+      "memory/device_reserved (GiB)": 50.15,
+      "memory/max_active (GiB)": 41.3,
+      "memory/max_allocated (GiB)": 41.3,
+      "ppl": 4.05579,
+      "step": 1,
+      "tokens/total": 262144,
+      "tokens/train_per_sec_per_gpu": 388.72,
+      "tokens/trainable": 261631
+    },
+    {
+      "epoch": 0.13114754098360656,
+      "grad_norm": 5.28125,
+      "learning_rate": 5e-06,
+      "loss": 1.5341796875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 4.63752,
+      "step": 2,
+      "tokens/total": 524288,
+      "tokens/train_per_sec_per_gpu": 500.35,
+      "tokens/trainable": 523336
+    },
+    {
+      "epoch": 0.19672131147540983,
+      "grad_norm": 4.46875,
+      "learning_rate": 1e-05,
+      "loss": 1.3289794921875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.77719,
+      "step": 3,
+      "tokens/total": 786432,
+      "tokens/train_per_sec_per_gpu": 508.68,
+      "tokens/trainable": 784892
+    },
+    {
+      "epoch": 0.26229508196721313,
+      "grad_norm": 4.34375,
+      "learning_rate": 9.994224282269737e-06,
+      "loss": 1.38037109375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.97638,
+      "step": 4,
+      "tokens/total": 1048576,
+      "tokens/train_per_sec_per_gpu": 509.08,
+      "tokens/trainable": 1046436
+    },
+    {
+      "epoch": 0.32786885245901637,
+      "grad_norm": 6.3125,
+      "learning_rate": 9.976911955263528e-06,
+      "loss": 1.3056640625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.69014,
+      "step": 5,
+      "tokens/total": 1310720,
+      "tokens/train_per_sec_per_gpu": 504.87,
+      "tokens/trainable": 1308128
+    },
+    {
+      "epoch": 0.39344262295081966,
+      "grad_norm": 2.609375,
+      "learning_rate": 9.948107459476502e-06,
+      "loss": 1.3333740234375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.79382,
+      "step": 6,
+      "tokens/total": 1572864,
+      "tokens/train_per_sec_per_gpu": 509.51,
+      "tokens/trainable": 1569781
+    },
+    {
+      "epoch": 0.45901639344262296,
+      "grad_norm": 2.03125,
+      "learning_rate": 9.907884735636226e-06,
+      "loss": 1.2593994140625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.5233,
+      "step": 7,
+      "tokens/total": 1835008,
+      "tokens/train_per_sec_per_gpu": 507.84,
+      "tokens/trainable": 1831462
+    },
+    {
+      "epoch": 0.5245901639344263,
+      "grad_norm": 2.140625,
+      "learning_rate": 9.85634703489792e-06,
+      "loss": 1.2254638671875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.40575,
+      "step": 8,
+      "tokens/total": 2097152,
+      "tokens/train_per_sec_per_gpu": 508.22,
+      "tokens/trainable": 2093222
+    },
+    {
+      "epoch": 0.5901639344262295,
+      "grad_norm": 3.921875,
+      "learning_rate": 9.79362665380022e-06,
+      "loss": 1.3134765625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.71908,
+      "step": 9,
+      "tokens/total": 2359296,
+      "tokens/train_per_sec_per_gpu": 507.58,
+      "tokens/trainable": 2354821
+    },
+    {
+      "epoch": 0.6557377049180327,
+      "grad_norm": 1.46875,
+      "learning_rate": 9.719884594661864e-06,
+      "loss": 1.259521484375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.52373,
+      "step": 10,
+      "tokens/total": 2621440,
+      "tokens/train_per_sec_per_gpu": 509.45,
+      "tokens/trainable": 2616354
+    },
+    {
+      "epoch": 0.7213114754098361,
+      "grad_norm": 1.3046875,
+      "learning_rate": 9.635310152291038e-06,
+      "loss": 0.96728515625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.63079,
+      "step": 11,
+      "tokens/total": 2883584,
+      "tokens/train_per_sec_per_gpu": 514.08,
+      "tokens/trainable": 2877941
+    },
+    {
+      "epoch": 0.7868852459016393,
+      "grad_norm": 1.1875,
+      "learning_rate": 9.540120428068337e-06,
+      "loss": 1.3236083984375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.75695,
+      "step": 12,
+      "tokens/total": 3145728,
+      "tokens/train_per_sec_per_gpu": 479.44,
+      "tokens/trainable": 3139558
+    },
+    {
+      "epoch": 0.8524590163934426,
+      "grad_norm": 1.3984375,
+      "learning_rate": 9.43455977265062e-06,
+      "loss": 1.2686767578125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.55614,
+      "step": 13,
+      "tokens/total": 3407872,
+      "tokens/train_per_sec_per_gpu": 509.41,
+      "tokens/trainable": 3400945
+    },
+    {
+      "epoch": 0.9180327868852459,
+      "grad_norm": 1.421875,
+      "learning_rate": 9.31889915872638e-06,
+      "loss": 1.2362060546875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.44253,
+      "step": 14,
+      "tokens/total": 3670016,
+      "tokens/train_per_sec_per_gpu": 508.05,
+      "tokens/trainable": 3662179
+    },
+    {
+      "epoch": 0.9836065573770492,
+      "grad_norm": 2.03125,
+      "learning_rate": 9.193435485432744e-06,
+      "loss": 1.1353759765625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.11234,
+      "step": 15,
+      "tokens/total": 3932160,
+      "tokens/train_per_sec_per_gpu": 508.59,
+      "tokens/trainable": 3923553
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 1.59375,
+      "learning_rate": 9.058490816219644e-06,
+      "loss": 1.072265625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.92199,
+      "step": 16,
+      "tokens/total": 3997696,
+      "tokens/train_per_sec_per_gpu": 1972.13,
+      "tokens/trainable": 3988784
+    },
+    {
+      "epoch": 1.0655737704918034,
+      "grad_norm": 1.671875,
+      "learning_rate": 8.91441155211756e-06,
+      "loss": 1.0592041015625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.88407,
+      "step": 17,
+      "tokens/total": 4259840,
+      "tokens/train_per_sec_per_gpu": 502.25,
+      "tokens/trainable": 4250415
+    },
+    {
+      "epoch": 1.1311475409836065,
+      "grad_norm": 4.21875,
+      "learning_rate": 8.76156754253104e-06,
+      "loss": 1.227294921875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.41199,
+      "step": 18,
+      "tokens/total": 4521984,
+      "tokens/train_per_sec_per_gpu": 505.73,
+      "tokens/trainable": 4512120
+    },
+    {
+      "epoch": 1.1967213114754098,
+      "grad_norm": 1.3671875,
+      "learning_rate": 8.60035113584059e-06,
+      "loss": 1.0728759765625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.92378,
+      "step": 19,
+      "tokens/total": 4784128,
+      "tokens/train_per_sec_per_gpu": 507.27,
+      "tokens/trainable": 4773676
+    },
+    {
+      "epoch": 1.2622950819672132,
+      "grad_norm": 1.09375,
+      "learning_rate": 8.431176172250002e-06,
+      "loss": 1.2071533203125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.34395,
+      "step": 20,
+      "tokens/total": 5046272,
+      "tokens/train_per_sec_per_gpu": 508.15,
+      "tokens/trainable": 5035220
+    },
+    {
+      "epoch": 1.3278688524590163,
+      "grad_norm": 1.1953125,
+      "learning_rate": 8.254476921464483e-06,
+      "loss": 1.1644287109375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.20409,
+      "step": 21,
+      "tokens/total": 5308416,
+      "tokens/train_per_sec_per_gpu": 503.86,
+      "tokens/trainable": 5296912
+    },
+    {
+      "epoch": 1.3934426229508197,
+      "grad_norm": 1.515625,
+      "learning_rate": 8.070706967926565e-06,
+      "loss": 1.21142578125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.35827,
+      "step": 22,
+      "tokens/total": 5570560,
+      "tokens/train_per_sec_per_gpu": 508.38,
+      "tokens/trainable": 5558565
+    },
+    {
+      "epoch": 1.459016393442623,
+      "grad_norm": 1.8671875,
+      "learning_rate": 7.880338046471331e-06,
+      "loss": 1.14385986328125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.13886,
+      "step": 23,
+      "tokens/total": 5832704,
+      "tokens/train_per_sec_per_gpu": 507.07,
+      "tokens/trainable": 5820246
+    },
+    {
+      "epoch": 1.5245901639344264,
+      "grad_norm": 0.98828125,
+      "learning_rate": 7.683858831389866e-06,
+      "loss": 1.11944580078125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.06316,
+      "step": 24,
+      "tokens/total": 6094848,
+      "tokens/train_per_sec_per_gpu": 507.51,
+      "tokens/trainable": 6082006
+    },
+    {
+      "epoch": 1.5901639344262295,
+      "grad_norm": 1.6484375,
+      "learning_rate": 7.481773682009356e-06,
+      "loss": 1.2337646484375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.43413,
+      "step": 25,
+      "tokens/total": 6356992,
+      "tokens/train_per_sec_per_gpu": 506.44,
+      "tokens/trainable": 6343605
+    },
+    {
+      "epoch": 1.6557377049180326,
+      "grad_norm": 1.1484375,
+      "learning_rate": 7.274601348009934e-06,
+      "loss": 1.15863037109375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.18557,
+      "step": 26,
+      "tokens/total": 6619136,
+      "tokens/train_per_sec_per_gpu": 508.54,
+      "tokens/trainable": 6605138
+    },
+    {
+      "epoch": 1.721311475409836,
+      "grad_norm": 1.0625,
+      "learning_rate": 7.062873637801692e-06,
+      "loss": 0.871826171875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.39127,
+      "step": 27,
+      "tokens/total": 6881280,
+      "tokens/train_per_sec_per_gpu": 513.21,
+      "tokens/trainable": 6866725
+    },
+    {
+      "epoch": 1.7868852459016393,
+      "grad_norm": 1.0703125,
+      "learning_rate": 6.847134053380113e-06,
+      "loss": 1.23193359375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.42785,
+      "step": 28,
+      "tokens/total": 7143424,
+      "tokens/train_per_sec_per_gpu": 502.27,
+      "tokens/trainable": 7128342
+    },
+    {
+      "epoch": 1.8524590163934427,
+      "grad_norm": 1.015625,
+      "learning_rate": 6.627936395164243e-06,
+      "loss": 1.188720703125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.28288,
+      "step": 29,
+      "tokens/total": 7405568,
+      "tokens/train_per_sec_per_gpu": 508.53,
+      "tokens/trainable": 7389729
+    },
+    {
+      "epoch": 1.918032786885246,
+      "grad_norm": 1.0625,
+      "learning_rate": 6.405843340398971e-06,
+      "loss": 1.16455078125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.20448,
+      "step": 30,
+      "tokens/total": 7667712,
+      "tokens/train_per_sec_per_gpu": 507.88,
+      "tokens/trainable": 7650963
+    },
+    {
+      "epoch": 1.9836065573770492,
+      "grad_norm": 0.9375,
+      "learning_rate": 6.181424998770595e-06,
+      "loss": 1.07537841796875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.9311,
+      "step": 31,
+      "tokens/total": 7929856,
+      "tokens/train_per_sec_per_gpu": 508.56,
+      "tokens/trainable": 7912337
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 1.765625,
+      "learning_rate": 5.955257448943446e-06,
+      "loss": 0.951171875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.58874,
+      "step": 32,
+      "tokens/total": 7995392,
+      "tokens/train_per_sec_per_gpu": 1977.53,
+      "tokens/trainable": 7977568
+    },
+    {
+      "epoch": 2.0655737704918034,
+      "grad_norm": 0.9375,
+      "learning_rate": 5.727921259774208e-06,
+      "loss": 1.00445556640625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.73042,
+      "step": 33,
+      "tokens/total": 8257536,
+      "tokens/train_per_sec_per_gpu": 501.3,
+      "tokens/trainable": 8239199
+    },
+    {
+      "epoch": 2.1311475409836067,
+      "grad_norm": 0.9765625,
+      "learning_rate": 5.500000000000001e-06,
+      "loss": 1.191650390625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.29251,
+      "step": 34,
+      "tokens/total": 8519680,
+      "tokens/train_per_sec_per_gpu": 507.05,
+      "tokens/trainable": 8500904
+    },
+    {
+      "epoch": 2.19672131147541,
+      "grad_norm": 0.90625,
+      "learning_rate": 5.272078740225793e-06,
+      "loss": 1.0179443359375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.7675,
+      "step": 35,
+      "tokens/total": 8781824,
+      "tokens/train_per_sec_per_gpu": 507.87,
+      "tokens/trainable": 8762460
+    },
+    {
+      "epoch": 2.262295081967213,
+      "grad_norm": 1.59375,
+      "learning_rate": 5.0447425510565564e-06,
+      "loss": 1.1546630859375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.17295,
+      "step": 36,
+      "tokens/total": 9043968,
+      "tokens/train_per_sec_per_gpu": 508.23,
+      "tokens/trainable": 9024004
+    },
+    {
+      "epoch": 2.3278688524590163,
+      "grad_norm": 1.0,
+      "learning_rate": 4.818575001229406e-06,
+      "loss": 1.115234375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.05028,
+      "step": 37,
+      "tokens/total": 9306112,
+      "tokens/train_per_sec_per_gpu": 504.46,
+      "tokens/trainable": 9285696
+    },
+    {
+      "epoch": 2.3934426229508197,
+      "grad_norm": 0.96875,
+      "learning_rate": 4.594156659601029e-06,
+      "loss": 1.171630859375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.22725,
+      "step": 38,
+      "tokens/total": 9568256,
+      "tokens/train_per_sec_per_gpu": 508.83,
+      "tokens/trainable": 9547349
+    },
+    {
+      "epoch": 2.459016393442623,
+      "grad_norm": 1.2734375,
+      "learning_rate": 4.372063604835758e-06,
+      "loss": 1.1068115234375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.0247,
+      "step": 39,
+      "tokens/total": 9830400,
+      "tokens/train_per_sec_per_gpu": 507.44,
+      "tokens/trainable": 9809030
+    },
+    {
+      "epoch": 2.5245901639344264,
+      "grad_norm": 0.9375,
+      "learning_rate": 4.15286594661989e-06,
+      "loss": 1.075439453125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.93128,
+      "step": 40,
+      "tokens/total": 10092544,
+      "tokens/train_per_sec_per_gpu": 507.07,
+      "tokens/trainable": 10070790
+    },
+    {
+      "epoch": 2.5901639344262293,
+      "grad_norm": 1.3125,
+      "learning_rate": 3.93712636219831e-06,
+      "loss": 1.20166015625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.32563,
+      "step": 41,
+      "tokens/total": 10354688,
+      "tokens/train_per_sec_per_gpu": 506.75,
+      "tokens/trainable": 10332389
+    },
+    {
+      "epoch": 2.6557377049180326,
+      "grad_norm": 0.89453125,
+      "learning_rate": 3.7253986519900674e-06,
+      "loss": 1.12188720703125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.07064,
+      "step": 42,
+      "tokens/total": 10616832,
+      "tokens/train_per_sec_per_gpu": 508.88,
+      "tokens/trainable": 10593922
+    },
+    {
+      "epoch": 2.721311475409836,
+      "grad_norm": 0.859375,
+      "learning_rate": 3.5182263179906462e-06,
+      "loss": 0.8382568359375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.31233,
+      "step": 43,
+      "tokens/total": 10878976,
+      "tokens/train_per_sec_per_gpu": 512.92,
+      "tokens/trainable": 10855509
+    },
+    {
+      "epoch": 2.7868852459016393,
+      "grad_norm": 1.2109375,
+      "learning_rate": 3.316141168610136e-06,
+      "loss": 1.197021484375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.31024,
+      "step": 44,
+      "tokens/total": 11141120,
+      "tokens/train_per_sec_per_gpu": 502.17,
+      "tokens/trainable": 11117126
+    },
+    {
+      "epoch": 2.8524590163934427,
+      "grad_norm": 5.46875,
+      "learning_rate": 3.119661953528671e-06,
+      "loss": 1.1572265625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.1811,
+      "step": 45,
+      "tokens/total": 11403264,
+      "tokens/train_per_sec_per_gpu": 508.51,
+      "tokens/trainable": 11378513
+    },
+    {
+      "epoch": 2.918032786885246,
+      "grad_norm": 0.83984375,
+      "learning_rate": 2.9292930320734337e-06,
+      "loss": 1.13720703125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.11805,
+      "step": 46,
+      "tokens/total": 11665408,
+      "tokens/train_per_sec_per_gpu": 507.19,
+      "tokens/trainable": 11639747
+    },
+    {
+      "epoch": 2.9836065573770494,
+      "grad_norm": 0.8359375,
+      "learning_rate": 2.745523078535517e-06,
+      "loss": 1.04937744140625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.85587,
+      "step": 47,
+      "tokens/total": 11927552,
+      "tokens/train_per_sec_per_gpu": 486.72,
+      "tokens/trainable": 11901121
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 1.6171875,
+      "learning_rate": 2.568823827750001e-06,
+      "loss": 0.9130859375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.492,
+      "step": 48,
+      "tokens/total": 11993088,
+      "tokens/train_per_sec_per_gpu": 1970.5,
+      "tokens/trainable": 11966352
+    },
+    {
+      "epoch": 3.0655737704918034,
+      "grad_norm": 0.78125,
+      "learning_rate": 2.3996488641594104e-06,
+      "loss": 0.98260498046875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.67141,
+      "step": 49,
+      "tokens/total": 12255232,
+      "tokens/train_per_sec_per_gpu": 491.2,
+      "tokens/trainable": 12227983
+    },
+    {
+      "epoch": 3.1311475409836067,
+      "grad_norm": 0.859375,
+      "learning_rate": 2.2384324574689614e-06,
+      "loss": 1.17425537109375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.23573,
+      "step": 50,
+      "tokens/total": 12517376,
+      "tokens/train_per_sec_per_gpu": 506.27,
+      "tokens/trainable": 12489688
+    },
+    {
+      "epoch": 3.19672131147541,
+      "grad_norm": 0.8984375,
+      "learning_rate": 2.085588447882441e-06,
+      "loss": 1.000244140625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.71895,
+      "step": 51,
+      "tokens/total": 12779520,
+      "tokens/train_per_sec_per_gpu": 507.08,
+      "tokens/trainable": 12751244
+    },
+    {
+      "epoch": 3.262295081967213,
+      "grad_norm": 1.015625,
+      "learning_rate": 1.9415091837803574e-06,
+      "loss": 1.1417236328125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.13216,
+      "step": 52,
+      "tokens/total": 13041664,
+      "tokens/train_per_sec_per_gpu": 506.66,
+      "tokens/trainable": 13012788
+    },
+    {
+      "epoch": 3.3278688524590163,
+      "grad_norm": 1.4609375,
+      "learning_rate": 1.8065645145672578e-06,
+      "loss": 1.1019287109375,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.00997,
+      "step": 53,
+      "tokens/total": 13303808,
+      "tokens/train_per_sec_per_gpu": 504.11,
+      "tokens/trainable": 13274480
+    },
+    {
+      "epoch": 3.3934426229508197,
+      "grad_norm": 0.88671875,
+      "learning_rate": 1.6811008412736207e-06,
+      "loss": 1.1590576171875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.18693,
+      "step": 54,
+      "tokens/total": 13565952,
+      "tokens/train_per_sec_per_gpu": 508.5,
+      "tokens/trainable": 13536133
+    },
+    {
+      "epoch": 3.459016393442623,
+      "grad_norm": 0.8203125,
+      "learning_rate": 1.5654402273493808e-06,
+      "loss": 1.09600830078125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.9922,
+      "step": 55,
+      "tokens/total": 13828096,
+      "tokens/train_per_sec_per_gpu": 507.19,
+      "tokens/trainable": 13797814
+    },
+    {
+      "epoch": 3.5245901639344264,
+      "grad_norm": 0.76953125,
+      "learning_rate": 1.459879571931663e-06,
+      "loss": 1.06561279296875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.90262,
+      "step": 56,
+      "tokens/total": 14090240,
+      "tokens/train_per_sec_per_gpu": 507.33,
+      "tokens/trainable": 14059574
+    },
+    {
+      "epoch": 3.5901639344262293,
+      "grad_norm": 1.2421875,
+      "learning_rate": 1.3646898477089627e-06,
+      "loss": 1.1900634765625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.28729,
+      "step": 57,
+      "tokens/total": 14352384,
+      "tokens/train_per_sec_per_gpu": 506.26,
+      "tokens/trainable": 14321173
+    },
+    {
+      "epoch": 3.6557377049180326,
+      "grad_norm": 0.7890625,
+      "learning_rate": 1.2801154053381386e-06,
+      "loss": 1.11334228515625,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.04452,
+      "step": 58,
+      "tokens/total": 14614528,
+      "tokens/train_per_sec_per_gpu": 508.49,
+      "tokens/trainable": 14582706
+    },
+    {
+      "epoch": 3.721311475409836,
+      "grad_norm": 1.3359375,
+      "learning_rate": 1.2063733461997804e-06,
+      "loss": 0.83184814453125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.29756,
+      "step": 59,
+      "tokens/total": 14876672,
+      "tokens/train_per_sec_per_gpu": 512.9,
+      "tokens/trainable": 14844293
+    },
+    {
+      "epoch": 3.7868852459016393,
+      "grad_norm": 0.97265625,
+      "learning_rate": 1.1436529651020813e-06,
+      "loss": 1.1912841796875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.29131,
+      "step": 60,
+      "tokens/total": 15138816,
+      "tokens/train_per_sec_per_gpu": 502.3,
+      "tokens/trainable": 15105910
+    },
+    {
+      "epoch": 3.8524590163934427,
+      "grad_norm": 0.8203125,
+      "learning_rate": 1.092115264363775e-06,
+      "loss": 1.1541748046875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.17141,
+      "step": 61,
+      "tokens/total": 15400960,
+      "tokens/train_per_sec_per_gpu": 507.77,
+      "tokens/trainable": 15367297
+    },
+    {
+      "epoch": 3.918032786885246,
+      "grad_norm": 0.9140625,
+      "learning_rate": 1.0518925405234988e-06,
+      "loss": 1.13330078125,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 3.10589,
+      "step": 62,
+      "tokens/total": 15663104,
+      "tokens/train_per_sec_per_gpu": 507.72,
+      "tokens/trainable": 15628531
+    },
+    {
+      "epoch": 3.9836065573770494,
+      "grad_norm": 0.8359375,
+      "learning_rate": 1.0230880447364719e-06,
+      "loss": 1.04559326171875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.84509,
+      "step": 63,
+      "tokens/total": 15925248,
+      "tokens/train_per_sec_per_gpu": 508.53,
+      "tokens/trainable": 15889905
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 1.6171875,
+      "learning_rate": 1.0057757177302628e-06,
+      "loss": 0.90576171875,
+      "memory/device_reserved (GiB)": 61.11,
+      "memory/max_active (GiB)": 52.26,
+      "memory/max_allocated (GiB)": 52.26,
+      "ppl": 2.47382,
+      "step": 64,
+      "tokens/total": 15990784,
+      "tokens/train_per_sec_per_gpu": 1945.14,
+      "tokens/trainable": 15955136
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 64,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 64,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.0722518944763085e+18,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

sdf/4x/shared/post_dolmino/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1124ab66a09103d042321fefc5053143a36ee4eee8bb7bfb3810d4ea7f4e95bd
+size 7377