Training in progress, step 500, checkpoint

last-checkpoint/config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 18,
+  "num_key_value_heads": 16,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.8.0",
+  "use_cache": false,
+  "vocab_size": 50257
+}

last-checkpoint/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.8.0",
+  "use_cache": true
+}

last-checkpoint/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c357e9c2944702a41bcf9e0c2ac136e48a71e118595736666655bcc876eb4677
+size 4523108832

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2e1912645084bd8899cf84ece2cfd87234cfbcc7c5298a2f7b71be0c49ca68b
+size 2911851147

last-checkpoint/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878
+size 14645

last-checkpoint/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f77569c2e850b04af982cc8c1389f1430851448915c593b69e5da36ce05b71d7
+size 1383

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f61cd911648f3cff02d47bc6b9e58cb3242dd8a153313085c99fe548e711bd9c
+size 1465

last-checkpoint/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

last-checkpoint/trainer_state.json

@@ -0,0 +1,384 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0188,
+  "eval_steps": 500,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.001,
+      "grad_norm": 5.9411797523498535,
+      "learning_rate": 0.00019982,
+      "loss": 10.402878570556641,
+      "step": 10
+    },
+    {
+      "epoch": 0.002,
+      "grad_norm": 2.5087270736694336,
+      "learning_rate": 0.00019962000000000002,
+      "loss": 5.9839332580566404,
+      "step": 20
+    },
+    {
+      "epoch": 0.003,
+      "grad_norm": 1.3757283687591553,
+      "learning_rate": 0.00019942,
+      "loss": 5.513664627075196,
+      "step": 30
+    },
+    {
+      "epoch": 0.004,
+      "grad_norm": 1.2012906074523926,
+      "learning_rate": 0.00019922,
+      "loss": 5.281090545654297,
+      "step": 40
+    },
+    {
+      "epoch": 0.005,
+      "grad_norm": 2.0055949687957764,
+      "learning_rate": 0.00019902,
+      "loss": 5.105018615722656,
+      "step": 50
+    },
+    {
+      "epoch": 0.006,
+      "grad_norm": 2.674002170562744,
+      "learning_rate": 0.00019882,
+      "loss": 4.82950439453125,
+      "step": 60
+    },
+    {
+      "epoch": 0.007,
+      "grad_norm": 1.7347458600997925,
+      "learning_rate": 0.00019862000000000002,
+      "loss": 4.698889923095703,
+      "step": 70
+    },
+    {
+      "epoch": 0.008,
+      "grad_norm": 2.345951795578003,
+      "learning_rate": 0.00019842000000000001,
+      "loss": 4.519314956665039,
+      "step": 80
+    },
+    {
+      "epoch": 0.009,
+      "grad_norm": 1.3420459032058716,
+      "learning_rate": 0.00019822,
+      "loss": 4.280438995361328,
+      "step": 90
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 1.4221423864364624,
+      "learning_rate": 0.00019802,
+      "loss": 4.258197784423828,
+      "step": 100
+    },
+    {
+      "epoch": 0.011,
+      "grad_norm": 1.4795005321502686,
+      "learning_rate": 0.00019782,
+      "loss": 4.103477478027344,
+      "step": 110
+    },
+    {
+      "epoch": 0.012,
+      "grad_norm": 2.135676860809326,
+      "learning_rate": 0.00019762,
+      "loss": 3.9324199676513674,
+      "step": 120
+    },
+    {
+      "epoch": 0.013,
+      "grad_norm": 0.9774000644683838,
+      "learning_rate": 0.00019742000000000002,
+      "loss": 3.872743606567383,
+      "step": 130
+    },
+    {
+      "epoch": 0.014,
+      "grad_norm": 1.0677322149276733,
+      "learning_rate": 0.00019722,
+      "loss": 3.8740089416503904,
+      "step": 140
+    },
+    {
+      "epoch": 0.015,
+      "grad_norm": 0.956723153591156,
+      "learning_rate": 0.00019702,
+      "loss": 3.7239048004150392,
+      "step": 150
+    },
+    {
+      "epoch": 0.016,
+      "grad_norm": 1.0390738248825073,
+      "learning_rate": 0.00019682,
+      "loss": 3.7527164459228515,
+      "step": 160
+    },
+    {
+      "epoch": 0.017,
+      "grad_norm": 1.084784984588623,
+      "learning_rate": 0.00019662,
+      "loss": 3.5590255737304686,
+      "step": 170
+    },
+    {
+      "epoch": 0.018,
+      "grad_norm": 1.1409659385681152,
+      "learning_rate": 0.00019642,
+      "loss": 3.6691215515136717,
+      "step": 180
+    },
+    {
+      "epoch": 0.019,
+      "grad_norm": 0.8642674088478088,
+      "learning_rate": 0.00019622000000000002,
+      "loss": 3.5040924072265627,
+      "step": 190
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.8515411019325256,
+      "learning_rate": 0.00019602,
+      "loss": 3.5071990966796873,
+      "step": 200
+    },
+    {
+      "epoch": 0.021,
+      "grad_norm": 1.0194536447525024,
+      "learning_rate": 0.00019582,
+      "loss": 3.3893787384033205,
+      "step": 210
+    },
+    {
+      "epoch": 0.022,
+      "grad_norm": 0.9459556937217712,
+      "learning_rate": 0.00019562,
+      "loss": 3.3273845672607423,
+      "step": 220
+    },
+    {
+      "epoch": 0.023,
+      "grad_norm": 0.8981431722640991,
+      "learning_rate": 0.00019542,
+      "loss": 3.3746875762939452,
+      "step": 230
+    },
+    {
+      "epoch": 0.024,
+      "grad_norm": 0.9751044511795044,
+      "learning_rate": 0.00019522,
+      "loss": 3.1989873886108398,
+      "step": 240
+    },
+    {
+      "epoch": 0.025,
+      "grad_norm": 0.8557232022285461,
+      "learning_rate": 0.00019502,
+      "loss": 3.1978750228881836,
+      "step": 250
+    },
+    {
+      "epoch": 0.026,
+      "grad_norm": 0.7834332585334778,
+      "learning_rate": 0.00019482,
+      "loss": 3.28112678527832,
+      "step": 260
+    },
+    {
+      "epoch": 0.027,
+      "grad_norm": 0.8482623100280762,
+      "learning_rate": 0.00019462,
+      "loss": 3.2512683868408203,
+      "step": 270
+    },
+    {
+      "epoch": 0.028,
+      "grad_norm": 1.1247769594192505,
+      "learning_rate": 0.00019442,
+      "loss": 3.167398452758789,
+      "step": 280
+    },
+    {
+      "epoch": 0.029,
+      "grad_norm": 0.7613039612770081,
+      "learning_rate": 0.00019422,
+      "loss": 3.1816900253295897,
+      "step": 290
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.7643451690673828,
+      "learning_rate": 0.00019402,
+      "loss": 2.998531723022461,
+      "step": 300
+    },
+    {
+      "epoch": 0.031,
+      "grad_norm": 0.8855999112129211,
+      "learning_rate": 0.00019382,
+      "loss": 3.131449317932129,
+      "step": 310
+    },
+    {
+      "epoch": 1.0008,
+      "grad_norm": 0.8182146549224854,
+      "learning_rate": 0.00019362,
+      "loss": 3.362815093994141,
+      "step": 320
+    },
+    {
+      "epoch": 1.0018,
+      "grad_norm": 0.8580504655838013,
+      "learning_rate": 0.00019342,
+      "loss": 3.060775947570801,
+      "step": 330
+    },
+    {
+      "epoch": 1.0028,
+      "grad_norm": 0.691214382648468,
+      "learning_rate": 0.00019322,
+      "loss": 2.930637741088867,
+      "step": 340
+    },
+    {
+      "epoch": 1.0038,
+      "grad_norm": 0.8206065893173218,
+      "learning_rate": 0.00019302,
+      "loss": 2.9964399337768555,
+      "step": 350
+    },
+    {
+      "epoch": 1.0048,
+      "grad_norm": 0.7985368371009827,
+      "learning_rate": 0.00019282000000000001,
+      "loss": 2.869978141784668,
+      "step": 360
+    },
+    {
+      "epoch": 1.0058,
+      "grad_norm": 0.8223126530647278,
+      "learning_rate": 0.00019262,
+      "loss": 2.929924964904785,
+      "step": 370
+    },
+    {
+      "epoch": 1.0068,
+      "grad_norm": 0.7868787050247192,
+      "learning_rate": 0.00019242,
+      "loss": 2.8226070404052734,
+      "step": 380
+    },
+    {
+      "epoch": 1.0078,
+      "grad_norm": 0.6478514671325684,
+      "learning_rate": 0.00019222,
+      "loss": 2.8945112228393555,
+      "step": 390
+    },
+    {
+      "epoch": 1.0088,
+      "grad_norm": 0.8428190350532532,
+      "learning_rate": 0.00019202,
+      "loss": 2.852457618713379,
+      "step": 400
+    },
+    {
+      "epoch": 1.0098,
+      "grad_norm": 0.7077657580375671,
+      "learning_rate": 0.00019182,
+      "loss": 2.887688636779785,
+      "step": 410
+    },
+    {
+      "epoch": 1.0108,
+      "grad_norm": 0.6806844472885132,
+      "learning_rate": 0.00019162,
+      "loss": 2.8496984481811523,
+      "step": 420
+    },
+    {
+      "epoch": 1.0118,
+      "grad_norm": 0.6535109877586365,
+      "learning_rate": 0.00019142,
+      "loss": 2.7463850021362304,
+      "step": 430
+    },
+    {
+      "epoch": 1.0128,
+      "grad_norm": 0.611242949962616,
+      "learning_rate": 0.00019122,
+      "loss": 2.7267787933349608,
+      "step": 440
+    },
+    {
+      "epoch": 1.0138,
+      "grad_norm": 0.7190207839012146,
+      "learning_rate": 0.00019102,
+      "loss": 2.7335268020629884,
+      "step": 450
+    },
+    {
+      "epoch": 1.0148,
+      "grad_norm": 0.6520172357559204,
+      "learning_rate": 0.00019082,
+      "loss": 2.705989646911621,
+      "step": 460
+    },
+    {
+      "epoch": 1.0158,
+      "grad_norm": 0.7822602987289429,
+      "learning_rate": 0.00019062,
+      "loss": 2.7904048919677735,
+      "step": 470
+    },
+    {
+      "epoch": 1.0168,
+      "grad_norm": 0.6247469186782837,
+      "learning_rate": 0.00019042,
+      "loss": 2.627030372619629,
+      "step": 480
+    },
+    {
+      "epoch": 1.0178,
+      "grad_norm": 0.7010595798492432,
+      "learning_rate": 0.00019022,
+      "loss": 2.7214162826538084,
+      "step": 490
+    },
+    {
+      "epoch": 1.0188,
+      "grad_norm": 0.5612587928771973,
+      "learning_rate": 0.00019002,
+      "loss": 2.7336082458496094,
+      "step": 500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.23467821564969e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6613ae7e039f69e0c219d13c85d932cd4ee709153ecf7a7918efd897af4b506a
+size 5329