Training in progress, step 2000, checkpoint

last-checkpoint/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "dtype": "bfloat16",
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 1536,
+  "n_head": 24,
+  "n_inner": null,
+  "n_layer": 24,
+  "n_positions": 2048,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "transformers_version": "4.57.3",
+  "use_cache": false,
+  "vocab_size": 50257
+}

last-checkpoint/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.57.3",
+  "use_cache": false
+}

last-checkpoint/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d307d63e2bd5721810a84f6c0aded4fa80bef46ee1e253293ea2ea57f13b2b35
+size 1520630616

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec9a01e5ca2a705c9e40bcd141b1b30c981cdb1ecde544437140b15a37cec10c
+size 3041448587

last-checkpoint/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:181e60f1f01e165fe1ea237e5ea1bc5e876c0b1a74e9355ac894a44ab5895cb2
+size 14645

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e91a2aad683dbb34b2ed1315719473de36da58f4dbb5b8ab53f09c5f23b65cac
+size 1465

last-checkpoint/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

last-checkpoint/tokenizer_config.json

@@ -0,0 +1,21 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

last-checkpoint/trainer_state.json

@@ -0,0 +1,314 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.02185792349726776,
+  "eval_steps": 500,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.000546448087431694,
+      "grad_norm": 5.3125,
+      "learning_rate": 6.124999999999999e-06,
+      "loss": 9.6445,
+      "step": 50
+    },
+    {
+      "epoch": 0.001092896174863388,
+      "grad_norm": 5.40625,
+      "learning_rate": 1.2375e-05,
+      "loss": 8.1001,
+      "step": 100
+    },
+    {
+      "epoch": 0.001639344262295082,
+      "grad_norm": 4.0,
+      "learning_rate": 1.8625e-05,
+      "loss": 7.5007,
+      "step": 150
+    },
+    {
+      "epoch": 0.002185792349726776,
+      "grad_norm": 6.125,
+      "learning_rate": 2.4874999999999998e-05,
+      "loss": 7.0884,
+      "step": 200
+    },
+    {
+      "epoch": 0.00273224043715847,
+      "grad_norm": 3.703125,
+      "learning_rate": 3.1125e-05,
+      "loss": 6.6637,
+      "step": 250
+    },
+    {
+      "epoch": 0.003278688524590164,
+      "grad_norm": 3.25,
+      "learning_rate": 3.7375e-05,
+      "loss": 6.3275,
+      "step": 300
+    },
+    {
+      "epoch": 0.003825136612021858,
+      "grad_norm": 2.578125,
+      "learning_rate": 4.3624999999999997e-05,
+      "loss": 6.0834,
+      "step": 350
+    },
+    {
+      "epoch": 0.004371584699453552,
+      "grad_norm": 2.640625,
+      "learning_rate": 4.9875e-05,
+      "loss": 5.8539,
+      "step": 400
+    },
+    {
+      "epoch": 0.004918032786885246,
+      "grad_norm": 2.578125,
+      "learning_rate": 5.6124999999999995e-05,
+      "loss": 5.7353,
+      "step": 450
+    },
+    {
+      "epoch": 0.00546448087431694,
+      "grad_norm": 2.75,
+      "learning_rate": 6.2375e-05,
+      "loss": 5.6411,
+      "step": 500
+    },
+    {
+      "epoch": 0.006010928961748634,
+      "grad_norm": 1.8359375,
+      "learning_rate": 6.8625e-05,
+      "loss": 5.488,
+      "step": 550
+    },
+    {
+      "epoch": 0.006557377049180328,
+      "grad_norm": 2.609375,
+      "learning_rate": 7.487499999999999e-05,
+      "loss": 5.3441,
+      "step": 600
+    },
+    {
+      "epoch": 0.007103825136612022,
+      "grad_norm": 1.765625,
+      "learning_rate": 8.112500000000001e-05,
+      "loss": 5.2922,
+      "step": 650
+    },
+    {
+      "epoch": 0.007650273224043716,
+      "grad_norm": 1.7265625,
+      "learning_rate": 8.7375e-05,
+      "loss": 5.1776,
+      "step": 700
+    },
+    {
+      "epoch": 0.00819672131147541,
+      "grad_norm": 2.28125,
+      "learning_rate": 9.362499999999999e-05,
+      "loss": 5.0849,
+      "step": 750
+    },
+    {
+      "epoch": 0.008743169398907104,
+      "grad_norm": 1.171875,
+      "learning_rate": 9.9875e-05,
+      "loss": 4.9607,
+      "step": 800
+    },
+    {
+      "epoch": 0.009289617486338797,
+      "grad_norm": 1.5234375,
+      "learning_rate": 0.00010612499999999999,
+      "loss": 4.8847,
+      "step": 850
+    },
+    {
+      "epoch": 0.009836065573770493,
+      "grad_norm": 1.15625,
+      "learning_rate": 0.000112375,
+      "loss": 4.8233,
+      "step": 900
+    },
+    {
+      "epoch": 0.010382513661202186,
+      "grad_norm": 1.3359375,
+      "learning_rate": 0.000118625,
+      "loss": 4.7077,
+      "step": 950
+    },
+    {
+      "epoch": 0.01092896174863388,
+      "grad_norm": 0.9921875,
+      "learning_rate": 0.00012487499999999999,
+      "loss": 4.6274,
+      "step": 1000
+    },
+    {
+      "epoch": 0.011475409836065573,
+      "grad_norm": 1.15625,
+      "learning_rate": 0.00013112499999999998,
+      "loss": 4.5506,
+      "step": 1050
+    },
+    {
+      "epoch": 0.012021857923497269,
+      "grad_norm": 1.03125,
+      "learning_rate": 0.000137375,
+      "loss": 4.4801,
+      "step": 1100
+    },
+    {
+      "epoch": 0.012568306010928962,
+      "grad_norm": 1.3046875,
+      "learning_rate": 0.00014362499999999998,
+      "loss": 4.425,
+      "step": 1150
+    },
+    {
+      "epoch": 0.013114754098360656,
+      "grad_norm": 1.046875,
+      "learning_rate": 0.000149875,
+      "loss": 4.3204,
+      "step": 1200
+    },
+    {
+      "epoch": 0.01366120218579235,
+      "grad_norm": 7.5,
+      "learning_rate": 0.000156125,
+      "loss": 4.2244,
+      "step": 1250
+    },
+    {
+      "epoch": 0.014207650273224045,
+      "grad_norm": 0.99609375,
+      "learning_rate": 0.00016237499999999998,
+      "loss": 4.1705,
+      "step": 1300
+    },
+    {
+      "epoch": 0.014754098360655738,
+      "grad_norm": 0.83984375,
+      "learning_rate": 0.000168625,
+      "loss": 4.1256,
+      "step": 1350
+    },
+    {
+      "epoch": 0.015300546448087432,
+      "grad_norm": 0.93359375,
+      "learning_rate": 0.00017487499999999998,
+      "loss": 4.0456,
+      "step": 1400
+    },
+    {
+      "epoch": 0.015846994535519125,
+      "grad_norm": 0.90234375,
+      "learning_rate": 0.000181125,
+      "loss": 3.9429,
+      "step": 1450
+    },
+    {
+      "epoch": 0.01639344262295082,
+      "grad_norm": 0.8671875,
+      "learning_rate": 0.000187375,
+      "loss": 3.8733,
+      "step": 1500
+    },
+    {
+      "epoch": 0.016939890710382512,
+      "grad_norm": 0.9140625,
+      "learning_rate": 0.00019362499999999998,
+      "loss": 3.8094,
+      "step": 1550
+    },
+    {
+      "epoch": 0.017486338797814208,
+      "grad_norm": 0.890625,
+      "learning_rate": 0.000199875,
+      "loss": 3.8117,
+      "step": 1600
+    },
+    {
+      "epoch": 0.018032786885245903,
+      "grad_norm": 1.0390625,
+      "learning_rate": 0.00020612499999999998,
+      "loss": 3.6919,
+      "step": 1650
+    },
+    {
+      "epoch": 0.018579234972677595,
+      "grad_norm": 0.94921875,
+      "learning_rate": 0.00021237499999999997,
+      "loss": 3.6148,
+      "step": 1700
+    },
+    {
+      "epoch": 0.01912568306010929,
+      "grad_norm": 1.0546875,
+      "learning_rate": 0.000218625,
+      "loss": 3.5881,
+      "step": 1750
+    },
+    {
+      "epoch": 0.019672131147540985,
+      "grad_norm": 0.953125,
+      "learning_rate": 0.000224875,
+      "loss": 3.46,
+      "step": 1800
+    },
+    {
+      "epoch": 0.020218579234972677,
+      "grad_norm": 0.9765625,
+      "learning_rate": 0.00023112499999999997,
+      "loss": 3.4731,
+      "step": 1850
+    },
+    {
+      "epoch": 0.020765027322404372,
+      "grad_norm": 0.984375,
+      "learning_rate": 0.00023737499999999998,
+      "loss": 3.3781,
+      "step": 1900
+    },
+    {
+      "epoch": 0.021311475409836064,
+      "grad_norm": 1.0078125,
+      "learning_rate": 0.000243625,
+      "loss": 3.3166,
+      "step": 1950
+    },
+    {
+      "epoch": 0.02185792349726776,
+      "grad_norm": 1.0859375,
+      "learning_rate": 0.000249875,
+      "loss": 3.3105,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 91500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 2000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.069483900796928e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88aaa412e5cf86b09db29ee797f5f159150045903c62c1723d48edb775dde54f
+size 5841