Training in progress, step 50

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
     "o_proj",
     "q_proj",
-    "up_proj",
     "down_proj",
     "gate_proj",
-    "v_proj"
+    "v_proj",
+    "k_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ba183c1c97dd8fd79734a2fb1011526a1bc0695f6a38a908cc3cc23a2265e55
+oid sha256:2bf964742964d2ec503f25da4ead7a6c8a5691fc8d1cb60aa46b3a6fe8d444f8
 size 35668592

metrics.json

@@ -1,18 +1 @@
-{"Step":20,"eval_loss":1.9001051188,"eval_runtime":1521.8531,"eval_samples_per_second":3.285,"eval_steps_per_second":0.411,"epoch":0.032}
-{"Step":40,"eval_loss":1.8370074034,"eval_runtime":1516.9668,"eval_samples_per_second":3.296,"eval_steps_per_second":0.412,"epoch":0.064}
-{"Step":60,"eval_loss":1.6439222097,"eval_runtime":1500.7581,"eval_samples_per_second":3.332,"eval_steps_per_second":0.416,"epoch":0.096}
-{"Step":80,"eval_loss":1.550334692,"eval_runtime":1506.1101,"eval_samples_per_second":3.32,"eval_steps_per_second":0.415,"epoch":0.128}
-{"Step":100,"eval_loss":1.519143939,"eval_runtime":1499.8012,"eval_samples_per_second":3.334,"eval_steps_per_second":0.417,"epoch":0.16}
-{"Step":120,"eval_loss":1.5071601868,"eval_runtime":1498.469,"eval_samples_per_second":3.337,"eval_steps_per_second":0.417,"epoch":0.192}
-{"Step":140,"eval_loss":1.4998296499,"eval_runtime":1486.0248,"eval_samples_per_second":3.365,"eval_steps_per_second":0.421,"epoch":0.224}
-{"Step":160,"eval_loss":1.4962984324,"eval_runtime":1491.4174,"eval_samples_per_second":3.353,"eval_steps_per_second":0.419,"epoch":0.256}
-{"Step":180,"eval_loss":1.4931029081,"eval_runtime":1501.5451,"eval_samples_per_second":3.33,"eval_steps_per_second":0.416,"epoch":0.288}
-{"Step":200,"eval_loss":1.4910326004,"eval_runtime":1501.9275,"eval_samples_per_second":3.329,"eval_steps_per_second":0.416,"epoch":0.32}
-{"Step":220,"eval_loss":1.488992691,"eval_runtime":1498.0439,"eval_samples_per_second":3.338,"eval_steps_per_second":0.417,"epoch":0.352}
-{"Step":240,"eval_loss":1.4867918491,"eval_runtime":1460.1808,"eval_samples_per_second":3.424,"eval_steps_per_second":0.428,"epoch":0.384}
-{"Step":260,"eval_loss":1.4859720469,"eval_runtime":1460.3853,"eval_samples_per_second":3.424,"eval_steps_per_second":0.428,"epoch":0.416}
-{"Step":280,"eval_loss":1.4846266508,"eval_runtime":1460.0793,"eval_samples_per_second":3.424,"eval_steps_per_second":0.428,"epoch":0.448}
-{"Step":300,"eval_loss":1.4849002361,"eval_runtime":1472.1795,"eval_samples_per_second":3.396,"eval_steps_per_second":0.425,"epoch":0.48}
-{"Step":320,"eval_loss":1.4825308323,"eval_runtime":1500.8065,"eval_samples_per_second":3.332,"eval_steps_per_second":0.416,"epoch":0.512}
-{"Step":340,"eval_loss":1.4830516577,"eval_runtime":1501.5622,"eval_samples_per_second":3.33,"eval_steps_per_second":0.416,"epoch":0.544}
-{"Step":360,"eval_loss":1.4818130732,"eval_runtime":1501.7616,"eval_samples_per_second":3.329,"eval_steps_per_second":0.416,"epoch":0.576}
+{"Step":50,"eval_loss":1.7156720161,"eval_runtime":149.1859,"eval_samples_per_second":3.352,"eval_steps_per_second":0.422,"epoch":0.08}

state.json

@@ -1,289 +1,34 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.592,
-  "eval_steps": 20,
-  "global_step": 370,
+  "epoch": 0.0816,
+  "eval_steps": 50,
+  "global_step": 51,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.032,
-      "grad_norm": 0.015696076676249504,
-      "learning_rate": 5.319148936170213e-05,
-      "loss": 1.9359,
-      "step": 20
+      "epoch": 0.08,
+      "grad_norm": 0.05632242560386658,
+      "learning_rate": 0.00013297872340425532,
+      "loss": 1.8852,
+      "step": 50
     },
     {
-      "epoch": 0.032,
-      "eval_loss": 1.9001051187515259,
-      "eval_runtime": 1521.8531,
-      "eval_samples_per_second": 3.285,
-      "eval_steps_per_second": 0.411,
-      "step": 20
-    },
-    {
-      "epoch": 0.064,
-      "grad_norm": 0.039251815527677536,
-      "learning_rate": 0.00010638297872340425,
-      "loss": 1.9075,
-      "step": 40
-    },
-    {
-      "epoch": 0.064,
-      "eval_loss": 1.8370074033737183,
-      "eval_runtime": 1516.9668,
-      "eval_samples_per_second": 3.296,
-      "eval_steps_per_second": 0.412,
-      "step": 40
-    },
-    {
-      "epoch": 0.096,
-      "grad_norm": NaN,
-      "learning_rate": 0.00015691489361702128,
-      "loss": 1.7374,
-      "step": 60
-    },
-    {
-      "epoch": 0.096,
-      "eval_loss": 1.643922209739685,
-      "eval_runtime": 1500.7581,
-      "eval_samples_per_second": 3.332,
-      "eval_steps_per_second": 0.416,
-      "step": 60
-    },
-    {
-      "epoch": 0.128,
-      "grad_norm": 0.10252918303012848,
-      "learning_rate": 0.0002101063829787234,
-      "loss": 1.6047,
-      "step": 80
-    },
-    {
-      "epoch": 0.128,
-      "eval_loss": 1.5503346920013428,
-      "eval_runtime": 1506.1101,
-      "eval_samples_per_second": 3.32,
-      "eval_steps_per_second": 0.415,
-      "step": 80
-    },
-    {
-      "epoch": 0.16,
-      "grad_norm": 0.026087351143360138,
-      "learning_rate": 0.0002632978723404255,
-      "loss": 1.539,
-      "step": 100
-    },
-    {
-      "epoch": 0.16,
-      "eval_loss": 1.5191439390182495,
-      "eval_runtime": 1499.8012,
-      "eval_samples_per_second": 3.334,
-      "eval_steps_per_second": 0.417,
-      "step": 100
-    },
-    {
-      "epoch": 0.192,
-      "grad_norm": 0.03835824504494667,
-      "learning_rate": 0.00031648936170212765,
-      "loss": 1.5149,
-      "step": 120
-    },
-    {
-      "epoch": 0.192,
-      "eval_loss": 1.5071601867675781,
-      "eval_runtime": 1498.469,
-      "eval_samples_per_second": 3.337,
-      "eval_steps_per_second": 0.417,
-      "step": 120
-    },
-    {
-      "epoch": 0.224,
-      "grad_norm": 0.03541890159249306,
-      "learning_rate": 0.0003696808510638298,
-      "loss": 1.5267,
-      "step": 140
-    },
-    {
-      "epoch": 0.224,
-      "eval_loss": 1.499829649925232,
-      "eval_runtime": 1486.0248,
-      "eval_samples_per_second": 3.365,
-      "eval_steps_per_second": 0.421,
-      "step": 140
-    },
-    {
-      "epoch": 0.256,
-      "grad_norm": 0.045721374452114105,
-      "learning_rate": 0.0004228723404255319,
-      "loss": 1.4927,
-      "step": 160
-    },
-    {
-      "epoch": 0.256,
-      "eval_loss": 1.4962984323501587,
-      "eval_runtime": 1491.4174,
-      "eval_samples_per_second": 3.353,
-      "eval_steps_per_second": 0.419,
-      "step": 160
-    },
-    {
-      "epoch": 0.288,
-      "grad_norm": 0.04470464587211609,
-      "learning_rate": 0.0004760638297872341,
-      "loss": 1.4861,
-      "step": 180
-    },
-    {
-      "epoch": 0.288,
-      "eval_loss": 1.4931029081344604,
-      "eval_runtime": 1501.5451,
-      "eval_samples_per_second": 3.33,
-      "eval_steps_per_second": 0.416,
-      "step": 180
-    },
-    {
-      "epoch": 0.32,
-      "grad_norm": 0.04011245444417,
-      "learning_rate": 0.0004967397747480735,
-      "loss": 1.4815,
-      "step": 200
-    },
-    {
-      "epoch": 0.32,
-      "eval_loss": 1.491032600402832,
-      "eval_runtime": 1501.9275,
-      "eval_samples_per_second": 3.329,
-      "eval_steps_per_second": 0.416,
-      "step": 200
-    },
-    {
-      "epoch": 0.352,
-      "grad_norm": 0.03507564216852188,
-      "learning_rate": 0.0004908120924718435,
-      "loss": 1.5256,
-      "step": 220
-    },
-    {
-      "epoch": 0.352,
-      "eval_loss": 1.488992691040039,
-      "eval_runtime": 1498.0439,
-      "eval_samples_per_second": 3.338,
-      "eval_steps_per_second": 0.417,
-      "step": 220
-    },
-    {
-      "epoch": 0.384,
-      "grad_norm": 0.03816624730825424,
-      "learning_rate": 0.0004848844101956135,
-      "loss": 1.4932,
-      "step": 240
-    },
-    {
-      "epoch": 0.384,
-      "eval_loss": 1.4867918491363525,
-      "eval_runtime": 1460.1808,
-      "eval_samples_per_second": 3.424,
-      "eval_steps_per_second": 0.428,
-      "step": 240
-    },
-    {
-      "epoch": 0.416,
-      "grad_norm": 0.03537523001432419,
-      "learning_rate": 0.00047895672791938357,
-      "loss": 1.4747,
-      "step": 260
-    },
-    {
-      "epoch": 0.416,
-      "eval_loss": 1.4859720468521118,
-      "eval_runtime": 1460.3853,
-      "eval_samples_per_second": 3.424,
-      "eval_steps_per_second": 0.428,
-      "step": 260
-    },
-    {
-      "epoch": 0.448,
-      "grad_norm": 0.037848278880119324,
-      "learning_rate": 0.0004730290456431535,
-      "loss": 1.4635,
-      "step": 280
-    },
-    {
-      "epoch": 0.448,
-      "eval_loss": 1.4846266508102417,
-      "eval_runtime": 1460.0793,
-      "eval_samples_per_second": 3.424,
-      "eval_steps_per_second": 0.428,
-      "step": 280
-    },
-    {
-      "epoch": 0.48,
-      "grad_norm": 0.038794029504060745,
-      "learning_rate": 0.00046710136336692356,
-      "loss": 1.5055,
-      "step": 300
-    },
-    {
-      "epoch": 0.48,
-      "eval_loss": 1.4849002361297607,
-      "eval_runtime": 1472.1795,
-      "eval_samples_per_second": 3.396,
-      "eval_steps_per_second": 0.425,
-      "step": 300
-    },
-    {
-      "epoch": 0.512,
-      "grad_norm": 0.03938526660203934,
-      "learning_rate": 0.00046117368109069355,
-      "loss": 1.4996,
-      "step": 320
-    },
-    {
-      "epoch": 0.512,
-      "eval_loss": 1.4825308322906494,
-      "eval_runtime": 1500.8065,
-      "eval_samples_per_second": 3.332,
-      "eval_steps_per_second": 0.416,
-      "step": 320
-    },
-    {
-      "epoch": 0.544,
-      "grad_norm": 0.041331056505441666,
-      "learning_rate": 0.00045524599881446355,
-      "loss": 1.4742,
-      "step": 340
-    },
-    {
-      "epoch": 0.544,
-      "eval_loss": 1.4830516576766968,
-      "eval_runtime": 1501.5622,
-      "eval_samples_per_second": 3.33,
-      "eval_steps_per_second": 0.416,
-      "step": 340
-    },
-    {
-      "epoch": 0.576,
-      "grad_norm": 0.03991026058793068,
-      "learning_rate": 0.0004493183165382336,
-      "loss": 1.4809,
-      "step": 360
-    },
-    {
-      "epoch": 0.576,
-      "eval_loss": 1.4818130731582642,
-      "eval_runtime": 1501.7616,
-      "eval_samples_per_second": 3.329,
-      "eval_steps_per_second": 0.416,
-      "step": 360
+      "epoch": 0.08,
+      "eval_loss": 1.7156720161437988,
+      "eval_runtime": 149.1859,
+      "eval_samples_per_second": 3.352,
+      "eval_steps_per_second": 0.422,
+      "step": 50
     }
   ],
-  "logging_steps": 20,
+  "logging_steps": 50,
   "max_steps": 1875,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
-  "save_steps": 10,
+  "save_steps": 50,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
@@ -296,7 +41,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.892365895794688e+16,
+  "total_flos": 4002786222735360.0,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null

tokenizer_config.json

@@ -121,11 +121,15 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "legacy": false,
+  "max_length": 512,
   "model_max_length": 4096,
   "pad_token": "<|endoftext|>",
   "padding_side": "right",
   "sp_model_kwargs": {},
+  "stride": 0,
   "tokenizer_class": "LlamaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>",
   "use_default_system_prompt": false
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4898537838146fbf12c74d87a475327b6a5606d9348748214e9ab9ceaa968a02
+oid sha256:f6aadb86e7c11eaf53076fa580bc16d46538da6411436f54844013f1933386b8
 size 5432