Training in progress, step 1000, checkpoint

last-checkpoint/README.md

@@ -206,4 +206,4 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
 [More Information Needed]
 ### Framework versions
 
-- PEFT 0.19.1
+- PEFT 0.18.1

last-checkpoint/adapter_config.json

@@ -19,29 +19,27 @@
   "lora_alpha": 256,
   "lora_bias": false,
   "lora_dropout": 0.05,
-  "lora_ga_config": null,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "peft_version": "0.19.1",
+  "peft_version": "0.18.1",
   "qalora_group_size": 16,
   "r": 128,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "q_proj",
+    "down_proj",
     "v_proj",
-    "gate_proj",
+    "o_proj",
     "up_proj",
-    "down_proj",
-    "k_proj",
-    "q_proj",
-    "o_proj"
+    "gate_proj",
+    "k_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,
-  "use_bdlora": null,
   "use_dora": false,
   "use_qalora": false,
   "use_rslora": false

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bbc31def6fc1183573b4ca53bf82830b3e675c174bc8741079dee8f82ace3c3d
+oid sha256:ece9918ba6ae7ff5c6354d8843a6ff5e36f4fd5dcd0c2fc4f171e781fa6c7b95
 size 363365712

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:618454c138b7364f8dc01f3db6c85573dc0fe52fe5fefec5e12a9d9ba2f00f2f
+oid sha256:2c790bf95a1b16a87fcf911b7884e6a62ad5b0af02cfb8b9640661805d06a06a
 size 184804245

last-checkpoint/trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "entropy": 2.937763540148735,
+      "entropy": 2.933625968694687,
       "epoch": 0.02127659574468085,
-      "grad_norm": 0.4686773121356964,
+      "grad_norm": 0.4740275740623474,
       "learning_rate": 9.951e-05,
-      "loss": 4.138067016601562,
-      "mean_token_accuracy": 0.4264019676297903,
+      "loss": 4.140991821289062,
+      "mean_token_accuracy": 0.4264977302402258,
       "num_tokens": 988301.0,
       "step": 50
     },
     {
-      "entropy": 2.962523487210274,
+      "entropy": 2.962103115916252,
       "epoch": 0.0425531914893617,
-      "grad_norm": 0.6529152989387512,
+      "grad_norm": 0.654786229133606,
       "learning_rate": 9.901e-05,
-      "loss": 3.1858258056640625,
-      "mean_token_accuracy": 0.491901636198163,
+      "loss": 3.1859881591796877,
+      "mean_token_accuracy": 0.4919606515020132,
       "num_tokens": 1981516.0,
       "step": 100
     },
     {
-      "entropy": 2.8207009065151216,
+      "entropy": 2.8202617847919464,
       "epoch": 0.06382978723404255,
-      "grad_norm": 0.6847605109214783,
+      "grad_norm": 0.6856227517127991,
       "learning_rate": 9.851e-05,
-      "loss": 2.97326904296875,
-      "mean_token_accuracy": 0.5133598280698061,
+      "loss": 2.973466796875,
+      "mean_token_accuracy": 0.5133317831903697,
       "num_tokens": 2962772.0,
       "step": 150
     },
     {
-      "entropy": 2.6926355504989625,
+      "entropy": 2.692441967725754,
       "epoch": 0.0851063829787234,
-      "grad_norm": 0.7111132740974426,
+      "grad_norm": 0.7118680477142334,
       "learning_rate": 9.801e-05,
-      "loss": 2.8132635498046876,
-      "mean_token_accuracy": 0.5304626803100109,
+      "loss": 2.8135525512695314,
+      "mean_token_accuracy": 0.5304926330596209,
       "num_tokens": 3952403.0,
       "step": 200
     },
     {
-      "entropy": 2.6115103197097778,
+      "entropy": 2.6121814751625063,
       "epoch": 0.10638297872340426,
-      "grad_norm": 0.6989225149154663,
+      "grad_norm": 0.7034955620765686,
       "learning_rate": 9.751e-05,
-      "loss": 2.7149212646484373,
-      "mean_token_accuracy": 0.5415211926400662,
+      "loss": 2.715215148925781,
+      "mean_token_accuracy": 0.5413537514209747,
       "num_tokens": 4939231.0,
       "step": 250
     },
     {
-      "entropy": 2.552058919072151,
+      "entropy": 2.551229443252087,
       "epoch": 0.1276595744680851,
-      "grad_norm": 0.6323665976524353,
+      "grad_norm": 0.6356984972953796,
       "learning_rate": 9.701e-05,
-      "loss": 2.6398919677734374,
-      "mean_token_accuracy": 0.5516271162033081,
+      "loss": 2.6399871826171877,
+      "mean_token_accuracy": 0.55161267362535,
       "num_tokens": 5931036.0,
       "step": 300
     },
     {
-      "entropy": 2.4714413863420486,
+      "entropy": 2.471001845598221,
       "epoch": 0.14893617021276595,
-      "grad_norm": 0.6492835283279419,
+      "grad_norm": 0.653047502040863,
       "learning_rate": 9.651e-05,
-      "loss": 2.546295471191406,
-      "mean_token_accuracy": 0.5627167555689812,
+      "loss": 2.5464208984375,
+      "mean_token_accuracy": 0.5625544948130846,
       "num_tokens": 6919236.0,
       "step": 350
     },
     {
-      "entropy": 2.452097176015377,
+      "entropy": 2.451621271967888,
       "epoch": 0.1702127659574468,
-      "grad_norm": 0.7032881379127502,
+      "grad_norm": 0.7060537934303284,
       "learning_rate": 9.601e-05,
-      "loss": 2.5202024841308592,
-      "mean_token_accuracy": 0.5672398428618908,
+      "loss": 2.520094299316406,
+      "mean_token_accuracy": 0.5672792405635119,
       "num_tokens": 7901385.0,
       "step": 400
     },
     {
-      "entropy": 2.38740619212389,
+      "entropy": 2.386715810596943,
       "epoch": 0.19148936170212766,
-      "grad_norm": 0.715317964553833,
+      "grad_norm": 0.7115684747695923,
       "learning_rate": 9.551e-05,
-      "loss": 2.449654541015625,
-      "mean_token_accuracy": 0.5774810115993023,
+      "loss": 2.449301452636719,
+      "mean_token_accuracy": 0.5778872921317816,
       "num_tokens": 8891410.0,
       "step": 450
     },
     {
-      "entropy": 2.339343198239803,
+      "entropy": 2.3399247616529464,
       "epoch": 0.2127659574468085,
-      "grad_norm": 0.7024112939834595,
+      "grad_norm": 0.7020632028579712,
       "learning_rate": 9.501e-05,
-      "loss": 2.399354248046875,
-      "mean_token_accuracy": 0.584610120356083,
+      "loss": 2.3999610900878907,
+      "mean_token_accuracy": 0.5845850779861211,
       "num_tokens": 9872429.0,
       "step": 500
     },
     {
-      "entropy": 2.3019780376553536,
+      "entropy": 2.301297716200352,
       "epoch": 0.23404255319148937,
-      "grad_norm": 0.7314584851264954,
+      "grad_norm": 0.7258976101875305,
       "learning_rate": 9.451000000000002e-05,
-      "loss": 2.3609829711914063,
-      "mean_token_accuracy": 0.588694809526205,
+      "loss": 2.3609393310546873,
+      "mean_token_accuracy": 0.5889028573036194,
       "num_tokens": 10852406.0,
       "step": 550
     },
     {
-      "entropy": 2.2801691934466364,
+      "entropy": 2.28171086281538,
       "epoch": 0.2553191489361702,
-      "grad_norm": 0.7630459070205688,
+      "grad_norm": 0.7619220018386841,
       "learning_rate": 9.401e-05,
-      "loss": 2.322369384765625,
-      "mean_token_accuracy": 0.5943487723916769,
+      "loss": 2.32339111328125,
+      "mean_token_accuracy": 0.5941300053894519,
       "num_tokens": 11847983.0,
       "step": 600
     },
     {
-      "entropy": 2.231162509918213,
+      "entropy": 2.2323903796076774,
       "epoch": 0.2765957446808511,
-      "grad_norm": 0.7081300616264343,
+      "grad_norm": 0.7111139893531799,
       "learning_rate": 9.351e-05,
-      "loss": 2.2724359130859373,
-      "mean_token_accuracy": 0.6005339217931032,
+      "loss": 2.2743270874023436,
+      "mean_token_accuracy": 0.6003150211274624,
       "num_tokens": 12839631.0,
       "step": 650
     },
     {
-      "entropy": 2.195046606659889,
+      "entropy": 2.195645292699337,
       "epoch": 0.2978723404255319,
-      "grad_norm": 0.8117260336875916,
+      "grad_norm": 0.8029466867446899,
       "learning_rate": 9.301e-05,
-      "loss": 2.2360572814941406,
-      "mean_token_accuracy": 0.6055160685628652,
+      "loss": 2.2373281860351564,
+      "mean_token_accuracy": 0.605772587954998,
       "num_tokens": 13825421.0,
       "step": 700
     },
     {
-      "entropy": 2.17775638371706,
+      "entropy": 2.1788447910547255,
       "epoch": 0.3191489361702128,
-      "grad_norm": 0.7083834409713745,
+      "grad_norm": 0.7066243290901184,
       "learning_rate": 9.251000000000001e-05,
-      "loss": 2.2109527587890625,
-      "mean_token_accuracy": 0.610633347928524,
+      "loss": 2.2124517822265624,
+      "mean_token_accuracy": 0.6105387426912785,
       "num_tokens": 14811783.0,
       "step": 750
     },
     {
-      "entropy": 2.1384602162241935,
+      "entropy": 2.140341859459877,
       "epoch": 0.3404255319148936,
-      "grad_norm": 0.6845762729644775,
+      "grad_norm": 0.6823806166648865,
       "learning_rate": 9.201000000000001e-05,
-      "loss": 2.1738687133789063,
-      "mean_token_accuracy": 0.6158557101339102,
+      "loss": 2.175841064453125,
+      "mean_token_accuracy": 0.615707865729928,
       "num_tokens": 15799402.0,
       "step": 800
     },
     {
-      "entropy": 2.1188551610708237,
+      "entropy": 2.121108899116516,
       "epoch": 0.3617021276595745,
-      "grad_norm": 0.7421966791152954,
+      "grad_norm": 0.7394977807998657,
       "learning_rate": 9.151000000000001e-05,
-      "loss": 2.146589660644531,
-      "mean_token_accuracy": 0.6187865848094225,
+      "loss": 2.1485189819335937,
+      "mean_token_accuracy": 0.6186434020847082,
       "num_tokens": 16787125.0,
       "step": 850
     },
     {
-      "entropy": 2.0914336186647415,
+      "entropy": 2.092526486814022,
       "epoch": 0.3829787234042553,
-      "grad_norm": 0.706660807132721,
+      "grad_norm": 0.7025715112686157,
       "learning_rate": 9.101000000000001e-05,
-      "loss": 2.1187652587890624,
-      "mean_token_accuracy": 0.6228364047408104,
+      "loss": 2.1199916076660155,
+      "mean_token_accuracy": 0.6228454371541738,
       "num_tokens": 17783798.0,
       "step": 900
     },
     {
-      "entropy": 2.0830547219514846,
+      "entropy": 2.0856992295384407,
       "epoch": 0.40425531914893614,
-      "grad_norm": 0.7128080725669861,
+      "grad_norm": 0.726789653301239,
       "learning_rate": 9.051000000000001e-05,
-      "loss": 2.10831298828125,
-      "mean_token_accuracy": 0.6251034809648991,
+      "loss": 2.1102699279785155,
+      "mean_token_accuracy": 0.6247553788125515,
       "num_tokens": 18760192.0,
       "step": 950
     },
     {
-      "entropy": 2.042051522433758,
+      "entropy": 2.0423195973038673,
       "epoch": 0.425531914893617,
-      "grad_norm": 0.7421649098396301,
+      "grad_norm": 0.7402950525283813,
       "learning_rate": 9.001e-05,
-      "loss": 2.059340362548828,
-      "mean_token_accuracy": 0.6305920536071062,
+      "loss": 2.059694519042969,
+      "mean_token_accuracy": 0.6303426054865122,
       "num_tokens": 19742108.0,
       "step": 1000
     }