Training in progress, epoch 2, checkpoint

last-checkpoint/adapter_config.json

@@ -29,13 +29,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "o_proj",
-    "k_proj",
     "down_proj",
-    "gate_proj",
     "q_proj",
-    "up_proj"
+    "o_proj",
+    "v_proj",
+    "gate_proj",
+    "up_proj",
+    "k_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b28a611f18c5bf6bd4e94b537636210f4683b5a33960138bf4b7f8759dfcb59e
+oid sha256:df1c1aa2916d5972f5efeb6284e8eab0c9c72f3782a534e72ebed006d4a326dc
 size 228140600

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b0a7a51782095db67faf17d4424456c08cb55127408d12014dc3c304ffbdab7
-size 116484839
+oid sha256:84a3341db1ea0703a3a5bfb4c84d9c6f9c629d7ec814a41d47f66338b4b4316a
+size 117931203

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b0b09bbaedfdbe2893d036820ab6e355fea8a9aab8a443615445767baabde29
+oid sha256:5c823b36aa64ec6d5ba470435413c8fa628bdc36879db73fd6bcc786691658d3
 size 14645

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee67534ce5c31fc7fdd40446bc6096b050048bea81431627ee7eb7a4e0420fce
+oid sha256:7db9931cd2bdb0cce107e4058673881f0e4939f11f21f05dabe6ed2ca0118fd7
 size 1383

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba3d9ccd5719a6149375c0d8ad46aab7eeed3ae74f7933879cf60a287f920385
+oid sha256:53f409af08acb24ba2f85422d6d830e93fdc97a01268b4582a53eec3cbfeb20a
 size 1465

last-checkpoint/trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 2.0,
   "eval_steps": 500,
-  "global_step": 438,
+  "global_step": 876,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -450,6 +450,458 @@
       "eval_samples_per_second": 1.56,
       "eval_steps_per_second": 0.391,
       "step": 438
+    },
+    {
+      "entropy": 0.19485942274332047,
+      "epoch": 1.0045714285714287,
+      "grad_norm": 0.5989819169044495,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.1729,
+      "mean_token_accuracy": 0.9562485031783581,
+      "num_tokens": 46756.0,
+      "step": 440
+    },
+    {
+      "entropy": 0.15740223932079972,
+      "epoch": 1.0274285714285714,
+      "grad_norm": 0.9629825949668884,
+      "learning_rate": 7.95433789954338e-06,
+      "loss": 0.1453,
+      "mean_token_accuracy": 0.9640702843666077,
+      "num_tokens": 162230.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.18924359129741788,
+      "epoch": 1.0502857142857143,
+      "grad_norm": 0.8555030226707458,
+      "learning_rate": 7.908675799086758e-06,
+      "loss": 0.1647,
+      "mean_token_accuracy": 0.9583326201885939,
+      "num_tokens": 231685.0,
+      "step": 460
+    },
+    {
+      "entropy": 0.22046699812635778,
+      "epoch": 1.0731428571428572,
+      "grad_norm": 0.6105676293373108,
+      "learning_rate": 7.863013698630137e-06,
+      "loss": 0.1864,
+      "mean_token_accuracy": 0.9536379788070917,
+      "num_tokens": 276927.0,
+      "step": 470
+    },
+    {
+      "entropy": 0.26470216070301833,
+      "epoch": 1.096,
+      "grad_norm": 0.6477329134941101,
+      "learning_rate": 7.817351598173517e-06,
+      "loss": 0.2354,
+      "mean_token_accuracy": 0.9424127731472254,
+      "num_tokens": 309026.0,
+      "step": 480
+    },
+    {
+      "entropy": 0.25112292610574516,
+      "epoch": 1.1188571428571428,
+      "grad_norm": 0.8889337778091431,
+      "learning_rate": 7.771689497716896e-06,
+      "loss": 0.222,
+      "mean_token_accuracy": 0.9455349139869214,
+      "num_tokens": 375120.0,
+      "step": 490
+    },
+    {
+      "entropy": 0.15172108153346925,
+      "epoch": 1.1417142857142857,
+      "grad_norm": 1.6094820499420166,
+      "learning_rate": 7.726027397260276e-06,
+      "loss": 0.1358,
+      "mean_token_accuracy": 0.9661709513515234,
+      "num_tokens": 486745.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.167777626728639,
+      "epoch": 1.1645714285714286,
+      "grad_norm": 0.7636669278144836,
+      "learning_rate": 7.680365296803653e-06,
+      "loss": 0.1456,
+      "mean_token_accuracy": 0.964617732539773,
+      "num_tokens": 548794.0,
+      "step": 510
+    },
+    {
+      "entropy": 0.20727068707346916,
+      "epoch": 1.1874285714285715,
+      "grad_norm": 0.6218438148498535,
+      "learning_rate": 7.634703196347033e-06,
+      "loss": 0.1779,
+      "mean_token_accuracy": 0.9553128611296415,
+      "num_tokens": 590048.0,
+      "step": 520
+    },
+    {
+      "entropy": 0.24928416800685227,
+      "epoch": 1.2102857142857144,
+      "grad_norm": 0.5786827802658081,
+      "learning_rate": 7.589041095890411e-06,
+      "loss": 0.2181,
+      "mean_token_accuracy": 0.945609737932682,
+      "num_tokens": 620670.0,
+      "step": 530
+    },
+    {
+      "entropy": 0.23164914632216096,
+      "epoch": 1.233142857142857,
+      "grad_norm": 0.9021991491317749,
+      "learning_rate": 7.543378995433791e-06,
+      "loss": 0.2029,
+      "mean_token_accuracy": 0.9481291055679322,
+      "num_tokens": 678029.0,
+      "step": 540
+    },
+    {
+      "entropy": 0.11451007889118045,
+      "epoch": 1.256,
+      "grad_norm": 1.143864631652832,
+      "learning_rate": 7.497716894977169e-06,
+      "loss": 0.105,
+      "mean_token_accuracy": 0.9750772431492806,
+      "num_tokens": 789478.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.14542806874960662,
+      "epoch": 1.278857142857143,
+      "grad_norm": 0.6672413349151611,
+      "learning_rate": 7.452054794520549e-06,
+      "loss": 0.1256,
+      "mean_token_accuracy": 0.9681380245834589,
+      "num_tokens": 853276.0,
+      "step": 560
+    },
+    {
+      "entropy": 0.19598664692603052,
+      "epoch": 1.3017142857142856,
+      "grad_norm": 0.5779910683631897,
+      "learning_rate": 7.406392694063927e-06,
+      "loss": 0.1742,
+      "mean_token_accuracy": 0.9562454361468553,
+      "num_tokens": 894797.0,
+      "step": 570
+    },
+    {
+      "entropy": 0.23272629571147263,
+      "epoch": 1.3245714285714285,
+      "grad_norm": 0.4893546998500824,
+      "learning_rate": 7.360730593607307e-06,
+      "loss": 0.2026,
+      "mean_token_accuracy": 0.9472691085189581,
+      "num_tokens": 925575.0,
+      "step": 580
+    },
+    {
+      "entropy": 0.22576562578324227,
+      "epoch": 1.3474285714285714,
+      "grad_norm": 0.47898435592651367,
+      "learning_rate": 7.315068493150685e-06,
+      "loss": 0.1976,
+      "mean_token_accuracy": 0.9479547172784806,
+      "num_tokens": 983597.0,
+      "step": 590
+    },
+    {
+      "entropy": 0.11407975524198263,
+      "epoch": 1.3702857142857143,
+      "grad_norm": 1.4250750541687012,
+      "learning_rate": 7.269406392694065e-06,
+      "loss": 0.1095,
+      "mean_token_accuracy": 0.9737830355763435,
+      "num_tokens": 1090322.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.1459290421102196,
+      "epoch": 1.3931428571428572,
+      "grad_norm": 0.6979950666427612,
+      "learning_rate": 7.223744292237444e-06,
+      "loss": 0.1302,
+      "mean_token_accuracy": 0.9667541589587927,
+      "num_tokens": 1152923.0,
+      "step": 610
+    },
+    {
+      "entropy": 0.18885702546685934,
+      "epoch": 1.416,
+      "grad_norm": 0.5068536996841431,
+      "learning_rate": 7.178082191780823e-06,
+      "loss": 0.1636,
+      "mean_token_accuracy": 0.959310057759285,
+      "num_tokens": 1193481.0,
+      "step": 620
+    },
+    {
+      "entropy": 0.23108526985161007,
+      "epoch": 1.4388571428571428,
+      "grad_norm": 0.8436884880065918,
+      "learning_rate": 7.132420091324202e-06,
+      "loss": 0.2072,
+      "mean_token_accuracy": 0.9471893258392811,
+      "num_tokens": 1222847.0,
+      "step": 630
+    },
+    {
+      "entropy": 0.21841485593467952,
+      "epoch": 1.4617142857142857,
+      "grad_norm": 0.8229106068611145,
+      "learning_rate": 7.086757990867581e-06,
+      "loss": 0.1863,
+      "mean_token_accuracy": 0.9520192969590425,
+      "num_tokens": 1290269.0,
+      "step": 640
+    },
+    {
+      "entropy": 0.10626114641781896,
+      "epoch": 1.4845714285714287,
+      "grad_norm": 1.5995644330978394,
+      "learning_rate": 7.0410958904109596e-06,
+      "loss": 0.0994,
+      "mean_token_accuracy": 0.976220278069377,
+      "num_tokens": 1402634.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.13410865939222277,
+      "epoch": 1.5074285714285716,
+      "grad_norm": 0.4029393196105957,
+      "learning_rate": 6.995433789954339e-06,
+      "loss": 0.1213,
+      "mean_token_accuracy": 0.9693065240979195,
+      "num_tokens": 1466175.0,
+      "step": 660
+    },
+    {
+      "entropy": 0.17447674251161516,
+      "epoch": 1.5302857142857142,
+      "grad_norm": 0.418222576379776,
+      "learning_rate": 6.9497716894977175e-06,
+      "loss": 0.1528,
+      "mean_token_accuracy": 0.9606836523860693,
+      "num_tokens": 1507803.0,
+      "step": 670
+    },
+    {
+      "entropy": 0.21088513871654868,
+      "epoch": 1.5531428571428572,
+      "grad_norm": 0.5335624814033508,
+      "learning_rate": 6.904109589041097e-06,
+      "loss": 0.1855,
+      "mean_token_accuracy": 0.9512796506285668,
+      "num_tokens": 1538690.0,
+      "step": 680
+    },
+    {
+      "entropy": 0.21284959067124873,
+      "epoch": 1.576,
+      "grad_norm": 0.7531531453132629,
+      "learning_rate": 6.858447488584475e-06,
+      "loss": 0.1834,
+      "mean_token_accuracy": 0.9524108562618494,
+      "num_tokens": 1599601.0,
+      "step": 690
+    },
+    {
+      "entropy": 0.08681527464650571,
+      "epoch": 1.5988571428571428,
+      "grad_norm": 0.6462493538856506,
+      "learning_rate": 6.812785388127855e-06,
+      "loss": 0.078,
+      "mean_token_accuracy": 0.980168628692627,
+      "num_tokens": 1713685.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.1219312352128327,
+      "epoch": 1.6217142857142857,
+      "grad_norm": 0.40015217661857605,
+      "learning_rate": 6.767123287671233e-06,
+      "loss": 0.1132,
+      "mean_token_accuracy": 0.9700204558670521,
+      "num_tokens": 1776814.0,
+      "step": 710
+    },
+    {
+      "entropy": 0.16716388445347546,
+      "epoch": 1.6445714285714286,
+      "grad_norm": 0.5019240975379944,
+      "learning_rate": 6.721461187214613e-06,
+      "loss": 0.1529,
+      "mean_token_accuracy": 0.9590244695544243,
+      "num_tokens": 1818772.0,
+      "step": 720
+    },
+    {
+      "entropy": 0.20658068330958484,
+      "epoch": 1.6674285714285715,
+      "grad_norm": 0.48935461044311523,
+      "learning_rate": 6.675799086757991e-06,
+      "loss": 0.186,
+      "mean_token_accuracy": 0.9494243700057268,
+      "num_tokens": 1848794.0,
+      "step": 730
+    },
+    {
+      "entropy": 0.21933096905704588,
+      "epoch": 1.6902857142857144,
+      "grad_norm": 1.0664595365524292,
+      "learning_rate": 6.630136986301371e-06,
+      "loss": 0.195,
+      "mean_token_accuracy": 0.9486182644963265,
+      "num_tokens": 1912529.0,
+      "step": 740
+    },
+    {
+      "entropy": 0.09219505588989704,
+      "epoch": 1.713142857142857,
+      "grad_norm": 0.9666043519973755,
+      "learning_rate": 6.584474885844749e-06,
+      "loss": 0.0842,
+      "mean_token_accuracy": 0.9790573690086604,
+      "num_tokens": 2023382.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.12722355276346206,
+      "epoch": 1.736,
+      "grad_norm": 0.325158953666687,
+      "learning_rate": 6.538812785388129e-06,
+      "loss": 0.1151,
+      "mean_token_accuracy": 0.9699444197118282,
+      "num_tokens": 2088409.0,
+      "step": 760
+    },
+    {
+      "entropy": 0.16778573733754457,
+      "epoch": 1.758857142857143,
+      "grad_norm": 0.4155607521533966,
+      "learning_rate": 6.493150684931508e-06,
+      "loss": 0.1528,
+      "mean_token_accuracy": 0.9595557443797589,
+      "num_tokens": 2130039.0,
+      "step": 770
+    },
+    {
+      "entropy": 0.2043486479204148,
+      "epoch": 1.7817142857142856,
+      "grad_norm": 0.4696311354637146,
+      "learning_rate": 6.447488584474887e-06,
+      "loss": 0.1915,
+      "mean_token_accuracy": 0.9497469838708639,
+      "num_tokens": 2160612.0,
+      "step": 780
+    },
+    {
+      "entropy": 0.2095549178076908,
+      "epoch": 1.8045714285714287,
+      "grad_norm": 0.49995991587638855,
+      "learning_rate": 6.401826484018266e-06,
+      "loss": 0.1917,
+      "mean_token_accuracy": 0.951378521323204,
+      "num_tokens": 2219878.0,
+      "step": 790
+    },
+    {
+      "entropy": 0.09752151321154087,
+      "epoch": 1.8274285714285714,
+      "grad_norm": 0.7513600587844849,
+      "learning_rate": 6.356164383561645e-06,
+      "loss": 0.0877,
+      "mean_token_accuracy": 0.9781307391822338,
+      "num_tokens": 2328503.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.1315026845317334,
+      "epoch": 1.8502857142857143,
+      "grad_norm": 0.4865649938583374,
+      "learning_rate": 6.3105022831050235e-06,
+      "loss": 0.1191,
+      "mean_token_accuracy": 0.9693873535841704,
+      "num_tokens": 2393270.0,
+      "step": 810
+    },
+    {
+      "entropy": 0.15703398073092104,
+      "epoch": 1.8731428571428572,
+      "grad_norm": 0.46761906147003174,
+      "learning_rate": 6.264840182648403e-06,
+      "loss": 0.1418,
+      "mean_token_accuracy": 0.9627573467791081,
+      "num_tokens": 2436335.0,
+      "step": 820
+    },
+    {
+      "entropy": 0.19582971301861107,
+      "epoch": 1.896,
+      "grad_norm": 0.5706267356872559,
+      "learning_rate": 6.219178082191781e-06,
+      "loss": 0.1771,
+      "mean_token_accuracy": 0.9532948363572359,
+      "num_tokens": 2467211.0,
+      "step": 830
+    },
+    {
+      "entropy": 0.19951584844384343,
+      "epoch": 1.9188571428571428,
+      "grad_norm": 0.1575632095336914,
+      "learning_rate": 6.173515981735161e-06,
+      "loss": 0.1733,
+      "mean_token_accuracy": 0.9532737210392952,
+      "num_tokens": 2527699.0,
+      "step": 840
+    },
+    {
+      "entropy": 0.09860867839306593,
+      "epoch": 1.9417142857142857,
+      "grad_norm": 0.37800732254981995,
+      "learning_rate": 6.127853881278539e-06,
+      "loss": 0.0922,
+      "mean_token_accuracy": 0.9767971355468035,
+      "num_tokens": 2624761.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.14002426667138934,
+      "epoch": 1.9645714285714284,
+      "grad_norm": 0.8500357866287231,
+      "learning_rate": 6.082191780821919e-06,
+      "loss": 0.1284,
+      "mean_token_accuracy": 0.9653151527047157,
+      "num_tokens": 2674387.0,
+      "step": 860
+    },
+    {
+      "entropy": 0.20728676998987794,
+      "epoch": 1.9874285714285715,
+      "grad_norm": 0.6311262845993042,
+      "learning_rate": 6.036529680365297e-06,
+      "loss": 0.1892,
+      "mean_token_accuracy": 0.9492694169282914,
+      "num_tokens": 2705462.0,
+      "step": 870
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.001506220691455712,
+      "eval_entropy": 0.38320175457645106,
+      "eval_loss": 1.0289124250411987,
+      "eval_mean_token_accuracy": 0.8326533791181203,
+      "eval_num_tokens": 2716693.0,
+      "eval_runtime": 734.0738,
+      "eval_samples_per_second": 1.409,
+      "eval_steps_per_second": 0.353,
+      "step": 876
     }
   ],
   "logging_steps": 10,
@@ -469,7 +921,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.8963131243019264e+17,
+  "total_flos": 3.792626248603853e+17,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:58259b31d8c49fec76c9044575d2a0dc11fa8080720bce2e7820a1dfbfb8174f
+oid sha256:dd36270ff585b2d668c6df7d7ada51207c25255f0fc66fa207d06a8a67152786
 size 6353