Upload trainer_state.json with huggingface_hub

trainer_state.json

@@ -1,1507 +1,915 @@
 {
-  "epoch": 3.5724508050089447,
-  "global_step": 1000,
-  "max_steps": 2232,
+  "epoch": 1.185633176993335,
+  "global_step": 600,
+  "max_steps": 4048,
   "logging_steps": 5,
   "eval_steps": 100,
   "save_steps": 100,
   "train_batch_size": 16,
   "num_train_epochs": 8,
   "num_input_tokens_seen": 0,
-  "total_flos": 4.722006162277663e+17,
+  "total_flos": 2.946671339427594e+17,
   "log_history": [
     {
-      "loss": 10.1939,
-      "grad_norm": 74.8128433227539,
+      "loss": 10.7217,
+      "grad_norm": 35.45301055908203,
       "learning_rate": 3.75e-05,
-      "epoch": 0.017889087656529516,
+      "epoch": 0.009874105159219946,
       "step": 5
     },
     {
-      "loss": 3.9273,
-      "grad_norm": 12.053423881530762,
+      "loss": 4.3632,
+      "grad_norm": 12.563997268676758,
       "learning_rate": 7.5e-05,
-      "epoch": 0.03577817531305903,
+      "epoch": 0.01974821031843989,
       "step": 10
     },
     {
-      "loss": 1.4898,
-      "grad_norm": 40.130340576171875,
+      "loss": 1.6282,
+      "grad_norm": 10.4967041015625,
       "learning_rate": 0.0001125,
-      "epoch": 0.05366726296958855,
+      "epoch": 0.02962231547765984,
       "step": 15
     },
     {
-      "loss": 0.9064,
-      "grad_norm": 3.677988290786743,
+      "loss": 1.1701,
+      "grad_norm": 2.9293179512023926,
       "learning_rate": 0.00015,
-      "epoch": 0.07155635062611806,
+      "epoch": 0.03949642063687978,
       "step": 20
     },
     {
-      "loss": 0.8129,
-      "grad_norm": 1.335487961769104,
+      "loss": 0.9779,
+      "grad_norm": 1.5904552936553955,
       "learning_rate": 0.00018749999999999998,
-      "epoch": 0.08944543828264759,
+      "epoch": 0.049370525796099726,
       "step": 25
     },
     {
-      "loss": 1.1672,
-      "grad_norm": 29.67264175415039,
+      "loss": 0.8266,
+      "grad_norm": 1.494022011756897,
       "learning_rate": 0.000225,
-      "epoch": 0.1073345259391771,
+      "epoch": 0.05924463095531968,
       "step": 30
     },
     {
-      "loss": 0.7385,
-      "grad_norm": 1.039961576461792,
+      "loss": 0.7704,
+      "grad_norm": 0.42715126276016235,
       "learning_rate": 0.0002625,
-      "epoch": 0.1252236135957066,
+      "epoch": 0.06911873611453961,
       "step": 35
     },
     {
-      "loss": 0.6884,
-      "grad_norm": 0.4314720928668976,
+      "loss": 0.735,
+      "grad_norm": 0.3201207220554352,
       "learning_rate": 0.0003,
-      "epoch": 0.14311270125223613,
+      "epoch": 0.07899284127375956,
       "step": 40
     },
     {
-      "loss": 0.6748,
-      "grad_norm": 0.40387117862701416,
-      "learning_rate": 0.0002999961486050259,
-      "epoch": 0.16100178890876565,
+      "loss": 0.7419,
+      "grad_norm": 2.504087209701538,
+      "learning_rate": 0.00029999884801974356,
+      "epoch": 0.08886694643297952,
       "step": 45
     },
     {
-      "loss": 0.6628,
-      "grad_norm": 0.448535293340683,
-      "learning_rate": 0.00029998459461788026,
-      "epoch": 0.17889087656529518,
+      "loss": 0.8097,
+      "grad_norm": 0.5647741556167603,
+      "learning_rate": 0.00029999539209666856,
+      "epoch": 0.09874105159219945,
       "step": 50
     },
     {
-      "loss": 0.6659,
-      "grad_norm": 0.5291304588317871,
-      "learning_rate": 0.0002999653386318826,
-      "epoch": 0.1967799642218247,
+      "loss": 0.7196,
+      "grad_norm": 0.5168487429618835,
+      "learning_rate": 0.00029998963228385685,
+      "epoch": 0.1086151567514194,
       "step": 55
     },
     {
-      "loss": 0.6517,
-      "grad_norm": 0.6947611570358276,
-      "learning_rate": 0.0002999383816358651,
-      "epoch": 0.2146690518783542,
+      "loss": 0.707,
+      "grad_norm": 0.6618458032608032,
+      "learning_rate": 0.0002999815686697778,
+      "epoch": 0.11848926191063935,
       "step": 60
     },
     {
-      "loss": 0.7263,
-      "grad_norm": 0.5037692785263062,
-      "learning_rate": 0.0002999037250141215,
-      "epoch": 0.23255813953488372,
+      "loss": 0.7013,
+      "grad_norm": 4.516430854797363,
+      "learning_rate": 0.0002999712013782864,
+      "epoch": 0.1283633670698593,
       "step": 65
     },
     {
-      "loss": 0.6464,
-      "grad_norm": 0.35925614833831787,
-      "learning_rate": 0.0002998613705463364,
-      "epoch": 0.2504472271914132,
+      "loss": 0.8418,
+      "grad_norm": 0.38112834095954895,
+      "learning_rate": 0.0002999585305686214,
+      "epoch": 0.13823747222907923,
       "step": 70
     },
     {
-      "loss": 0.6531,
-      "grad_norm": 0.7011401653289795,
-      "learning_rate": 0.0002998113204074936,
-      "epoch": 0.26833631484794274,
+      "loss": 0.6765,
+      "grad_norm": 0.4867613613605499,
+      "learning_rate": 0.00029994355643540323,
+      "epoch": 0.1481115773882992,
       "step": 75
     },
     {
-      "loss": 0.6563,
-      "grad_norm": 0.29620787501335144,
-      "learning_rate": 0.0002997535771677644,
-      "epoch": 0.28622540250447226,
+      "loss": 0.675,
+      "grad_norm": 0.4741807281970978,
+      "learning_rate": 0.00029992627920863056,
+      "epoch": 0.15798568254751913,
       "step": 80
     },
     {
-      "loss": 0.6555,
-      "grad_norm": 0.24026329815387726,
-      "learning_rate": 0.00029968814379237584,
-      "epoch": 0.3041144901610018,
+      "loss": 0.6441,
+      "grad_norm": 0.38224297761917114,
+      "learning_rate": 0.000299906699153677,
+      "epoch": 0.16785978770673907,
       "step": 85
     },
     {
-      "loss": 0.6213,
-      "grad_norm": 0.17133238911628723,
-      "learning_rate": 0.00029961502364145824,
-      "epoch": 0.3220035778175313,
+      "loss": 0.64,
+      "grad_norm": 0.40791240334510803,
+      "learning_rate": 0.00029988481657128714,
+      "epoch": 0.17773389286595903,
       "step": 90
     },
     {
-      "loss": 0.6216,
-      "grad_norm": 0.24976064264774323,
-      "learning_rate": 0.0002995342204698726,
-      "epoch": 0.33989266547406083,
+      "loss": 0.6528,
+      "grad_norm": 0.2506788969039917,
+      "learning_rate": 0.00029986063179757166,
+      "epoch": 0.18760799802517897,
       "step": 95
     },
     {
-      "loss": 0.6357,
-      "grad_norm": 0.527025043964386,
-      "learning_rate": 0.00029944573842701806,
-      "epoch": 0.35778175313059035,
+      "loss": 0.6278,
+      "grad_norm": 0.2861074209213257,
+      "learning_rate": 0.0002998341452040023,
+      "epoch": 0.1974821031843989,
       "step": 100
     },
     {
-      "eval_loss": 0.5987367630004883,
-      "eval_runtime": 20.6719,
-      "eval_samples_per_second": 162.83,
-      "eval_steps_per_second": 10.207,
-      "epoch": 0.35778175313059035,
+      "eval_loss": 0.6399099230766296,
+      "eval_runtime": 46.6556,
+      "eval_samples_per_second": 154.301,
+      "eval_steps_per_second": 9.645,
+      "epoch": 0.1974821031843989,
       "step": 100
     },
     {
-      "loss": 0.6205,
-      "grad_norm": 0.3432077467441559,
-      "learning_rate": 0.00029934958205661853,
-      "epoch": 0.3756708407871199,
+      "loss": 0.6385,
+      "grad_norm": 0.2758176624774933,
+      "learning_rate": 0.00029980535719740614,
+      "epoch": 0.20735620834361887,
       "step": 105
     },
     {
-      "loss": 0.6071,
-      "grad_norm": 0.22968943417072296,
-      "learning_rate": 0.00029924575629648953,
-      "epoch": 0.3935599284436494,
+      "loss": 0.6354,
+      "grad_norm": 0.3389268219470978,
+      "learning_rate": 0.0002997742682199594,
+      "epoch": 0.2172303135028388,
       "step": 110
     },
     {
-      "loss": 0.6096,
-      "grad_norm": 0.402422696352005,
-      "learning_rate": 0.0002991342664782845,
-      "epoch": 0.41144901610017887,
+      "loss": 0.6263,
+      "grad_norm": 0.35775822401046753,
+      "learning_rate": 0.0002997408787491806,
+      "epoch": 0.22710441866205874,
       "step": 115
     },
     {
-      "loss": 0.597,
-      "grad_norm": 0.3375358283519745,
-      "learning_rate": 0.00029901511832722104,
-      "epoch": 0.4293381037567084,
+      "loss": 0.6245,
+      "grad_norm": 0.3307900130748749,
+      "learning_rate": 0.0002997051892979233,
+      "epoch": 0.2369785238212787,
       "step": 120
     },
     {
-      "loss": 0.5821,
-      "grad_norm": 0.321890652179718,
-      "learning_rate": 0.00029888831796178714,
-      "epoch": 0.4472271914132379,
+      "loss": 0.589,
+      "grad_norm": 0.49249130487442017,
+      "learning_rate": 0.0002996672004143679,
+      "epoch": 0.24685262898049865,
       "step": 125
     },
     {
-      "loss": 0.5982,
-      "grad_norm": 0.20958741009235382,
-      "learning_rate": 0.0002987538718934266,
-      "epoch": 0.46511627906976744,
+      "loss": 0.6172,
+      "grad_norm": 0.48136135935783386,
+      "learning_rate": 0.00029962691268201383,
+      "epoch": 0.2567267341397186,
       "step": 130
     },
     {
-      "loss": 0.5672,
-      "grad_norm": 0.2855433225631714,
-      "learning_rate": 0.000298611787026205,
-      "epoch": 0.48300536672629696,
+      "loss": 0.5992,
+      "grad_norm": 0.3200276494026184,
+      "learning_rate": 0.0002995843267196699,
+      "epoch": 0.2666008392989385,
       "step": 135
     },
     {
-      "loss": 0.5756,
-      "grad_norm": 0.3264622688293457,
-      "learning_rate": 0.0002984620706564548,
-      "epoch": 0.5008944543828264,
+      "loss": 0.5685,
+      "grad_norm": 0.39301344752311707,
+      "learning_rate": 0.00029953944318144535,
+      "epoch": 0.27647494445815846,
       "step": 140
     },
     {
-      "loss": 0.5643,
-      "grad_norm": 0.3137723505496979,
-      "learning_rate": 0.0002983047304724011,
-      "epoch": 0.518783542039356,
+      "loss": 0.5749,
+      "grad_norm": 0.27214545011520386,
+      "learning_rate": 0.00029949226275673954,
+      "epoch": 0.28634904961737845,
       "step": 145
     },
     {
-      "loss": 0.5854,
-      "grad_norm": 0.2823493480682373,
-      "learning_rate": 0.00029813977455376634,
-      "epoch": 0.5366726296958855,
+      "loss": 0.5586,
+      "grad_norm": 0.43858101963996887,
+      "learning_rate": 0.00029944278617023137,
+      "epoch": 0.2962231547765984,
       "step": 150
     },
     {
-      "loss": 0.5702,
-      "grad_norm": 0.24358761310577393,
-      "learning_rate": 0.00029796721137135597,
-      "epoch": 0.554561717352415,
+      "loss": 0.5515,
+      "grad_norm": 0.541606605052948,
+      "learning_rate": 0.0002993910141818681,
+      "epoch": 0.3060972599358183,
       "step": 155
     },
     {
-      "loss": 0.5487,
-      "grad_norm": 0.31340864300727844,
-      "learning_rate": 0.00029778704978662284,
-      "epoch": 0.5724508050089445,
+      "loss": 0.556,
+      "grad_norm": 0.48988083004951477,
+      "learning_rate": 0.00029933694758685396,
+      "epoch": 0.31597136509503826,
       "step": 160
     },
     {
-      "loss": 0.5478,
-      "grad_norm": 0.3177396357059479,
-      "learning_rate": 0.0002975992990512127,
-      "epoch": 0.590339892665474,
+      "loss": 0.5383,
+      "grad_norm": 0.3207962214946747,
+      "learning_rate": 0.0002992805872156376,
+      "epoch": 0.3258454702542582,
       "step": 165
     },
     {
-      "loss": 0.5501,
-      "grad_norm": 0.3493712842464447,
-      "learning_rate": 0.0002974039688064886,
-      "epoch": 0.6082289803220036,
+      "loss": 0.5315,
+      "grad_norm": 0.3307562470436096,
+      "learning_rate": 0.00029922193393389936,
+      "epoch": 0.33571957541347813,
       "step": 170
     },
     {
-      "loss": 0.5411,
-      "grad_norm": 0.2914203107357025,
-      "learning_rate": 0.00029720106908303625,
-      "epoch": 0.6261180679785331,
+      "loss": 0.5041,
+      "grad_norm": 0.4709561765193939,
+      "learning_rate": 0.0002991609886425383,
+      "epoch": 0.3455936805726981,
       "step": 175
     },
     {
-      "loss": 0.5306,
-      "grad_norm": 0.29958376288414,
-      "learning_rate": 0.0002969906103001486,
-      "epoch": 0.6440071556350626,
+      "loss": 0.4966,
+      "grad_norm": 0.39687275886535645,
+      "learning_rate": 0.00029909775227765804,
+      "epoch": 0.35546778573191806,
       "step": 180
     },
     {
-      "loss": 0.5304,
-      "grad_norm": 0.26977163553237915,
-      "learning_rate": 0.00029677260326529107,
-      "epoch": 0.6618962432915921,
+      "loss": 0.5063,
+      "grad_norm": 0.5469186305999756,
+      "learning_rate": 0.0002990322258105525,
+      "epoch": 0.365341890891138,
       "step": 185
     },
     {
-      "loss": 0.5231,
-      "grad_norm": 0.2629687786102295,
-      "learning_rate": 0.0002965470591735462,
-      "epoch": 0.6797853309481217,
+      "loss": 0.5162,
+      "grad_norm": 0.46143925189971924,
+      "learning_rate": 0.000298964410247691,
+      "epoch": 0.37521599605035794,
       "step": 190
     },
     {
-      "loss": 0.524,
-      "grad_norm": 0.3607335388660431,
-      "learning_rate": 0.0002963139896070391,
-      "epoch": 0.6976744186046512,
+      "loss": 0.5,
+      "grad_norm": 0.454875111579895,
+      "learning_rate": 0.00029889430663070267,
+      "epoch": 0.3850901012095779,
       "step": 195
     },
     {
-      "loss": 0.522,
-      "grad_norm": 0.36577939987182617,
-      "learning_rate": 0.0002960734065343426,
-      "epoch": 0.7155635062611807,
+      "loss": 0.4925,
+      "grad_norm": 0.5251834392547607,
+      "learning_rate": 0.0002988219160363607,
+      "epoch": 0.3949642063687978,
       "step": 200
     },
     {
-      "eval_loss": 0.5033101439476013,
-      "eval_runtime": 20.6749,
-      "eval_samples_per_second": 162.806,
-      "eval_steps_per_second": 10.206,
-      "epoch": 0.7155635062611807,
+      "eval_loss": 0.525215208530426,
+      "eval_runtime": 46.6605,
+      "eval_samples_per_second": 154.285,
+      "eval_steps_per_second": 9.644,
+      "epoch": 0.3949642063687978,
       "step": 200
     },
     {
-      "loss": 0.5033,
-      "grad_norm": 0.30699774622917175,
-      "learning_rate": 0.00029582532230986244,
-      "epoch": 0.7334525939177102,
+      "loss": 0.4747,
+      "grad_norm": 0.4320356249809265,
+      "learning_rate": 0.00029874723957656543,
+      "epoch": 0.40483831152801775,
       "step": 205
     },
     {
-      "loss": 0.5037,
-      "grad_norm": 0.27759912610054016,
-      "learning_rate": 0.0002955697496732031,
-      "epoch": 0.7513416815742398,
+      "loss": 0.4638,
+      "grad_norm": 0.43524378538131714,
+      "learning_rate": 0.00029867027839832773,
+      "epoch": 0.41471241668723774,
       "step": 210
     },
     {
-      "loss": 0.5085,
-      "grad_norm": 0.32028377056121826,
-      "learning_rate": 0.0002953067017485136,
-      "epoch": 0.7692307692307693,
+      "loss": 0.4736,
+      "grad_norm": 0.4714348018169403,
+      "learning_rate": 0.000298591033683751,
+      "epoch": 0.4245865218464577,
       "step": 215
     },
     {
-      "loss": 0.493,
-      "grad_norm": 0.5791727304458618,
-      "learning_rate": 0.00029503619204381313,
-      "epoch": 0.7871198568872988,
+      "loss": 0.4653,
+      "grad_norm": 0.3850748836994171,
+      "learning_rate": 0.0002985095066500132,
+      "epoch": 0.4344606270056776,
       "step": 220
     },
     {
-      "loss": 0.5089,
-      "grad_norm": 0.44022566080093384,
-      "learning_rate": 0.0002947582344502981,
-      "epoch": 0.8050089445438283,
+      "loss": 0.4525,
+      "grad_norm": 0.41485950350761414,
+      "learning_rate": 0.0002984256985493481,
+      "epoch": 0.44433473216489755,
       "step": 225
     },
     {
-      "loss": 0.4986,
-      "grad_norm": 0.304227739572525,
-      "learning_rate": 0.0002944728432416282,
-      "epoch": 0.8228980322003577,
+      "loss": 0.4559,
+      "grad_norm": 0.4150899052619934,
+      "learning_rate": 0.0002983396106690261,
+      "epoch": 0.4542088373241175,
       "step": 230
     },
     {
-      "loss": 0.4991,
-      "grad_norm": 0.2987827956676483,
-      "learning_rate": 0.0002941800330731936,
-      "epoch": 0.8407871198568873,
+      "loss": 0.4523,
+      "grad_norm": 0.5892351269721985,
+      "learning_rate": 0.00029825124433133433,
+      "epoch": 0.4640829424833374,
       "step": 235
     },
     {
-      "loss": 0.4798,
-      "grad_norm": 0.32087433338165283,
-      "learning_rate": 0.0002938798189813625,
-      "epoch": 0.8586762075134168,
+      "loss": 0.4423,
+      "grad_norm": 0.5627350807189941,
+      "learning_rate": 0.0002981606008935565,
+      "epoch": 0.4739570476425574,
       "step": 240
     },
     {
-      "loss": 0.4839,
-      "grad_norm": 0.354798287153244,
-      "learning_rate": 0.0002935722163827087,
-      "epoch": 0.8765652951699463,
+      "loss": 0.4596,
+      "grad_norm": 0.5459627509117126,
+      "learning_rate": 0.00029806768174795193,
+      "epoch": 0.48383115280177735,
       "step": 245
     },
     {
-      "loss": 0.4874,
-      "grad_norm": 0.5133390426635742,
-      "learning_rate": 0.0002932572410732204,
-      "epoch": 0.8944543828264758,
+      "loss": 0.4426,
+      "grad_norm": 0.5007842183113098,
+      "learning_rate": 0.00029797248832173426,
+      "epoch": 0.4937052579609973,
       "step": 250
     },
     {
-      "loss": 0.4749,
-      "grad_norm": 0.3183499574661255,
-      "learning_rate": 0.00029293490922748856,
-      "epoch": 0.9123434704830053,
+      "loss": 0.4299,
+      "grad_norm": 0.5660277009010315,
+      "learning_rate": 0.0002978750220770494,
+      "epoch": 0.5035793631202172,
       "step": 255
     },
     {
-      "loss": 0.471,
-      "grad_norm": 0.3271007537841797,
-      "learning_rate": 0.0002926052373978764,
-      "epoch": 0.9302325581395349,
+      "loss": 0.4043,
+      "grad_norm": 0.5617759227752686,
+      "learning_rate": 0.0002977752845109534,
+      "epoch": 0.5134534682794372,
       "step": 260
     },
     {
-      "loss": 0.459,
-      "grad_norm": 1.1660605669021606,
-      "learning_rate": 0.00029226824251366967,
-      "epoch": 0.9481216457960644,
+      "loss": 0.4117,
+      "grad_norm": 10.37293529510498,
+      "learning_rate": 0.00029767327715538873,
+      "epoch": 0.5233275734386571,
       "step": 265
     },
     {
-      "loss": 0.4369,
-      "grad_norm": 0.48370498418807983,
-      "learning_rate": 0.00029192394188020716,
-      "epoch": 0.9660107334525939,
+      "loss": 0.4237,
+      "grad_norm": 0.5515385270118713,
+      "learning_rate": 0.00029756900157716176,
+      "epoch": 0.533201678597877,
       "step": 270
     },
     {
-      "loss": 0.443,
-      "grad_norm": 0.3688034117221832,
-      "learning_rate": 0.0002915723531779919,
-      "epoch": 0.9838998211091234,
+      "loss": 0.4268,
+      "grad_norm": 0.5526891946792603,
+      "learning_rate": 0.00029746245937791783,
+      "epoch": 0.543075783757097,
       "step": 275
     },
     {
-      "loss": 0.4541,
-      "grad_norm": 0.7489350438117981,
-      "learning_rate": 0.00029121349446178333,
-      "epoch": 1.0,
+      "loss": 0.4088,
+      "grad_norm": 0.6997721195220947,
+      "learning_rate": 0.00029735365219411706,
+      "epoch": 0.5529498889163169,
       "step": 280
     },
     {
-      "loss": 0.4349,
-      "grad_norm": 0.3898984491825104,
-      "learning_rate": 0.00029084738415967017,
-      "epoch": 1.0178890876565294,
+      "loss": 0.4173,
+      "grad_norm": 1.0989447832107544,
+      "learning_rate": 0.0002972425816970091,
+      "epoch": 0.562823994075537,
       "step": 285
     },
     {
-      "loss": 0.4455,
-      "grad_norm": 0.488726407289505,
-      "learning_rate": 0.0002904740410721242,
-      "epoch": 1.035778175313059,
+      "loss": 0.3924,
+      "grad_norm": 0.45413172245025635,
+      "learning_rate": 0.0002971292495926077,
+      "epoch": 0.5726980992347569,
       "step": 290
     },
     {
-      "loss": 0.4431,
-      "grad_norm": 0.4695197641849518,
-      "learning_rate": 0.00029009348437103455,
-      "epoch": 1.0536672629695885,
+      "loss": 0.3877,
+      "grad_norm": 0.53785640001297,
+      "learning_rate": 0.000297013657621664,
+      "epoch": 0.5825722043939768,
       "step": 295
     },
     {
-      "loss": 0.4223,
-      "grad_norm": 0.3967965543270111,
-      "learning_rate": 0.0002897057335987235,
-      "epoch": 1.071556350626118,
+      "loss": 0.3714,
+      "grad_norm": 0.390790194272995,
+      "learning_rate": 0.0002968958075596403,
+      "epoch": 0.5924463095531968,
       "step": 300
     },
     {
-      "eval_loss": 0.4258047640323639,
-      "eval_runtime": 20.6743,
-      "eval_samples_per_second": 162.811,
-      "eval_steps_per_second": 10.206,
-      "epoch": 1.071556350626118,
+      "eval_loss": 0.4794607162475586,
+      "eval_runtime": 47.1461,
+      "eval_samples_per_second": 152.696,
+      "eval_steps_per_second": 9.545,
+      "epoch": 0.5924463095531968,
       "step": 300
     },
     {
-      "loss": 0.4262,
-      "grad_norm": 0.502098023891449,
-      "learning_rate": 0.00028931080866694274,
-      "epoch": 1.0894454382826475,
+      "loss": 0.381,
+      "grad_norm": 8.648950576782227,
+      "learning_rate": 0.00029677570121668253,
+      "epoch": 0.6023204147124167,
       "step": 305
     },
     {
-      "loss": 0.4155,
-      "grad_norm": 0.3177882134914398,
-      "learning_rate": 0.0002889087298558508,
-      "epoch": 1.1073345259391771,
+      "loss": 0.3731,
+      "grad_norm": 0.8738763332366943,
+      "learning_rate": 0.0002966533404375925,
+      "epoch": 0.6121945198716366,
       "step": 310
     },
     {
-      "loss": 0.4151,
-      "grad_norm": 0.39488786458969116,
-      "learning_rate": 0.0002884995178129719,
-      "epoch": 1.1252236135957066,
+      "loss": 0.4041,
+      "grad_norm": 1.4086726903915405,
+      "learning_rate": 0.00029652872710179967,
+      "epoch": 0.6220686250308566,
       "step": 315
     },
     {
-      "loss": 0.4025,
-      "grad_norm": 0.5109437704086304,
-      "learning_rate": 0.0002880831935521355,
-      "epoch": 1.1431127012522362,
+      "loss": 0.3719,
+      "grad_norm": 0.5402569770812988,
+      "learning_rate": 0.0002964018631233319,
+      "epoch": 0.6319427301900765,
       "step": 320
     },
     {
-      "loss": 0.4054,
-      "grad_norm": 0.5653850436210632,
-      "learning_rate": 0.00028765977845239706,
-      "epoch": 1.1610017889087656,
+      "loss": 0.3624,
+      "grad_norm": 0.6380510926246643,
+      "learning_rate": 0.00029627275045078657,
+      "epoch": 0.6418168353492965,
       "step": 325
     },
     {
-      "loss": 0.3982,
-      "grad_norm": 0.7002792358398438,
-      "learning_rate": 0.00028722929425694044,
-      "epoch": 1.1788908765652952,
+      "loss": 0.3707,
+      "grad_norm": 0.9881039261817932,
+      "learning_rate": 0.0002961413910673003,
+      "epoch": 0.6516909405085164,
       "step": 330
     },
     {
-      "loss": 0.4375,
-      "grad_norm": 0.33770787715911865,
-      "learning_rate": 0.0002867917630719612,
-      "epoch": 1.1967799642218246,
+      "loss": 0.3544,
+      "grad_norm": 0.5078769326210022,
+      "learning_rate": 0.00029600778699051875,
+      "epoch": 0.6615650456677363,
       "step": 335
     },
     {
-      "loss": 0.4053,
-      "grad_norm": 0.4120868444442749,
-      "learning_rate": 0.00028634720736553143,
-      "epoch": 1.2146690518783543,
+      "loss": 0.3498,
+      "grad_norm": 0.5671535134315491,
+      "learning_rate": 0.0002958719402725653,
+      "epoch": 0.6714391508269563,
       "step": 340
     },
     {
-      "loss": 0.411,
-      "grad_norm": 0.4081322252750397,
-      "learning_rate": 0.00028589564996644594,
-      "epoch": 1.2325581395348837,
+      "loss": 0.337,
+      "grad_norm": 0.6489645838737488,
+      "learning_rate": 0.0002957338530000097,
+      "epoch": 0.6813132559861762,
       "step": 345
     },
     {
-      "loss": 0.4167,
-      "grad_norm": 0.6889946460723877,
-      "learning_rate": 0.0002854371140630501,
-      "epoch": 1.250447227191413,
+      "loss": 0.3578,
+      "grad_norm": 0.8148907423019409,
+      "learning_rate": 0.0002955935272938363,
+      "epoch": 0.6911873611453963,
       "step": 350
     },
     {
-      "loss": 0.4196,
-      "grad_norm": 0.3247791826725006,
-      "learning_rate": 0.00028497162320204885,
-      "epoch": 1.2683363148479427,
+      "loss": 0.3598,
+      "grad_norm": 0.4891752004623413,
+      "learning_rate": 0.0002954509653094108,
+      "epoch": 0.7010614663046162,
       "step": 355
     },
     {
-      "loss": 0.4091,
-      "grad_norm": 0.41395092010498047,
-      "learning_rate": 0.00028449920128729766,
-      "epoch": 1.2862254025044724,
+      "loss": 0.3502,
+      "grad_norm": 6.3048176765441895,
+      "learning_rate": 0.0002953061692364479,
+      "epoch": 0.7109355714638361,
       "step": 360
     },
     {
-      "loss": 0.3958,
-      "grad_norm": 0.3653516173362732,
-      "learning_rate": 0.00028401987257857514,
-      "epoch": 1.3041144901610018,
+      "loss": 0.3308,
+      "grad_norm": 0.5603845715522766,
+      "learning_rate": 0.0002951591412989771,
+      "epoch": 0.7208096766230561,
       "step": 365
     },
     {
-      "loss": 0.3898,
-      "grad_norm": 0.3595719337463379,
-      "learning_rate": 0.0002835336616903369,
-      "epoch": 1.3220035778175312,
+      "loss": 0.3173,
+      "grad_norm": 0.5107300877571106,
+      "learning_rate": 0.0002950098837553088,
+      "epoch": 0.730683781782276,
       "step": 370
     },
     {
-      "loss": 0.3604,
-      "grad_norm": 0.7938292622566223,
-      "learning_rate": 0.000283040593590452,
-      "epoch": 1.3398926654740608,
+      "loss": 0.3348,
+      "grad_norm": 0.6624521613121033,
+      "learning_rate": 0.0002948583988979996,
+      "epoch": 0.7405578869414959,
       "step": 375
     },
     {
-      "loss": 0.382,
-      "grad_norm": 0.562586784362793,
-      "learning_rate": 0.00028254069359892034,
-      "epoch": 1.3577817531305905,
+      "loss": 0.3463,
+      "grad_norm": 1.0949567556381226,
+      "learning_rate": 0.00029470468905381706,
+      "epoch": 0.7504319921007159,
       "step": 380
     },
     {
-      "loss": 0.385,
-      "grad_norm": 0.29362237453460693,
-      "learning_rate": 0.0002820339873865729,
-      "epoch": 1.3756708407871199,
+      "loss": 0.3428,
+      "grad_norm": 0.6831812262535095,
+      "learning_rate": 0.00029454875658370384,
+      "epoch": 0.7603060972599358,
       "step": 385
     },
     {
-      "loss": 0.3776,
-      "grad_norm": 0.38946282863616943,
-      "learning_rate": 0.00028152050097375304,
-      "epoch": 1.3935599284436493,
+      "loss": 0.3245,
+      "grad_norm": 0.8317500352859497,
+      "learning_rate": 0.0002943906038827417,
+      "epoch": 0.7701802024191557,
       "step": 390
     },
     {
-      "loss": 0.3756,
-      "grad_norm": 0.3675141930580139,
-      "learning_rate": 0.00028100026072898067,
-      "epoch": 1.411449016100179,
+      "loss": 0.3336,
+      "grad_norm": 0.5925719141960144,
+      "learning_rate": 0.00029423023338011453,
+      "epoch": 0.7800543075783757,
       "step": 395
     },
     {
-      "loss": 0.3627,
-      "grad_norm": 0.4281761050224304,
-      "learning_rate": 0.00028047329336759806,
-      "epoch": 1.4293381037567083,
+      "loss": 0.3138,
+      "grad_norm": 0.5986014604568481,
+      "learning_rate": 0.00029406764753907094,
+      "epoch": 0.7899284127375956,
       "step": 400
     },
     {
-      "eval_loss": 0.3722744286060333,
-      "eval_runtime": 20.6775,
-      "eval_samples_per_second": 162.785,
-      "eval_steps_per_second": 10.204,
-      "epoch": 1.4293381037567083,
+      "eval_loss": 0.48380041122436523,
+      "eval_runtime": 46.653,
+      "eval_samples_per_second": 154.31,
+      "eval_steps_per_second": 9.646,
+      "epoch": 0.7899284127375956,
       "step": 400
     },
     {
-      "loss": 0.3637,
-      "grad_norm": 0.3468483090400696,
-      "learning_rate": 0.0002799396259503977,
-      "epoch": 1.447227191413238,
+      "loss": 0.3153,
+      "grad_norm": 1.081153392791748,
+      "learning_rate": 0.0002939028488568867,
+      "epoch": 0.7998025178968156,
       "step": 405
     },
     {
-      "loss": 0.3729,
-      "grad_norm": 0.4413630962371826,
-      "learning_rate": 0.00027939928588223314,
-      "epoch": 1.4651162790697674,
+      "loss": 0.3295,
+      "grad_norm": 1.3051981925964355,
+      "learning_rate": 0.00029373583986482617,
+      "epoch": 0.8096766230560355,
       "step": 410
     },
     {
-      "loss": 0.3511,
-      "grad_norm": 0.41624078154563904,
-      "learning_rate": 0.00027885230091061127,
-      "epoch": 1.483005366726297,
+      "loss": 0.3226,
+      "grad_norm": 0.7439870834350586,
+      "learning_rate": 0.0002935666231281035,
+      "epoch": 0.8195507282152555,
       "step": 415
     },
     {
-      "loss": 0.3599,
-      "grad_norm": 0.42356589436531067,
-      "learning_rate": 0.00027829869912426777,
-      "epoch": 1.5008944543828264,
+      "loss": 0.2997,
+      "grad_norm": 0.5314565896987915,
+      "learning_rate": 0.00029339520124584324,
+      "epoch": 0.8294248333744755,
       "step": 420
     },
     {
-      "loss": 0.3586,
-      "grad_norm": 0.4348715543746948,
-      "learning_rate": 0.0002777385089517244,
-      "epoch": 1.518783542039356,
+      "loss": 0.2957,
+      "grad_norm": 0.6831716299057007,
+      "learning_rate": 0.0002932215768510404,
+      "epoch": 0.8392989385336954,
       "step": 425
     },
     {
-      "loss": 0.3487,
-      "grad_norm": 0.5163382291793823,
-      "learning_rate": 0.00027717175915982945,
-      "epoch": 1.5366726296958855,
+      "loss": 0.2846,
+      "grad_norm": 0.49296361207962036,
+      "learning_rate": 0.0002930457526105199,
+      "epoch": 0.8491730436929154,
       "step": 430
     },
     {
-      "loss": 0.3551,
-      "grad_norm": 0.47645968198776245,
-      "learning_rate": 0.0002765984788522801,
-      "epoch": 1.5545617173524149,
+      "loss": 0.285,
+      "grad_norm": 0.5440292954444885,
+      "learning_rate": 0.00029286773122489583,
+      "epoch": 0.8590471488521353,
       "step": 435
     },
     {
-      "loss": 0.3523,
-      "grad_norm": 0.5956697463989258,
-      "learning_rate": 0.0002760186974681285,
-      "epoch": 1.5724508050089445,
+      "loss": 0.2861,
+      "grad_norm": 0.777549147605896,
+      "learning_rate": 0.0002926875154285298,
+      "epoch": 0.8689212540113552,
       "step": 440
     },
     {
-      "loss": 0.3842,
-      "grad_norm": 0.3199741244316101,
-      "learning_rate": 0.0002754324447802693,
-      "epoch": 1.5903398926654742,
+      "loss": 0.2683,
+      "grad_norm": 0.5027139186859131,
+      "learning_rate": 0.0002925051079894891,
+      "epoch": 0.8787953591705752,
       "step": 445
     },
     {
-      "loss": 0.363,
-      "grad_norm": 0.4985601007938385,
-      "learning_rate": 0.00027483975089391126,
-      "epoch": 1.6082289803220036,
+      "loss": 0.2897,
+      "grad_norm": 0.833881139755249,
+      "learning_rate": 0.0002923205117095037,
+      "epoch": 0.8886694643297951,
       "step": 450
     },
     {
-      "loss": 0.3573,
-      "grad_norm": 0.43190059065818787,
-      "learning_rate": 0.0002742406462450311,
-      "epoch": 1.626118067978533,
+      "loss": 0.2792,
+      "grad_norm": 0.7060643434524536,
+      "learning_rate": 0.0002921337294239242,
+      "epoch": 0.898543569489015,
       "step": 455
     },
     {
-      "loss": 0.3399,
-      "grad_norm": 0.3566244840621948,
-      "learning_rate": 0.00027363516159881066,
-      "epoch": 1.6440071556350626,
+      "loss": 0.2663,
+      "grad_norm": 0.6060872673988342,
+      "learning_rate": 0.00029194476400167717,
+      "epoch": 0.908417674648235,
       "step": 460
     },
     {
-      "loss": 0.3465,
-      "grad_norm": 0.4065592586994171,
-      "learning_rate": 0.0002730233280480569,
-      "epoch": 1.6618962432915922,
+      "loss": 0.2677,
+      "grad_norm": 0.7497969269752502,
+      "learning_rate": 0.0002917536183452217,
+      "epoch": 0.9182917798074549,
       "step": 465
     },
     {
-      "loss": 0.3324,
-      "grad_norm": 0.36722782254219055,
-      "learning_rate": 0.0002724051770116052,
-      "epoch": 1.6797853309481217,
+      "loss": 0.2585,
+      "grad_norm": 1.225437879562378,
+      "learning_rate": 0.00029156029539050487,
+      "epoch": 0.9281658849666748,
       "step": 470
     },
     {
-      "loss": 0.325,
-      "grad_norm": 0.3565702736377716,
-      "learning_rate": 0.00027178074023270635,
-      "epoch": 1.697674418604651,
+      "loss": 0.2614,
+      "grad_norm": 0.48100578784942627,
+      "learning_rate": 0.0002913647981069164,
+      "epoch": 0.9380399901258949,
       "step": 475
     },
     {
-      "loss": 0.3679,
-      "grad_norm": 0.5317059755325317,
-      "learning_rate": 0.00027115004977739586,
-      "epoch": 1.7155635062611807,
+      "loss": 0.2488,
+      "grad_norm": 4.289666175842285,
+      "learning_rate": 0.0002911671294972429,
+      "epoch": 0.9479140952851148,
       "step": 480
     },
     {
-      "loss": 0.3433,
-      "grad_norm": 0.30274462699890137,
-      "learning_rate": 0.00027051313803284774,
-      "epoch": 1.7334525939177103,
+      "loss": 0.2548,
+      "grad_norm": 0.7958076596260071,
+      "learning_rate": 0.0002909672925976224,
+      "epoch": 0.9577882004443348,
       "step": 485
     },
     {
-      "loss": 0.3492,
-      "grad_norm": 4.536398887634277,
-      "learning_rate": 0.00026987003770571124,
-      "epoch": 1.7513416815742398,
+      "loss": 0.2583,
+      "grad_norm": 0.5692360997200012,
+      "learning_rate": 0.00029076529047749694,
+      "epoch": 0.9676623056035547,
       "step": 490
     },
     {
-      "loss": 0.3506,
-      "grad_norm": 0.4281180202960968,
-      "learning_rate": 0.0002692207818204313,
-      "epoch": 1.7692307692307692,
+      "loss": 0.2643,
+      "grad_norm": 0.6238569021224976,
+      "learning_rate": 0.00029056112623956595,
+      "epoch": 0.9775364107627746,
       "step": 495
     },
     {
-      "loss": 0.3417,
-      "grad_norm": 0.3851430416107178,
-      "learning_rate": 0.0002685654037175525,
-      "epoch": 1.7871198568872988,
+      "loss": 0.2548,
+      "grad_norm": 1.2753865718841553,
+      "learning_rate": 0.00029035480301973837,
+      "epoch": 0.9874105159219946,
       "step": 500
     },
     {
-      "eval_loss": 0.3463773727416992,
-      "eval_runtime": 20.6737,
-      "eval_samples_per_second": 162.815,
-      "eval_steps_per_second": 10.206,
-      "epoch": 1.7871198568872988,
+      "eval_loss": 0.49340417981147766,
+      "eval_runtime": 46.5345,
+      "eval_samples_per_second": 154.702,
+      "eval_steps_per_second": 9.67,
+      "epoch": 0.9874105159219946,
       "step": 500
     },
     {
-      "loss": 0.3287,
-      "grad_norm": 0.41302162408828735,
-      "learning_rate": 0.0002679039370520074,
-      "epoch": 1.8050089445438284,
+      "loss": 0.259,
+      "grad_norm": 0.5439327359199524,
+      "learning_rate": 0.0002901463239870846,
+      "epoch": 0.9972846210812145,
       "step": 505
     },
     {
-      "loss": 0.3292,
-      "grad_norm": 0.3512372076511383,
-      "learning_rate": 0.00026723641579138777,
-      "epoch": 1.8228980322003578,
+      "loss": 0.3047,
+      "grad_norm": 0.7135459184646606,
+      "learning_rate": 0.00028993569234378756,
+      "epoch": 1.007899284127376,
       "step": 510
     },
     {
-      "loss": 0.3241,
-      "grad_norm": 0.4498589336872101,
-      "learning_rate": 0.0002665628742142008,
-      "epoch": 1.8407871198568873,
+      "loss": 0.2138,
+      "grad_norm": 0.6349644064903259,
+      "learning_rate": 0.00028972291132509396,
+      "epoch": 1.0177733892865959,
       "step": 515
     },
     {
-      "loss": 0.3302,
-      "grad_norm": 0.532430112361908,
-      "learning_rate": 0.0002658833469081082,
-      "epoch": 1.8586762075134167,
+      "loss": 0.2317,
+      "grad_norm": 0.6580847501754761,
+      "learning_rate": 0.0002895079841992642,
+      "epoch": 1.0276474944458158,
       "step": 520
     },
     {
-      "loss": 0.3254,
-      "grad_norm": 0.40190622210502625,
-      "learning_rate": 0.0002651978687681509,
-      "epoch": 1.8765652951699463,
+      "loss": 0.2267,
+      "grad_norm": 0.6198315024375916,
+      "learning_rate": 0.00028929091426752236,
+      "epoch": 1.0375215996050358,
       "step": 525
     },
     {
-      "loss": 0.3212,
-      "grad_norm": 0.4508027732372284,
-      "learning_rate": 0.00026450647499495626,
-      "epoch": 1.894454382826476,
+      "loss": 0.2394,
+      "grad_norm": 0.4667389690876007,
+      "learning_rate": 0.0002890717048640055,
+      "epoch": 1.0473957047642557,
       "step": 530
     },
     {
-      "loss": 0.3191,
-      "grad_norm": 0.48150140047073364,
-      "learning_rate": 0.00026380920109293104,
-      "epoch": 1.9123434704830053,
+      "loss": 0.2017,
+      "grad_norm": 0.6087087988853455,
+      "learning_rate": 0.00028885035935571223,
+      "epoch": 1.0572698099234756,
       "step": 535
     },
     {
-      "loss": 0.3374,
-      "grad_norm": 0.34987515211105347,
-      "learning_rate": 0.00026310608286843795,
-      "epoch": 1.9302325581395348,
+      "loss": 0.2006,
+      "grad_norm": 0.6860356330871582,
+      "learning_rate": 0.00028862688114245137,
+      "epoch": 1.0671439150826956,
       "step": 540
     },
     {
-      "loss": 0.3193,
-      "grad_norm": 0.2754597067832947,
-      "learning_rate": 0.00026239715642795684,
-      "epoch": 1.9481216457960644,
+      "loss": 0.2155,
+      "grad_norm": 0.6059162020683289,
+      "learning_rate": 0.00028840127365678943,
+      "epoch": 1.0770180202419155,
       "step": 545
     },
     {
-      "loss": 0.3276,
-      "grad_norm": 0.35444873571395874,
-      "learning_rate": 0.00026168245817623085,
-      "epoch": 1.966010733452594,
+      "loss": 0.1982,
+      "grad_norm": 0.5954364538192749,
+      "learning_rate": 0.000288173540363998,
+      "epoch": 1.0868921254011354,
       "step": 550
     },
     {
-      "loss": 0.3028,
-      "grad_norm": 0.3781956434249878,
-      "learning_rate": 0.00026096202481439674,
-      "epoch": 1.9838998211091234,
+      "loss": 0.2235,
+      "grad_norm": 0.5217412710189819,
+      "learning_rate": 0.0002879436847620005,
+      "epoch": 1.0967662305603554,
       "step": 555
     },
     {
-      "loss": 0.3163,
-      "grad_norm": 0.43553584814071655,
-      "learning_rate": 0.00026023589333810015,
-      "epoch": 2.0,
+      "loss": 0.2331,
+      "grad_norm": 0.7081472277641296,
+      "learning_rate": 0.0002877117103813185,
+      "epoch": 1.1066403357195753,
       "step": 560
     },
     {
-      "loss": 0.3041,
-      "grad_norm": 0.3745397627353668,
-      "learning_rate": 0.00025950410103559604,
-      "epoch": 2.0178890876565294,
+      "loss": 0.2262,
+      "grad_norm": 0.6892974376678467,
+      "learning_rate": 0.00028747762078501737,
+      "epoch": 1.1165144408787953,
       "step": 565
     },
     {
-      "loss": 0.314,
-      "grad_norm": 0.4507661759853363,
-      "learning_rate": 0.00025876668548583374,
-      "epoch": 2.035778175313059,
+      "loss": 0.2107,
+      "grad_norm": 0.688247561454773,
+      "learning_rate": 0.0002872414195686517,
+      "epoch": 1.1263885460380152,
       "step": 570
     },
     {
-      "loss": 0.2911,
-      "grad_norm": 0.45311978459358215,
-      "learning_rate": 0.00025802368455652704,
-      "epoch": 2.0536672629695887,
+      "loss": 0.2076,
+      "grad_norm": 0.6474739909172058,
+      "learning_rate": 0.00028700311036021004,
+      "epoch": 1.1362626511972351,
       "step": 575
     },
     {
-      "loss": 0.2791,
-      "grad_norm": 0.3784424066543579,
-      "learning_rate": 0.00025727513640220985,
-      "epoch": 2.071556350626118,
+      "loss": 0.204,
+      "grad_norm": 0.5220997333526611,
+      "learning_rate": 0.00028676269682005904,
+      "epoch": 1.1461367563564553,
       "step": 580
     },
     {
-      "loss": 0.2781,
-      "grad_norm": 0.37560757994651794,
-      "learning_rate": 0.000256521079462277,
-      "epoch": 2.0894454382826475,
+      "loss": 0.2003,
+      "grad_norm": 0.5570688843727112,
+      "learning_rate": 0.0002865201826408874,
+      "epoch": 1.1560108615156752,
       "step": 585
     },
     {
-      "loss": 0.2705,
-      "grad_norm": 0.4536455571651459,
-      "learning_rate": 0.0002557615524590097,
-      "epoch": 2.107334525939177,
+      "loss": 0.2028,
+      "grad_norm": 0.5454896092414856,
+      "learning_rate": 0.0002862755715476491,
+      "epoch": 1.1658849666748952,
       "step": 590
     },
     {
-      "loss": 0.2845,
-      "grad_norm": 0.8856727480888367,
-      "learning_rate": 0.000254996594395588,
-      "epoch": 2.1252236135957068,
+      "loss": 0.2011,
+      "grad_norm": 0.5824912190437317,
+      "learning_rate": 0.00028602886729750613,
+      "epoch": 1.175759071834115,
       "step": 595
     },
     {
-      "loss": 0.2894,
-      "grad_norm": 0.5242048501968384,
-      "learning_rate": 0.00025422624455408686,
-      "epoch": 2.143112701252236,
+      "loss": 0.1875,
+      "grad_norm": 0.612477719783783,
+      "learning_rate": 0.00028578007367977074,
+      "epoch": 1.185633176993335,
       "step": 600
     },
     {
-      "eval_loss": 0.33662545680999756,
-      "eval_runtime": 20.6734,
-      "eval_samples_per_second": 162.818,
-      "eval_steps_per_second": 10.206,
-      "epoch": 2.143112701252236,
+      "eval_loss": 0.5274343490600586,
+      "eval_runtime": 46.5304,
+      "eval_samples_per_second": 154.716,
+      "eval_steps_per_second": 9.671,
+      "epoch": 1.185633176993335,
       "step": 600
     },
     {
-      "loss": 0.2955,
-      "grad_norm": 0.37391841411590576,
-      "learning_rate": 0.0002534505424934599,
-      "epoch": 2.1610017889087656,
-      "step": 605
-    },
-    {
-      "loss": 0.2869,
-      "grad_norm": 0.3889912962913513,
-      "learning_rate": 0.00025266952804750727,
-      "epoch": 2.178890876565295,
-      "step": 610
-    },
-    {
-      "loss": 0.2805,
-      "grad_norm": 0.37954917550086975,
-      "learning_rate": 0.0002518832413228304,
-      "epoch": 2.196779964221825,
-      "step": 615
-    },
-    {
-      "loss": 0.2834,
-      "grad_norm": 0.3385255038738251,
-      "learning_rate": 0.00025109172269677265,
-      "epoch": 2.2146690518783543,
-      "step": 620
-    },
-    {
-      "loss": 0.2741,
-      "grad_norm": 0.33614033460617065,
-      "learning_rate": 0.00025029501281534534,
-      "epoch": 2.2325581395348837,
-      "step": 625
-    },
-    {
-      "loss": 0.2803,
-      "grad_norm": 0.4882681369781494,
-      "learning_rate": 0.00024949315259114094,
-      "epoch": 2.250447227191413,
-      "step": 630
-    },
-    {
-      "loss": 0.305,
-      "grad_norm": 0.39836975932121277,
-      "learning_rate": 0.00024868618320123193,
-      "epoch": 2.268336314847943,
-      "step": 635
-    },
-    {
-      "loss": 0.2794,
-      "grad_norm": 0.30090266466140747,
-      "learning_rate": 0.00024787414608505636,
-      "epoch": 2.2862254025044724,
-      "step": 640
-    },
-    {
-      "loss": 0.2923,
-      "grad_norm": 0.40115466713905334,
-      "learning_rate": 0.0002470570829422898,
-      "epoch": 2.304114490161002,
-      "step": 645
-    },
-    {
-      "loss": 0.2752,
-      "grad_norm": 0.35384082794189453,
-      "learning_rate": 0.00024623503573070407,
-      "epoch": 2.322003577817531,
-      "step": 650
-    },
-    {
-      "loss": 0.2647,
-      "grad_norm": 0.5324997901916504,
-      "learning_rate": 0.00024540804666401235,
-      "epoch": 2.3398926654740606,
-      "step": 655
-    },
-    {
-      "loss": 0.2667,
-      "grad_norm": 0.3377424478530884,
-      "learning_rate": 0.00024457615820970194,
-      "epoch": 2.3577817531305905,
-      "step": 660
-    },
-    {
-      "loss": 0.273,
-      "grad_norm": 0.3497815728187561,
-      "learning_rate": 0.0002437394130868529,
-      "epoch": 2.37567084078712,
-      "step": 665
-    },
-    {
-      "loss": 0.2674,
-      "grad_norm": 0.4024289548397064,
-      "learning_rate": 0.00024289785426394471,
-      "epoch": 2.3935599284436493,
-      "step": 670
-    },
-    {
-      "loss": 0.2552,
-      "grad_norm": 0.2747831642627716,
-      "learning_rate": 0.00024205152495664965,
-      "epoch": 2.4114490161001787,
-      "step": 675
-    },
-    {
-      "loss": 0.2775,
-      "grad_norm": 0.5090282559394836,
-      "learning_rate": 0.00024120046862561364,
-      "epoch": 2.4293381037567086,
-      "step": 680
-    },
-    {
-      "loss": 0.2638,
-      "grad_norm": 0.423360675573349,
-      "learning_rate": 0.0002403447289742243,
-      "epoch": 2.447227191413238,
-      "step": 685
-    },
-    {
-      "loss": 0.2629,
-      "grad_norm": 0.3922378122806549,
-      "learning_rate": 0.0002394843499463669,
-      "epoch": 2.4651162790697674,
-      "step": 690
-    },
-    {
-      "loss": 0.2642,
-      "grad_norm": 0.381626158952713,
-      "learning_rate": 0.00023861937572416757,
-      "epoch": 2.483005366726297,
-      "step": 695
-    },
-    {
-      "loss": 0.2741,
-      "grad_norm": 0.41004201769828796,
-      "learning_rate": 0.0002377498507257247,
-      "epoch": 2.500894454382826,
-      "step": 700
-    },
-    {
-      "eval_loss": 0.3169912099838257,
-      "eval_runtime": 20.6812,
-      "eval_samples_per_second": 162.756,
-      "eval_steps_per_second": 10.202,
-      "epoch": 2.500894454382826,
-      "step": 700
-    },
-    {
-      "loss": 0.2669,
-      "grad_norm": 0.36008498072624207,
-      "learning_rate": 0.00023687581960282763,
-      "epoch": 2.518783542039356,
-      "step": 705
-    },
-    {
-      "loss": 0.2529,
-      "grad_norm": 0.47046661376953125,
-      "learning_rate": 0.00023599732723866414,
-      "epoch": 2.5366726296958855,
-      "step": 710
-    },
-    {
-      "loss": 0.2882,
-      "grad_norm": 0.6661579012870789,
-      "learning_rate": 0.00023511441874551512,
-      "epoch": 2.554561717352415,
-      "step": 715
-    },
-    {
-      "loss": 0.2805,
-      "grad_norm": 0.3578526973724365,
-      "learning_rate": 0.00023422713946243842,
-      "epoch": 2.5724508050089447,
-      "step": 720
-    },
-    {
-      "loss": 0.2615,
-      "grad_norm": 0.42732375860214233,
-      "learning_rate": 0.0002333355349529403,
-      "epoch": 2.590339892665474,
-      "step": 725
-    },
-    {
-      "loss": 0.2635,
-      "grad_norm": 0.3077392876148224,
-      "learning_rate": 0.0002324396510026358,
-      "epoch": 2.6082289803220036,
-      "step": 730
-    },
-    {
-      "loss": 0.2575,
-      "grad_norm": 0.3254878520965576,
-      "learning_rate": 0.00023153953361689753,
-      "epoch": 2.626118067978533,
-      "step": 735
-    },
-    {
-      "loss": 0.2539,
-      "grad_norm": 0.3971039056777954,
-      "learning_rate": 0.00023063522901849303,
-      "epoch": 2.6440071556350624,
-      "step": 740
-    },
-    {
-      "loss": 0.2675,
-      "grad_norm": 0.36017006635665894,
-      "learning_rate": 0.00022972678364521155,
-      "epoch": 2.6618962432915922,
-      "step": 745
-    },
-    {
-      "loss": 0.2752,
-      "grad_norm": 0.3148644268512726,
-      "learning_rate": 0.00022881424414747902,
-      "epoch": 2.6797853309481217,
-      "step": 750
-    },
-    {
-      "loss": 0.257,
-      "grad_norm": 0.46191850304603577,
-      "learning_rate": 0.00022789765738596258,
-      "epoch": 2.697674418604651,
-      "step": 755
-    },
-    {
-      "loss": 0.2602,
-      "grad_norm": 0.5142350792884827,
-      "learning_rate": 0.00022697707042916413,
-      "epoch": 2.715563506261181,
-      "step": 760
-    },
-    {
-      "loss": 0.2619,
-      "grad_norm": 0.4233485162258148,
-      "learning_rate": 0.00022605253055100346,
-      "epoch": 2.7334525939177103,
-      "step": 765
-    },
-    {
-      "loss": 0.2699,
-      "grad_norm": 0.36081498861312866,
-      "learning_rate": 0.00022512408522839034,
-      "epoch": 2.7513416815742398,
-      "step": 770
-    },
-    {
-      "loss": 0.2589,
-      "grad_norm": 0.3283044695854187,
-      "learning_rate": 0.0002241917821387868,
-      "epoch": 2.769230769230769,
-      "step": 775
-    },
-    {
-      "loss": 0.2569,
-      "grad_norm": 0.36509019136428833,
-      "learning_rate": 0.00022325566915775872,
-      "epoch": 2.7871198568872986,
-      "step": 780
-    },
-    {
-      "loss": 0.2631,
-      "grad_norm": 0.30786919593811035,
-      "learning_rate": 0.000222315794356517,
-      "epoch": 2.8050089445438284,
-      "step": 785
-    },
-    {
-      "loss": 0.2596,
-      "grad_norm": 0.26746758818626404,
-      "learning_rate": 0.0002213722059994496,
-      "epoch": 2.822898032200358,
-      "step": 790
-    },
-    {
-      "loss": 0.2462,
-      "grad_norm": 0.3422580659389496,
-      "learning_rate": 0.00022042495254164248,
-      "epoch": 2.8407871198568873,
-      "step": 795
-    },
-    {
-      "loss": 0.2604,
-      "grad_norm": 0.3908096253871918,
-      "learning_rate": 0.0002194740826263918,
-      "epoch": 2.8586762075134167,
-      "step": 800
-    },
-    {
-      "eval_loss": 0.31991705298423767,
-      "eval_runtime": 20.6861,
-      "eval_samples_per_second": 162.718,
-      "eval_steps_per_second": 10.2,
-      "epoch": 2.8586762075134167,
-      "step": 800
-    },
-    {
-      "loss": 0.259,
-      "grad_norm": 0.3868574798107147,
-      "learning_rate": 0.00021851964508270573,
-      "epoch": 2.8765652951699465,
-      "step": 805
-    },
-    {
-      "loss": 0.2627,
-      "grad_norm": 0.3655846416950226,
-      "learning_rate": 0.00021756168892279705,
-      "epoch": 2.894454382826476,
-      "step": 810
-    },
-    {
-      "loss": 0.249,
-      "grad_norm": 0.6286638379096985,
-      "learning_rate": 0.00021660026333956625,
-      "epoch": 2.9123434704830053,
-      "step": 815
-    },
-    {
-      "loss": 0.2517,
-      "grad_norm": 0.5203955769538879,
-      "learning_rate": 0.0002156354177040755,
-      "epoch": 2.9302325581395348,
-      "step": 820
-    },
-    {
-      "loss": 0.2665,
-      "grad_norm": 0.38546180725097656,
-      "learning_rate": 0.00021466720156301314,
-      "epoch": 2.948121645796064,
-      "step": 825
-    },
-    {
-      "loss": 0.2651,
-      "grad_norm": 0.3447847366333008,
-      "learning_rate": 0.00021369566463614968,
-      "epoch": 2.966010733452594,
-      "step": 830
-    },
-    {
-      "loss": 0.2494,
-      "grad_norm": 0.6365504264831543,
-      "learning_rate": 0.00021272085681378418,
-      "epoch": 2.9838998211091234,
-      "step": 835
-    },
-    {
-      "loss": 0.2448,
-      "grad_norm": 0.6940473914146423,
-      "learning_rate": 0.0002117428281541827,
-      "epoch": 3.0,
-      "step": 840
-    },
-    {
-      "loss": 0.1991,
-      "grad_norm": 0.3761572539806366,
-      "learning_rate": 0.00021076162888100734,
-      "epoch": 3.0178890876565294,
-      "step": 845
-    },
-    {
-      "loss": 0.1976,
-      "grad_norm": 0.3070697784423828,
-      "learning_rate": 0.00020977730938073747,
-      "epoch": 3.035778175313059,
-      "step": 850
-    },
-    {
-      "loss": 0.188,
-      "grad_norm": 0.5102139115333557,
-      "learning_rate": 0.0002087899202000821,
-      "epoch": 3.0536672629695887,
-      "step": 855
-    },
-    {
-      "loss": 0.2003,
-      "grad_norm": 0.4111884534358978,
-      "learning_rate": 0.00020779951204338422,
-      "epoch": 3.071556350626118,
-      "step": 860
-    },
-    {
-      "loss": 0.1914,
-      "grad_norm": 0.46625199913978577,
-      "learning_rate": 0.00020680613577001724,
-      "epoch": 3.0894454382826475,
-      "step": 865
-    },
-    {
-      "loss": 0.2037,
-      "grad_norm": 0.4094313383102417,
-      "learning_rate": 0.0002058098423917729,
-      "epoch": 3.107334525939177,
-      "step": 870
-    },
-    {
-      "loss": 0.2,
-      "grad_norm": 0.4287453591823578,
-      "learning_rate": 0.000204810683070242,
-      "epoch": 3.1252236135957068,
-      "step": 875
-    },
-    {
-      "loss": 0.1918,
-      "grad_norm": 0.4345555901527405,
-      "learning_rate": 0.00020380870911418709,
-      "epoch": 3.143112701252236,
-      "step": 880
-    },
-    {
-      "loss": 0.1911,
-      "grad_norm": 0.43231672048568726,
-      "learning_rate": 0.00020280397197690753,
-      "epoch": 3.1610017889087656,
-      "step": 885
-    },
-    {
-      "loss": 0.196,
-      "grad_norm": 0.4027317464351654,
-      "learning_rate": 0.00020179652325359756,
-      "epoch": 3.178890876565295,
-      "step": 890
-    },
-    {
-      "loss": 0.1964,
-      "grad_norm": 0.4894247353076935,
-      "learning_rate": 0.00020078641467869652,
-      "epoch": 3.196779964221825,
-      "step": 895
-    },
-    {
-      "loss": 0.1847,
-      "grad_norm": 0.4106491804122925,
-      "learning_rate": 0.00019977369812323214,
-      "epoch": 3.2146690518783543,
-      "step": 900
-    },
-    {
-      "eval_loss": 0.3646461069583893,
-      "eval_runtime": 20.6747,
-      "eval_samples_per_second": 162.808,
-      "eval_steps_per_second": 10.206,
-      "epoch": 3.2146690518783543,
-      "step": 900
-    },
-    {
-      "loss": 0.1918,
-      "grad_norm": 0.5217190980911255,
-      "learning_rate": 0.00019875842559215724,
-      "epoch": 3.2325581395348837,
-      "step": 905
-    },
-    {
-      "loss": 0.1856,
-      "grad_norm": 0.3551487922668457,
-      "learning_rate": 0.00019774064922167876,
-      "epoch": 3.250447227191413,
-      "step": 910
-    },
-    {
-      "loss": 0.1978,
-      "grad_norm": 0.44846031069755554,
-      "learning_rate": 0.00019672042127658062,
-      "epoch": 3.268336314847943,
-      "step": 915
-    },
-    {
-      "loss": 0.1912,
-      "grad_norm": 0.45646387338638306,
-      "learning_rate": 0.00019569779414753999,
-      "epoch": 3.2862254025044724,
-      "step": 920
-    },
-    {
-      "loss": 0.1988,
-      "grad_norm": 0.42022913694381714,
-      "learning_rate": 0.00019467282034843654,
-      "epoch": 3.304114490161002,
-      "step": 925
-    },
-    {
-      "loss": 0.188,
-      "grad_norm": 0.44034436345100403,
-      "learning_rate": 0.00019364555251365627,
-      "epoch": 3.322003577817531,
-      "step": 930
-    },
-    {
-      "loss": 0.2039,
-      "grad_norm": 0.5154224634170532,
-      "learning_rate": 0.00019261604339538809,
-      "epoch": 3.3398926654740606,
-      "step": 935
-    },
-    {
-      "loss": 0.2013,
-      "grad_norm": 0.6147205829620361,
-      "learning_rate": 0.0001915843458609152,
-      "epoch": 3.3577817531305905,
-      "step": 940
-    },
-    {
-      "loss": 0.2017,
-      "grad_norm": 0.460700660943985,
-      "learning_rate": 0.0001905505128899004,
-      "epoch": 3.37567084078712,
-      "step": 945
-    },
-    {
-      "loss": 0.1908,
-      "grad_norm": 0.5751897692680359,
-      "learning_rate": 0.000189514597571665,
-      "epoch": 3.3935599284436493,
-      "step": 950
-    },
-    {
-      "loss": 0.1965,
-      "grad_norm": 0.4614831507205963,
-      "learning_rate": 0.00018847665310246312,
-      "epoch": 3.4114490161001787,
-      "step": 955
-    },
-    {
-      "loss": 0.2014,
-      "grad_norm": 0.3863570988178253,
-      "learning_rate": 0.00018743673278274954,
-      "epoch": 3.4293381037567086,
-      "step": 960
-    },
-    {
-      "loss": 0.189,
-      "grad_norm": 0.3508825898170471,
-      "learning_rate": 0.0001863948900144428,
-      "epoch": 3.447227191413238,
-      "step": 965
-    },
-    {
-      "loss": 0.1793,
-      "grad_norm": 0.3714524805545807,
-      "learning_rate": 0.00018535117829818293,
-      "epoch": 3.4651162790697674,
-      "step": 970
-    },
-    {
-      "loss": 0.2049,
-      "grad_norm": 0.43370577692985535,
-      "learning_rate": 0.00018430565123058409,
-      "epoch": 3.483005366726297,
-      "step": 975
-    },
-    {
-      "loss": 0.1803,
-      "grad_norm": 0.34576115012168884,
-      "learning_rate": 0.00018325836250148215,
-      "epoch": 3.500894454382826,
-      "step": 980
-    },
-    {
-      "loss": 0.1891,
-      "grad_norm": 0.41390174627304077,
-      "learning_rate": 0.00018220936589117775,
-      "epoch": 3.518783542039356,
-      "step": 985
-    },
-    {
-      "loss": 0.1909,
-      "grad_norm": 0.5129839777946472,
-      "learning_rate": 0.00018115871526767455,
-      "epoch": 3.5366726296958855,
-      "step": 990
-    },
-    {
-      "loss": 0.1919,
-      "grad_norm": 0.537712812423706,
-      "learning_rate": 0.00018010646458391293,
-      "epoch": 3.554561717352415,
-      "step": 995
-    },
-    {
-      "loss": 0.1909,
-      "grad_norm": 0.3729874789714813,
-      "learning_rate": 0.00017905266787499947,
-      "epoch": 3.5724508050089447,
-      "step": 1000
-    },
-    {
-      "eval_loss": 0.36307844519615173,
-      "eval_runtime": 20.6777,
-      "eval_samples_per_second": 162.784,
-      "eval_steps_per_second": 10.204,
-      "epoch": 3.5724508050089447,
-      "step": 1000
-    },
-    {
-      "train_runtime": 2976.681,
-      "train_samples_per_second": 96.118,
-      "train_steps_per_second": 0.75,
-      "total_flos": 4.722006162277663e+17,
-      "train_loss": 0.444128730237484,
-      "epoch": 3.5724508050089447,
-      "step": 1000
+      "train_runtime": 1998.6484,
+      "train_samples_per_second": 259.379,
+      "train_steps_per_second": 2.025,
+      "total_flos": 2.946671339427594e+17,
+      "train_loss": 0.5542280136545499,
+      "epoch": 1.185633176993335,
+      "step": 600
     }
   ],
-  "best_metric": 0.3169912099838257,
-  "best_model_checkpoint": "saves/adpr-llama/checkpoint-700",
+  "best_metric": 0.4794607162475586,
+  "best_model_checkpoint": "saves/adpr-llama/checkpoint-300",
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "is_hyper_param_search": false,
@@ -1529,7 +937,7 @@
     }
   },
   "model": "PeftModelForCausalLM(\n  (base_model): LoraModel(\n    (model): LlamaForCausalLM(\n      (model): LlamaModel(\n        (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n        (layers): ModuleList(\n          (0-31): 32 x LlamaDecoderLayer(\n            (self_attn): LlamaAttention(\n              (q_proj): lora.Linear(\n                (base_layer): Linear(in_features=4096, out_features=4096, bias=False)\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=4096, out_features=64, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=64, out_features=4096, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n                (lora_magnitude_vector): ModuleDict()\n              )\n              (k_proj): lora.Linear(\n                (base_layer): Linear(in_features=4096, out_features=4096, bias=False)\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=4096, out_features=64, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=64, out_features=4096, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n                (lora_magnitude_vector): ModuleDict()\n              )\n              (v_proj): lora.Linear(\n                (base_layer): Linear(in_features=4096, out_features=4096, bias=False)\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=4096, out_features=64, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=64, out_features=4096, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n                (lora_magnitude_vector): ModuleDict()\n              )\n              (o_proj): lora.Linear(\n                (base_layer): Linear(in_features=4096, out_features=4096, bias=False)\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=4096, out_features=64, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=64, out_features=4096, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n                (lora_magnitude_vector): ModuleDict()\n              )\n            )\n            (mlp): LlamaMLP(\n              (gate_proj): lora.Linear(\n                (base_layer): Linear(in_features=4096, out_features=11008, bias=False)\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=4096, out_features=64, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=64, out_features=11008, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n                (lora_magnitude_vector): ModuleDict()\n              )\n              (up_proj): lora.Linear(\n                (base_layer): Linear(in_features=4096, out_features=11008, bias=False)\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=4096, out_features=64, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=64, out_features=11008, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n                (lora_magnitude_vector): ModuleDict()\n              )\n              (down_proj): lora.Linear(\n                (base_layer): Linear(in_features=11008, out_features=4096, bias=False)\n                (lora_dropout): ModuleDict(\n                  (default): Dropout(p=0.05, inplace=False)\n                )\n                (lora_A): ModuleDict(\n                  (default): Linear(in_features=11008, out_features=64, bias=False)\n                )\n                (lora_B): ModuleDict(\n                  (default): Linear(in_features=64, out_features=4096, bias=False)\n                )\n                (lora_embedding_A): ParameterDict()\n                (lora_embedding_B): ParameterDict()\n                (lora_magnitude_vector): ModuleDict()\n              )\n              (act_fn): SiLU()\n            )\n            (input_layernorm): LlamaRMSNorm((4096,), eps=1e-05)\n            (post_attention_layernorm): LlamaRMSNorm((4096,), eps=1e-05)\n          )\n        )\n        (norm): LlamaRMSNorm((4096,), eps=1e-05)\n        (rotary_emb): LlamaRotaryEmbedding()\n      )\n      (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n    )\n  )\n)",
-  "optimizer": "AcceleratedOptimizer (\nParameter Group 0\n    amsgrad: False\n    betas: (0.9, 0.999)\n    capturable: False\n    decoupled_weight_decay: True\n    differentiable: False\n    eps: 1e-08\n    foreach: None\n    fused: None\n    initial_lr: 0.0003\n    lr: 0.00017905266787499947\n    maximize: False\n    weight_decay: 0.0\n\nParameter Group 1\n    amsgrad: False\n    betas: (0.9, 0.999)\n    capturable: False\n    decoupled_weight_decay: True\n    differentiable: False\n    eps: 1e-08\n    foreach: None\n    fused: None\n    initial_lr: 0.0003\n    lr: 0.00017905266787499947\n    maximize: False\n    weight_decay: 0.0\n)",
-  "lr_scheduler": "<torch.optim.lr_scheduler.LambdaLR object at 0x7c03193e23f0>",
-  "train_dataloader": "<accelerate.data_loader.DataLoaderShard object at 0x7c03195b34d0>"
+  "optimizer": "AcceleratedOptimizer (\nParameter Group 0\n    amsgrad: False\n    betas: (0.9, 0.999)\n    capturable: False\n    decoupled_weight_decay: True\n    differentiable: False\n    eps: 1e-08\n    foreach: None\n    fused: None\n    initial_lr: 0.0003\n    lr: 0.00028578007367977074\n    maximize: False\n    weight_decay: 0.0\n\nParameter Group 1\n    amsgrad: False\n    betas: (0.9, 0.999)\n    capturable: False\n    decoupled_weight_decay: True\n    differentiable: False\n    eps: 1e-08\n    foreach: None\n    fused: None\n    initial_lr: 0.0003\n    lr: 0.00028578007367977074\n    maximize: False\n    weight_decay: 0.0\n)",
+  "lr_scheduler": "<torch.optim.lr_scheduler.LambdaLR object at 0x7de720bc1580>",
+  "train_dataloader": "<accelerate.data_loader.DataLoaderShard object at 0x7de619edf170>"
 }