| { |
| "epoch": 1.185633176993335, |
| "global_step": 600, |
| "max_steps": 4048, |
| "logging_steps": 5, |
| "eval_steps": 100, |
| "save_steps": 100, |
| "train_batch_size": 16, |
| "num_train_epochs": 8, |
| "num_input_tokens_seen": 0, |
| "total_flos": 2.946671339427594e+17, |
| "log_history": [ |
| { |
| "loss": 10.7217, |
| "grad_norm": 35.45301055908203, |
| "learning_rate": 3.75e-05, |
| "epoch": 0.009874105159219946, |
| "step": 5 |
| }, |
| { |
| "loss": 4.3632, |
| "grad_norm": 12.563997268676758, |
| "learning_rate": 7.5e-05, |
| "epoch": 0.01974821031843989, |
| "step": 10 |
| }, |
| { |
| "loss": 1.6282, |
| "grad_norm": 10.4967041015625, |
| "learning_rate": 0.0001125, |
| "epoch": 0.02962231547765984, |
| "step": 15 |
| }, |
| { |
| "loss": 1.1701, |
| "grad_norm": 2.9293179512023926, |
| "learning_rate": 0.00015, |
| "epoch": 0.03949642063687978, |
| "step": 20 |
| }, |
| { |
| "loss": 0.9779, |
| "grad_norm": 1.5904552936553955, |
| "learning_rate": 0.00018749999999999998, |
| "epoch": 0.049370525796099726, |
| "step": 25 |
| }, |
| { |
| "loss": 0.8266, |
| "grad_norm": 1.494022011756897, |
| "learning_rate": 0.000225, |
| "epoch": 0.05924463095531968, |
| "step": 30 |
| }, |
| { |
| "loss": 0.7704, |
| "grad_norm": 0.42715126276016235, |
| "learning_rate": 0.0002625, |
| "epoch": 0.06911873611453961, |
| "step": 35 |
| }, |
| { |
| "loss": 0.735, |
| "grad_norm": 0.3201207220554352, |
| "learning_rate": 0.0003, |
| "epoch": 0.07899284127375956, |
| "step": 40 |
| }, |
| { |
| "loss": 0.7419, |
| "grad_norm": 2.504087209701538, |
| "learning_rate": 0.00029999884801974356, |
| "epoch": 0.08886694643297952, |
| "step": 45 |
| }, |
| { |
| "loss": 0.8097, |
| "grad_norm": 0.5647741556167603, |
| "learning_rate": 0.00029999539209666856, |
| "epoch": 0.09874105159219945, |
| "step": 50 |
| }, |
| { |
| "loss": 0.7196, |
| "grad_norm": 0.5168487429618835, |
| "learning_rate": 0.00029998963228385685, |
| "epoch": 0.1086151567514194, |
| "step": 55 |
| }, |
| { |
| "loss": 0.707, |
| "grad_norm": 0.6618458032608032, |
| "learning_rate": 0.0002999815686697778, |
| "epoch": 0.11848926191063935, |
| "step": 60 |
| }, |
| { |
| "loss": 0.7013, |
| "grad_norm": 4.516430854797363, |
| "learning_rate": 0.0002999712013782864, |
| "epoch": 0.1283633670698593, |
| "step": 65 |
| }, |
| { |
| "loss": 0.8418, |
| "grad_norm": 0.38112834095954895, |
| "learning_rate": 0.0002999585305686214, |
| "epoch": 0.13823747222907923, |
| "step": 70 |
| }, |
| { |
| "loss": 0.6765, |
| "grad_norm": 0.4867613613605499, |
| "learning_rate": 0.00029994355643540323, |
| "epoch": 0.1481115773882992, |
| "step": 75 |
| }, |
| { |
| "loss": 0.675, |
| "grad_norm": 0.4741807281970978, |
| "learning_rate": 0.00029992627920863056, |
| "epoch": 0.15798568254751913, |
| "step": 80 |
| }, |
| { |
| "loss": 0.6441, |
| "grad_norm": 0.38224297761917114, |
| "learning_rate": 0.000299906699153677, |
| "epoch": 0.16785978770673907, |
| "step": 85 |
| }, |
| { |
| "loss": 0.64, |
| "grad_norm": 0.40791240334510803, |
| "learning_rate": 0.00029988481657128714, |
| "epoch": 0.17773389286595903, |
| "step": 90 |
| }, |
| { |
| "loss": 0.6528, |
| "grad_norm": 0.2506788969039917, |
| "learning_rate": 0.00029986063179757166, |
| "epoch": 0.18760799802517897, |
| "step": 95 |
| }, |
| { |
| "loss": 0.6278, |
| "grad_norm": 0.2861074209213257, |
| "learning_rate": 0.0002998341452040023, |
| "epoch": 0.1974821031843989, |
| "step": 100 |
| }, |
| { |
| "eval_loss": 0.6399099230766296, |
| "eval_runtime": 46.6556, |
| "eval_samples_per_second": 154.301, |
| "eval_steps_per_second": 9.645, |
| "epoch": 0.1974821031843989, |
| "step": 100 |
| }, |
| { |
| "loss": 0.6385, |
| "grad_norm": 0.2758176624774933, |
| "learning_rate": 0.00029980535719740614, |
| "epoch": 0.20735620834361887, |
| "step": 105 |
| }, |
| { |
| "loss": 0.6354, |
| "grad_norm": 0.3389268219470978, |
| "learning_rate": 0.0002997742682199594, |
| "epoch": 0.2172303135028388, |
| "step": 110 |
| }, |
| { |
| "loss": 0.6263, |
| "grad_norm": 0.35775822401046753, |
| "learning_rate": 0.0002997408787491806, |
| "epoch": 0.22710441866205874, |
| "step": 115 |
| }, |
| { |
| "loss": 0.6245, |
| "grad_norm": 0.3307900130748749, |
| "learning_rate": 0.0002997051892979233, |
| "epoch": 0.2369785238212787, |
| "step": 120 |
| }, |
| { |
| "loss": 0.589, |
| "grad_norm": 0.49249130487442017, |
| "learning_rate": 0.0002996672004143679, |
| "epoch": 0.24685262898049865, |
| "step": 125 |
| }, |
| { |
| "loss": 0.6172, |
| "grad_norm": 0.48136135935783386, |
| "learning_rate": 0.00029962691268201383, |
| "epoch": 0.2567267341397186, |
| "step": 130 |
| }, |
| { |
| "loss": 0.5992, |
| "grad_norm": 0.3200276494026184, |
| "learning_rate": 0.0002995843267196699, |
| "epoch": 0.2666008392989385, |
| "step": 135 |
| }, |
| { |
| "loss": 0.5685, |
| "grad_norm": 0.39301344752311707, |
| "learning_rate": 0.00029953944318144535, |
| "epoch": 0.27647494445815846, |
| "step": 140 |
| }, |
| { |
| "loss": 0.5749, |
| "grad_norm": 0.27214545011520386, |
| "learning_rate": 0.00029949226275673954, |
| "epoch": 0.28634904961737845, |
| "step": 145 |
| }, |
| { |
| "loss": 0.5586, |
| "grad_norm": 0.43858101963996887, |
| "learning_rate": 0.00029944278617023137, |
| "epoch": 0.2962231547765984, |
| "step": 150 |
| }, |
| { |
| "loss": 0.5515, |
| "grad_norm": 0.541606605052948, |
| "learning_rate": 0.0002993910141818681, |
| "epoch": 0.3060972599358183, |
| "step": 155 |
| }, |
| { |
| "loss": 0.556, |
| "grad_norm": 0.48988083004951477, |
| "learning_rate": 0.00029933694758685396, |
| "epoch": 0.31597136509503826, |
| "step": 160 |
| }, |
| { |
| "loss": 0.5383, |
| "grad_norm": 0.3207962214946747, |
| "learning_rate": 0.0002992805872156376, |
| "epoch": 0.3258454702542582, |
| "step": 165 |
| }, |
| { |
| "loss": 0.5315, |
| "grad_norm": 0.3307562470436096, |
| "learning_rate": 0.00029922193393389936, |
| "epoch": 0.33571957541347813, |
| "step": 170 |
| }, |
| { |
| "loss": 0.5041, |
| "grad_norm": 0.4709561765193939, |
| "learning_rate": 0.0002991609886425383, |
| "epoch": 0.3455936805726981, |
| "step": 175 |
| }, |
| { |
| "loss": 0.4966, |
| "grad_norm": 0.39687275886535645, |
| "learning_rate": 0.00029909775227765804, |
| "epoch": 0.35546778573191806, |
| "step": 180 |
| }, |
| { |
| "loss": 0.5063, |
| "grad_norm": 0.5469186305999756, |
| "learning_rate": 0.0002990322258105525, |
| "epoch": 0.365341890891138, |
| "step": 185 |
| }, |
| { |
| "loss": 0.5162, |
| "grad_norm": 0.46143925189971924, |
| "learning_rate": 0.000298964410247691, |
| "epoch": 0.37521599605035794, |
| "step": 190 |
| }, |
| { |
| "loss": 0.5, |
| "grad_norm": 0.454875111579895, |
| "learning_rate": 0.00029889430663070267, |
| "epoch": 0.3850901012095779, |
| "step": 195 |
| }, |
| { |
| "loss": 0.4925, |
| "grad_norm": 0.5251834392547607, |
| "learning_rate": 0.0002988219160363607, |
| "epoch": 0.3949642063687978, |
| "step": 200 |
| }, |
| { |
| "eval_loss": 0.525215208530426, |
| "eval_runtime": 46.6605, |
| "eval_samples_per_second": 154.285, |
| "eval_steps_per_second": 9.644, |
| "epoch": 0.3949642063687978, |
| "step": 200 |
| }, |
| { |
| "loss": 0.4747, |
| "grad_norm": 0.4320356249809265, |
| "learning_rate": 0.00029874723957656543, |
| "epoch": 0.40483831152801775, |
| "step": 205 |
| }, |
| { |
| "loss": 0.4638, |
| "grad_norm": 0.43524378538131714, |
| "learning_rate": 0.00029867027839832773, |
| "epoch": 0.41471241668723774, |
| "step": 210 |
| }, |
| { |
| "loss": 0.4736, |
| "grad_norm": 0.4714348018169403, |
| "learning_rate": 0.000298591033683751, |
| "epoch": 0.4245865218464577, |
| "step": 215 |
| }, |
| { |
| "loss": 0.4653, |
| "grad_norm": 0.3850748836994171, |
| "learning_rate": 0.0002985095066500132, |
| "epoch": 0.4344606270056776, |
| "step": 220 |
| }, |
| { |
| "loss": 0.4525, |
| "grad_norm": 0.41485950350761414, |
| "learning_rate": 0.0002984256985493481, |
| "epoch": 0.44433473216489755, |
| "step": 225 |
| }, |
| { |
| "loss": 0.4559, |
| "grad_norm": 0.4150899052619934, |
| "learning_rate": 0.0002983396106690261, |
| "epoch": 0.4542088373241175, |
| "step": 230 |
| }, |
| { |
| "loss": 0.4523, |
| "grad_norm": 0.5892351269721985, |
| "learning_rate": 0.00029825124433133433, |
| "epoch": 0.4640829424833374, |
| "step": 235 |
| }, |
| { |
| "loss": 0.4423, |
| "grad_norm": 0.5627350807189941, |
| "learning_rate": 0.0002981606008935565, |
| "epoch": 0.4739570476425574, |
| "step": 240 |
| }, |
| { |
| "loss": 0.4596, |
| "grad_norm": 0.5459627509117126, |
| "learning_rate": 0.00029806768174795193, |
| "epoch": 0.48383115280177735, |
| "step": 245 |
| }, |
| { |
| "loss": 0.4426, |
| "grad_norm": 0.5007842183113098, |
| "learning_rate": 0.00029797248832173426, |
| "epoch": 0.4937052579609973, |
| "step": 250 |
| }, |
| { |
| "loss": 0.4299, |
| "grad_norm": 0.5660277009010315, |
| "learning_rate": 0.0002978750220770494, |
| "epoch": 0.5035793631202172, |
| "step": 255 |
| }, |
| { |
| "loss": 0.4043, |
| "grad_norm": 0.5617759227752686, |
| "learning_rate": 0.0002977752845109534, |
| "epoch": 0.5134534682794372, |
| "step": 260 |
| }, |
| { |
| "loss": 0.4117, |
| "grad_norm": 10.37293529510498, |
| "learning_rate": 0.00029767327715538873, |
| "epoch": 0.5233275734386571, |
| "step": 265 |
| }, |
| { |
| "loss": 0.4237, |
| "grad_norm": 0.5515385270118713, |
| "learning_rate": 0.00029756900157716176, |
| "epoch": 0.533201678597877, |
| "step": 270 |
| }, |
| { |
| "loss": 0.4268, |
| "grad_norm": 0.5526891946792603, |
| "learning_rate": 0.00029746245937791783, |
| "epoch": 0.543075783757097, |
| "step": 275 |
| }, |
| { |
| "loss": 0.4088, |
| "grad_norm": 0.6997721195220947, |
| "learning_rate": 0.00029735365219411706, |
| "epoch": 0.5529498889163169, |
| "step": 280 |
| }, |
| { |
| "loss": 0.4173, |
| "grad_norm": 1.0989447832107544, |
| "learning_rate": 0.0002972425816970091, |
| "epoch": 0.562823994075537, |
| "step": 285 |
| }, |
| { |
| "loss": 0.3924, |
| "grad_norm": 0.45413172245025635, |
| "learning_rate": 0.0002971292495926077, |
| "epoch": 0.5726980992347569, |
| "step": 290 |
| }, |
| { |
| "loss": 0.3877, |
| "grad_norm": 0.53785640001297, |
| "learning_rate": 0.000297013657621664, |
| "epoch": 0.5825722043939768, |
| "step": 295 |
| }, |
| { |
| "loss": 0.3714, |
| "grad_norm": 0.390790194272995, |
| "learning_rate": 0.0002968958075596403, |
| "epoch": 0.5924463095531968, |
| "step": 300 |
| }, |
| { |
| "eval_loss": 0.4794607162475586, |
| "eval_runtime": 47.1461, |
| "eval_samples_per_second": 152.696, |
| "eval_steps_per_second": 9.545, |
| "epoch": 0.5924463095531968, |
| "step": 300 |
| }, |
| { |
| "loss": 0.381, |
| "grad_norm": 8.648950576782227, |
| "learning_rate": 0.00029677570121668253, |
| "epoch": 0.6023204147124167, |
| "step": 305 |
| }, |
| { |
| "loss": 0.3731, |
| "grad_norm": 0.8738763332366943, |
| "learning_rate": 0.0002966533404375925, |
| "epoch": 0.6121945198716366, |
| "step": 310 |
| }, |
| { |
| "loss": 0.4041, |
| "grad_norm": 1.4086726903915405, |
| "learning_rate": 0.00029652872710179967, |
| "epoch": 0.6220686250308566, |
| "step": 315 |
| }, |
| { |
| "loss": 0.3719, |
| "grad_norm": 0.5402569770812988, |
| "learning_rate": 0.0002964018631233319, |
| "epoch": 0.6319427301900765, |
| "step": 320 |
| }, |
| { |
| "loss": 0.3624, |
| "grad_norm": 0.6380510926246643, |
| "learning_rate": 0.00029627275045078657, |
| "epoch": 0.6418168353492965, |
| "step": 325 |
| }, |
| { |
| "loss": 0.3707, |
| "grad_norm": 0.9881039261817932, |
| "learning_rate": 0.0002961413910673003, |
| "epoch": 0.6516909405085164, |
| "step": 330 |
| }, |
| { |
| "loss": 0.3544, |
| "grad_norm": 0.5078769326210022, |
| "learning_rate": 0.00029600778699051875, |
| "epoch": 0.6615650456677363, |
| "step": 335 |
| }, |
| { |
| "loss": 0.3498, |
| "grad_norm": 0.5671535134315491, |
| "learning_rate": 0.0002958719402725653, |
| "epoch": 0.6714391508269563, |
| "step": 340 |
| }, |
| { |
| "loss": 0.337, |
| "grad_norm": 0.6489645838737488, |
| "learning_rate": 0.0002957338530000097, |
| "epoch": 0.6813132559861762, |
| "step": 345 |
| }, |
| { |
| "loss": 0.3578, |
| "grad_norm": 0.8148907423019409, |
| "learning_rate": 0.0002955935272938363, |
| "epoch": 0.6911873611453963, |
| "step": 350 |
| }, |
| { |
| "loss": 0.3598, |
| "grad_norm": 0.4891752004623413, |
| "learning_rate": 0.0002954509653094108, |
| "epoch": 0.7010614663046162, |
| "step": 355 |
| }, |
| { |
| "loss": 0.3502, |
| "grad_norm": 6.3048176765441895, |
| "learning_rate": 0.0002953061692364479, |
| "epoch": 0.7109355714638361, |
| "step": 360 |
| }, |
| { |
| "loss": 0.3308, |
| "grad_norm": 0.5603845715522766, |
| "learning_rate": 0.0002951591412989771, |
| "epoch": 0.7208096766230561, |
| "step": 365 |
| }, |
| { |
| "loss": 0.3173, |
| "grad_norm": 0.5107300877571106, |
| "learning_rate": 0.0002950098837553088, |
| "epoch": 0.730683781782276, |
| "step": 370 |
| }, |
| { |
| "loss": 0.3348, |
| "grad_norm": 0.6624521613121033, |
| "learning_rate": 0.0002948583988979996, |
| "epoch": 0.7405578869414959, |
| "step": 375 |
| }, |
| { |
| "loss": 0.3463, |
| "grad_norm": 1.0949567556381226, |
| "learning_rate": 0.00029470468905381706, |
| "epoch": 0.7504319921007159, |
| "step": 380 |
| }, |
| { |
| "loss": 0.3428, |
| "grad_norm": 0.6831812262535095, |
| "learning_rate": 0.00029454875658370384, |
| "epoch": 0.7603060972599358, |
| "step": 385 |
| }, |
| { |
| "loss": 0.3245, |
| "grad_norm": 0.8317500352859497, |
| "learning_rate": 0.0002943906038827417, |
| "epoch": 0.7701802024191557, |
| "step": 390 |
| }, |
| { |
| "loss": 0.3336, |
| "grad_norm": 0.5925719141960144, |
| "learning_rate": 0.00029423023338011453, |
| "epoch": 0.7800543075783757, |
| "step": 395 |
| }, |
| { |
| "loss": 0.3138, |
| "grad_norm": 0.5986014604568481, |
| "learning_rate": 0.00029406764753907094, |
| "epoch": 0.7899284127375956, |
| "step": 400 |
| }, |
| { |
| "eval_loss": 0.48380041122436523, |
| "eval_runtime": 46.653, |
| "eval_samples_per_second": 154.31, |
| "eval_steps_per_second": 9.646, |
| "epoch": 0.7899284127375956, |
| "step": 400 |
| }, |
| { |
| "loss": 0.3153, |
| "grad_norm": 1.081153392791748, |
| "learning_rate": 0.0002939028488568867, |
| "epoch": 0.7998025178968156, |
| "step": 405 |
| }, |
| { |
| "loss": 0.3295, |
| "grad_norm": 1.3051981925964355, |
| "learning_rate": 0.00029373583986482617, |
| "epoch": 0.8096766230560355, |
| "step": 410 |
| }, |
| { |
| "loss": 0.3226, |
| "grad_norm": 0.7439870834350586, |
| "learning_rate": 0.0002935666231281035, |
| "epoch": 0.8195507282152555, |
| "step": 415 |
| }, |
| { |
| "loss": 0.2997, |
| "grad_norm": 0.5314565896987915, |
| "learning_rate": 0.00029339520124584324, |
| "epoch": 0.8294248333744755, |
| "step": 420 |
| }, |
| { |
| "loss": 0.2957, |
| "grad_norm": 0.6831716299057007, |
| "learning_rate": 0.0002932215768510404, |
| "epoch": 0.8392989385336954, |
| "step": 425 |
| }, |
| { |
| "loss": 0.2846, |
| "grad_norm": 0.49296361207962036, |
| "learning_rate": 0.0002930457526105199, |
| "epoch": 0.8491730436929154, |
| "step": 430 |
| }, |
| { |
| "loss": 0.285, |
| "grad_norm": 0.5440292954444885, |
| "learning_rate": 0.00029286773122489583, |
| "epoch": 0.8590471488521353, |
| "step": 435 |
| }, |
| { |
| "loss": 0.2861, |
| "grad_norm": 0.777549147605896, |
| "learning_rate": 0.0002926875154285298, |
| "epoch": 0.8689212540113552, |
| "step": 440 |
| }, |
| { |
| "loss": 0.2683, |
| "grad_norm": 0.5027139186859131, |
| "learning_rate": 0.0002925051079894891, |
| "epoch": 0.8787953591705752, |
| "step": 445 |
| }, |
| { |
| "loss": 0.2897, |
| "grad_norm": 0.833881139755249, |
| "learning_rate": 0.0002923205117095037, |
| "epoch": 0.8886694643297951, |
| "step": 450 |
| }, |
| { |
| "loss": 0.2792, |
| "grad_norm": 0.7060643434524536, |
| "learning_rate": 0.0002921337294239242, |
| "epoch": 0.898543569489015, |
| "step": 455 |
| }, |
| { |
| "loss": 0.2663, |
| "grad_norm": 0.6060872673988342, |
| "learning_rate": 0.00029194476400167717, |
| "epoch": 0.908417674648235, |
| "step": 460 |
| }, |
| { |
| "loss": 0.2677, |
| "grad_norm": 0.7497969269752502, |
| "learning_rate": 0.0002917536183452217, |
| "epoch": 0.9182917798074549, |
| "step": 465 |
| }, |
| { |
| "loss": 0.2585, |
| "grad_norm": 1.225437879562378, |
| "learning_rate": 0.00029156029539050487, |
| "epoch": 0.9281658849666748, |
| "step": 470 |
| }, |
| { |
| "loss": 0.2614, |
| "grad_norm": 0.48100578784942627, |
| "learning_rate": 0.0002913647981069164, |
| "epoch": 0.9380399901258949, |
| "step": 475 |
| }, |
| { |
| "loss": 0.2488, |
| "grad_norm": 4.289666175842285, |
| "learning_rate": 0.0002911671294972429, |
| "epoch": 0.9479140952851148, |
| "step": 480 |
| }, |
| { |
| "loss": 0.2548, |
| "grad_norm": 0.7958076596260071, |
| "learning_rate": 0.0002909672925976224, |
| "epoch": 0.9577882004443348, |
| "step": 485 |
| }, |
| { |
| "loss": 0.2583, |
| "grad_norm": 0.5692360997200012, |
| "learning_rate": 0.00029076529047749694, |
| "epoch": 0.9676623056035547, |
| "step": 490 |
| }, |
| { |
| "loss": 0.2643, |
| "grad_norm": 0.6238569021224976, |
| "learning_rate": 0.00029056112623956595, |
| "epoch": 0.9775364107627746, |
| "step": 495 |
| }, |
| { |
| "loss": 0.2548, |
| "grad_norm": 1.2753865718841553, |
| "learning_rate": 0.00029035480301973837, |
| "epoch": 0.9874105159219946, |
| "step": 500 |
| }, |
| { |
| "eval_loss": 0.49340417981147766, |
| "eval_runtime": 46.5345, |
| "eval_samples_per_second": 154.702, |
| "eval_steps_per_second": 9.67, |
| "epoch": 0.9874105159219946, |
| "step": 500 |
| }, |
| { |
| "loss": 0.259, |
| "grad_norm": 0.5439327359199524, |
| "learning_rate": 0.0002901463239870846, |
| "epoch": 0.9972846210812145, |
| "step": 505 |
| }, |
| { |
| "loss": 0.3047, |
| "grad_norm": 0.7135459184646606, |
| "learning_rate": 0.00028993569234378756, |
| "epoch": 1.007899284127376, |
| "step": 510 |
| }, |
| { |
| "loss": 0.2138, |
| "grad_norm": 0.6349644064903259, |
| "learning_rate": 0.00028972291132509396, |
| "epoch": 1.0177733892865959, |
| "step": 515 |
| }, |
| { |
| "loss": 0.2317, |
| "grad_norm": 0.6580847501754761, |
| "learning_rate": 0.0002895079841992642, |
| "epoch": 1.0276474944458158, |
| "step": 520 |
| }, |
| { |
| "loss": 0.2267, |
| "grad_norm": 0.6198315024375916, |
| "learning_rate": 0.00028929091426752236, |
| "epoch": 1.0375215996050358, |
| "step": 525 |
| }, |
| { |
| "loss": 0.2394, |
| "grad_norm": 0.4667389690876007, |
| "learning_rate": 0.0002890717048640055, |
| "epoch": 1.0473957047642557, |
| "step": 530 |
| }, |
| { |
| "loss": 0.2017, |
| "grad_norm": 0.6087087988853455, |
| "learning_rate": 0.00028885035935571223, |
| "epoch": 1.0572698099234756, |
| "step": 535 |
| }, |
| { |
| "loss": 0.2006, |
| "grad_norm": 0.6860356330871582, |
| "learning_rate": 0.00028862688114245137, |
| "epoch": 1.0671439150826956, |
| "step": 540 |
| }, |
| { |
| "loss": 0.2155, |
| "grad_norm": 0.6059162020683289, |
| "learning_rate": 0.00028840127365678943, |
| "epoch": 1.0770180202419155, |
| "step": 545 |
| }, |
| { |
| "loss": 0.1982, |
| "grad_norm": 0.5954364538192749, |
| "learning_rate": 0.000288173540363998, |
| "epoch": 1.0868921254011354, |
| "step": 550 |
| }, |
| { |
| "loss": 0.2235, |
| "grad_norm": 0.5217412710189819, |
| "learning_rate": 0.0002879436847620005, |
| "epoch": 1.0967662305603554, |
| "step": 555 |
| }, |
| { |
| "loss": 0.2331, |
| "grad_norm": 0.7081472277641296, |
| "learning_rate": 0.0002877117103813185, |
| "epoch": 1.1066403357195753, |
| "step": 560 |
| }, |
| { |
| "loss": 0.2262, |
| "grad_norm": 0.6892974376678467, |
| "learning_rate": 0.00028747762078501737, |
| "epoch": 1.1165144408787953, |
| "step": 565 |
| }, |
| { |
| "loss": 0.2107, |
| "grad_norm": 0.688247561454773, |
| "learning_rate": 0.0002872414195686517, |
| "epoch": 1.1263885460380152, |
| "step": 570 |
| }, |
| { |
| "loss": 0.2076, |
| "grad_norm": 0.6474739909172058, |
| "learning_rate": 0.00028700311036021004, |
| "epoch": 1.1362626511972351, |
| "step": 575 |
| }, |
| { |
| "loss": 0.204, |
| "grad_norm": 0.5220997333526611, |
| "learning_rate": 0.00028676269682005904, |
| "epoch": 1.1461367563564553, |
| "step": 580 |
| }, |
| { |
| "loss": 0.2003, |
| "grad_norm": 0.5570688843727112, |
| "learning_rate": 0.0002865201826408874, |
| "epoch": 1.1560108615156752, |
| "step": 585 |
| }, |
| { |
| "loss": 0.2028, |
| "grad_norm": 0.5454896092414856, |
| "learning_rate": 0.0002862755715476491, |
| "epoch": 1.1658849666748952, |
| "step": 590 |
| }, |
| { |
| "loss": 0.2011, |
| "grad_norm": 0.5824912190437317, |
| "learning_rate": 0.00028602886729750613, |
| "epoch": 1.175759071834115, |
| "step": 595 |
| }, |
| { |
| "loss": 0.1875, |
| "grad_norm": 0.612477719783783, |
| "learning_rate": 0.00028578007367977074, |
| "epoch": 1.185633176993335, |
| "step": 600 |
| }, |
| { |
| "eval_loss": 0.5274343490600586, |
| "eval_runtime": 46.5304, |
| "eval_samples_per_second": 154.716, |
| "eval_steps_per_second": 9.671, |
| "epoch": 1.185633176993335, |
| "step": 600 |
| }, |
| { |
| "train_runtime": 1998.6484, |
| "train_samples_per_second": 259.379, |
| "train_steps_per_second": 2.025, |
| "total_flos": 2.946671339427594e+17, |
| "train_loss": 0.5542280136545499, |
| "epoch": 1.185633176993335, |
| "step": 600 |
| } |
| ], |
| "best_metric": 0.4794607162475586, |
| "best_model_checkpoint": "saves/adpr-llama/checkpoint-300", |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "is_hyper_param_search": false, |
| "trial_name": null, |
| "trial_params": null, |
| "stateful_callbacks": { |
| "EarlyStoppingCallback": { |
| "args": { |
| "early_stopping_patience": 3, |
| "early_stopping_threshold": 0.0 |
| }, |
| "attributes": { |
| "early_stopping_patience_counter": 3 |
| } |
| }, |
| "TrainerControl": { |
| "args": { |
| "should_training_stop": true, |
| "should_epoch_stop": false, |
| "should_save": true, |
| "should_evaluate": false, |
| "should_log": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "model": "PeftModelForCausalLM(\n (base_model): LoraModel(\n (model): LlamaForCausalLM(\n (model): LlamaModel(\n (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n (layers): ModuleList(\n (0-31): 32 x LlamaDecoderLayer(\n (self_attn): LlamaAttention(\n (q_proj): lora.Linear(\n (base_layer): Linear(in_features=4096, out_features=4096, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=4096, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=4096, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n (k_proj): lora.Linear(\n (base_layer): Linear(in_features=4096, out_features=4096, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=4096, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=4096, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n (v_proj): lora.Linear(\n (base_layer): Linear(in_features=4096, out_features=4096, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=4096, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=4096, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n (o_proj): lora.Linear(\n (base_layer): Linear(in_features=4096, out_features=4096, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=4096, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=4096, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n )\n (mlp): LlamaMLP(\n (gate_proj): lora.Linear(\n (base_layer): Linear(in_features=4096, out_features=11008, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=4096, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=11008, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n (up_proj): lora.Linear(\n (base_layer): Linear(in_features=4096, out_features=11008, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=4096, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=11008, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n (down_proj): lora.Linear(\n (base_layer): Linear(in_features=11008, out_features=4096, bias=False)\n (lora_dropout): ModuleDict(\n (default): Dropout(p=0.05, inplace=False)\n )\n (lora_A): ModuleDict(\n (default): Linear(in_features=11008, out_features=64, bias=False)\n )\n (lora_B): ModuleDict(\n (default): Linear(in_features=64, out_features=4096, bias=False)\n )\n (lora_embedding_A): ParameterDict()\n (lora_embedding_B): ParameterDict()\n (lora_magnitude_vector): ModuleDict()\n )\n (act_fn): SiLU()\n )\n (input_layernorm): LlamaRMSNorm((4096,), eps=1e-05)\n (post_attention_layernorm): LlamaRMSNorm((4096,), eps=1e-05)\n )\n )\n (norm): LlamaRMSNorm((4096,), eps=1e-05)\n (rotary_emb): LlamaRotaryEmbedding()\n )\n (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n )\n )\n)", |
| "optimizer": "AcceleratedOptimizer (\nParameter Group 0\n amsgrad: False\n betas: (0.9, 0.999)\n capturable: False\n decoupled_weight_decay: True\n differentiable: False\n eps: 1e-08\n foreach: None\n fused: None\n initial_lr: 0.0003\n lr: 0.00028578007367977074\n maximize: False\n weight_decay: 0.0\n\nParameter Group 1\n amsgrad: False\n betas: (0.9, 0.999)\n capturable: False\n decoupled_weight_decay: True\n differentiable: False\n eps: 1e-08\n foreach: None\n fused: None\n initial_lr: 0.0003\n lr: 0.00028578007367977074\n maximize: False\n weight_decay: 0.0\n)", |
| "lr_scheduler": "<torch.optim.lr_scheduler.LambdaLR object at 0x7de720bc1580>", |
| "train_dataloader": "<accelerate.data_loader.DataLoaderShard object at 0x7de619edf170>" |
| } |
