| { |
| "iterations": 31400, |
| "parameter_count": 730164480, |
| "tokens_per_step": 262144, |
| "processed_tokens": 8231321600, |
| "elapsed_seconds": 271423.6548615545, |
| "best_validation_loss": 3.712334632873535, |
| "history": [ |
| { |
| "iteration": 500, |
| "train_loss": 6.657673358917236, |
| "validation_loss": 6.6738409996032715, |
| "learning_rate": 0.0002499772703356698, |
| "elapsed_seconds": 4271.6415923163295, |
| "projected_total_hours": 74.5164144437404, |
| "projected_total_cost_usd": 103.57781607679915 |
| }, |
| { |
| "iteration": 1000, |
| "train_loss": 5.294595718383789, |
| "validation_loss": 5.350976467132568, |
| "learning_rate": 0.0002497196662435132, |
| "elapsed_seconds": 8576.508910141885, |
| "projected_total_hours": 74.80621660512645, |
| "projected_total_cost_usd": 103.98064108112575 |
| }, |
| { |
| "iteration": 1500, |
| "train_loss": 4.833080291748047, |
| "validation_loss": 4.898420810699463, |
| "learning_rate": 0.00024917584502227825, |
| "elapsed_seconds": 12891.454528726637, |
| "projected_total_hours": 74.9614207781512, |
| "projected_total_cost_usd": 104.19637488163015 |
| }, |
| { |
| "iteration": 2000, |
| "train_loss": 4.620269775390625, |
| "validation_loss": 4.684819221496582, |
| "learning_rate": 0.0002483471936922074, |
| "elapsed_seconds": 17209.910989679396, |
| "projected_total_hours": 75.05433403832403, |
| "projected_total_cost_usd": 104.32552431327039 |
| }, |
| { |
| "iteration": 2500, |
| "train_loss": 4.481897354125977, |
| "validation_loss": 4.551168441772461, |
| "learning_rate": 0.0002472358257349446, |
| "elapsed_seconds": 21535.88317859918, |
| "projected_total_hours": 75.13630353422381, |
| "projected_total_cost_usd": 104.43946191257109 |
| }, |
| { |
| "iteration": 3000, |
| "train_loss": 4.3804826736450195, |
| "validation_loss": 4.452548503875732, |
| "learning_rate": 0.0002458445757030842, |
| "elapsed_seconds": 25854.85624050349, |
| "projected_total_hours": 75.17060055109349, |
| "projected_total_cost_usd": 104.48713476601993 |
| }, |
| { |
| "iteration": 3500, |
| "train_loss": 4.3059492111206055, |
| "validation_loss": 4.380916118621826, |
| "learning_rate": 0.0002441769919906223, |
| "elapsed_seconds": 30173.152176752687, |
| "projected_total_hours": 75.19341098016146, |
| "projected_total_cost_usd": 104.51884126242442 |
| }, |
| { |
| "iteration": 4000, |
| "train_loss": 4.242997169494629, |
| "validation_loss": 4.31741189956665, |
| "learning_rate": 0.00024223732778275053, |
| "elapsed_seconds": 34493.88701976091, |
| "projected_total_hours": 75.21583697364531, |
| "projected_total_cost_usd": 104.55001339336697 |
| }, |
| { |
| "iteration": 4500, |
| "train_loss": 4.189329147338867, |
| "validation_loss": 4.266604423522949, |
| "learning_rate": 0.0002400305302080742, |
| "elapsed_seconds": 38816.21281320602, |
| "projected_total_hours": 75.23636310707833, |
| "projected_total_cost_usd": 104.57854471883887 |
| }, |
| { |
| "iteration": 5000, |
| "train_loss": 4.1476969718933105, |
| "validation_loss": 4.227367877960205, |
| "learning_rate": 0.00023756222772092247, |
| "elapsed_seconds": 43158.014198705554, |
| "projected_total_hours": 75.28675810218637, |
| "projected_total_cost_usd": 104.64859376203904 |
| }, |
| { |
| "iteration": 5500, |
| "train_loss": 4.109938621520996, |
| "validation_loss": 4.190089225769043, |
| "learning_rate": 0.00023483871574593183, |
| "elapsed_seconds": 47477.29582044482, |
| "projected_total_hours": 75.29227721020038, |
| "projected_total_cost_usd": 104.65626532217853 |
| }, |
| { |
| "iteration": 6000, |
| "train_loss": 4.076807022094727, |
| "validation_loss": 4.157365798950195, |
| "learning_rate": 0.0002318669406215168, |
| "elapsed_seconds": 51794.34619618952, |
| "projected_total_hours": 75.29363289631254, |
| "projected_total_cost_usd": 104.65814972587442 |
| }, |
| { |
| "iteration": 6500, |
| "train_loss": 4.042527675628662, |
| "validation_loss": 4.129131317138672, |
| "learning_rate": 0.0002286544818831799, |
| "elapsed_seconds": 56113.59111300856, |
| "projected_total_hours": 75.29772482685765, |
| "projected_total_cost_usd": 104.66383750933213 |
| }, |
| { |
| "iteration": 7000, |
| "train_loss": 4.053408622741699, |
| "validation_loss": 4.135789394378662, |
| "learning_rate": 0.00022520953293184766, |
| "elapsed_seconds": 60422.58037369698, |
| "projected_total_hours": 75.28845332278115, |
| "projected_total_cost_usd": 104.65095011866579 |
| }, |
| { |
| "iteration": 7500, |
| "train_loss": 3.9998574256896973, |
| "validation_loss": 4.0869526863098145, |
| "learning_rate": 0.0002215408801365379, |
| "elapsed_seconds": 64723.61913320422, |
| "projected_total_hours": 75.2711718808375, |
| "projected_total_cost_usd": 104.62692891436411 |
| }, |
| { |
| "iteration": 8000, |
| "train_loss": 3.9767398834228516, |
| "validation_loss": 4.06421422958374, |
| "learning_rate": 0.00021765788042465737, |
| "elapsed_seconds": 69042.05848243833, |
| "projected_total_hours": 75.27502209543623, |
| "projected_total_cost_usd": 104.63228071265635 |
| }, |
| { |
| "iteration": 8500, |
| "train_loss": 3.9555294513702393, |
| "validation_loss": 4.046372890472412, |
| "learning_rate": 0.00021357043741708606, |
| "elapsed_seconds": 73355.51514386386, |
| "projected_total_hours": 75.27330638945507, |
| "projected_total_cost_usd": 104.62989588134255 |
| }, |
| { |
| "iteration": 9000, |
| "train_loss": 3.934286117553711, |
| "validation_loss": 4.028558254241943, |
| "learning_rate": 0.00020928897616891504, |
| "elapsed_seconds": 77672.20504949987, |
| "projected_total_hours": 75.27491477019431, |
| "projected_total_cost_usd": 104.63213153057009 |
| }, |
| { |
| "iteration": 9500, |
| "train_loss": 3.9189178943634033, |
| "validation_loss": 4.014111518859863, |
| "learning_rate": 0.0002048244165802629, |
| "elapsed_seconds": 81986.55616268516, |
| "projected_total_hours": 75.27420653533082, |
| "projected_total_cost_usd": 104.63114708410983 |
| }, |
| { |
| "iteration": 10000, |
| "train_loss": 3.896970272064209, |
| "validation_loss": 3.996415138244629, |
| "learning_rate": 0.00020018814554498553, |
| "elapsed_seconds": 86303.21427984536, |
| "projected_total_hours": 75.27558134408734, |
| "projected_total_cost_usd": 104.6330580682814 |
| }, |
| { |
| "iteration": 10500, |
| "train_loss": 3.882066011428833, |
| "validation_loss": 3.9830996990203857, |
| "learning_rate": 0.00019539198790831518, |
| "elapsed_seconds": 90618.27073761076, |
| "projected_total_hours": 75.2754947397084, |
| "projected_total_cost_usd": 104.63293768819467 |
| }, |
| { |
| "iteration": 11000, |
| "train_loss": 3.8664541244506836, |
| "validation_loss": 3.9677159786224365, |
| "learning_rate": 0.000190448176307501, |
| "elapsed_seconds": 94942.73910321295, |
| "projected_total_hours": 75.28287898588098, |
| "projected_total_cost_usd": 104.64320179037456 |
| }, |
| { |
| "iteration": 11500, |
| "train_loss": 3.851578950881958, |
| "validation_loss": 3.9544689655303955, |
| "learning_rate": 0.0001853693199723728, |
| "elapsed_seconds": 99264.58329997957, |
| "projected_total_hours": 75.2876308120618, |
| "projected_total_cost_usd": 104.6498068287659 |
| }, |
| { |
| "iteration": 12000, |
| "train_loss": 3.8364357948303223, |
| "validation_loss": 3.9412567615509033, |
| "learning_rate": 0.00018016837256540257, |
| "elapsed_seconds": 103622.94289319962, |
| "projected_total_hours": 75.31852793626084, |
| "projected_total_cost_usd": 104.69275383140256 |
| }, |
| { |
| "iteration": 12500, |
| "train_loss": 3.8234899044036865, |
| "validation_loss": 3.929971694946289, |
| "learning_rate": 0.00017485859914328803, |
| "elapsed_seconds": 107943.69804026932, |
| "projected_total_hours": 75.32071374365458, |
| "projected_total_cost_usd": 104.69579210367986 |
| }, |
| { |
| "iteration": 13000, |
| "train_loss": 3.8084628582000732, |
| "validation_loss": 3.917842388153076, |
| "learning_rate": 0.00016945354232432232, |
| "elapsed_seconds": 112258.32612586766, |
| "projected_total_hours": 75.3186205203471, |
| "projected_total_cost_usd": 104.69288252328245 |
| }, |
| { |
| "iteration": 13500, |
| "train_loss": 3.7961957454681396, |
| "validation_loss": 3.906834125518799, |
| "learning_rate": 0.00016396698774784145, |
| "elapsed_seconds": 116572.56532915682, |
| "projected_total_hours": 75.31643109743877, |
| "projected_total_cost_usd": 104.68983922543988 |
| }, |
| { |
| "iteration": 14000, |
| "train_loss": 3.7818515300750732, |
| "validation_loss": 3.8985838890075684, |
| "learning_rate": 0.0001584129289138439, |
| "elapsed_seconds": 120888.49704167992, |
| "projected_total_hours": 75.31545252199899, |
| "projected_total_cost_usd": 104.68847900557859 |
| }, |
| { |
| "iteration": 14500, |
| "train_loss": 3.7692983150482178, |
| "validation_loss": 3.889788866043091, |
| "learning_rate": 0.00015280553149246053, |
| "elapsed_seconds": 125203.99943609536, |
| "projected_total_hours": 75.3142831856972, |
| "projected_total_cost_usd": 104.68685362811911 |
| }, |
| { |
| "iteration": 15000, |
| "train_loss": 3.7596471309661865, |
| "validation_loss": 3.879643201828003, |
| "learning_rate": 0.00014715909719430296, |
| "elapsed_seconds": 129519.89085417241, |
| "projected_total_hours": 75.31341801520396, |
| "projected_total_cost_usd": 104.6856510411335 |
| }, |
| { |
| "iteration": 15500, |
| "train_loss": 3.748656988143921, |
| "validation_loss": 3.8702828884124756, |
| "learning_rate": 0.00014148802729383982, |
| "elapsed_seconds": 133837.80417640507, |
| "projected_total_hours": 75.31374643618493, |
| "projected_total_cost_usd": 104.68610754629704 |
| }, |
| { |
| "iteration": 16000, |
| "train_loss": 3.735333204269409, |
| "validation_loss": 3.8588755130767822, |
| "learning_rate": 0.00013580678589883435, |
| "elapsed_seconds": 138154.23407498747, |
| "projected_total_hours": 75.31324565893414, |
| "projected_total_cost_usd": 104.68541146591843 |
| }, |
| { |
| "iteration": 16500, |
| "train_loss": 3.7218117713928223, |
| "validation_loss": 3.8520736694335938, |
| "learning_rate": 0.00013012986305952577, |
| "elapsed_seconds": 142472.6207278967, |
| "projected_total_hours": 75.31380961036963, |
| "projected_total_cost_usd": 104.68619535841378 |
| }, |
| { |
| "iteration": 17000, |
| "train_loss": 3.7100892066955566, |
| "validation_loss": 3.8433520793914795, |
| "learning_rate": 0.00012447173781164324, |
| "elapsed_seconds": 146782.17834874988, |
| "projected_total_hours": 75.3098104599795, |
| "projected_total_cost_usd": 104.6806365393715 |
| }, |
| { |
| "iteration": 17500, |
| "train_loss": 3.7014331817626953, |
| "validation_loss": 3.8347644805908203, |
| "learning_rate": 0.00011884684124751335, |
| "elapsed_seconds": 151125.83998079598, |
| "projected_total_hours": 75.3230377047142, |
| "projected_total_cost_usd": 104.69902240955273 |
| }, |
| { |
| "iteration": 18000, |
| "train_loss": 3.6897997856140137, |
| "validation_loss": 3.826525926589966, |
| "learning_rate": 0.00011326951970944703, |
| "elapsed_seconds": 155471.40847074986, |
| "projected_total_hours": 75.33645410465348, |
| "projected_total_cost_usd": 104.71767120546833 |
| }, |
| { |
| "iteration": 18500, |
| "train_loss": 3.679842472076416, |
| "validation_loss": 3.819714307785034, |
| "learning_rate": 0.0001077539981992822, |
| "elapsed_seconds": 159820.10016273707, |
| "projected_total_hours": 75.3506177944436, |
| "projected_total_cost_usd": 104.73735873427658 |
| }, |
| { |
| "iteration": 19000, |
| "train_loss": 3.6696887016296387, |
| "validation_loss": 3.8133578300476074, |
| "learning_rate": 0.00010231434409740646, |
| "elapsed_seconds": 164168.3918394372, |
| "projected_total_hours": 75.3638523941276, |
| "projected_total_cost_usd": 104.75575482783736 |
| }, |
| { |
| "iteration": 19500, |
| "train_loss": 3.661100149154663, |
| "validation_loss": 3.805586576461792, |
| "learning_rate": 9.696443128379405e-05, |
| "elapsed_seconds": 168500.78201993555, |
| "projected_total_hours": 75.3692956613387, |
| "projected_total_cost_usd": 104.76332096926077 |
| }, |
| { |
| "iteration": 20000, |
| "train_loss": 3.6516928672790527, |
| "validation_loss": 3.79829740524292, |
| "learning_rate": 9.171790475256764e-05, |
| "elapsed_seconds": 172862.07756865025, |
| "projected_total_hours": 75.38707271743914, |
| "projected_total_cost_usd": 104.78803107724039 |
| }, |
| { |
| "iteration": 20500, |
| "train_loss": 3.64292573928833, |
| "validation_loss": 3.791306257247925, |
| "learning_rate": 8.658814581033465e-05, |
| "elapsed_seconds": 177173.67764238268, |
| "projected_total_hours": 75.3828384548891, |
| "projected_total_cost_usd": 104.78214545229584 |
| }, |
| { |
| "iteration": 21000, |
| "train_loss": 3.630474328994751, |
| "validation_loss": 3.7852611541748047, |
| "learning_rate": 8.158823794706145e-05, |
| "elapsed_seconds": 181491.15980650485, |
| "projected_total_hours": 75.38124891434197, |
| "projected_total_cost_usd": 104.77993599093533 |
| }, |
| { |
| "iteration": 21500, |
| "train_loss": 3.621816396713257, |
| "validation_loss": 3.7775344848632812, |
| "learning_rate": 7.673093346653014e-05, |
| "elapsed_seconds": 185801.37050130963, |
| "projected_total_hours": 75.37678338166826, |
| "projected_total_cost_usd": 104.77372890051888 |
| }, |
| { |
| "iteration": 22000, |
| "train_loss": 3.6151795387268066, |
| "validation_loss": 3.7732739448547363, |
| "learning_rate": 7.20286209614895e-05, |
| "elapsed_seconds": 190149.34689292312, |
| "projected_total_hours": 75.38749359138619, |
| "projected_total_cost_usd": 104.7886160920268 |
| }, |
| { |
| "iteration": 22500, |
| "train_loss": 3.606862783432007, |
| "validation_loss": 3.767625093460083, |
| "learning_rate": 6.749329371645251e-05, |
| "elapsed_seconds": 194459.4843392372, |
| "projected_total_hours": 75.3830593611364, |
| "projected_total_cost_usd": 104.7824525119796 |
| }, |
| { |
| "iteration": 23000, |
| "train_loss": 3.598644495010376, |
| "validation_loss": 3.762594699859619, |
| "learning_rate": 6.313651911873158e-05, |
| "elapsed_seconds": 198774.46855475008, |
| "projected_total_hours": 75.38065594950666, |
| "projected_total_cost_usd": 104.77911176981425 |
| }, |
| { |
| "iteration": 23500, |
| "train_loss": 3.5893373489379883, |
| "validation_loss": 3.7594027519226074, |
| "learning_rate": 5.8969409155727336e-05, |
| "elapsed_seconds": 203100.5320912972, |
| "projected_total_hours": 75.3824669936966, |
| "projected_total_cost_usd": 104.78162912123827 |
| }, |
| { |
| "iteration": 24000, |
| "train_loss": 3.583808660507202, |
| "validation_loss": 3.754646062850952, |
| "learning_rate": 5.500259207371864e-05, |
| "elapsed_seconds": 207457.90694285184, |
| "projected_total_hours": 75.3955819213605, |
| "projected_total_cost_usd": 104.7998588706911 |
| }, |
| { |
| "iteration": 24500, |
| "train_loss": 3.57780385017395, |
| "validation_loss": 3.7487640380859375, |
| "learning_rate": 5.1246185270437925e-05, |
| "elapsed_seconds": 211774.69773548096, |
| "projected_total_hours": 75.3937132527676, |
| "projected_total_cost_usd": 104.79726142134695 |
| }, |
| { |
| "iteration": 25000, |
| "train_loss": 3.5740554332733154, |
| "validation_loss": 3.7441320419311523, |
| "learning_rate": 4.7709769490569937e-05, |
| "elapsed_seconds": 216085.9324932173, |
| "projected_total_hours": 75.38998089207803, |
| "projected_total_cost_usd": 104.79207343998846 |
| }, |
| { |
| "iteration": 25500, |
| "train_loss": 3.5674097537994385, |
| "validation_loss": 3.7407143115997314, |
| "learning_rate": 4.440236438998808e-05, |
| "elapsed_seconds": 220397.60610739142, |
| "projected_total_hours": 75.38654500841056, |
| "projected_total_cost_usd": 104.78729756169068 |
| }, |
| { |
| "iteration": 26000, |
| "train_loss": 3.5626490116119385, |
| "validation_loss": 3.7369534969329834, |
| "learning_rate": 4.1332405531052075e-05, |
| "elapsed_seconds": 224747.3975746855, |
| "projected_total_hours": 75.39602867355903, |
| "projected_total_cost_usd": 104.80047985624705 |
| }, |
| { |
| "iteration": 26500, |
| "train_loss": 3.557927131652832, |
| "validation_loss": 3.7329952716827393, |
| "learning_rate": 3.850772286764083e-05, |
| "elapsed_seconds": 229064.31341329962, |
| "projected_total_hours": 75.39433376496444, |
| "projected_total_cost_usd": 104.79812393330057 |
| }, |
| { |
| "iteration": 27000, |
| "train_loss": 3.5516343116760254, |
| "validation_loss": 3.7303378582000732, |
| "learning_rate": 3.593552077479432e-05, |
| "elapsed_seconds": 233381.3956354186, |
| "projected_total_hours": 75.3927553801661, |
| "projected_total_cost_usd": 104.79592997843086 |
| }, |
| { |
| "iteration": 27500, |
| "train_loss": 3.545631170272827, |
| "validation_loss": 3.7279391288757324, |
| "learning_rate": 3.362235967389917e-05, |
| "elapsed_seconds": 237694.1610769406, |
| "projected_total_hours": 75.38986523046398, |
| "projected_total_cost_usd": 104.79191267034493 |
| }, |
| { |
| "iteration": 28000, |
| "train_loss": 3.542006731033325, |
| "validation_loss": 3.7245471477508545, |
| "learning_rate": 3.157413930028304e-05, |
| "elapsed_seconds": 242045.88678893447, |
| "projected_total_hours": 75.39921473385459, |
| "projected_total_cost_usd": 104.80490848005788 |
| }, |
| { |
| "iteration": 28500, |
| "train_loss": 3.5386886596679688, |
| "validation_loss": 3.7228081226348877, |
| "learning_rate": 2.979608365589387e-05, |
| "elapsed_seconds": 246361.35709098727, |
| "projected_total_hours": 75.39714047423976, |
| "projected_total_cost_usd": 104.80202525919326 |
| }, |
| { |
| "iteration": 29000, |
| "train_loss": 3.536064386367798, |
| "validation_loss": 3.7195823192596436, |
| "learning_rate": 2.8292727685442474e-05, |
| "elapsed_seconds": 250676.92014835775, |
| "projected_total_hours": 75.39516563849074, |
| "projected_total_cost_usd": 104.79928023750212 |
| }, |
| { |
| "iteration": 29500, |
| "train_loss": 3.5316381454467773, |
| "validation_loss": 3.7183539867401123, |
| "learning_rate": 2.7067905709990822e-05, |
| "elapsed_seconds": 254991.04394229501, |
| "projected_total_hours": 75.3928322013942, |
| "projected_total_cost_usd": 104.79603675993793 |
| }, |
| { |
| "iteration": 30000, |
| "train_loss": 3.5296361446380615, |
| "validation_loss": 3.716928720474243, |
| "learning_rate": 2.612474164748647e-05, |
| "elapsed_seconds": 259309.55208150297, |
| "projected_total_hours": 75.39185125332585, |
| "projected_total_cost_usd": 104.79467324212293 |
| }, |
| { |
| "iteration": 30500, |
| "train_loss": 3.526557207107544, |
| "validation_loss": 3.71480131149292, |
| "learning_rate": 2.5465641045185573e-05, |
| "elapsed_seconds": 263647.62443690747, |
| "projected_total_hours": 75.39649733441617, |
| "projected_total_cost_usd": 104.80113129483847 |
| }, |
| { |
| "iteration": 31000, |
| "train_loss": 3.5243618488311768, |
| "validation_loss": 3.712334632873535, |
| "learning_rate": 2.5092284944285984e-05, |
| "elapsed_seconds": 267962.22376234084, |
| "projected_total_hours": 75.39438912309589, |
| "projected_total_cost_usd": 104.79820088110328 |
| }, |
| { |
| "iteration": 31400, |
| "train_loss": 3.52117919921875, |
| "validation_loss": 3.712394952774048, |
| "learning_rate": 2.500000057398626e-05, |
| "elapsed_seconds": 271423.6443994045, |
| "projected_total_hours": 75.39545677761237, |
| "projected_total_cost_usd": 104.79968492088118 |
| } |
| ], |
| "environment": { |
| "python": "3.11.10", |
| "platform": "Linux-6.5.0-45-generic-x86_64-with-glibc2.35", |
| "torch": "2.5.1+cu124", |
| "cuda": "12.4", |
| "gpu": "NVIDIA A100 80GB PCIe" |
| }, |
| "data_manifest_sha256": "c05b79cf5753e25fe9276f092a64c5079c5a206fd790d96841477e8cca51530f", |
| "peak_vram_bytes": 43817826304, |
| "gpu_total_bytes": 84974239744, |
| "estimated_training_cost_usd": 104.79968896043354 |
| } |
|
|
