penfever's picture
Add parsed metric surface and redacted trainer log for run 1ae770
b713018 verified
Raw
History Blame Contribute Delete
9.18 kB
log_file,async/discard_rate,async/discarded_count,async/effective_batch_groups,async/effective_batch_samples,async/staleness_max,async/staleness_mean,async/staleness_min,async/staleness_ratio,generate/avg_num_tokens,generate/avg_tokens_non_zero_rewards,generate/avg_tokens_zero_rewards,generate/max_num_tokens,generate/min_num_tokens,generate/std_num_tokens,generate/tis/aligned_tokens,generate/tis/alignment_fail_count,generate/tis/exact_match_fraction,generate/tis/lcs_fallback_alert,generate/tis/lcs_fallback_fraction,generate/tis/lcs_fallback_messages,generate/tis/unaligned_fraction,loss/avg_final_rewards,loss/avg_raw_advantages,loss/avg_raw_advantages_abs,policy/final_loss,policy/policy_entropy,policy/policy_loss,policy/policy_lr,policy/policy_update_steps,policy/ppo_clip_ratio,policy/raw_grad_norm,policy/rollout_train_prob_diff_mean,policy/rollout_train_prob_diff_std,reward/avg_pass_at_8,reward/avg_raw_reward,system/process_rss_gb,system/process_vms_gb,system/ram_available_gb,system/ram_percent,system/ram_total_gb,system/ram_used_gb,timing/cleanup_old_checkpoints,timing/compute_advantages_and_returns,timing/convert_to_training_input,timing/fwd_logprobs_values_reward,timing/policy_train,timing/run_training,timing/save_checkpoints,timing/step,timing/sync_weights,timing/train_critic_and_policy,timing/wait_for_generation_buffer,tis/batch_skipped_no_logprobs,tis/skipped_fraction,trainer/epoch,trainer/global_step,timing/save_hf_model,global_step
finelog,0.0,0,64,512,1,0.03125,0,0.03125,2352.162109375,2012.2652173913043,2629.3829787234044,21648,271,2308.2586931329156,580442.0,0.0,1.0,0.0,0.0,0.0,0.0,0.44921875,-0.022331397980451584,0.10573633015155792,-0.0001103550021070987,0.916983806702774,-0.0001103550021070987,7.999999979801942e-06,1,0.0,0.07379782199859619,1.0393970012664795,0.7227853536605835,0.515625,0.44921875,17.535274505615234,120.92271041870117,1872.6025466918945,7.0,2014.4287147521973,141.82616806030273,5.63318657875061,0.0873723030090332,3.595928192138672,69.1614830493927,183.64072036743164,253.0149073600769,479.92912912368774,877.4440121650696,82.811190366745,183.74106550216675,538.0217173099518,0.0,0.0,0,11,,11
finelog,0.0,0,64,512,1,1.0,1,1.0,2537.443359375,2154.151515151515,2670.5868421052633,23204,460,2391.8542421188295,621973.0,0.0,1.0,0.0,0.0,0.0,0.0,0.2578125,-0.004582593217492104,0.09677589684724808,-1.4968361938372254e-05,0.9270317238988355,-1.4968361938372254e-05,7.999999979801942e-06,1,0.0,0.07234907150268555,1.3672617673873901,189.98556518554688,0.453125,0.2578125,18.890262603759766,125.78672409057617,1869.3302154541016,7.2,2014.4287147521973,145.0984992980957,0.006937503814697266,0.08305644989013672,3.0499768257141113,46.31576132774353,148.18072295188904,194.67333555221558,463.34481477737427,462.7308204174042,264.97486186027527,148.25221467018127,0.000518798828125,0.0,0.0,0,12,,12
finelog,0.0,0,64,512,2,1.828125,1,1.0,3358.01953125,3274.4132653061224,3409.8765822784812,25674,247,3377.537795809031,850578.0,0.0,1.0,0.0,0.0,0.0,0.0,0.3828125,-0.021296631544828415,0.1237059086561203,0.00020468563889153302,0.9662019034731202,0.00020468563889153302,7.999999979801942e-06,1,0.0,0.07480604201555252,32.1727294921875,26229.494140625,0.5,0.3828125,20.376602172851562,130.76318359375,1866.640037536621,7.3,2014.4287147521973,147.78867721557617,0.006958484649658203,0.08489203453063965,3.6398375034332275,48.58984732627869,159.764310836792,208.5389130115509,461.6352093219757,293.3591275215149,81.14270424842834,159.84030389785767,0.0006954669952392578,0.0,0.0,0,13,,13
finelog,0.0,0,64,512,2,1.65625,1,1.0,3771.751953125,3321.8522167487686,4067.3171521035597,24369,300,3173.1146787729913,955649.0,0.0,1.0,0.0,0.0,0.0,0.0,0.396484375,-0.02060784213244915,0.13694418966770172,4.18531708419323e-05,1.013929680455476,4.18531708419323e-05,7.999999979801942e-06,1,0.0,0.06541100889444351,11.94090461730957,8152.9208984375,0.53125,0.396484375,21.203872680664062,135.46282958984375,1865.4100379943848,7.4,2014.4287147521973,149.0186767578125,0.009932994842529297,0.07995367050170898,3.4873104095458984,46.4379518032074,152.79619932174683,199.44171118736267,462.82099890708923,312.19322419166565,109.21838808059692,152.8915433883667,0.0005068778991699219,0.0,0.0,0,14,,14
finelog,0.0,0,64,512,4,1.265625,0,0.90625,3671.82421875,3321.214659685864,3880.442367601246,21768,376,2500.7308635503846,887909.0,0.0,1.0,0.0,0.0,0.0,0.0,0.373046875,-0.027924705296754837,0.15362447500228882,-0.00017171968647744507,1.001769600668922,-0.00017171968647744507,7.999999979801942e-06,1,0.0,0.0640101507306099,5253.3203125,4885459.0,0.53125,0.373046875,21.476531982421875,140.16432571411133,1864.5351448059082,7.4,2014.4287147521973,149.89356994628906,0.006436586380004883,0.0720057487487793,2.9676759243011475,43.895183086395264,144.5767424106598,188.62816667556763,472.7359070777893,672.1971838474274,216.38741636276245,144.6401011943817,264.1604299545288,0.0,0.0,0,15,35.39909744262695,15
finelog,0.0,0,64,512,2,0.734375,0,0.6875,3629.326171875,3374.0632411067195,3878.675675675676,25577,307,2985.3460391870876,875476.0,0.0,1.0,0.0,0.0,0.0,0.0,0.494140625,0.0013031653361395001,0.11755329370498657,-0.0003406114992685616,1.0002137462724932,-0.0003406114992685616,7.999999979801942e-06,1,0.0,0.058628324419260025,1.9664862155914307,573.5465698242188,0.625,0.494140625,21.878799438476562,140.45945739746094,1863.2931518554688,7.5,2014.4287147521973,151.13556289672852,0.0063478946685791016,0.08103275299072266,3.3881232738494873,44.876765966415405,147.02753973007202,192.07576894760132,469.7542881965637,985.7916593551636,82.15019392967224,147.09987020492554,708.1371586322784,0.0,0.0,0,16,,16
finelog,0.0,0,64,512,4,0.953125,0,0.890625,3002.22265625,2752.0490797546013,3119.0659025787963,26854,651,2790.015845685145,719255.0,1.0,0.9999944386900335,0.0,0.0,0.0,5.561309966562623e-06,0.318359375,-0.0008369925781153142,0.09486871212720871,-0.00022500994964502752,0.9017434206325561,-0.00022500994964502752,7.999999979801942e-06,1,0.0,0.05405773967504501,1.348465919494629,201.1986083984375,0.46875,0.318359375,21.203689575195312,140.65093612670898,1863.7885856628418,7.5,2014.4287147521973,150.64012908935547,0.0067882537841796875,0.09145879745483398,3.419133186340332,44.97348093986511,147.18843388557434,192.34168219566345,465.55749225616455,428.29968667030334,81.63673996925354,147.25872898101807,150.85458874702454,0.0,0.0,0,17,,17
finelog,0.0,0,64,512,2,0.71875,0,0.703125,2478.958984375,2188.1207729468597,2676.3475409836065,23859,319,2411.9658079799956,598075.0,0.0,1.0,0.0,0.0,0.0,0.0,0.404296875,-0.023721639066934586,0.12942376732826233,-8.077389793470502e-06,0.8676014154916629,-8.077389793470502e-06,7.999999979801942e-06,1,0.0,0.07506733387708664,1.1716746091842651,28.08149528503418,0.5625,0.404296875,21.832080841064453,140.57871627807617,1863.176570892334,7.5,2014.4287147521973,151.25214385986328,0.00798487663269043,0.07138228416442871,2.9453682899475098,43.19208216667175,138.68992280960083,182.03820514678955,461.42279839515686,693.5649092197418,81.85265922546387,138.75854396820068,426.6822907924652,0.0,0.0,0,18,,18
finelog,0.0,0,64,512,4,0.8125,0,0.765625,2556.279296875,2093.8447488584475,2901.921501706485,20704,473,2699.305868189104,610144.0,0.0,1.0,0.0,0.0,0.0,0.0,0.427734375,-0.029633240774273872,0.14690423011779785,-6.726852734573185e-05,0.8370082812616602,-6.726852734573185e-05,7.999999979801942e-06,1,0.0,0.05800345540046692,1.3862186670303345,100.81414794921875,0.546875,0.427734375,22.089065551757812,149.64979553222656,1862.121425628662,7.6,2014.4287147521973,152.30728912353516,0.00617218017578125,0.06627368927001953,2.776560068130493,43.89715623855591,140.9724726676941,185.03911590576172,460.6599955558777,836.5690820217133,192.18002724647522,141.05319905281067,456.53414034843445,0.0,0.0,0,19,,19
finelog,0.0,0,64,512,4,0.609375,0,0.546875,2340.130859375,1941.570446735395,2864.9321266968327,22150,378,2765.9895078407653,583260.0,0.0,1.0,0.0,0.0,0.0,0.0,0.568359375,1.252095080417348e-05,0.06844891607761383,-0.00018067998462356627,0.7792838950990699,-0.00018067998462356627,7.999999979801942e-06,1,0.0,0.043685052543878555,4.070535659790039,1505.0963134765625,0.625,0.568359375,21.034622192382812,149.92840194702148,1863.1474533081055,7.5,2014.4287147521973,151.2812614440918,0.006177186965942383,0.07135415077209473,2.839724063873291,43.34166693687439,138.39206886291504,181.88161945343018,455.8801598548889,1160.3470537662506,81.77262330055237,138.45159482955933,893.8027949333191,0.0,0.0,0,20,38.88026189804077,20
finelog,0.0,0,64,512,7,0.5,0,0.40625,2234.611328125,1952.3653846153845,2427.7269736842104,20143,394,2265.943199919693,566081.0,0.0,1.0,0.0,0.0,0.0,0.0,0.40625,-0.019281430169939995,0.11397971957921982,-0.00015442364383488894,0.7739015080878744,-0.00015442364383488894,7.999999979801942e-06,1,0.0,0.06565333157777786,159.1624298095703,88023.6875,0.5,0.40625,21.767887115478516,149.9852638244629,1863.4467658996582,7.5,2014.4287147521973,150.98194885253906,0.006381511688232422,0.07059240341186523,2.5255768299102783,41.81991696357727,135.55581903457642,177.515766620636,459.9064302444458,1965.6425108909607,81.717937707901,135.60997200012207,1703.8462085723877,0.0,0.0,0,21,,21