File size: 41,388 Bytes

903307f

timestamp,event,step,epoch,key,value
1773761894.8111923,train_step,10,1,train/step_loss,1.8352766107110416
1773761894.8111923,train_step,10,1,train/step_real_loss,1.028106451034546
1773761894.8111923,train_step,10,1,train/lr,5.2631578947368424e-05
1773761894.8111923,train_step,10,1,train/step_canary_loss,14.75
1773761894.8111923,train_step,10,1,perf/step_duration_sec,6.234770041890442
1773761894.8111923,train_step,10,1,perf/samples_per_sec,5.453288536956348
1773761894.8111923,train_step,10,1,perf/tokens_per_sec,3980.098677781523
1773761894.8111923,train_step,10,1,perf/logical_batch_size,34.0
1773761894.8111923,train_step,10,1,perf/logical_token_count,24815.0
1773761894.8111923,train_step,10,1,perf/gradient_accumulation_steps,4.0
1773761894.8111923,train_step,10,1,system/cuda_memory_allocated_gb,15.915565013885498
1773761894.8111923,train_step,10,1,system/cuda_max_memory_allocated_gb,94.4762544631958
1773761950.7987902,train_step,20,1,train/step_loss,1.0323970019817352
1773761950.7987902,train_step,20,1,train/step_real_loss,1.0323970019817352
1773761950.7987902,train_step,20,1,train/lr,9.999797424944042e-05
1773761950.7987902,train_step,20,1,perf/step_duration_sec,5.150427320972085
1773761950.7987902,train_step,20,1,perf/samples_per_sec,6.213076703305535
1773761950.7987902,train_step,20,1,perf/tokens_per_sec,4927.9406189561805
1773761950.7987902,train_step,20,1,perf/logical_batch_size,32.0
1773761950.7987902,train_step,20,1,perf/logical_token_count,25381.0
1773761950.7987902,train_step,20,1,perf/gradient_accumulation_steps,4.0
1773761950.7987902,train_step,20,1,system/cuda_memory_allocated_gb,15.915565013885498
1773761950.7987902,train_step,20,1,system/cuda_max_memory_allocated_gb,94.4762544631958
1773762007.3004794,train_step,30,1,train/step_loss,0.8551503717899323
1773762007.3004794,train_step,30,1,train/step_real_loss,0.8551503717899323
1773762007.3004794,train_step,30,1,train/lr,9.975508273693644e-05
1773762007.3004794,train_step,30,1,perf/step_duration_sec,5.69609066285193
1773762007.3004794,train_step,30,1,perf/samples_per_sec,5.617888108539722
1773762007.3004794,train_step,30,1,perf/tokens_per_sec,4432.689276641233
1773762007.3004794,train_step,30,1,perf/logical_batch_size,32.0
1773762007.3004794,train_step,30,1,perf/logical_token_count,25249.0
1773762007.3004794,train_step,30,1,perf/gradient_accumulation_steps,4.0
1773762007.3004794,train_step,30,1,system/cuda_memory_allocated_gb,15.915565013885498
1773762007.3004794,train_step,30,1,system/cuda_max_memory_allocated_gb,94.4762544631958
1773762065.1568909,train_step,40,1,train/step_loss,0.8950656801462173
1773762065.1568909,train_step,40,1,train/step_real_loss,0.8950656801462173
1773762065.1568909,train_step,40,1,train/lr,9.910929512300672e-05
1773762065.1568909,train_step,40,1,perf/step_duration_sec,6.2338299779221416
1773762065.1568909,train_step,40,1,perf/samples_per_sec,5.133280842328368
1773762065.1568909,train_step,40,1,perf/tokens_per_sec,4016.4714290693023
1773762065.1568909,train_step,40,1,perf/logical_batch_size,32.0
1773762065.1568909,train_step,40,1,perf/logical_token_count,25038.0
1773762065.1568909,train_step,40,1,perf/gradient_accumulation_steps,4.0
1773762065.1568909,train_step,40,1,system/cuda_memory_allocated_gb,15.915565013885498
1773762065.1568909,train_step,40,1,system/cuda_max_memory_allocated_gb,94.4762544631958
1773762121.2732794,train_step,50,1,train/step_loss,0.8450518101453781
1773762121.2732794,train_step,50,1,train/step_real_loss,0.8450518101453781
1773762121.2732794,train_step,50,1,train/lr,9.806584072891234e-05
1773762121.2732794,train_step,50,1,perf/step_duration_sec,5.423216213937849
1773762121.2732794,train_step,50,1,perf/samples_per_sec,5.900557664980961
1773762121.2732794,train_step,50,1,perf/tokens_per_sec,5482.724425329497
1773762121.2732794,train_step,50,1,perf/logical_batch_size,32.0
1773762121.2732794,train_step,50,1,perf/logical_token_count,29734.0
1773762121.2732794,train_step,50,1,perf/gradient_accumulation_steps,4.0
1773762121.2732794,train_step,50,1,system/cuda_memory_allocated_gb,15.915565013885498
1773762121.2732794,train_step,50,1,system/cuda_max_memory_allocated_gb,94.4762544631958
1773762134.073646,eval_step,50,1,eval/loss,0.8533683589253671
1773762134.073646,eval_step,50,1,eval/duration_sec,12.798461285419762
1773762190.2760499,train_step,60,1,train/step_loss,1.114606170943289
1773762190.2760499,train_step,60,1,train/step_real_loss,0.8427969664335251
1773762190.2760499,train_step,60,1,train/lr,9.663316901718597e-05
1773762190.2760499,train_step,60,1,train/step_canary_loss,9.8125
1773762190.2760499,train_step,60,1,perf/step_duration_sec,5.968041606713086
1773762190.2760499,train_step,60,1,perf/samples_per_sec,5.5294520673046765
1773762190.2760499,train_step,60,1,perf/tokens_per_sec,4259.353683360148
1773762190.2760499,train_step,60,1,perf/logical_batch_size,33.0
1773762190.2760499,train_step,60,1,perf/logical_token_count,25420.0
1773762190.2760499,train_step,60,1,perf/gradient_accumulation_steps,4.0
1773762190.2760499,train_step,60,1,system/cuda_memory_allocated_gb,15.915565013885498
1773762190.2760499,train_step,60,1,system/cuda_max_memory_allocated_gb,94.4762544631958
1773762249.207255,train_step,70,1,train/step_loss,1.1624658794114084
1773762249.207255,train_step,70,1,train/step_real_loss,0.8745741844177246
1773762249.207255,train_step,70,1,train/lr,9.48228811713756e-05
1773762249.207255,train_step,70,1,train/step_canary_loss,10.375
1773762249.207255,train_step,70,1,perf/step_duration_sec,6.334186799824238
1773762249.207255,train_step,70,1,perf/samples_per_sec,5.209824250986045
1773762249.207255,train_step,70,1,perf/tokens_per_sec,3829.378697936894
1773762249.207255,train_step,70,1,perf/logical_batch_size,33.0
1773762249.207255,train_step,70,1,perf/logical_token_count,24256.0
1773762249.207255,train_step,70,1,perf/gradient_accumulation_steps,4.0
1773762249.207255,train_step,70,1,system/cuda_memory_allocated_gb,16.205660820007324
1773762249.207255,train_step,70,1,system/cuda_max_memory_allocated_gb,94.4762544631958
1773762305.719841,train_step,80,1,train/step_loss,1.157600255573497
1773762305.719841,train_step,80,1,train/step_real_loss,0.8803409039974213
1773762305.719841,train_step,80,1,train/lr,9.26496361544538e-05
1773762305.719841,train_step,80,1,train/step_canary_loss,5.59375
1773762305.719841,train_step,80,1,perf/step_duration_sec,5.699764240998775
1773762305.719841,train_step,80,1,perf/samples_per_sec,5.9651590070052
1773762305.719841,train_step,80,1,perf/tokens_per_sec,4690.720329743854
1773762305.719841,train_step,80,1,perf/logical_batch_size,34.0
1773762305.719841,train_step,80,1,perf/logical_token_count,26736.0
1773762305.719841,train_step,80,1,perf/gradient_accumulation_steps,4.0
1773762305.719841,train_step,80,1,system/cuda_memory_allocated_gb,16.205660820007324
1773762305.719841,train_step,80,1,system/cuda_max_memory_allocated_gb,94.4762544631958
1773762363.2025864,train_step,90,1,train/step_loss,0.8746908158063889
1773762363.2025864,train_step,90,1,train/step_real_loss,0.8746908158063889
1773762363.2025864,train_step,90,1,train/lr,9.013103200659241e-05
1773762363.2025864,train_step,90,1,perf/step_duration_sec,5.422750173136592
1773762363.2025864,train_step,90,1,perf/samples_per_sec,5.901064769408466
1773762363.2025864,train_step,90,1,perf/tokens_per_sec,4211.147346069116
1773762363.2025864,train_step,90,1,perf/logical_batch_size,32.0
1773762363.2025864,train_step,90,1,perf/logical_token_count,22836.0
1773762363.2025864,train_step,90,1,perf/gradient_accumulation_steps,4.0
1773762363.2025864,train_step,90,1,system/cuda_memory_allocated_gb,15.915565013885498
1773762363.2025864,train_step,90,1,system/cuda_max_memory_allocated_gb,94.4762544631958
1773762419.340239,train_step,100,1,train/step_loss,1.227062124194521
1773762419.340239,train_step,100,1,train/step_real_loss,0.9060328304767609
1773762419.340239,train_step,100,1,train/lr,8.728746334350483e-05
1773762419.340239,train_step,100,1,train/step_canary_loss,11.5
1773762419.340239,train_step,100,1,perf/step_duration_sec,5.688522285781801
1773762419.340239,train_step,100,1,perf/samples_per_sec,5.801155087760134
1773762419.340239,train_step,100,1,perf/tokens_per_sec,4329.068036096396
1773762419.340239,train_step,100,1,perf/logical_batch_size,33.0
1773762419.340239,train_step,100,1,perf/logical_token_count,24626.0
1773762419.340239,train_step,100,1,perf/gradient_accumulation_steps,4.0
1773762419.340239,train_step,100,1,system/cuda_memory_allocated_gb,16.205660820007324
1773762419.340239,train_step,100,1,system/cuda_max_memory_allocated_gb,94.4762544631958
1773762432.1276581,eval_step,100,1,eval/loss,0.829470864473245
1773762432.1276581,eval_step,100,1,eval/duration_sec,12.785310188308358
1773762489.3303852,train_step,110,1,train/step_loss,1.011215921604272
1773762489.3303852,train_step,110,1,train/step_real_loss,0.9119570553302765
1773762489.3303852,train_step,110,1,train/lr,8.414195620927492e-05
1773762489.3303852,train_step,110,1,train/step_canary_loss,4.1875
1773762489.3303852,train_step,110,1,perf/step_duration_sec,5.6974792359396815
1773762489.3303852,train_step,110,1,perf/samples_per_sec,5.792035149831895
1773762489.3303852,train_step,110,1,perf/tokens_per_sec,4259.603062159705
1773762489.3303852,train_step,110,1,perf/logical_batch_size,33.0
1773762489.3303852,train_step,110,1,perf/logical_token_count,24269.0
1773762489.3303852,train_step,110,1,perf/gradient_accumulation_steps,4.0
1773762489.3303852,train_step,110,1,system/cuda_memory_allocated_gb,15.915565013885498
1773762489.3303852,train_step,110,1,system/cuda_max_memory_allocated_gb,94.4762544631958
1773762544.9202547,train_step,120,1,train/step_loss,0.7180200964212418
1773762544.9202547,train_step,120,1,train/step_real_loss,0.7180200964212418
1773762544.9202547,train_step,120,1,train/lr,8.071998162096612e-05
1773762544.9202547,train_step,120,1,perf/step_duration_sec,5.694211829919368
1773762544.9202547,train_step,120,1,perf/samples_per_sec,5.619741758088605
1773762544.9202547,train_step,120,1,perf/tokens_per_sec,4598.00245969612
1773762544.9202547,train_step,120,1,perf/logical_batch_size,32.0
1773762544.9202547,train_step,120,1,perf/logical_token_count,26182.0
1773762544.9202547,train_step,120,1,perf/gradient_accumulation_steps,4.0
1773762544.9202547,train_step,120,1,system/cuda_memory_allocated_gb,15.915565013885498
1773762544.9202547,train_step,120,1,system/cuda_max_memory_allocated_gb,94.4762544631958
1773762601.86658,train_step,130,1,train/step_loss,0.8463245183229446
1773762601.86658,train_step,130,1,train/step_real_loss,0.8463245183229446
1773762601.86658,train_step,130,1,train/lr,7.704924931484997e-05
1773762601.86658,train_step,130,1,perf/step_duration_sec,5.1536158989183605
1773762601.86658,train_step,130,1,perf/samples_per_sec,6.209232629602092
1773762601.86658,train_step,130,1,perf/tokens_per_sec,4267.683201733388
1773762601.86658,train_step,130,1,perf/logical_batch_size,32.0
1773762601.86658,train_step,130,1,perf/logical_token_count,21994.0
1773762601.86658,train_step,130,1,perf/gradient_accumulation_steps,4.0
1773762601.86658,train_step,130,1,system/cuda_memory_allocated_gb,15.915565013885498
1773762601.86658,train_step,130,1,system/cuda_max_memory_allocated_gb,101.70386934280396
1773762659.5638738,train_step,140,1,train/step_loss,0.9742496162652969
1773762659.5638738,train_step,140,1,train/step_real_loss,0.9742496162652969
1773762659.5638738,train_step,140,1,train/lr,7.315948336441117e-05
1773762659.5638738,train_step,140,1,perf/step_duration_sec,5.969049285165966
1773762659.5638738,train_step,140,1,perf/samples_per_sec,5.360987733762741
1773762659.5638738,train_step,140,1,perf/tokens_per_sec,3927.091045847888
1773762659.5638738,train_step,140,1,perf/logical_batch_size,32.0
1773762659.5638738,train_step,140,1,perf/logical_token_count,23441.0
1773762659.5638738,train_step,140,1,perf/gradient_accumulation_steps,4.0
1773762659.5638738,train_step,140,1,system/cuda_memory_allocated_gb,15.915565013885498
1773762659.5638738,train_step,140,1,system/cuda_max_memory_allocated_gb,101.70386934280396
1773762717.6212678,train_step,150,1,train/step_loss,0.914526179432869
1773762717.6212678,train_step,150,1,train/step_real_loss,0.914526179432869
1773762717.6212678,train_step,150,1,train/lr,6.908218148708247e-05
1773762717.6212678,train_step,150,1,perf/step_duration_sec,5.703813333064318
1773762717.6212678,train_step,150,1,perf/samples_per_sec,5.610281776666824
1773762717.6212678,train_step,150,1,perf/tokens_per_sec,4645.31331108013
1773762717.6212678,train_step,150,1,perf/logical_batch_size,32.0
1773762717.6212678,train_step,150,1,perf/logical_token_count,26496.0
1773762717.6212678,train_step,150,1,perf/gradient_accumulation_steps,4.0
1773762717.6212678,train_step,150,1,system/cuda_memory_allocated_gb,15.915565013885498
1773762717.6212678,train_step,150,1,system/cuda_max_memory_allocated_gb,101.70386934280396
1773762730.4167268,eval_step,150,1,eval/loss,0.8174186625923866
1773762730.4167268,eval_step,150,1,eval/duration_sec,12.793565314263105
1773762785.9700294,train_step,160,1,train/step_loss,0.9113822728395462
1773762785.9700294,train_step,160,1,train/step_real_loss,0.9113822728395462
1773762785.9700294,train_step,160,1,train/lr,6.485035998874356e-05
1773762785.9700294,train_step,160,1,perf/step_duration_sec,5.425368802621961
1773762785.9700294,train_step,160,1,perf/samples_per_sec,5.898216538668322
1773762785.9700294,train_step,160,1,perf/tokens_per_sec,4679.128907832313
1773762785.9700294,train_step,160,1,perf/logical_batch_size,32.0
1773762785.9700294,train_step,160,1,perf/logical_token_count,25386.0
1773762785.9700294,train_step,160,1,perf/gradient_accumulation_steps,4.0
1773762785.9700294,train_step,160,1,system/cuda_memory_allocated_gb,15.915565013885498
1773762785.9700294,train_step,160,1,system/cuda_max_memory_allocated_gb,101.70386934280396
1773762841.0786932,train_step,170,1,train/step_loss,0.8910564256436897
1773762841.0786932,train_step,170,1,train/step_real_loss,0.8441949039697647
1773762841.0786932,train_step,170,1,train/lr,6.049828641131825e-05
1773762841.0786932,train_step,170,1,train/step_canary_loss,2.390625
1773762841.0786932,train_step,170,1,perf/step_duration_sec,5.690398690290749
1773762841.0786932,train_step,170,1,perf/samples_per_sec,5.799242161415912
1773762841.0786932,train_step,170,1,perf/tokens_per_sec,4355.582332445254
1773762841.0786932,train_step,170,1,perf/logical_batch_size,33.0
1773762841.0786932,train_step,170,1,perf/logical_token_count,24785.0
1773762841.0786932,train_step,170,1,perf/gradient_accumulation_steps,4.0
1773762841.0786932,train_step,170,1,system/cuda_memory_allocated_gb,16.205660820007324
1773762841.0786932,train_step,170,1,system/cuda_max_memory_allocated_gb,101.70386934280396
1773762898.8441253,train_step,180,1,train/step_loss,1.0836328773787527
1773762898.8441253,train_step,180,1,train/step_real_loss,0.8538245260715485
1773762898.8441253,train_step,180,1,train/lr,5.6061202048379124e-05
1773762898.8441253,train_step,180,1,train/step_canary_loss,8.4375
1773762898.8441253,train_step,180,1,perf/step_duration_sec,5.692985306028277
1773762898.8441253,train_step,180,1,perf/samples_per_sec,5.7966072677293665
1773762898.8441253,train_step,180,1,perf/tokens_per_sec,4476.7373583439585
1773762898.8441253,train_step,180,1,perf/logical_batch_size,33.0
1773762898.8441253,train_step,180,1,perf/logical_token_count,25486.0
1773762898.8441253,train_step,180,1,perf/gradient_accumulation_steps,4.0
1773762898.8441253,train_step,180,1,system/cuda_memory_allocated_gb,15.915565013885498
1773762898.8441253,train_step,180,1,system/cuda_max_memory_allocated_gb,101.70386934280396
1773762934.6675656,train_epoch,184,1,train/epoch_loss,0.9805822407983543
1773762934.6675656,train_epoch,184,1,train/epoch_real_loss,0.9071368900552877
1773762934.6675656,train_epoch,184,1,train/epoch_canary_loss,8.42140839386602
1773762934.6675656,train_epoch,184,1,perf/epoch_duration_sec,1085.6805565529503
1773762934.6675656,train_epoch,184,1,perf/epoch_samples_per_sec,43.826887856473604
1773762934.6675656,train_epoch,184,1,perf/epoch_tokens_per_sec,34574.90951038253
1773762934.6675656,train_epoch,184,1,perf/epoch_samples,47582.0
1773762934.6675656,train_epoch,184,1,perf/epoch_tokens,37537307.0
1773762934.6675656,train_epoch,184,1,system/cuda_epoch_peak_memory_gb,101.70386934280396
1773762934.6675656,train_epoch,184,1,eval/loss,0.8121764394335258
1773762934.6675656,train_epoch,184,1,eval/duration_sec,12.826699289958924
1773762949.1512606,audit_epoch,184,1,audit/delta,1e-05
1773762949.1512606,audit_epoch,184,1,audit/num_canaries,500.0
1773762949.1512606,audit_epoch,184,1,audit/num_members,250.0
1773762949.1512606,audit_epoch,184,1,audit/paper_guess_fraction,0.2
1773762949.1512606,audit_epoch,184,1,audit/paper_guess_steps,20.0
1773762949.1512606,audit_epoch,184,1,audit/loss/auc,0.907944
1773762949.1512606,audit_epoch,184,1,audit/loss/empirical_epsilon/0.05,3.4791953936219215
1773762949.1512606,audit_epoch,184,1,audit/loss/empirical_epsilon/0.01,3.023197554051876
1773762949.1512606,audit_epoch,184,1,audit/loss/empirical_epsilon_details/0.05/epsilon,3.4791953936219215
1773762949.1512606,audit_epoch,184,1,audit/loss/empirical_epsilon_details/0.05/num_guesses,100.0
1773762949.1512606,audit_epoch,184,1,audit/loss/empirical_epsilon_details/0.05/correct_guesses,100.0
1773762949.1512606,audit_epoch,184,1,audit/loss/empirical_epsilon_details/0.01/epsilon,3.023197554051876
1773762949.1512606,audit_epoch,184,1,audit/loss/empirical_epsilon_details/0.01/num_guesses,100.0
1773762949.1512606,audit_epoch,184,1,audit/loss/empirical_epsilon_details/0.01/correct_guesses,100.0
1773762949.1512606,audit_epoch,184,1,audit/embedding/auc,0.876048
1773762949.1512606,audit_epoch,184,1,audit/embedding/empirical_epsilon/0.05,3.4791953936219215
1773762949.1512606,audit_epoch,184,1,audit/embedding/empirical_epsilon/0.01,3.023197554051876
1773762949.1512606,audit_epoch,184,1,audit/embedding/empirical_epsilon_details/0.05/epsilon,3.4791953936219215
1773762949.1512606,audit_epoch,184,1,audit/embedding/empirical_epsilon_details/0.05/num_guesses,100.0
1773762949.1512606,audit_epoch,184,1,audit/embedding/empirical_epsilon_details/0.05/correct_guesses,100.0
1773762949.1512606,audit_epoch,184,1,audit/embedding/empirical_epsilon_details/0.01/epsilon,3.023197554051876
1773762949.1512606,audit_epoch,184,1,audit/embedding/empirical_epsilon_details/0.01/num_guesses,100.0
1773762949.1512606,audit_epoch,184,1,audit/embedding/empirical_epsilon_details/0.01/correct_guesses,100.0
1773762949.1512606,audit_epoch,184,1,perf/audit_duration_sec,8.130579099990427
1773762984.332577,train_step,190,2,train/step_loss,0.8655764758586884
1773762984.332577,train_step,190,2,train/step_real_loss,0.8655764758586884
1773762984.332577,train_step,190,2,train/lr,5.157503657571385e-05
1773762984.332577,train_step,190,2,perf/step_duration_sec,5.690848938189447
1773762984.332577,train_step,190,2,perf/samples_per_sec,5.623062630472995
1773762984.332577,train_step,190,2,perf/tokens_per_sec,4602.301042334944
1773762984.332577,train_step,190,2,perf/logical_batch_size,32.0
1773762984.332577,train_step,190,2,perf/logical_token_count,26191.0
1773762984.332577,train_step,190,2,perf/gradient_accumulation_steps,4.0
1773762984.332577,train_step,190,2,system/cuda_memory_allocated_gb,15.915565013885498
1773762984.332577,train_step,190,2,system/cuda_max_memory_allocated_gb,87.30217599868774
1773763040.7157884,train_step,200,2,train/step_loss,0.8589679941986547
1773763040.7157884,train_step,200,2,train/step_real_loss,0.8308791071176529
1773763040.7157884,train_step,200,2,train/lr,4.7076117107656534e-05
1773763040.7157884,train_step,200,2,train/step_canary_loss,1.7578125
1773763040.7157884,train_step,200,2,perf/step_duration_sec,5.69332688068971
1773763040.7157884,train_step,200,2,perf/samples_per_sec,5.796259496697344
1773763040.7157884,train_step,200,2,perf/tokens_per_sec,5059.0806752537455
1773763040.7157884,train_step,200,2,perf/logical_batch_size,33.0
1773763040.7157884,train_step,200,2,perf/logical_token_count,28803.0
1773763040.7157884,train_step,200,2,perf/gradient_accumulation_steps,4.0
1773763040.7157884,train_step,200,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763040.7157884,train_step,200,2,system/cuda_max_memory_allocated_gb,87.30217599868774
1773763053.5208263,eval_step,200,2,eval/loss,0.8110936123591204
1773763053.5208263,eval_step,200,2,eval/duration_sec,12.803085402119905
1773763110.1544352,train_step,210,2,train/step_loss,0.7684839069843292
1773763110.1544352,train_step,210,2,train/step_real_loss,0.7684839069843292
1773763110.1544352,train_step,210,2,train/lr,4.2600874035126046e-05
1773763110.1544352,train_step,210,2,perf/step_duration_sec,5.4159107422456145
1773763110.1544352,train_step,210,2,perf/samples_per_sec,5.908516872405425
1773763110.1544352,train_step,210,2,perf/tokens_per_sec,4630.061534138701
1773763110.1544352,train_step,210,2,perf/logical_batch_size,32.0
1773763110.1544352,train_step,210,2,perf/logical_token_count,25076.0
1773763110.1544352,train_step,210,2,perf/gradient_accumulation_steps,4.0
1773763110.1544352,train_step,210,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763110.1544352,train_step,210,2,system/cuda_max_memory_allocated_gb,87.30229806900024
1773763168.1105735,train_step,220,2,train/step_loss,0.8040641099214554
1773763168.1105735,train_step,220,2,train/step_real_loss,0.8040641099214554
1773763168.1105735,train_step,220,2,train/lr,3.818554602737332e-05
1773763168.1105735,train_step,220,2,perf/step_duration_sec,5.967140641994774
1773763168.1105735,train_step,220,2,perf/samples_per_sec,5.36270249351834
1773763168.1105735,train_step,220,2,perf/tokens_per_sec,4430.932935269528
1773763168.1105735,train_step,220,2,perf/logical_batch_size,32.0
1773763168.1105735,train_step,220,2,perf/logical_token_count,26440.0
1773763168.1105735,train_step,220,2,perf/gradient_accumulation_steps,4.0
1773763168.1105735,train_step,220,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763168.1105735,train_step,220,2,system/cuda_max_memory_allocated_gb,87.30229806900024
1773763224.221138,train_step,230,2,train/step_loss,0.806927278637886
1773763224.221138,train_step,230,2,train/step_real_loss,0.806927278637886
1773763224.221138,train_step,230,2,train/lr,3.386588658621128e-05
1773763224.221138,train_step,230,2,perf/step_duration_sec,5.424597659613937
1773763224.221138,train_step,230,2,perf/samples_per_sec,5.899055009782496
1773763224.221138,train_step,230,2,perf/tokens_per_sec,4482.175734620363
1773763224.221138,train_step,230,2,perf/logical_batch_size,32.0
1773763224.221138,train_step,230,2,perf/logical_token_count,24314.0
1773763224.221138,train_step,230,2,perf/gradient_accumulation_steps,4.0
1773763224.221138,train_step,230,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763224.221138,train_step,230,2,system/cuda_max_memory_allocated_gb,87.30229806900024
1773763282.06351,train_step,240,2,train/step_loss,0.9124463796615601
1773763282.06351,train_step,240,2,train/step_real_loss,0.9124463796615601
1773763282.06351,train_step,240,2,train/lr,2.967687452893051e-05
1773763282.06351,train_step,240,2,perf/step_duration_sec,5.692487298045307
1773763282.06351,train_step,240,2,perf/samples_per_sec,5.621444251792743
1773763282.06351,train_step,240,2,perf/tokens_per_sec,4766.282044988772
1773763282.06351,train_step,240,2,perf/logical_batch_size,32.0
1773763282.06351,train_step,240,2,perf/logical_token_count,27132.0
1773763282.06351,train_step,240,2,perf/gradient_accumulation_steps,4.0
1773763282.06351,train_step,240,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763282.06351,train_step,240,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773763337.7170568,train_step,250,2,train/step_loss,0.7962393760681152
1773763337.7170568,train_step,250,2,train/step_real_loss,0.7962393760681152
1773763337.7170568,train_step,250,2,train/lr,2.5652430744289756e-05
1773763337.7170568,train_step,250,2,perf/step_duration_sec,5.419141778722405
1773763337.7170568,train_step,250,2,perf/samples_per_sec,5.904994057480479
1773763337.7170568,train_step,250,2,perf/tokens_per_sec,4963.885629569528
1773763337.7170568,train_step,250,2,perf/logical_batch_size,32.0
1773763337.7170568,train_step,250,2,perf/logical_token_count,26900.0
1773763337.7170568,train_step,250,2,perf/gradient_accumulation_steps,4.0
1773763337.7170568,train_step,250,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763337.7170568,train_step,250,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773763350.5270941,eval_step,250,2,eval/loss,0.8086002505360506
1773763350.5270941,eval_step,250,2,eval/duration_sec,12.808165564201772
1773763406.908792,train_step,260,2,train/step_loss,0.7549401223659515
1773763406.908792,train_step,260,2,train/step_real_loss,0.7549401223659515
1773763406.908792,train_step,260,2,train/lr,2.1825143515174878e-05
1773763406.908792,train_step,260,2,perf/step_duration_sec,5.96535021904856
1773763406.908792,train_step,260,2,perf/samples_per_sec,5.364312039520762
1773763406.908792,train_step,260,2,perf/tokens_per_sec,4828.551374573625
1773763406.908792,train_step,260,2,perf/logical_batch_size,32.0
1773763406.908792,train_step,260,2,perf/logical_token_count,28804.0
1773763406.908792,train_step,260,2,perf/gradient_accumulation_steps,4.0
1773763406.908792,train_step,260,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763406.908792,train_step,260,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773763462.8593152,train_step,270,2,train/step_loss,0.8835187554359436
1773763462.8593152,train_step,270,2,train/step_real_loss,0.8835187554359436
1773763462.8593152,train_step,270,2,train/lr,1.822600463214922e-05
1773763462.8593152,train_step,270,2,perf/step_duration_sec,6.238990655634552
1773763462.8593152,train_step,270,2,perf/samples_per_sec,5.129034769606553
1773763462.8593152,train_step,270,2,perf/tokens_per_sec,4065.8820312690445
1773763462.8593152,train_step,270,2,perf/logical_batch_size,32.0
1773763462.8593152,train_step,270,2,perf/logical_token_count,25367.0
1773763462.8593152,train_step,270,2,perf/gradient_accumulation_steps,4.0
1773763462.8593152,train_step,270,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763462.8593152,train_step,270,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773763519.993293,train_step,280,2,train/step_loss,0.9217604398727417
1773763519.993293,train_step,280,2,train/step_real_loss,0.9217604398727417
1773763519.993293,train_step,280,2,train/lr,1.488415843473942e-05
1773763519.993293,train_step,280,2,perf/step_duration_sec,5.145696292165667
1773763519.993293,train_step,280,2,perf/samples_per_sec,6.2187890973511335
1773763519.993293,train_step,280,2,perf/tokens_per_sec,4732.109828765628
1773763519.993293,train_step,280,2,perf/logical_batch_size,32.0
1773763519.993293,train_step,280,2,perf/logical_token_count,24350.0
1773763519.993293,train_step,280,2,perf/gradient_accumulation_steps,4.0
1773763519.993293,train_step,280,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763519.993293,train_step,280,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773763577.139687,train_step,290,2,train/step_loss,0.9289029836654663
1773763577.139687,train_step,290,2,train/step_real_loss,0.9481655806303024
1773763577.139687,train_step,290,2,train/lr,1.1826665812616183e-05
1773763577.139687,train_step,290,2,train/step_canary_loss,0.3125
1773763577.139687,train_step,290,2,perf/step_duration_sec,5.694677841849625
1773763577.139687,train_step,290,2,perf/samples_per_sec,5.794884437094274
1773763577.139687,train_step,290,2,perf/tokens_per_sec,4061.3359776095867
1773763577.139687,train_step,290,2,perf/logical_batch_size,33.0
1773763577.139687,train_step,290,2,perf/logical_token_count,23128.0
1773763577.139687,train_step,290,2,perf/gradient_accumulation_steps,4.0
1773763577.139687,train_step,290,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763577.139687,train_step,290,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773763634.8883243,train_step,300,2,train/step_loss,0.9466440713766849
1773763634.8883243,train_step,300,2,train/step_real_loss,0.8473204374313354
1773763634.8883243,train_step,300,2,train/lr,9.078285077691178e-06
1773763634.8883243,train_step,300,2,train/step_canary_loss,4.125
1773763634.8883243,train_step,300,2,perf/step_duration_sec,6.2339927861467
1773763634.8883243,train_step,300,2,perf/samples_per_sec,5.293557617412269
1773763634.8883243,train_step,300,2,perf/tokens_per_sec,4104.752905210987
1773763634.8883243,train_step,300,2,perf/logical_batch_size,33.0
1773763634.8883243,train_step,300,2,perf/logical_token_count,25589.0
1773763634.8883243,train_step,300,2,perf/gradient_accumulation_steps,4.0
1773763634.8883243,train_step,300,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763634.8883243,train_step,300,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773763647.6901762,eval_step,300,2,eval/loss,0.8076710475560945
1773763647.6901762,eval_step,300,2,eval/duration_sec,12.799929299857467
1773763704.81611,train_step,310,2,train/step_loss,0.9043312668800354
1773763704.81611,train_step,310,2,train/step_real_loss,0.9043312668800354
1773763704.81611,train_step,310,2,train/lr,6.661271481537157e-06
1773763704.81611,train_step,310,2,perf/step_duration_sec,5.423097257036716
1773763704.81611,train_step,310,2,perf/samples_per_sec,5.900687095087322
1773763704.81611,train_step,310,2,perf/tokens_per_sec,4037.360748341778
1773763704.81611,train_step,310,2,perf/logical_batch_size,32.0
1773763704.81611,train_step,310,2,perf/logical_token_count,21895.0
1773763704.81611,train_step,310,2,perf/gradient_accumulation_steps,4.0
1773763704.81611,train_step,310,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763704.81611,train_step,310,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773763760.6698298,train_step,320,2,train/step_loss,0.8735850304365158
1773763760.6698298,train_step,320,2,train/step_real_loss,0.8735850304365158
1773763760.6698298,train_step,320,2,train/lr,4.595197001556562e-06
1773763760.6698298,train_step,320,2,perf/step_duration_sec,5.15081740077585
1773763760.6698298,train_step,320,2,perf/samples_per_sec,6.212606176872034
1773763760.6698298,train_step,320,2,perf/tokens_per_sec,4842.532370928723
1773763760.6698298,train_step,320,2,perf/logical_batch_size,32.0
1773763760.6698298,train_step,320,2,perf/logical_token_count,24943.0
1773763760.6698298,train_step,320,2,perf/gradient_accumulation_steps,4.0
1773763760.6698298,train_step,320,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763760.6698298,train_step,320,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773763818.6010072,train_step,330,2,train/step_loss,0.7620985209941864
1773763818.6010072,train_step,330,2,train/step_real_loss,0.7620985209941864
1773763818.6010072,train_step,330,2,train/lr,2.8967918551955297e-06
1773763818.6010072,train_step,330,2,perf/step_duration_sec,6.058840225916356
1773763818.6010072,train_step,330,2,perf/samples_per_sec,5.281538843543317
1773763818.6010072,train_step,330,2,perf/tokens_per_sec,4943.025213289962
1773763818.6010072,train_step,330,2,perf/logical_batch_size,32.0
1773763818.6010072,train_step,330,2,perf/logical_token_count,29949.0
1773763818.6010072,train_step,330,2,perf/gradient_accumulation_steps,4.0
1773763818.6010072,train_step,330,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763818.6010072,train_step,330,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773763875.2762098,train_step,340,2,train/step_loss,0.9000806212425232
1773763875.2762098,train_step,340,2,train/step_real_loss,0.9000806212425232
1773763875.2762098,train_step,340,2,train/lr,1.5798090255558617e-06
1773763875.2762098,train_step,340,2,perf/step_duration_sec,5.154371100012213
1773763875.2762098,train_step,340,2,perf/samples_per_sec,6.2083228737496565
1773763875.2762098,train_step,340,2,perf/tokens_per_sec,4376.285595724094
1773763875.2762098,train_step,340,2,perf/logical_batch_size,32.0
1773763875.2762098,train_step,340,2,perf/logical_token_count,22557.0
1773763875.2762098,train_step,340,2,perf/gradient_accumulation_steps,4.0
1773763875.2762098,train_step,340,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763875.2762098,train_step,340,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773763932.3082285,train_step,350,2,train/step_loss,0.7404757142066956
1773763932.3082285,train_step,350,2,train/step_real_loss,0.7404757142066956
1773763932.3082285,train_step,350,2,train/lr,6.54912895420573e-07
1773763932.3082285,train_step,350,2,perf/step_duration_sec,5.694617530796677
1773763932.3082285,train_step,350,2,perf/samples_per_sec,5.619341391575985
1773763932.3082285,train_step,350,2,perf/tokens_per_sec,4990.501968974935
1773763932.3082285,train_step,350,2,perf/logical_batch_size,32.0
1773763932.3082285,train_step,350,2,perf/logical_token_count,28419.0
1773763932.3082285,train_step,350,2,perf/gradient_accumulation_steps,4.0
1773763932.3082285,train_step,350,2,system/cuda_memory_allocated_gb,15.915565013885498
1773763932.3082285,train_step,350,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773763945.1132307,eval_step,350,2,eval/loss,0.807514699605795
1773763945.1132307,eval_step,350,2,eval/duration_sec,12.803151289001107
1773764000.5064592,train_step,360,2,train/step_loss,0.8157700151205063
1773764000.5064592,train_step,360,2,train/step_real_loss,0.8157700151205063
1773764000.5064592,train_step,360,2,train/lr,1.295928914885336e-07
1773764000.5064592,train_step,360,2,perf/step_duration_sec,5.695594378747046
1773764000.5064592,train_step,360,2,perf/samples_per_sec,5.618377621729371
1773764000.5064592,train_step,360,2,perf/tokens_per_sec,4667.116060650316
1773764000.5064592,train_step,360,2,perf/logical_batch_size,32.0
1773764000.5064592,train_step,360,2,perf/logical_token_count,26582.0
1773764000.5064592,train_step,360,2,perf/gradient_accumulation_steps,4.0
1773764000.5064592,train_step,360,2,system/cuda_memory_allocated_gb,15.915565013885498
1773764000.5064592,train_step,360,2,system/cuda_max_memory_allocated_gb,94.47624206542969
1773764058.7973218,train_epoch,368,2,train/epoch_loss,0.856036927981092
1773764058.7973218,train_epoch,368,2,train/epoch_real_loss,0.8280306565727147
1773764058.7973218,train_epoch,368,2,train/epoch_canary_loss,3.6806401156922846
1773764058.7973218,train_epoch,368,2,perf/epoch_duration_sec,1096.7485609338619
1773764058.7973218,train_epoch,368,2,perf/epoch_samples_per_sec,43.38460217306761
1773764058.7973218,train_epoch,368,2,perf/epoch_tokens_per_sec,34225.847506959835
1773764058.7973218,train_epoch,368,2,perf/epoch_samples,47582.0
1773764058.7973218,train_epoch,368,2,perf/epoch_tokens,37537149.0
1773764058.7973218,train_epoch,368,2,system/cuda_epoch_peak_memory_gb,94.47624206542969
1773764058.7973218,train_epoch,368,2,eval/loss,0.8075161480750794
1773764058.7973218,train_epoch,368,2,eval/duration_sec,12.857198356185108
1773764072.6744637,audit_epoch,368,2,audit/delta,1e-05
1773764072.6744637,audit_epoch,368,2,audit/num_canaries,500.0
1773764072.6744637,audit_epoch,368,2,audit/num_members,250.0
1773764072.6744637,audit_epoch,368,2,audit/paper_guess_fraction,0.2
1773764072.6744637,audit_epoch,368,2,audit/paper_guess_steps,20.0
1773764072.6744637,audit_epoch,368,2,audit/loss/auc,0.968584
1773764072.6744637,audit_epoch,368,2,audit/loss/empirical_epsilon/0.05,3.4791953936219215
1773764072.6744637,audit_epoch,368,2,audit/loss/empirical_epsilon/0.01,3.023197554051876
1773764072.6744637,audit_epoch,368,2,audit/loss/empirical_epsilon_details/0.05/epsilon,3.4791953936219215
1773764072.6744637,audit_epoch,368,2,audit/loss/empirical_epsilon_details/0.05/num_guesses,100.0
1773764072.6744637,audit_epoch,368,2,audit/loss/empirical_epsilon_details/0.05/correct_guesses,100.0
1773764072.6744637,audit_epoch,368,2,audit/loss/empirical_epsilon_details/0.01/epsilon,3.023197554051876
1773764072.6744637,audit_epoch,368,2,audit/loss/empirical_epsilon_details/0.01/num_guesses,100.0
1773764072.6744637,audit_epoch,368,2,audit/loss/empirical_epsilon_details/0.01/correct_guesses,100.0
1773764072.6744637,audit_epoch,368,2,audit/embedding/auc,0.883776
1773764072.6744637,audit_epoch,368,2,audit/embedding/empirical_epsilon/0.05,3.4791953936219215
1773764072.6744637,audit_epoch,368,2,audit/embedding/empirical_epsilon/0.01,3.023197554051876
1773764072.6744637,audit_epoch,368,2,audit/embedding/empirical_epsilon_details/0.05/epsilon,3.4791953936219215
1773764072.6744637,audit_epoch,368,2,audit/embedding/empirical_epsilon_details/0.05/num_guesses,100.0
1773764072.6744637,audit_epoch,368,2,audit/embedding/empirical_epsilon_details/0.05/correct_guesses,100.0
1773764072.6744637,audit_epoch,368,2,audit/embedding/empirical_epsilon_details/0.01/epsilon,3.023197554051876
1773764072.6744637,audit_epoch,368,2,audit/embedding/empirical_epsilon_details/0.01/num_guesses,100.0
1773764072.6744637,audit_epoch,368,2,audit/embedding/empirical_epsilon_details/0.01/correct_guesses,100.0
1773764072.6744637,audit_epoch,368,2,perf/audit_duration_sec,7.556974642910063
1773764086.367738,audit_final,368,2,audit/delta,1e-05
1773764086.367738,audit_final,368,2,audit/num_canaries,500.0
1773764086.367738,audit_final,368,2,audit/num_members,250.0
1773764086.367738,audit_final,368,2,audit/paper_guess_fraction,0.2
1773764086.367738,audit_final,368,2,audit/paper_guess_steps,20.0
1773764086.367738,audit_final,368,2,audit/loss/auc,0.968584
1773764086.367738,audit_final,368,2,audit/loss/empirical_epsilon/0.05,3.4791953936219215
1773764086.367738,audit_final,368,2,audit/loss/empirical_epsilon/0.01,3.023197554051876
1773764086.367738,audit_final,368,2,audit/loss/empirical_epsilon_details/0.05/epsilon,3.4791953936219215
1773764086.367738,audit_final,368,2,audit/loss/empirical_epsilon_details/0.05/num_guesses,100.0
1773764086.367738,audit_final,368,2,audit/loss/empirical_epsilon_details/0.05/correct_guesses,100.0
1773764086.367738,audit_final,368,2,audit/loss/empirical_epsilon_details/0.01/epsilon,3.023197554051876
1773764086.367738,audit_final,368,2,audit/loss/empirical_epsilon_details/0.01/num_guesses,100.0
1773764086.367738,audit_final,368,2,audit/loss/empirical_epsilon_details/0.01/correct_guesses,100.0
1773764086.367738,audit_final,368,2,audit/embedding/auc,0.883776
1773764086.367738,audit_final,368,2,audit/embedding/empirical_epsilon/0.05,3.4791953936219215
1773764086.367738,audit_final,368,2,audit/embedding/empirical_epsilon/0.01,3.023197554051876
1773764086.367738,audit_final,368,2,audit/embedding/empirical_epsilon_details/0.05/epsilon,3.4791953936219215
1773764086.367738,audit_final,368,2,audit/embedding/empirical_epsilon_details/0.05/num_guesses,100.0
1773764086.367738,audit_final,368,2,audit/embedding/empirical_epsilon_details/0.05/correct_guesses,100.0
1773764086.367738,audit_final,368,2,audit/embedding/empirical_epsilon_details/0.01/epsilon,3.023197554051876
1773764086.367738,audit_final,368,2,audit/embedding/empirical_epsilon_details/0.01/num_guesses,100.0
1773764086.367738,audit_final,368,2,audit/embedding/empirical_epsilon_details/0.01/correct_guesses,100.0
1773764086.9161372,energy_final,368,,energy/codecarbon/duration,2345.9966679112986
1773764086.9161372,energy_final,368,,energy/codecarbon/emissions,0.09022432714096462
1773764086.9161372,energy_final,368,,energy/codecarbon/emissions_rate,3.8458847096868924e-05
1773764086.9161372,energy_final,368,,energy/codecarbon/cpu_power,72.02285277932866
1773764086.9161372,energy_final,368,,energy/codecarbon/gpu_power,3280.290622412428
1773764086.9161372,energy_final,368,,energy/codecarbon/ram_power,54.0
1773764086.9161372,energy_final,368,,energy/codecarbon/cpu_energy,0.045218986505725985
1773764086.9161372,energy_final,368,,energy/codecarbon/gpu_energy,2.137964725370466
1773764086.9161372,energy_final,368,,energy/codecarbon/ram_energy,0.03390259211605879
1773764086.9161372,energy_final,368,,energy/codecarbon/energy_consumed,2.2170863039922497
1773764086.9161372,energy_final,368,,energy/codecarbon/water_consumed,0.0
1773764086.9161372,energy_final,368,,energy/codecarbon/cpu_count,256.0
1773764086.9161372,energy_final,368,,energy/codecarbon/gpu_count,8.0
1773764086.9161372,energy_final,368,,energy/codecarbon/longitude,16.1885
1773764086.9161372,energy_final,368,,energy/codecarbon/latitude,58.594
1773764086.9161372,energy_final,368,,energy/codecarbon/ram_total_size,1511.49019241333
1773764086.9161372,energy_final,368,,energy/codecarbon/cpu_utilization_percent,3.3142796066695253
1773764086.9161372,energy_final,368,,energy/codecarbon/gpu_utilization_percent,88.58721675929884
1773764086.9161372,energy_final,368,,energy/codecarbon/ram_utilization_percent,5.287772552372644
1773764086.9161372,energy_final,368,,energy/codecarbon/ram_used_gb,79.7947571596665
1773764086.9161372,energy_final,368,,energy/codecarbon/pue,1.0
1773764086.9161372,energy_final,368,,energy/codecarbon/wue,0.0