File size: 42,563 Bytes

903307f

timestamp,event,step,epoch,key,value
1773845500.3890991,train_step,10,1,train/step_loss,1.5491975618131233
1773845500.3890991,train_step,10,1,train/step_real_loss,1.1542506143450737
1773845500.3890991,train_step,10,1,train/lr,5.2631578947368424e-05
1773845500.3890991,train_step,10,1,train/step_canary_loss,14.1875
1773845500.3890991,train_step,10,1,perf/step_duration_sec,7.20499980985187
1773845500.3890991,train_step,10,1,perf/samples_per_sec,9.160305585262314
1773845500.3890991,train_step,10,1,perf/tokens_per_sec,6590.423491069632
1773845500.3890991,train_step,10,1,perf/logical_batch_size,66.0
1773845500.3890991,train_step,10,1,perf/logical_token_count,47484.0
1773845500.3890991,train_step,10,1,perf/gradient_accumulation_steps,8.0
1773845500.3890991,train_step,10,1,system/cuda_memory_allocated_gb,16.85233783721924
1773845500.3890991,train_step,10,1,system/cuda_max_memory_allocated_gb,60.84869050979614
1773845576.5618713,train_step,20,1,train/step_loss,1.749317905796108
1773845576.5618713,train_step,20,1,train/step_real_loss,1.195575013756752
1773845576.5618713,train_step,20,1,train/lr,9.999795083071328e-05
1773845576.5618713,train_step,20,1,train/step_canary_loss,13.5625
1773845576.5618713,train_step,20,1,perf/step_duration_sec,7.156173000810668
1773845576.5618713,train_step,20,1,perf/samples_per_sec,9.362546153147791
1773845576.5618713,train_step,20,1,perf/tokens_per_sec,6812.859330605484
1773845576.5618713,train_step,20,1,perf/logical_batch_size,67.0
1773845576.5618713,train_step,20,1,perf/logical_token_count,48754.0
1773845576.5618713,train_step,20,1,perf/gradient_accumulation_steps,8.0
1773845576.5618713,train_step,20,1,system/cuda_memory_allocated_gb,17.044722080230713
1773845576.5618713,train_step,20,1,system/cuda_max_memory_allocated_gb,60.84869050979614
1773845647.392718,train_step,30,1,train/step_loss,1.0699390769004822
1773845647.392718,train_step,30,1,train/step_real_loss,1.0699390769004822
1773845647.392718,train_step,30,1,train/lr,9.975225368753412e-05
1773845647.392718,train_step,30,1,perf/step_duration_sec,6.728532065171748
1773845647.392718,train_step,30,1,perf/samples_per_sec,9.511732927792234
1773845647.392718,train_step,30,1,perf/tokens_per_sec,7888.793496987684
1773845647.392718,train_step,30,1,perf/logical_batch_size,64.0
1773845647.392718,train_step,30,1,perf/logical_token_count,53080.0
1773845647.392718,train_step,30,1,perf/gradient_accumulation_steps,8.0
1773845647.392718,train_step,30,1,system/cuda_memory_allocated_gb,16.85233783721924
1773845647.392718,train_step,30,1,system/cuda_max_memory_allocated_gb,60.84869050979614
1773845720.600243,train_step,40,1,train/step_loss,1.095807023346424
1773845720.600243,train_step,40,1,train/step_real_loss,1.095807023346424
1773845720.600243,train_step,40,1,train/lr,9.909902907826884e-05
1773845720.600243,train_step,40,1,perf/step_duration_sec,6.983004813082516
1773845720.600243,train_step,40,1,perf/samples_per_sec,9.165108962848961
1773845720.600243,train_step,40,1,perf/tokens_per_sec,7604.176342613747
1773845720.600243,train_step,40,1,perf/logical_batch_size,64.0
1773845720.600243,train_step,40,1,perf/logical_token_count,53100.0
1773845720.600243,train_step,40,1,perf/gradient_accumulation_steps,8.0
1773845720.600243,train_step,40,1,system/cuda_memory_allocated_gb,16.85233783721924
1773845720.600243,train_step,40,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773845792.8609428,train_step,50,1,train/step_loss,1.0978227560336773
1773845792.8609428,train_step,50,1,train/step_real_loss,1.0427106022834778
1773845792.8609428,train_step,50,1,train/lr,9.804362765439688e-05
1773845792.8609428,train_step,50,1,train/step_canary_loss,4.625
1773845792.8609428,train_step,50,1,perf/step_duration_sec,7.137449341127649
1773845792.8609428,train_step,50,1,perf/samples_per_sec,9.106894759372206
1773845792.8609428,train_step,50,1,perf/tokens_per_sec,6859.453238832368
1773845792.8609428,train_step,50,1,perf/logical_batch_size,65.0
1773845792.8609428,train_step,50,1,perf/logical_token_count,48959.0
1773845792.8609428,train_step,50,1,perf/gradient_accumulation_steps,8.0
1773845792.8609428,train_step,50,1,system/cuda_memory_allocated_gb,16.85233783721924
1773845792.8609428,train_step,50,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773845810.9490905,eval_step,50,1,eval/loss,0.9920933229195606
1773845810.9490905,eval_step,50,1,eval/duration_sec,18.071402312023565
1773845883.288467,train_step,60,1,train/step_loss,1.3474635359118967
1773845883.288467,train_step,60,1,train/step_real_loss,1.0449612587690353
1773845883.288467,train_step,60,1,train/lr,9.659469435229992e-05
1773845883.288467,train_step,60,1,train/step_canary_loss,6.1875
1773845883.288467,train_step,60,1,perf/step_duration_sec,7.404544448945671
1773845883.288467,train_step,60,1,perf/samples_per_sec,9.183549436276593
1773845883.288467,train_step,60,1,perf/tokens_per_sec,7116.845656521583
1773845883.288467,train_step,60,1,perf/logical_batch_size,68.0
1773845883.288467,train_step,60,1,perf/logical_token_count,52697.0
1773845883.288467,train_step,60,1,perf/gradient_accumulation_steps,8.0
1773845883.288467,train_step,60,1,system/cuda_memory_allocated_gb,16.85233783721924
1773845883.288467,train_step,60,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773845955.9986777,train_step,70,1,train/step_loss,1.2259749336676165
1773845955.9986777,train_step,70,1,train/step_real_loss,0.9742475748062134
1773845955.9986777,train_step,70,1,train/lr,9.476409758141405e-05
1773845955.9986777,train_step,70,1,train/step_canary_loss,9.28125
1773845955.9986777,train_step,70,1,perf/step_duration_sec,7.936319092987105
1773845955.9986777,train_step,70,1,perf/samples_per_sec,8.316197877970987
1773845955.9986777,train_step,70,1,perf/tokens_per_sec,6486.50834181922
1773845955.9986777,train_step,70,1,perf/logical_batch_size,66.0
1773845955.9986777,train_step,70,1,perf/logical_token_count,51479.0
1773845955.9986777,train_step,70,1,perf/gradient_accumulation_steps,8.0
1773845955.9986777,train_step,70,1,system/cuda_memory_allocated_gb,17.044722080230713
1773845955.9986777,train_step,70,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773846033.6489208,train_step,80,1,train/step_loss,1.075090453028679
1773846033.6489208,train_step,80,1,train/step_real_loss,1.075090453028679
1773846033.6489208,train_step,80,1,train/lr,9.256683200847638e-05
1773846033.6489208,train_step,80,1,perf/step_duration_sec,8.235574607970193
1773846033.6489208,train_step,80,1,perf/samples_per_sec,7.771163889167165
1773846033.6489208,train_step,80,1,perf/tokens_per_sec,6101.2136240466025
1773846033.6489208,train_step,80,1,perf/logical_batch_size,64.0
1773846033.6489208,train_step,80,1,perf/logical_token_count,50247.0
1773846033.6489208,train_step,80,1,perf/gradient_accumulation_steps,8.0
1773846033.6489208,train_step,80,1,system/cuda_memory_allocated_gb,16.85233783721924
1773846033.6489208,train_step,80,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773846114.691344,train_step,90,1,train/step_loss,1.0333750322461128
1773846114.691344,train_step,90,1,train/step_real_loss,1.0333750322461128
1773846114.691344,train_step,90,1,train/lr,9.002089573417356e-05
1773846114.691344,train_step,90,1,perf/step_duration_sec,8.228110728086904
1773846114.691344,train_step,90,1,perf/samples_per_sec,7.778213263651652
1773846114.691344,train_step,90,1,perf/tokens_per_sec,6766.3163318834695
1773846114.691344,train_step,90,1,perf/logical_batch_size,64.0
1773846114.691344,train_step,90,1,perf/logical_token_count,55674.0
1773846114.691344,train_step,90,1,perf/gradient_accumulation_steps,8.0
1773846114.691344,train_step,90,1,system/cuda_memory_allocated_gb,16.85233783721924
1773846114.691344,train_step,90,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773846196.510411,train_step,100,1,train/step_loss,1.3118896195382783
1773846196.510411,train_step,100,1,train/step_real_loss,1.0975150763988495
1773846196.510411,train_step,100,1,train/lr,8.714714286825512e-05
1773846196.510411,train_step,100,1,train/step_canary_loss,8.171875
1773846196.510411,train_step,100,1,perf/step_duration_sec,8.322382139973342
1773846196.510411,train_step,100,1,perf/samples_per_sec,7.930421709788419
1773846196.510411,train_step,100,1,perf/tokens_per_sec,5789.32800605024
1773846196.510411,train_step,100,1,perf/logical_batch_size,66.0
1773846196.510411,train_step,100,1,perf/logical_token_count,48181.0
1773846196.510411,train_step,100,1,perf/gradient_accumulation_steps,8.0
1773846196.510411,train_step,100,1,system/cuda_memory_allocated_gb,16.85233783721924
1773846196.510411,train_step,100,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773846218.1695988,eval_step,100,1,eval/loss,0.9644145566921729
1773846218.1695988,eval_step,100,1,eval/duration_sec,21.646080798935145
1773846300.9108367,train_step,110,1,train/step_loss,1.1254502712790646
1773846300.9108367,train_step,110,1,train/step_real_loss,0.9916823133826256
1773846300.9108367,train_step,110,1,train/lr,8.39691127106884e-05
1773846300.9108367,train_step,110,1,train/step_canary_loss,3.9791666666666665
1773846300.9108367,train_step,110,1,perf/step_duration_sec,8.406522131990641
1773846300.9108367,train_step,110,1,perf/samples_per_sec,7.970002213523535
1773846300.9108367,train_step,110,1,perf/tokens_per_sec,6148.7972301049485
1773846300.9108367,train_step,110,1,perf/logical_batch_size,67.0
1773846300.9108367,train_step,110,1,perf/logical_token_count,51690.0
1773846300.9108367,train_step,110,1,perf/gradient_accumulation_steps,8.0
1773846300.9108367,train_step,110,1,system/cuda_memory_allocated_gb,16.85233783721924
1773846300.9108367,train_step,110,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773846382.1696744,train_step,120,1,train/step_loss,0.9898425564169884
1773846382.1696744,train_step,120,1,train/step_real_loss,0.9898425564169884
1773846382.1696744,train_step,120,1,train/lr,8.051283693805624e-05
1773846382.1696744,train_step,120,1,perf/step_duration_sec,7.923252637963742
1773846382.1696744,train_step,120,1,perf/samples_per_sec,8.077490763496323
1773846382.1696744,train_step,120,1,perf/tokens_per_sec,6447.73079116776
1773846382.1696744,train_step,120,1,perf/logical_batch_size,64.0
1773846382.1696744,train_step,120,1,perf/logical_token_count,51087.0
1773846382.1696744,train_step,120,1,perf/gradient_accumulation_steps,8.0
1773846382.1696744,train_step,120,1,system/cuda_memory_allocated_gb,16.85233783721924
1773846382.1696744,train_step,120,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773846465.942783,train_step,130,1,train/step_loss,1.0856322862885215
1773846465.942783,train_step,130,1,train/step_real_loss,1.0018824934959412
1773846465.942783,train_step,130,1,train/lr,7.680662637455889e-05
1773846465.942783,train_step,130,1,train/step_canary_loss,3.765625
1773846465.942783,train_step,130,1,perf/step_duration_sec,8.345965349115431
1773846465.942783,train_step,130,1,perf/samples_per_sec,7.908012703047609
1773846465.942783,train_step,130,1,perf/tokens_per_sec,6140.811500665045
1773846465.942783,train_step,130,1,perf/logical_batch_size,66.0
1773846465.942783,train_step,130,1,perf/logical_token_count,51251.0
1773846465.942783,train_step,130,1,perf/gradient_accumulation_steps,8.0
1773846465.942783,train_step,130,1,system/cuda_memory_allocated_gb,16.85233783721924
1773846465.942783,train_step,130,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773846547.3686187,train_step,140,1,train/step_loss,1.11777756430886
1773846547.3686187,train_step,140,1,train/step_real_loss,0.9652081280946732
1773846547.3686187,train_step,140,1,train/lr,7.288083909420865e-05
1773846547.3686187,train_step,140,1,train/step_canary_loss,6.0
1773846547.3686187,train_step,140,1,perf/step_duration_sec,7.923310180194676
1773846547.3686187,train_step,140,1,perf/samples_per_sec,8.329851854717921
1773846547.3686187,train_step,140,1,perf/tokens_per_sec,6493.371940505792
1773846547.3686187,train_step,140,1,perf/logical_batch_size,66.0
1773846547.3686187,train_step,140,1,perf/logical_token_count,51449.0
1773846547.3686187,train_step,140,1,perf/gradient_accumulation_steps,8.0
1773846547.3686187,train_step,140,1,system/cuda_memory_allocated_gb,16.85233783721924
1773846547.3686187,train_step,140,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773846630.8803382,train_step,150,1,train/step_loss,1.0294065218705397
1773846630.8803382,train_step,150,1,train/step_real_loss,1.005940206348896
1773846630.8803382,train_step,150,1,train/lr,6.876763175372305e-05
1773846630.8803382,train_step,150,1,train/step_canary_loss,2.53125
1773846630.8803382,train_step,150,1,perf/step_duration_sec,8.249663470080122
1773846630.8803382,train_step,150,1,perf/samples_per_sec,7.879109279516914
1773846630.8803382,train_step,150,1,perf/tokens_per_sec,6361.229181084441
1773846630.8803382,train_step,150,1,perf/logical_batch_size,65.0
1773846630.8803382,train_step,150,1,perf/logical_token_count,52478.0
1773846630.8803382,train_step,150,1,perf/gradient_accumulation_steps,8.0
1773846630.8803382,train_step,150,1,system/cuda_memory_allocated_gb,16.85233783721924
1773846630.8803382,train_step,150,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773846652.1329367,eval_step,150,1,eval/loss,0.9538889107379046
1773846652.1329367,eval_step,150,1,eval/duration_sec,21.243608912918717
1773846734.8427546,train_step,160,1,train/step_loss,1.02191624045372
1773846734.8427546,train_step,160,1,train/step_real_loss,1.02191624045372
1773846734.8427546,train_step,160,1,train/lr,6.450069619298299e-05
1773846734.8427546,train_step,160,1,perf/step_duration_sec,8.291792104020715
1773846734.8427546,train_step,160,1,perf/samples_per_sec,7.718476198766031
1773846734.8427546,train_step,160,1,perf/tokens_per_sec,6133.052947063246
1773846734.8427546,train_step,160,1,perf/logical_batch_size,64.0
1773846734.8427546,train_step,160,1,perf/logical_token_count,50854.0
1773846734.8427546,train_step,160,1,perf/gradient_accumulation_steps,8.0
1773846734.8427546,train_step,160,1,system/cuda_memory_allocated_gb,16.85233783721924
1773846734.8427546,train_step,160,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773846816.398206,train_step,170,1,train/step_loss,0.9282315596938133
1773846816.398206,train_step,170,1,train/step_real_loss,0.9282315596938133
1773846816.398206,train_step,170,1,train/lr,6.011498346059712e-05
1773846816.398206,train_step,170,1,perf/step_duration_sec,7.786464124917984
1773846816.398206,train_step,170,1,perf/samples_per_sec,8.219391879709473
1773846816.398206,train_step,170,1,perf/tokens_per_sec,6340.875551201497
1773846816.398206,train_step,170,1,perf/logical_batch_size,64.0
1773846816.398206,train_step,170,1,perf/logical_token_count,49373.0
1773846816.398206,train_step,170,1,perf/gradient_accumulation_steps,8.0
1773846816.398206,train_step,170,1,system/cuda_memory_allocated_gb,16.85233783721924
1773846816.398206,train_step,170,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773846898.7244656,train_step,180,1,train/step_loss,1.1359728827620998
1773846898.7244656,train_step,180,1,train/step_real_loss,1.0057005435228348
1773846898.7244656,train_step,180,1,train/lr,5.564641752511637e-05
1773846898.7244656,train_step,180,1,train/step_canary_loss,5.3046875
1773846898.7244656,train_step,180,1,perf/step_duration_sec,8.632995563792065
1773846898.7244656,train_step,180,1,perf/samples_per_sec,7.645086750283158
1773846898.7244656,train_step,180,1,perf/tokens_per_sec,5906.756191775592
1773846898.7244656,train_step,180,1,perf/logical_batch_size,66.0
1773846898.7244656,train_step,180,1,perf/logical_token_count,50993.0
1773846898.7244656,train_step,180,1,perf/gradient_accumulation_steps,8.0
1773846898.7244656,train_step,180,1,system/cuda_memory_allocated_gb,16.85233783721924
1773846898.7244656,train_step,180,1,system/cuda_max_memory_allocated_gb,64.87937498092651
1773846945.409436,train_epoch,183,1,train/epoch_loss,1.1745113145391142
1773846945.409436,train_epoch,183,1,train/epoch_real_loss,1.045788711171007
1773846945.409436,train_epoch,183,1,train/epoch_canary_loss,7.361844605070935
1773846945.409436,train_epoch,183,1,perf/epoch_duration_sec,1529.87840606994
1773846945.409436,train_epoch,183,1,perf/epoch_samples_per_sec,31.26849807814928
1773846945.409436,train_epoch,183,1,perf/epoch_tokens_per_sec,24337.648568848304
1773846945.409436,train_epoch,183,1,perf/epoch_samples,47837.0
1773846945.409436,train_epoch,183,1,perf/epoch_tokens,37233643.0
1773846945.409436,train_epoch,183,1,system/cuda_epoch_peak_memory_gb,64.87937498092651
1773846945.409436,train_epoch,183,1,eval/loss,0.9498279011481768
1773846945.409436,train_epoch,183,1,eval/duration_sec,21.582291743019596
1773846954.5938265,audit_epoch,183,1,audit/delta,1e-05
1773846954.5938265,audit_epoch,183,1,audit/num_canaries,500.0
1773846954.5938265,audit_epoch,183,1,audit/num_members,250.0
1773846954.5938265,audit_epoch,183,1,audit/paper_guess_fraction,0.2
1773846954.5938265,audit_epoch,183,1,audit/paper_guess_steps,20.0
1773846954.5938265,audit_epoch,183,1,audit/loss/auc,0.996728
1773846954.5938265,audit_epoch,183,1,audit/loss/empirical_epsilon/0.05,3.4791953936219215
1773846954.5938265,audit_epoch,183,1,audit/loss/empirical_epsilon/0.01,3.023197554051876
1773846954.5938265,audit_epoch,183,1,audit/loss/empirical_epsilon_details/0.05/epsilon,3.4791953936219215
1773846954.5938265,audit_epoch,183,1,audit/loss/empirical_epsilon_details/0.05/num_guesses,100.0
1773846954.5938265,audit_epoch,183,1,audit/loss/empirical_epsilon_details/0.05/correct_guesses,100.0
1773846954.5938265,audit_epoch,183,1,audit/loss/empirical_epsilon_details/0.01/epsilon,3.023197554051876
1773846954.5938265,audit_epoch,183,1,audit/loss/empirical_epsilon_details/0.01/num_guesses,100.0
1773846954.5938265,audit_epoch,183,1,audit/loss/empirical_epsilon_details/0.01/correct_guesses,100.0
1773846954.5938265,audit_epoch,183,1,audit/embedding/auc,0.996
1773846954.5938265,audit_epoch,183,1,audit/embedding/empirical_epsilon/0.05,3.4791953936219215
1773846954.5938265,audit_epoch,183,1,audit/embedding/empirical_epsilon/0.01,3.023197554051876
1773846954.5938265,audit_epoch,183,1,audit/embedding/empirical_epsilon_details/0.05/epsilon,3.4791953936219215
1773846954.5938265,audit_epoch,183,1,audit/embedding/empirical_epsilon_details/0.05/num_guesses,100.0
1773846954.5938265,audit_epoch,183,1,audit/embedding/empirical_epsilon_details/0.05/correct_guesses,100.0
1773846954.5938265,audit_epoch,183,1,audit/embedding/empirical_epsilon_details/0.01/epsilon,3.023197554051876
1773846954.5938265,audit_epoch,183,1,audit/embedding/empirical_epsilon_details/0.01/num_guesses,100.0
1773846954.5938265,audit_epoch,183,1,audit/embedding/empirical_epsilon_details/0.01/correct_guesses,100.0
1773846954.5938265,audit_epoch,183,1,perf/audit_duration_sec,7.048272565938532
1773847012.740762,train_step,190,2,train/step_loss,1.0674721995989482
1773847012.740762,train_step,190,2,train/step_real_loss,1.0078131332993507
1773847012.740762,train_step,190,2,train/lr,5.113160101692939e-05
1773847012.740762,train_step,190,2,train/step_canary_loss,2.9765625
1773847012.740762,train_step,190,2,perf/step_duration_sec,8.316238466184586
1773847012.740762,train_step,190,2,perf/samples_per_sec,7.9362803590070925
1773847012.740762,train_step,190,2,perf/tokens_per_sec,5525.09408993421
1773847012.740762,train_step,190,2,perf/logical_batch_size,66.0
1773847012.740762,train_step,190,2,perf/logical_token_count,45948.0
1773847012.740762,train_step,190,2,perf/gradient_accumulation_steps,8.0
1773847012.740762,train_step,190,2,system/cuda_memory_allocated_gb,16.85233783721924
1773847012.740762,train_step,190,2,system/cuda_max_memory_allocated_gb,56.812254428863525
1773847096.9887106,train_step,200,2,train/step_loss,1.1259089543269232
1773847096.9887106,train_step,200,2,train/step_real_loss,1.0414504930377007
1773847096.9887106,train_step,200,2,train/lr,4.660751541114641e-05
1773847096.9887106,train_step,200,2,train/step_canary_loss,6.53125
1773847096.9887106,train_step,200,2,perf/step_duration_sec,8.248281789012253
1773847096.9887106,train_step,200,2,perf/samples_per_sec,7.880429119988136
1773847096.9887106,train_step,200,2,perf/tokens_per_sec,5337.475261653503
1773847096.9887106,train_step,200,2,perf/logical_batch_size,65.0
1773847096.9887106,train_step,200,2,perf/logical_token_count,44025.0
1773847096.9887106,train_step,200,2,perf/gradient_accumulation_steps,8.0
1773847096.9887106,train_step,200,2,system/cuda_memory_allocated_gb,16.85233783721924
1773847096.9887106,train_step,200,2,system/cuda_max_memory_allocated_gb,56.81271266937256
1773847117.9581919,eval_step,200,2,eval/loss,0.9487124373773477
1773847117.9581919,eval_step,200,2,eval/duration_sec,20.96032243501395
1773847200.0371358,train_step,210,2,train/step_loss,1.1445510784784954
1773847200.0371358,train_step,210,2,train/step_real_loss,1.0787557885050774
1773847200.0371358,train_step,210,2,train/lr,4.2111218107314846e-05
1773847200.0371358,train_step,210,2,train/step_canary_loss,3.25
1773847200.0371358,train_step,210,2,perf/step_duration_sec,8.08854101691395
1773847200.0371358,train_step,210,2,perf/samples_per_sec,8.159691576266644
1773847200.0371358,train_step,210,2,perf/tokens_per_sec,6864.773249451238
1773847200.0371358,train_step,210,2,perf/logical_batch_size,66.0
1773847200.0371358,train_step,210,2,perf/logical_token_count,55526.0
1773847200.0371358,train_step,210,2,perf/gradient_accumulation_steps,8.0
1773847200.0371358,train_step,210,2,system/cuda_memory_allocated_gb,17.044722080230713
1773847200.0371358,train_step,210,2,system/cuda_max_memory_allocated_gb,56.81271266937256
1773847279.7295897,train_step,220,2,train/step_loss,1.0096144080162048
1773847279.7295897,train_step,220,2,train/step_real_loss,0.9559597969055176
1773847279.7295897,train_step,220,2,train/lr,3.7679538887227244e-05
1773847279.7295897,train_step,220,2,train/step_canary_loss,2.7265625
1773847279.7295897,train_step,220,2,perf/step_duration_sec,8.052449005888775
1773847279.7295897,train_step,220,2,perf/samples_per_sec,8.196264260939007
1773847279.7295897,train_step,220,2,perf/tokens_per_sec,6205.4413462857765
1773847279.7295897,train_step,220,2,perf/logical_batch_size,66.0
1773847279.7295897,train_step,220,2,perf/logical_token_count,49969.0
1773847279.7295897,train_step,220,2,perf/gradient_accumulation_steps,8.0
1773847279.7295897,train_step,220,2,system/cuda_memory_allocated_gb,16.85233783721924
1773847279.7295897,train_step,220,2,system/cuda_max_memory_allocated_gb,60.84816789627075
1773847361.656875,train_step,230,2,train/step_loss,0.9799742924444603
1773847361.656875,train_step,230,2,train/step_real_loss,0.9675686806440353
1773847361.656875,train_step,230,2,train/lr,3.334877823717737e-05
1773847361.656875,train_step,230,2,train/step_canary_loss,1.376953125
1773847361.656875,train_step,230,2,perf/step_duration_sec,8.545294485986233
1773847361.656875,train_step,230,2,perf/samples_per_sec,7.7235489201964915
1773847361.656875,train_step,230,2,perf/tokens_per_sec,6140.923532367137
1773847361.656875,train_step,230,2,perf/logical_batch_size,66.0
1773847361.656875,train_step,230,2,perf/logical_token_count,52476.0
1773847361.656875,train_step,230,2,perf/gradient_accumulation_steps,8.0
1773847361.656875,train_step,230,2,system/cuda_memory_allocated_gb,16.85233783721924
1773847361.656875,train_step,230,2,system/cuda_max_memory_allocated_gb,60.84816789627075
1773847448.8259733,train_step,240,2,train/step_loss,1.0699740214781328
1773847448.8259733,train_step,240,2,train/step_real_loss,0.9959888234734535
1773847448.8259733,train_step,240,2,train/lr,2.9154410005747583e-05
1773847448.8259733,train_step,240,2,train/step_canary_loss,3.4375
1773847448.8259733,train_step,240,2,perf/step_duration_sec,8.351536879083142
1773847448.8259733,train_step,240,2,perf/samples_per_sec,7.902737059726149
1773847448.8259733,train_step,240,2,perf/tokens_per_sec,6316.322464206273
1773847448.8259733,train_step,240,2,perf/logical_batch_size,66.0
1773847448.8259733,train_step,240,2,perf/logical_token_count,52751.0
1773847448.8259733,train_step,240,2,perf/gradient_accumulation_steps,8.0
1773847448.8259733,train_step,240,2,system/cuda_memory_allocated_gb,16.85233783721924
1773847448.8259733,train_step,240,2,system/cuda_max_memory_allocated_gb,60.84816789627075
1773847527.7394445,train_step,250,2,train/step_loss,1.0023332100648146
1773847527.7394445,train_step,250,2,train/step_real_loss,0.9728286564350128
1773847527.7394445,train_step,250,2,train/lr,2.5130790832697737e-05
1773847527.7394445,train_step,250,2,train/step_canary_loss,2.890625
1773847527.7394445,train_step,250,2,perf/step_duration_sec,8.24364663194865
1773847527.7394445,train_step,250,2,perf/samples_per_sec,7.884860050658815
1773847527.7394445,train_step,250,2,perf/tokens_per_sec,5500.114454721866
1773847527.7394445,train_step,250,2,perf/logical_batch_size,65.0
1773847527.7394445,train_step,250,2,perf/logical_token_count,45341.0
1773847527.7394445,train_step,250,2,perf/gradient_accumulation_steps,8.0
1773847527.7394445,train_step,250,2,system/cuda_memory_allocated_gb,16.85233783721924
1773847527.7394445,train_step,250,2,system/cuda_max_memory_allocated_gb,60.84816789627075
1773847548.8306634,eval_step,250,2,eval/loss,0.9464517607317342
1773847548.8306634,eval_step,250,2,eval/duration_sec,21.085849778959528
1773847631.4214828,train_step,260,2,train/step_loss,1.0758904056115584
1773847631.4214828,train_step,260,2,train/step_real_loss,0.9906154796481133
1773847631.4214828,train_step,260,2,train/lr,2.1310878729063642e-05
1773847631.4214828,train_step,260,2,train/step_canary_loss,3.8046875
1773847631.4214828,train_step,260,2,perf/step_duration_sec,8.149904164019972
1773847631.4214828,train_step,260,2,perf/samples_per_sec,8.098254736709105
1773847631.4214828,train_step,260,2,perf/tokens_per_sec,6327.313666786037
1773847631.4214828,train_step,260,2,perf/logical_batch_size,66.0
1773847631.4214828,train_step,260,2,perf/logical_token_count,51567.0
1773847631.4214828,train_step,260,2,perf/gradient_accumulation_steps,8.0
1773847631.4214828,train_step,260,2,system/cuda_memory_allocated_gb,17.044722080230713
1773847631.4214828,train_step,260,2,system/cuda_max_memory_allocated_gb,60.84816789627075
1773847712.8631833,train_step,270,2,train/step_loss,1.1142344745722683
1773847712.8631833,train_step,270,2,train/step_real_loss,0.9749820455908775
1773847712.8631833,train_step,270,2,train/lr,1.7725963113612998e-05
1773847712.8631833,train_step,270,2,train/step_canary_loss,5.5703125
1773847712.8631833,train_step,270,2,perf/step_duration_sec,7.97826447407715
1773847712.8631833,train_step,270,2,perf/samples_per_sec,8.272475826596892
1773847712.8631833,train_step,270,2,perf/tokens_per_sec,6396.37858155898
1773847712.8631833,train_step,270,2,perf/logical_batch_size,66.0
1773847712.8631833,train_step,270,2,perf/logical_token_count,51032.0
1773847712.8631833,train_step,270,2,perf/gradient_accumulation_steps,8.0
1773847712.8631833,train_step,270,2,system/cuda_memory_allocated_gb,16.85233783721924
1773847712.8631833,train_step,270,2,system/cuda_max_memory_allocated_gb,60.84816789627075
1773847793.1259449,train_step,280,2,train/step_loss,1.0694914211088151
1773847793.1259449,train_step,280,2,train/step_real_loss,1.0146433636546135
1773847793.1259449,train_step,280,2,train/lr,1.4405408516967328e-05
1773847793.1259449,train_step,280,2,train/step_canary_loss,2.2395833333333335
1773847793.1259449,train_step,280,2,perf/step_duration_sec,8.121065079933032
1773847793.1259449,train_step,280,2,perf/samples_per_sec,8.25014937579499
1773847793.1259449,train_step,280,2,perf/tokens_per_sec,6357.909891349593
1773847793.1259449,train_step,280,2,perf/logical_batch_size,67.0
1773847793.1259449,train_step,280,2,perf/logical_token_count,51633.0
1773847793.1259449,train_step,280,2,perf/gradient_accumulation_steps,8.0
1773847793.1259449,train_step,280,2,system/cuda_memory_allocated_gb,16.85233783721924
1773847793.1259449,train_step,280,2,system/cuda_max_memory_allocated_gb,60.84869050979614
1773847869.4765995,train_step,290,2,train/step_loss,0.9731774203705065
1773847869.4765995,train_step,290,2,train/step_real_loss,0.9380374550819397
1773847869.4765995,train_step,290,2,train/lr,1.1376414052744056e-05
1773847869.4765995,train_step,290,2,train/step_canary_loss,2.09765625
1773847869.4765995,train_step,290,2,perf/step_duration_sec,7.720702049089596
1773847869.4765995,train_step,290,2,perf/samples_per_sec,8.54844541083962
1773847869.4765995,train_step,290,2,perf/tokens_per_sec,6774.642988090398
1773847869.4765995,train_step,290,2,perf/logical_batch_size,66.0
1773847869.4765995,train_step,290,2,perf/logical_token_count,52305.0
1773847869.4765995,train_step,290,2,perf/gradient_accumulation_steps,8.0
1773847869.4765995,train_step,290,2,system/cuda_memory_allocated_gb,16.85233783721924
1773847869.4765995,train_step,290,2,system/cuda_max_memory_allocated_gb,60.84869050979614
1773847946.6124372,train_step,300,2,train/step_loss,1.0307691061135493
1773847946.6124372,train_step,300,2,train/step_real_loss,1.0040206909179688
1773847946.6124372,train_step,300,2,train/lr,8.66379062592345e-06
1773847946.6124372,train_step,300,2,train/step_canary_loss,1.88671875
1773847946.6124372,train_step,300,2,perf/step_duration_sec,7.759747894015163
1773847946.6124372,train_step,300,2,perf/samples_per_sec,8.505430962635218
1773847946.6124372,train_step,300,2,perf/tokens_per_sec,6224.171282323573
1773847946.6124372,train_step,300,2,perf/logical_batch_size,66.0
1773847946.6124372,train_step,300,2,perf/logical_token_count,48298.0
1773847946.6124372,train_step,300,2,perf/gradient_accumulation_steps,8.0
1773847946.6124372,train_step,300,2,system/cuda_memory_allocated_gb,17.044722080230713
1773847946.6124372,train_step,300,2,system/cuda_max_memory_allocated_gb,60.84869050979614
1773847966.612822,eval_step,300,2,eval/loss,0.9457971912312817
1773847966.612822,eval_step,300,2,eval/duration_sec,19.98325987579301
1773848043.95005,train_step,310,2,train/step_loss,1.0016437012757828
1773848043.95005,train_step,310,2,train/step_real_loss,0.9807856976985931
1773848043.95005,train_step,310,2,train/lr,6.2897577033565965e-06
1773848043.95005,train_step,310,2,train/step_canary_loss,1.4466145833333333
1773848043.95005,train_step,310,2,perf/step_duration_sec,7.719462159788236
1773848043.95005,train_step,310,2,perf/samples_per_sec,8.679361154072678
1773848043.95005,train_step,310,2,perf/tokens_per_sec,6271.6804613921595
1773848043.95005,train_step,310,2,perf/logical_batch_size,67.0
1773848043.95005,train_step,310,2,perf/logical_token_count,48414.0
1773848043.95005,train_step,310,2,perf/gradient_accumulation_steps,8.0
1773848043.95005,train_step,310,2,system/cuda_memory_allocated_gb,16.85233783721924
1773848043.95005,train_step,310,2,system/cuda_max_memory_allocated_gb,60.84869050979614
1773848122.1587205,train_step,320,2,train/step_loss,0.9575878876906175
1773848122.1587205,train_step,320,2,train/step_real_loss,0.9505775347352028
1773848122.1587205,train_step,320,2,train/lr,4.2737613110951925e-06
1773848122.1587205,train_step,320,2,train/step_canary_loss,1.40625
1773848122.1587205,train_step,320,2,perf/step_duration_sec,7.7270688829012215
1773848122.1587205,train_step,320,2,perf/samples_per_sec,8.411986612910194
1773848122.1587205,train_step,320,2,perf/tokens_per_sec,6291.130665027544
1773848122.1587205,train_step,320,2,perf/logical_batch_size,65.0
1773848122.1587205,train_step,320,2,perf/logical_token_count,48612.0
1773848122.1587205,train_step,320,2,perf/gradient_accumulation_steps,8.0
1773848122.1587205,train_step,320,2,system/cuda_memory_allocated_gb,16.85233783721924
1773848122.1587205,train_step,320,2,system/cuda_max_memory_allocated_gb,60.84869050979614
1773848200.6989753,train_step,330,2,train/step_loss,1.0191495253489569
1773848200.6989753,train_step,330,2,train/step_real_loss,0.9818510636687279
1773848200.6989753,train_step,330,2,train/lr,2.6323147493514833e-06
1773848200.6989753,train_step,330,2,train/step_canary_loss,3.40625
1773848200.6989753,train_step,330,2,perf/step_duration_sec,7.586550263920799
1773848200.6989753,train_step,330,2,perf/samples_per_sec,8.567794022155125
1773848200.6989753,train_step,330,2,perf/tokens_per_sec,6317.495875290012
1773848200.6989753,train_step,330,2,perf/logical_batch_size,65.0
1773848200.6989753,train_step,330,2,perf/logical_token_count,47928.0
1773848200.6989753,train_step,330,2,perf/gradient_accumulation_steps,8.0
1773848200.6989753,train_step,330,2,system/cuda_memory_allocated_gb,16.85233783721924
1773848200.6989753,train_step,330,2,system/cuda_max_memory_allocated_gb,60.84869050979614
1773848281.642083,train_step,340,2,train/step_loss,1.0784487796552253
1773848281.642083,train_step,340,2,train/step_real_loss,1.0184003114700317
1773848281.642083,train_step,340,2,train/lr,1.3788633298137288e-06
1773848281.642083,train_step,340,2,train/step_canary_loss,3.0
1773848281.642083,train_step,340,2,perf/step_duration_sec,8.599153365008533
1773848281.642083,train_step,340,2,perf/samples_per_sec,7.675174194306803
1773848281.642083,train_step,340,2,perf/tokens_per_sec,5509.263294777043
1773848281.642083,train_step,340,2,perf/logical_batch_size,66.0
1773848281.642083,train_step,340,2,perf/logical_token_count,47375.0
1773848281.642083,train_step,340,2,perf/gradient_accumulation_steps,8.0
1773848281.642083,train_step,340,2,system/cuda_memory_allocated_gb,16.85233783721924
1773848281.642083,train_step,340,2,system/cuda_max_memory_allocated_gb,60.84869050979614
1773848363.9098017,train_step,350,2,train/step_loss,1.1414880389752595
1773848363.9098017,train_step,350,2,train/step_real_loss,0.950454406440258
1773848363.9098017,train_step,350,2,train/lr,5.236742432724262e-07
1773848363.9098017,train_step,350,2,train/step_canary_loss,3.58671875
1773848363.9098017,train_step,350,2,perf/step_duration_sec,8.310993053019047
1773848363.9098017,train_step,350,2,perf/samples_per_sec,8.302256969753463
1773848363.9098017,train_step,350,2,perf/tokens_per_sec,6413.433347852161
1773848363.9098017,train_step,350,2,perf/logical_batch_size,69.0
1773848363.9098017,train_step,350,2,perf/logical_token_count,53302.0
1773848363.9098017,train_step,350,2,perf/gradient_accumulation_steps,8.0
1773848363.9098017,train_step,350,2,system/cuda_memory_allocated_gb,16.85233783721924
1773848363.9098017,train_step,350,2,system/cuda_max_memory_allocated_gb,60.84869050979614
1773848384.8834276,eval_step,350,2,eval/loss,0.9456373546030614
1773848384.8834276,eval_step,350,2,eval/duration_sec,20.951530331978574
1773848463.9385724,train_step,360,2,train/step_loss,1.0201530621601984
1773848463.9385724,train_step,360,2,train/step_real_loss,0.9811613112688065
1773848463.9385724,train_step,360,2,train/lr,7.375245966623756e-08
1773848463.9385724,train_step,360,2,train/step_canary_loss,3.515625
1773848463.9385724,train_step,360,2,perf/step_duration_sec,7.831112121930346
1773848463.9385724,train_step,360,2,perf/samples_per_sec,8.300225943384616
1773848463.9385724,train_step,360,2,perf/tokens_per_sec,6854.454279830759
1773848463.9385724,train_step,360,2,perf/logical_batch_size,65.0
1773848463.9385724,train_step,360,2,perf/logical_token_count,53678.0
1773848463.9385724,train_step,360,2,perf/gradient_accumulation_steps,8.0
1773848463.9385724,train_step,360,2,system/cuda_memory_allocated_gb,16.85233783721924
1773848463.9385724,train_step,360,2,system/cuda_max_memory_allocated_gb,60.84869050979614
1773848532.3133028,train_epoch,366,2,train/epoch_loss,1.0387111434896155
1773848532.3133028,train_epoch,366,2,train/epoch_real_loss,0.9917015731131118
1773848532.3133028,train_epoch,366,2,train/epoch_canary_loss,3.26868664640822
1773848532.3133028,train_epoch,366,2,perf/epoch_duration_sec,1556.666436379077
1773848532.3133028,train_epoch,366,2,perf/epoch_samples_per_sec,30.74068977250494
1773848532.3133028,train_epoch,366,2,perf/epoch_tokens_per_sec,23919.13266056493
1773848532.3133028,train_epoch,366,2,perf/epoch_samples,47853.0
1773848532.3133028,train_epoch,366,2,perf/epoch_tokens,37234111.0
1773848532.3133028,train_epoch,366,2,system/cuda_epoch_peak_memory_gb,60.84869050979614
1773848532.3133028,train_epoch,366,2,eval/loss,0.9456853411801449
1773848532.3133028,train_epoch,366,2,eval/duration_sec,21.018074851948768
1773848539.6229737,audit_epoch,366,2,audit/delta,1e-05
1773848539.6229737,audit_epoch,366,2,audit/num_canaries,500.0
1773848539.6229737,audit_epoch,366,2,audit/num_members,250.0
1773848539.6229737,audit_epoch,366,2,audit/paper_guess_fraction,0.2
1773848539.6229737,audit_epoch,366,2,audit/paper_guess_steps,20.0
1773848539.6229737,audit_epoch,366,2,audit/loss/auc,1.0
1773848539.6229737,audit_epoch,366,2,audit/loss/empirical_epsilon/0.05,3.4791953936219215
1773848539.6229737,audit_epoch,366,2,audit/loss/empirical_epsilon/0.01,3.023197554051876
1773848539.6229737,audit_epoch,366,2,audit/loss/empirical_epsilon_details/0.05/epsilon,3.4791953936219215
1773848539.6229737,audit_epoch,366,2,audit/loss/empirical_epsilon_details/0.05/num_guesses,100.0
1773848539.6229737,audit_epoch,366,2,audit/loss/empirical_epsilon_details/0.05/correct_guesses,100.0
1773848539.6229737,audit_epoch,366,2,audit/loss/empirical_epsilon_details/0.01/epsilon,3.023197554051876
1773848539.6229737,audit_epoch,366,2,audit/loss/empirical_epsilon_details/0.01/num_guesses,100.0
1773848539.6229737,audit_epoch,366,2,audit/loss/empirical_epsilon_details/0.01/correct_guesses,100.0
1773848539.6229737,audit_epoch,366,2,audit/embedding/auc,1.0
1773848539.6229737,audit_epoch,366,2,audit/embedding/empirical_epsilon/0.05,3.4791953936219215
1773848539.6229737,audit_epoch,366,2,audit/embedding/empirical_epsilon/0.01,3.023197554051876
1773848539.6229737,audit_epoch,366,2,audit/embedding/empirical_epsilon_details/0.05/epsilon,3.4791953936219215
1773848539.6229737,audit_epoch,366,2,audit/embedding/empirical_epsilon_details/0.05/num_guesses,100.0
1773848539.6229737,audit_epoch,366,2,audit/embedding/empirical_epsilon_details/0.05/correct_guesses,100.0
1773848539.6229737,audit_epoch,366,2,audit/embedding/empirical_epsilon_details/0.01/epsilon,3.023197554051876
1773848539.6229737,audit_epoch,366,2,audit/embedding/empirical_epsilon_details/0.01/num_guesses,100.0
1773848539.6229737,audit_epoch,366,2,audit/embedding/empirical_epsilon_details/0.01/correct_guesses,100.0
1773848539.6229737,audit_epoch,366,2,perf/audit_duration_sec,5.547323588980362
1773848547.60095,audit_final,366,2,audit/delta,1e-05
1773848547.60095,audit_final,366,2,audit/num_canaries,500.0
1773848547.60095,audit_final,366,2,audit/num_members,250.0
1773848547.60095,audit_final,366,2,audit/paper_guess_fraction,0.2
1773848547.60095,audit_final,366,2,audit/paper_guess_steps,20.0
1773848547.60095,audit_final,366,2,audit/loss/auc,0.999984
1773848547.60095,audit_final,366,2,audit/loss/empirical_epsilon/0.05,3.4791953936219215
1773848547.60095,audit_final,366,2,audit/loss/empirical_epsilon/0.01,3.023197554051876
1773848547.60095,audit_final,366,2,audit/loss/empirical_epsilon_details/0.05/epsilon,3.4791953936219215
1773848547.60095,audit_final,366,2,audit/loss/empirical_epsilon_details/0.05/num_guesses,100.0
1773848547.60095,audit_final,366,2,audit/loss/empirical_epsilon_details/0.05/correct_guesses,100.0
1773848547.60095,audit_final,366,2,audit/loss/empirical_epsilon_details/0.01/epsilon,3.023197554051876
1773848547.60095,audit_final,366,2,audit/loss/empirical_epsilon_details/0.01/num_guesses,100.0
1773848547.60095,audit_final,366,2,audit/loss/empirical_epsilon_details/0.01/correct_guesses,100.0
1773848547.60095,audit_final,366,2,audit/embedding/auc,1.0
1773848547.60095,audit_final,366,2,audit/embedding/empirical_epsilon/0.05,3.4791953936219215
1773848547.60095,audit_final,366,2,audit/embedding/empirical_epsilon/0.01,3.023197554051876
1773848547.60095,audit_final,366,2,audit/embedding/empirical_epsilon_details/0.05/epsilon,3.4791953936219215
1773848547.60095,audit_final,366,2,audit/embedding/empirical_epsilon_details/0.05/num_guesses,100.0
1773848547.60095,audit_final,366,2,audit/embedding/empirical_epsilon_details/0.05/correct_guesses,100.0
1773848547.60095,audit_final,366,2,audit/embedding/empirical_epsilon_details/0.01/epsilon,3.023197554051876
1773848547.60095,audit_final,366,2,audit/embedding/empirical_epsilon_details/0.01/num_guesses,100.0
1773848547.60095,audit_final,366,2,audit/embedding/empirical_epsilon_details/0.01/correct_guesses,100.0
1773848548.142728,energy_final,366,,energy/codecarbon/duration,3257.8297634990886
1773848548.142728,energy_final,366,,energy/codecarbon/emissions,0.046521798865538776
1773848548.142728,energy_final,366,,energy/codecarbon/emissions_rate,1.427999688221026e-05
1773848548.142728,energy_final,366,,energy/codecarbon/cpu_power,84.78240267963766
1773848548.142728,energy_final,366,,energy/codecarbon/gpu_power,1356.0935826800528
1773848548.142728,energy_final,366,,energy/codecarbon/ram_power,38.0
1773848548.142728,energy_final,366,,energy/codecarbon/cpu_energy,0.0740143041223195
1773848548.142728,energy_final,366,,energy/codecarbon/gpu_energy,1.2279951440620138
1773848548.142728,energy_final,366,,energy/codecarbon/ram_energy,0.033174056839308075
1773848548.142728,energy_final,366,,energy/codecarbon/energy_consumed,1.3351835050236422
1773848548.142728,energy_final,366,,energy/codecarbon/water_consumed,0.0
1773848548.142728,energy_final,366,,energy/codecarbon/cpu_count,8.0
1773848548.142728,energy_final,366,,energy/codecarbon/gpu_count,5.0
1773848548.142728,energy_final,366,,energy/codecarbon/longitude,8.212
1773848548.142728,energy_final,366,,energy/codecarbon/latitude,47.4843
1773848548.142728,energy_final,366,,energy/codecarbon/ram_total_size,128.0
1773848548.142728,energy_final,366,,energy/codecarbon/cpu_utilization_percent,12.756927339901509
1773848548.142728,energy_final,366,,energy/codecarbon/gpu_utilization_percent,63.19950738916256
1773848548.142728,energy_final,366,,energy/codecarbon/ram_utilization_percent,10.355603448275856
1773848548.142728,energy_final,366,,energy/codecarbon/ram_used_gb,208.74299130651164
1773848548.142728,energy_final,366,,energy/codecarbon/pue,1.0
1773848548.142728,energy_final,366,,energy/codecarbon/wue,0.0