File size: 27,559 Bytes
903307f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 | timestamp,event,step,epoch,key,value
1773852387.8640254,train_step,10,1,train/step_loss,1.6353936302128123
1773852387.8640254,train_step,10,1,train/step_real_loss,1.1510175429284573
1773852387.8640254,train_step,10,1,train/lr,0.0002
1773852387.8640254,train_step,10,1,train/step_canary_loss,11.968750158945719
1773852387.8640254,train_step,10,1,perf/step_duration_sec,14.872664371039718
1773852387.8640254,train_step,10,1,perf/samples_per_sec,8.94258060842008
1773852387.8640254,train_step,10,1,perf/tokens_per_sec,6777.198589666931
1773852387.8640254,train_step,10,1,perf/logical_batch_size,133.0
1773852387.8640254,train_step,10,1,perf/logical_token_count,100795.0
1773852387.8640254,train_step,10,1,perf/physical_batches,18.0
1773852387.8640254,train_step,10,1,privacy/epsilon,2.524123361583601
1773852387.8640254,train_step,10,1,system/cuda_memory_allocated_gb,15.702028751373291
1773852387.8640254,train_step,10,1,system/cuda_max_memory_allocated_gb,54.06287908554077
1773852537.5150325,train_step,20,1,train/step_loss,1.857343329130298
1773852537.5150325,train_step,20,1,train/step_real_loss,1.114402625709772
1773852537.5150325,train_step,20,1,train/lr,0.00019837448439320027
1773852537.5150325,train_step,20,1,train/step_canary_loss,12.42361111111111
1773852537.5150325,train_step,20,1,perf/step_duration_sec,15.259514638921246
1773852537.5150325,train_step,20,1,perf/samples_per_sec,8.91247219968029
1773852537.5150325,train_step,20,1,perf/tokens_per_sec,7023.880020837742
1773852537.5150325,train_step,20,1,perf/logical_batch_size,136.0
1773852537.5150325,train_step,20,1,perf/logical_token_count,107181.0
1773852537.5150325,train_step,20,1,perf/physical_batches,18.0
1773852537.5150325,train_step,20,1,privacy/epsilon,3.1196418977343887
1773852537.5150325,train_step,20,1,system/cuda_memory_allocated_gb,16.279295921325684
1773852537.5150325,train_step,20,1,system/cuda_max_memory_allocated_gb,54.062880516052246
1773852684.6348226,train_step,30,1,train/step_loss,1.4355280977306943
1773852684.6348226,train_step,30,1,train/step_real_loss,1.148357093334198
1773852684.6348226,train_step,30,1,train/lr,0.0001935507835925601
1773852684.6348226,train_step,30,1,train/step_canary_loss,10.625000238418579
1773852684.6348226,train_step,30,1,perf/step_duration_sec,14.81383830588311
1773852684.6348226,train_step,30,1,perf/samples_per_sec,8.84308288608599
1773852684.6348226,train_step,30,1,perf/tokens_per_sec,6924.876448749963
1773852684.6348226,train_step,30,1,perf/logical_batch_size,131.0
1773852684.6348226,train_step,30,1,perf/logical_token_count,102584.0
1773852684.6348226,train_step,30,1,perf/physical_batches,18.0
1773852684.6348226,train_step,30,1,privacy/epsilon,3.599257677191776
1773852684.6348226,train_step,30,1,system/cuda_memory_allocated_gb,15.317307472229004
1773852684.6348226,train_step,30,1,system/cuda_max_memory_allocated_gb,54.062880516052246
1773852833.9397993,train_step,40,1,train/step_loss,1.6915703279000742
1773852833.9397993,train_step,40,1,train/step_real_loss,1.1175741739571095
1773852833.9397993,train_step,40,1,train/lr,0.00018568571761675893
1773852833.9397993,train_step,40,1,train/step_canary_loss,12.1875
1773852833.9397993,train_step,40,1,perf/step_duration_sec,14.987364898901433
1773852833.9397993,train_step,40,1,perf/samples_per_sec,8.940864581860026
1773852833.9397993,train_step,40,1,perf/tokens_per_sec,6826.283385380117
1773852833.9397993,train_step,40,1,perf/logical_batch_size,134.0
1773852833.9397993,train_step,40,1,perf/logical_token_count,102308.0
1773852833.9397993,train_step,40,1,perf/physical_batches,18.0
1773852833.9397993,train_step,40,1,privacy/epsilon,4.0171270793767535
1773852833.9397993,train_step,40,1,system/cuda_memory_allocated_gb,15.894390106201172
1773852833.9397993,train_step,40,1,system/cuda_max_memory_allocated_gb,54.062883377075195
1773852981.6699533,train_step,50,1,train/step_loss,1.730209876872875
1773852981.6699533,train_step,50,1,train/step_real_loss,1.1695572771131992
1773852981.6699533,train_step,50,1,train/lr,0.00017503498221564025
1773852981.6699533,train_step,50,1,train/step_canary_loss,11.982143129621234
1773852981.6699533,train_step,50,1,perf/step_duration_sec,15.535889676073566
1773852981.6699533,train_step,50,1,perf/samples_per_sec,8.625189982287917
1773852981.6699533,train_step,50,1,perf/tokens_per_sec,6820.401162038881
1773852981.6699533,train_step,50,1,perf/logical_batch_size,134.0
1773852981.6699533,train_step,50,1,perf/logical_token_count,105961.0
1773852981.6699533,train_step,50,1,perf/physical_batches,18.0
1773852981.6699533,train_step,50,1,privacy/epsilon,4.394850417905478
1773852981.6699533,train_step,50,1,system/cuda_memory_allocated_gb,15.894390106201172
1773852981.6699533,train_step,50,1,system/cuda_max_memory_allocated_gb,54.062883377075195
1773853001.9967575,eval_step,50,1,eval/loss,1.0429061587367738
1773853001.9967575,eval_step,50,1,eval/duration_sec,20.30736266821623
1773853155.8696322,train_step,60,1,train/step_loss,1.5988739917152806
1773853155.8696322,train_step,60,1,train/step_real_loss,1.106154203414917
1773853155.8696322,train_step,60,1,train/lr,0.0001619448361215723
1773853155.8696322,train_step,60,1,train/step_canary_loss,14.21250057220459
1773853155.8696322,train_step,60,1,perf/step_duration_sec,14.15656855283305
1773853155.8696322,train_step,60,1,perf/samples_per_sec,9.3949320771935
1773853155.8696322,train_step,60,1,perf/tokens_per_sec,7349.238596324901
1773853155.8696322,train_step,60,1,perf/logical_batch_size,133.0
1773853155.8696322,train_step,60,1,perf/logical_token_count,104040.0
1773853155.8696322,train_step,60,1,perf/physical_batches,17.0
1773853155.8696322,train_step,60,1,privacy/epsilon,4.743740138692227
1773853155.8696322,train_step,60,1,system/cuda_memory_allocated_gb,15.702028751373291
1773853155.8696322,train_step,60,1,system/cuda_max_memory_allocated_gb,54.062886238098145
1773853298.6477532,train_step,70,1,train/step_loss,1.5077286886446404
1773853298.6477532,train_step,70,1,train/step_real_loss,1.1227163039147854
1773853298.6477532,train_step,70,1,train/lr,0.00014684084406997903
1773853298.6477532,train_step,70,1,train/step_canary_loss,13.828125
1773853298.6477532,train_step,70,1,perf/step_duration_sec,14.256150827044621
1773853298.6477532,train_step,70,1,perf/samples_per_sec,9.25916129826499
1773853298.6477532,train_step,70,1,perf/tokens_per_sec,6918.066538192309
1773853298.6477532,train_step,70,1,perf/logical_batch_size,132.0
1773853298.6477532,train_step,70,1,perf/logical_token_count,98625.0
1773853298.6477532,train_step,70,1,perf/physical_batches,17.0
1773853298.6477532,train_step,70,1,privacy/epsilon,5.069830876941064
1773853298.6477532,train_step,70,1,system/cuda_memory_allocated_gb,15.509668827056885
1773853298.6477532,train_step,70,1,system/cuda_max_memory_allocated_gb,54.062886238098145
1773853446.331358,train_step,80,1,train/step_loss,1.611925903119539
1773853446.331358,train_step,80,1,train/step_real_loss,1.1202042661607265
1773853446.331358,train_step,80,1,train/lr,0.00013021404155695725
1773853446.331358,train_step,80,1,train/step_canary_loss,14.199999809265137
1773853446.331358,train_step,80,1,perf/step_duration_sec,14.11704233684577
1773853446.331358,train_step,80,1,perf/samples_per_sec,9.421236887054398
1773853446.331358,train_step,80,1,perf/tokens_per_sec,7214.046509883511
1773853446.331358,train_step,80,1,perf/logical_batch_size,133.0
1773853446.331358,train_step,80,1,perf/logical_token_count,101841.0
1773853446.331358,train_step,80,1,perf/physical_batches,17.0
1773853446.331358,train_step,80,1,privacy/epsilon,5.377982250509876
1773853446.331358,train_step,80,1,system/cuda_memory_allocated_gb,15.702028751373291
1773853446.331358,train_step,80,1,system/cuda_max_memory_allocated_gb,54.062886238098145
1773853593.7571762,train_step,90,1,train/step_loss,1.8932846375625498
1773853593.7571762,train_step,90,1,train/step_real_loss,1.1377343386411667
1773853593.7571762,train_step,90,1,train/lr,0.00011260497112202895
1773853593.7571762,train_step,90,1,train/step_canary_loss,12.63888888888889
1773853593.7571762,train_step,90,1,perf/step_duration_sec,14.762270654086024
1773853593.7571762,train_step,90,1,perf/samples_per_sec,9.21267487819408
1773853593.7571762,train_step,90,1,perf/tokens_per_sec,6934.841014560594
1773853593.7571762,train_step,90,1,perf/logical_batch_size,136.0
1773853593.7571762,train_step,90,1,perf/logical_token_count,102374.0
1773853593.7571762,train_step,90,1,perf/physical_batches,18.0
1773853593.7571762,train_step,90,1,privacy/epsilon,5.671744621870462
1773853593.7571762,train_step,90,1,system/cuda_memory_allocated_gb,16.279295921325684
1773853593.7571762,train_step,90,1,system/cuda_max_memory_allocated_gb,54.062886238098145
1773853638.1426697,train_epoch,92,1,train/epoch_loss,1.7500966633389758
1773853638.1426697,train_epoch,92,1,train/epoch_real_loss,1.133818452948967
1773853638.1426697,train_epoch,92,1,train/epoch_canary_loss,13.288580806015256
1773853638.1426697,train_epoch,92,1,perf/epoch_duration_sec,1412.2077000171412
1773853638.1426697,train_epoch,92,1,perf/epoch_samples_per_sec,34.83623549099956
1773853638.1426697,train_epoch,92,1,perf/epoch_tokens_per_sec,26418.67269208853
1773853638.1426697,train_epoch,92,1,perf/epoch_samples,49196.0
1773853638.1426697,train_epoch,92,1,perf/epoch_tokens,37308653.0
1773853638.1426697,train_epoch,92,1,system/cuda_epoch_peak_memory_gb,54.062886238098145
1773853638.1426697,train_epoch,92,1,eval/loss,1.03997564451261
1773853638.1426697,train_epoch,92,1,eval/duration_sec,20.698376037878916
1773853638.1426697,train_epoch,92,1,privacy/epsilon,5.729164243937991
1773853645.390464,audit_epoch,92,1,audit/delta,1e-05
1773853645.390464,audit_epoch,92,1,audit/num_canaries,500.0
1773853645.390464,audit_epoch,92,1,audit/num_members,250.0
1773853645.390464,audit_epoch,92,1,audit/paper_guess_fraction,0.2
1773853645.390464,audit_epoch,92,1,audit/paper_guess_steps,20.0
1773853645.390464,audit_epoch,92,1,audit/loss/auc,0.552536
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon/0.05,0.3197584319859743
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon/0.01,0.030605491250753403
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon_details/0.05/epsilon,0.3197584319859743
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon_details/0.05/num_guesses,25.0
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon_details/0.05/correct_guesses,19.0
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon_details/0.01/epsilon,0.030605491250753403
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon_details/0.01/num_guesses,25.0
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon_details/0.01/correct_guesses,19.0
1773853645.390464,audit_epoch,92,1,audit/embedding/auc,0.507808
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon/0.05,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon/0.01,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon_details/0.05/epsilon,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon_details/0.05/num_guesses,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon_details/0.05/correct_guesses,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon_details/0.01/epsilon,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon_details/0.01/num_guesses,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon_details/0.01/correct_guesses,0.0
1773853645.390464,audit_epoch,92,1,perf/audit_duration_sec,5.064277199096978
1773853767.2709792,train_step,100,2,train/step_loss,2.0532480357349785
1773853767.2709792,train_step,100,2,train/step_real_loss,1.1272322572767735
1773853767.2709792,train_step,100,2,train/lr,9.458610914145826e-05
1773853767.2709792,train_step,100,2,train/step_canary_loss,13.90625
1773853767.2709792,train_step,100,2,perf/step_duration_sec,14.751106533920392
1773853767.2709792,train_step,100,2,perf/samples_per_sec,9.355230381033918
1773853767.2709792,train_step,100,2,perf/tokens_per_sec,7110.44963025728
1773853767.2709792,train_step,100,2,perf/logical_batch_size,138.0
1773853767.2709792,train_step,100,2,perf/logical_token_count,104887.0
1773853767.2709792,train_step,100,2,perf/physical_batches,18.0
1773853767.2709792,train_step,100,2,privacy/epsilon,5.953156126903914
1773853767.2709792,train_step,100,2,system/cuda_memory_allocated_gb,15.12495470046997
1773853767.2709792,train_step,100,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773853787.107858,eval_step,100,2,eval/loss,1.0396132693662272
1773853787.107858,eval_step,100,2,eval/duration_sec,19.829364891862497
1773853935.771681,train_step,110,2,train/step_loss,1.647982077812081
1773853935.771681,train_step,110,2,train/step_real_loss,1.1710320189595222
1773853935.771681,train_step,110,2,train/lr,7.674325444256899e-05
1773853935.771681,train_step,110,2,train/step_canary_loss,11.822916666666666
1773853935.771681,train_step,110,2,perf/step_duration_sec,14.729006989859045
1773853935.771681,train_step,110,2,perf/samples_per_sec,9.029800861087974
1773853935.771681,train_step,110,2,perf/tokens_per_sec,6888.787551656326
1773853935.771681,train_step,110,2,perf/logical_batch_size,133.0
1773853935.771681,train_step,110,2,perf/logical_token_count,101465.0
1773853935.771681,train_step,110,2,perf/physical_batches,18.0
1773853935.771681,train_step,110,2,privacy/epsilon,6.2242325072714415
1773853935.771681,train_step,110,2,system/cuda_memory_allocated_gb,15.702028751373291
1773853935.771681,train_step,110,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854082.9588912,train_step,120,2,train/step_loss,1.8470192860512837
1773854082.9588912,train_step,120,2,train/step_real_loss,1.1087237671017647
1773854082.9588912,train_step,120,2,train/lr,5.965648380101916e-05
1773854082.9588912,train_step,120,2,train/step_canary_loss,12.347222222222221
1773854082.9588912,train_step,120,2,perf/step_duration_sec,14.77171956608072
1773854082.9588912,train_step,120,2,perf/samples_per_sec,9.206781877465872
1773854082.9588912,train_step,120,2,perf/tokens_per_sec,6978.266784640144
1773854082.9588912,train_step,120,2,perf/logical_batch_size,136.0
1773854082.9588912,train_step,120,2,perf/logical_token_count,103081.0
1773854082.9588912,train_step,120,2,perf/physical_batches,18.0
1773854082.9588912,train_step,120,2,privacy/epsilon,6.487004247951293
1773854082.9588912,train_step,120,2,system/cuda_memory_allocated_gb,16.279295921325684
1773854082.9588912,train_step,120,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854228.6299124,train_step,130,2,train/step_loss,1.4841852043614243
1773854228.6299124,train_step,130,2,train/step_real_loss,1.1038081794977188
1773854228.6299124,train_step,130,2,train/lr,4.388129346376178e-05
1773854228.6299124,train_step,130,2,train/step_canary_loss,13.65625
1773854228.6299124,train_step,130,2,perf/step_duration_sec,14.138089621905237
1773854228.6299124,train_step,130,2,perf/samples_per_sec,9.336480637064444
1773854228.6299124,train_step,130,2,perf/tokens_per_sec,7087.591229068504
1773854228.6299124,train_step,130,2,perf/logical_batch_size,132.0
1773854228.6299124,train_step,130,2,perf/logical_token_count,100205.0
1773854228.6299124,train_step,130,2,perf/physical_batches,17.0
1773854228.6299124,train_step,130,2,privacy/epsilon,6.738888126040064
1773854228.6299124,train_step,130,2,system/cuda_memory_allocated_gb,15.509668827056885
1773854228.6299124,train_step,130,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854374.1609344,train_step,140,2,train/step_loss,1.2703148426900384
1773854374.1609344,train_step,140,2,train/step_real_loss,1.083779253065586
1773854374.1609344,train_step,140,2,train/lr,2.993053979266577e-05
1773854374.1609344,train_step,140,2,train/step_canary_loss,9.229166666666666
1773854374.1609344,train_step,140,2,perf/step_duration_sec,14.757511151023209
1773854374.1609344,train_step,140,2,perf/samples_per_sec,8.809073472459241
1773854374.1609344,train_step,140,2,perf/tokens_per_sec,6979.293388022188
1773854374.1609344,train_step,140,2,perf/logical_batch_size,130.0
1773854374.1609344,train_step,140,2,perf/logical_token_count,102997.0
1773854374.1609344,train_step,140,2,perf/physical_batches,18.0
1773854374.1609344,train_step,140,2,privacy/epsilon,6.9846017598599595
1773854374.1609344,train_step,140,2,system/cuda_memory_allocated_gb,15.124946117401123
1773854374.1609344,train_step,140,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854520.2708924,train_step,150,2,train/step_loss,1.4772591663680914
1773854520.2708924,train_step,150,2,train/step_real_loss,1.1925464905798435
1773854520.2708924,train_step,150,2,train/lr,1.825776614411082e-05
1773854520.2708924,train_step,150,2,train/step_canary_loss,13.625
1773854520.2708924,train_step,150,2,perf/step_duration_sec,14.719724283087999
1773854520.2708924,train_step,150,2,perf/samples_per_sec,8.899623218521182
1773854520.2708924,train_step,150,2,perf/tokens_per_sec,6637.556391749427
1773854520.2708924,train_step,150,2,perf/logical_batch_size,131.0
1773854520.2708924,train_step,150,2,perf/logical_token_count,97703.0
1773854520.2708924,train_step,150,2,perf/physical_batches,17.0
1773854520.2708924,train_step,150,2,privacy/epsilon,7.225647671872536
1773854520.2708924,train_step,150,2,system/cuda_memory_allocated_gb,15.317307472229004
1773854520.2708924,train_step,150,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854538.402299,eval_step,150,2,eval/loss,1.0383169391712586
1773854538.402299,eval_step,150,2,eval/duration_sec,18.111375170992687
1773854691.1560457,train_step,160,2,train/step_loss,1.6241072825531462
1773854691.1560457,train_step,160,2,train/step_real_loss,1.152385726571083
1773854691.1560457,train_step,160,2,train/lr,9.242458032904311e-06
1773854691.1560457,train_step,160,2,train/step_canary_loss,11.687500476837158
1773854691.1560457,train_step,160,2,perf/step_duration_sec,14.878075499087572
1773854691.1560457,train_step,160,2,perf/samples_per_sec,8.93932820868912
1773854691.1560457,train_step,160,2,perf/tokens_per_sec,6843.62705420095
1773854691.1560457,train_step,160,2,perf/logical_batch_size,133.0
1773854691.1560457,train_step,160,2,perf/logical_token_count,101820.0
1773854691.1560457,train_step,160,2,perf/physical_batches,18.0
1773854691.1560457,train_step,160,2,privacy/epsilon,7.457279858735506
1773854691.1560457,train_step,160,2,system/cuda_memory_allocated_gb,15.702028751373291
1773854691.1560457,train_step,160,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854837.3778715,train_step,170,2,train/step_loss,1.382974067717108
1773854837.3778715,train_step,170,2,train/step_real_loss,1.0921453274786472
1773854837.3778715,train_step,170,2,train/lr,3.1777059397436692e-06
1773854837.3778715,train_step,170,2,train/step_canary_loss,13.791666984558105
1773854837.3778715,train_step,170,2,perf/step_duration_sec,14.022617863025516
1773854837.3778715,train_step,170,2,perf/samples_per_sec,9.342050199158423
1773854837.3778715,train_step,170,2,perf/tokens_per_sec,7510.50916660128
1773854837.3778715,train_step,170,2,perf/logical_batch_size,131.0
1773854837.3778715,train_step,170,2,perf/logical_token_count,105317.0
1773854837.3778715,train_step,170,2,perf/physical_batches,17.0
1773854837.3778715,train_step,170,2,privacy/epsilon,7.687885302283429
1773854837.3778715,train_step,170,2,system/cuda_memory_allocated_gb,15.317307472229004
1773854837.3778715,train_step,170,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854982.5852828,train_step,180,2,train/step_loss,1.7006171685677987
1773854982.5852828,train_step,180,2,train/step_real_loss,1.1432290449738503
1773854982.5852828,train_step,180,2,train/lr,2.606768482050215e-07
1773854982.5852828,train_step,180,2,train/step_canary_loss,11.892857142857142
1773854982.5852828,train_step,180,2,perf/step_duration_sec,14.94509603898041
1773854982.5852828,train_step,180,2,perf/samples_per_sec,8.966151816655827
1773854982.5852828,train_step,180,2,perf/tokens_per_sec,6624.1795798291805
1773854982.5852828,train_step,180,2,perf/logical_batch_size,134.0
1773854982.5852828,train_step,180,2,perf/logical_token_count,98999.0
1773854982.5852828,train_step,180,2,perf/physical_batches,18.0
1773854982.5852828,train_step,180,2,privacy/epsilon,7.908478086773533
1773854982.5852828,train_step,180,2,system/cuda_memory_allocated_gb,15.894390106201172
1773854982.5852828,train_step,180,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773855055.0742006,train_epoch,184,2,train/epoch_loss,1.7304604712060176
1773855055.0742006,train_epoch,184,2,train/epoch_real_loss,1.1275523616137826
1773855055.0742006,train_epoch,184,2,train/epoch_canary_loss,13.386261670760845
1773855055.0742006,train_epoch,184,2,perf/epoch_duration_sec,1389.9474938560743
1773855055.0742006,train_epoch,184,2,perf/epoch_samples_per_sec,35.36968138531038
1773855055.0742006,train_epoch,184,2,perf/epoch_tokens_per_sec,26840.30811588557
1773855055.0742006,train_epoch,184,2,perf/epoch_samples,49162.0
1773855055.0742006,train_epoch,184,2,perf/epoch_tokens,37306619.0
1773855055.0742006,train_epoch,184,2,system/cuda_epoch_peak_memory_gb,54.06288194656372
1773855055.0742006,train_epoch,184,2,eval/loss,1.0382178419983232
1773855055.0742006,train_epoch,184,2,eval/duration_sec,19.597126913955435
1773855055.0742006,train_epoch,184,2,privacy/epsilon,7.99669228530272
1773855064.748227,audit_epoch,184,2,audit/delta,1e-05
1773855064.748227,audit_epoch,184,2,audit/num_canaries,500.0
1773855064.748227,audit_epoch,184,2,audit/num_members,250.0
1773855064.748227,audit_epoch,184,2,audit/paper_guess_fraction,0.2
1773855064.748227,audit_epoch,184,2,audit/paper_guess_steps,20.0
1773855064.748227,audit_epoch,184,2,audit/loss/auc,0.562848
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon/0.05,0.42648759204894304
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon/0.01,0.16005427204072475
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon_details/0.05/epsilon,0.42648759204894304
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon_details/0.05/num_guesses,30.0
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon_details/0.05/correct_guesses,23.0
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon_details/0.01/epsilon,0.16005427204072475
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon_details/0.01/num_guesses,30.0
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon_details/0.01/correct_guesses,23.0
1773855064.748227,audit_epoch,184,2,audit/embedding/auc,0.50796
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon/0.05,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon/0.01,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon_details/0.05/epsilon,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon_details/0.05/num_guesses,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon_details/0.05/correct_guesses,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon_details/0.01/epsilon,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon_details/0.01/num_guesses,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon_details/0.01/correct_guesses,0.0
1773855064.748227,audit_epoch,184,2,perf/audit_duration_sec,7.245345794828609
1773855072.9264297,audit_final,184,2,audit/delta,1e-05
1773855072.9264297,audit_final,184,2,audit/num_canaries,500.0
1773855072.9264297,audit_final,184,2,audit/num_members,250.0
1773855072.9264297,audit_final,184,2,audit/paper_guess_fraction,0.2
1773855072.9264297,audit_final,184,2,audit/paper_guess_steps,20.0
1773855072.9264297,audit_final,184,2,audit/loss/auc,0.563808
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon/0.05,0.42648759204894304
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon/0.01,0.16005427204072475
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon_details/0.05/epsilon,0.42648759204894304
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon_details/0.05/num_guesses,30.0
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon_details/0.05/correct_guesses,23.0
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon_details/0.01/epsilon,0.16005427204072475
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon_details/0.01/num_guesses,30.0
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon_details/0.01/correct_guesses,23.0
1773855072.9264297,audit_final,184,2,audit/embedding/auc,0.50796
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon/0.05,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon/0.01,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon_details/0.05/epsilon,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon_details/0.05/num_guesses,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon_details/0.05/correct_guesses,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon_details/0.01/epsilon,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon_details/0.01/num_guesses,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon_details/0.01/correct_guesses,0.0
1773855073.4648237,energy_final,184,,energy/codecarbon/duration,2940.2213213280775
1773855073.4648237,energy_final,184,,energy/codecarbon/emissions,0.04336494635667721
1773855073.4648237,energy_final,184,,energy/codecarbon/emissions_rate,1.4748871468318434e-05
1773855073.4648237,energy_final,184,,energy/codecarbon/cpu_power,82.44163995833674
1773855073.4648237,energy_final,184,,energy/codecarbon/gpu_power,1409.9170137162785
1773855073.4648237,energy_final,184,,energy/codecarbon/ram_power,38.0
1773855073.4648237,energy_final,184,,energy/codecarbon/cpu_energy,0.06497743055379518
1773855073.4648237,energy_final,184,,energy/codecarbon/gpu_energy,1.1496539994446735
1773855073.4648237,energy_final,184,,energy/codecarbon/ram_energy,0.029949873467856485
1773855073.4648237,energy_final,184,,energy/codecarbon/energy_consumed,1.2445813034663262
1773855073.4648237,energy_final,184,,energy/codecarbon/water_consumed,0.0
1773855073.4648237,energy_final,184,,energy/codecarbon/cpu_count,8.0
1773855073.4648237,energy_final,184,,energy/codecarbon/gpu_count,5.0
1773855073.4648237,energy_final,184,,energy/codecarbon/longitude,8.212
1773855073.4648237,energy_final,184,,energy/codecarbon/latitude,47.4843
1773855073.4648237,energy_final,184,,energy/codecarbon/ram_total_size,128.0
1773855073.4648237,energy_final,184,,energy/codecarbon/cpu_utilization_percent,12.307671326287071
1773855073.4648237,energy_final,184,,energy/codecarbon/gpu_utilization_percent,66.43163995908625
1773855073.4648237,energy_final,184,,energy/codecarbon/ram_utilization_percent,8.260415956358626
1773855073.4648237,energy_final,184,,energy/codecarbon/ram_used_gb,166.47142205691932
1773855073.4648237,energy_final,184,,energy/codecarbon/pue,1.0
1773855073.4648237,energy_final,184,,energy/codecarbon/wue,0.0
|