File size: 27,559 Bytes
903307f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
timestamp,event,step,epoch,key,value
1773852387.8640254,train_step,10,1,train/step_loss,1.6353936302128123
1773852387.8640254,train_step,10,1,train/step_real_loss,1.1510175429284573
1773852387.8640254,train_step,10,1,train/lr,0.0002
1773852387.8640254,train_step,10,1,train/step_canary_loss,11.968750158945719
1773852387.8640254,train_step,10,1,perf/step_duration_sec,14.872664371039718
1773852387.8640254,train_step,10,1,perf/samples_per_sec,8.94258060842008
1773852387.8640254,train_step,10,1,perf/tokens_per_sec,6777.198589666931
1773852387.8640254,train_step,10,1,perf/logical_batch_size,133.0
1773852387.8640254,train_step,10,1,perf/logical_token_count,100795.0
1773852387.8640254,train_step,10,1,perf/physical_batches,18.0
1773852387.8640254,train_step,10,1,privacy/epsilon,2.524123361583601
1773852387.8640254,train_step,10,1,system/cuda_memory_allocated_gb,15.702028751373291
1773852387.8640254,train_step,10,1,system/cuda_max_memory_allocated_gb,54.06287908554077
1773852537.5150325,train_step,20,1,train/step_loss,1.857343329130298
1773852537.5150325,train_step,20,1,train/step_real_loss,1.114402625709772
1773852537.5150325,train_step,20,1,train/lr,0.00019837448439320027
1773852537.5150325,train_step,20,1,train/step_canary_loss,12.42361111111111
1773852537.5150325,train_step,20,1,perf/step_duration_sec,15.259514638921246
1773852537.5150325,train_step,20,1,perf/samples_per_sec,8.91247219968029
1773852537.5150325,train_step,20,1,perf/tokens_per_sec,7023.880020837742
1773852537.5150325,train_step,20,1,perf/logical_batch_size,136.0
1773852537.5150325,train_step,20,1,perf/logical_token_count,107181.0
1773852537.5150325,train_step,20,1,perf/physical_batches,18.0
1773852537.5150325,train_step,20,1,privacy/epsilon,3.1196418977343887
1773852537.5150325,train_step,20,1,system/cuda_memory_allocated_gb,16.279295921325684
1773852537.5150325,train_step,20,1,system/cuda_max_memory_allocated_gb,54.062880516052246
1773852684.6348226,train_step,30,1,train/step_loss,1.4355280977306943
1773852684.6348226,train_step,30,1,train/step_real_loss,1.148357093334198
1773852684.6348226,train_step,30,1,train/lr,0.0001935507835925601
1773852684.6348226,train_step,30,1,train/step_canary_loss,10.625000238418579
1773852684.6348226,train_step,30,1,perf/step_duration_sec,14.81383830588311
1773852684.6348226,train_step,30,1,perf/samples_per_sec,8.84308288608599
1773852684.6348226,train_step,30,1,perf/tokens_per_sec,6924.876448749963
1773852684.6348226,train_step,30,1,perf/logical_batch_size,131.0
1773852684.6348226,train_step,30,1,perf/logical_token_count,102584.0
1773852684.6348226,train_step,30,1,perf/physical_batches,18.0
1773852684.6348226,train_step,30,1,privacy/epsilon,3.599257677191776
1773852684.6348226,train_step,30,1,system/cuda_memory_allocated_gb,15.317307472229004
1773852684.6348226,train_step,30,1,system/cuda_max_memory_allocated_gb,54.062880516052246
1773852833.9397993,train_step,40,1,train/step_loss,1.6915703279000742
1773852833.9397993,train_step,40,1,train/step_real_loss,1.1175741739571095
1773852833.9397993,train_step,40,1,train/lr,0.00018568571761675893
1773852833.9397993,train_step,40,1,train/step_canary_loss,12.1875
1773852833.9397993,train_step,40,1,perf/step_duration_sec,14.987364898901433
1773852833.9397993,train_step,40,1,perf/samples_per_sec,8.940864581860026
1773852833.9397993,train_step,40,1,perf/tokens_per_sec,6826.283385380117
1773852833.9397993,train_step,40,1,perf/logical_batch_size,134.0
1773852833.9397993,train_step,40,1,perf/logical_token_count,102308.0
1773852833.9397993,train_step,40,1,perf/physical_batches,18.0
1773852833.9397993,train_step,40,1,privacy/epsilon,4.0171270793767535
1773852833.9397993,train_step,40,1,system/cuda_memory_allocated_gb,15.894390106201172
1773852833.9397993,train_step,40,1,system/cuda_max_memory_allocated_gb,54.062883377075195
1773852981.6699533,train_step,50,1,train/step_loss,1.730209876872875
1773852981.6699533,train_step,50,1,train/step_real_loss,1.1695572771131992
1773852981.6699533,train_step,50,1,train/lr,0.00017503498221564025
1773852981.6699533,train_step,50,1,train/step_canary_loss,11.982143129621234
1773852981.6699533,train_step,50,1,perf/step_duration_sec,15.535889676073566
1773852981.6699533,train_step,50,1,perf/samples_per_sec,8.625189982287917
1773852981.6699533,train_step,50,1,perf/tokens_per_sec,6820.401162038881
1773852981.6699533,train_step,50,1,perf/logical_batch_size,134.0
1773852981.6699533,train_step,50,1,perf/logical_token_count,105961.0
1773852981.6699533,train_step,50,1,perf/physical_batches,18.0
1773852981.6699533,train_step,50,1,privacy/epsilon,4.394850417905478
1773852981.6699533,train_step,50,1,system/cuda_memory_allocated_gb,15.894390106201172
1773852981.6699533,train_step,50,1,system/cuda_max_memory_allocated_gb,54.062883377075195
1773853001.9967575,eval_step,50,1,eval/loss,1.0429061587367738
1773853001.9967575,eval_step,50,1,eval/duration_sec,20.30736266821623
1773853155.8696322,train_step,60,1,train/step_loss,1.5988739917152806
1773853155.8696322,train_step,60,1,train/step_real_loss,1.106154203414917
1773853155.8696322,train_step,60,1,train/lr,0.0001619448361215723
1773853155.8696322,train_step,60,1,train/step_canary_loss,14.21250057220459
1773853155.8696322,train_step,60,1,perf/step_duration_sec,14.15656855283305
1773853155.8696322,train_step,60,1,perf/samples_per_sec,9.3949320771935
1773853155.8696322,train_step,60,1,perf/tokens_per_sec,7349.238596324901
1773853155.8696322,train_step,60,1,perf/logical_batch_size,133.0
1773853155.8696322,train_step,60,1,perf/logical_token_count,104040.0
1773853155.8696322,train_step,60,1,perf/physical_batches,17.0
1773853155.8696322,train_step,60,1,privacy/epsilon,4.743740138692227
1773853155.8696322,train_step,60,1,system/cuda_memory_allocated_gb,15.702028751373291
1773853155.8696322,train_step,60,1,system/cuda_max_memory_allocated_gb,54.062886238098145
1773853298.6477532,train_step,70,1,train/step_loss,1.5077286886446404
1773853298.6477532,train_step,70,1,train/step_real_loss,1.1227163039147854
1773853298.6477532,train_step,70,1,train/lr,0.00014684084406997903
1773853298.6477532,train_step,70,1,train/step_canary_loss,13.828125
1773853298.6477532,train_step,70,1,perf/step_duration_sec,14.256150827044621
1773853298.6477532,train_step,70,1,perf/samples_per_sec,9.25916129826499
1773853298.6477532,train_step,70,1,perf/tokens_per_sec,6918.066538192309
1773853298.6477532,train_step,70,1,perf/logical_batch_size,132.0
1773853298.6477532,train_step,70,1,perf/logical_token_count,98625.0
1773853298.6477532,train_step,70,1,perf/physical_batches,17.0
1773853298.6477532,train_step,70,1,privacy/epsilon,5.069830876941064
1773853298.6477532,train_step,70,1,system/cuda_memory_allocated_gb,15.509668827056885
1773853298.6477532,train_step,70,1,system/cuda_max_memory_allocated_gb,54.062886238098145
1773853446.331358,train_step,80,1,train/step_loss,1.611925903119539
1773853446.331358,train_step,80,1,train/step_real_loss,1.1202042661607265
1773853446.331358,train_step,80,1,train/lr,0.00013021404155695725
1773853446.331358,train_step,80,1,train/step_canary_loss,14.199999809265137
1773853446.331358,train_step,80,1,perf/step_duration_sec,14.11704233684577
1773853446.331358,train_step,80,1,perf/samples_per_sec,9.421236887054398
1773853446.331358,train_step,80,1,perf/tokens_per_sec,7214.046509883511
1773853446.331358,train_step,80,1,perf/logical_batch_size,133.0
1773853446.331358,train_step,80,1,perf/logical_token_count,101841.0
1773853446.331358,train_step,80,1,perf/physical_batches,17.0
1773853446.331358,train_step,80,1,privacy/epsilon,5.377982250509876
1773853446.331358,train_step,80,1,system/cuda_memory_allocated_gb,15.702028751373291
1773853446.331358,train_step,80,1,system/cuda_max_memory_allocated_gb,54.062886238098145
1773853593.7571762,train_step,90,1,train/step_loss,1.8932846375625498
1773853593.7571762,train_step,90,1,train/step_real_loss,1.1377343386411667
1773853593.7571762,train_step,90,1,train/lr,0.00011260497112202895
1773853593.7571762,train_step,90,1,train/step_canary_loss,12.63888888888889
1773853593.7571762,train_step,90,1,perf/step_duration_sec,14.762270654086024
1773853593.7571762,train_step,90,1,perf/samples_per_sec,9.21267487819408
1773853593.7571762,train_step,90,1,perf/tokens_per_sec,6934.841014560594
1773853593.7571762,train_step,90,1,perf/logical_batch_size,136.0
1773853593.7571762,train_step,90,1,perf/logical_token_count,102374.0
1773853593.7571762,train_step,90,1,perf/physical_batches,18.0
1773853593.7571762,train_step,90,1,privacy/epsilon,5.671744621870462
1773853593.7571762,train_step,90,1,system/cuda_memory_allocated_gb,16.279295921325684
1773853593.7571762,train_step,90,1,system/cuda_max_memory_allocated_gb,54.062886238098145
1773853638.1426697,train_epoch,92,1,train/epoch_loss,1.7500966633389758
1773853638.1426697,train_epoch,92,1,train/epoch_real_loss,1.133818452948967
1773853638.1426697,train_epoch,92,1,train/epoch_canary_loss,13.288580806015256
1773853638.1426697,train_epoch,92,1,perf/epoch_duration_sec,1412.2077000171412
1773853638.1426697,train_epoch,92,1,perf/epoch_samples_per_sec,34.83623549099956
1773853638.1426697,train_epoch,92,1,perf/epoch_tokens_per_sec,26418.67269208853
1773853638.1426697,train_epoch,92,1,perf/epoch_samples,49196.0
1773853638.1426697,train_epoch,92,1,perf/epoch_tokens,37308653.0
1773853638.1426697,train_epoch,92,1,system/cuda_epoch_peak_memory_gb,54.062886238098145
1773853638.1426697,train_epoch,92,1,eval/loss,1.03997564451261
1773853638.1426697,train_epoch,92,1,eval/duration_sec,20.698376037878916
1773853638.1426697,train_epoch,92,1,privacy/epsilon,5.729164243937991
1773853645.390464,audit_epoch,92,1,audit/delta,1e-05
1773853645.390464,audit_epoch,92,1,audit/num_canaries,500.0
1773853645.390464,audit_epoch,92,1,audit/num_members,250.0
1773853645.390464,audit_epoch,92,1,audit/paper_guess_fraction,0.2
1773853645.390464,audit_epoch,92,1,audit/paper_guess_steps,20.0
1773853645.390464,audit_epoch,92,1,audit/loss/auc,0.552536
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon/0.05,0.3197584319859743
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon/0.01,0.030605491250753403
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon_details/0.05/epsilon,0.3197584319859743
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon_details/0.05/num_guesses,25.0
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon_details/0.05/correct_guesses,19.0
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon_details/0.01/epsilon,0.030605491250753403
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon_details/0.01/num_guesses,25.0
1773853645.390464,audit_epoch,92,1,audit/loss/empirical_epsilon_details/0.01/correct_guesses,19.0
1773853645.390464,audit_epoch,92,1,audit/embedding/auc,0.507808
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon/0.05,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon/0.01,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon_details/0.05/epsilon,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon_details/0.05/num_guesses,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon_details/0.05/correct_guesses,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon_details/0.01/epsilon,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon_details/0.01/num_guesses,0.0
1773853645.390464,audit_epoch,92,1,audit/embedding/empirical_epsilon_details/0.01/correct_guesses,0.0
1773853645.390464,audit_epoch,92,1,perf/audit_duration_sec,5.064277199096978
1773853767.2709792,train_step,100,2,train/step_loss,2.0532480357349785
1773853767.2709792,train_step,100,2,train/step_real_loss,1.1272322572767735
1773853767.2709792,train_step,100,2,train/lr,9.458610914145826e-05
1773853767.2709792,train_step,100,2,train/step_canary_loss,13.90625
1773853767.2709792,train_step,100,2,perf/step_duration_sec,14.751106533920392
1773853767.2709792,train_step,100,2,perf/samples_per_sec,9.355230381033918
1773853767.2709792,train_step,100,2,perf/tokens_per_sec,7110.44963025728
1773853767.2709792,train_step,100,2,perf/logical_batch_size,138.0
1773853767.2709792,train_step,100,2,perf/logical_token_count,104887.0
1773853767.2709792,train_step,100,2,perf/physical_batches,18.0
1773853767.2709792,train_step,100,2,privacy/epsilon,5.953156126903914
1773853767.2709792,train_step,100,2,system/cuda_memory_allocated_gb,15.12495470046997
1773853767.2709792,train_step,100,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773853787.107858,eval_step,100,2,eval/loss,1.0396132693662272
1773853787.107858,eval_step,100,2,eval/duration_sec,19.829364891862497
1773853935.771681,train_step,110,2,train/step_loss,1.647982077812081
1773853935.771681,train_step,110,2,train/step_real_loss,1.1710320189595222
1773853935.771681,train_step,110,2,train/lr,7.674325444256899e-05
1773853935.771681,train_step,110,2,train/step_canary_loss,11.822916666666666
1773853935.771681,train_step,110,2,perf/step_duration_sec,14.729006989859045
1773853935.771681,train_step,110,2,perf/samples_per_sec,9.029800861087974
1773853935.771681,train_step,110,2,perf/tokens_per_sec,6888.787551656326
1773853935.771681,train_step,110,2,perf/logical_batch_size,133.0
1773853935.771681,train_step,110,2,perf/logical_token_count,101465.0
1773853935.771681,train_step,110,2,perf/physical_batches,18.0
1773853935.771681,train_step,110,2,privacy/epsilon,6.2242325072714415
1773853935.771681,train_step,110,2,system/cuda_memory_allocated_gb,15.702028751373291
1773853935.771681,train_step,110,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854082.9588912,train_step,120,2,train/step_loss,1.8470192860512837
1773854082.9588912,train_step,120,2,train/step_real_loss,1.1087237671017647
1773854082.9588912,train_step,120,2,train/lr,5.965648380101916e-05
1773854082.9588912,train_step,120,2,train/step_canary_loss,12.347222222222221
1773854082.9588912,train_step,120,2,perf/step_duration_sec,14.77171956608072
1773854082.9588912,train_step,120,2,perf/samples_per_sec,9.206781877465872
1773854082.9588912,train_step,120,2,perf/tokens_per_sec,6978.266784640144
1773854082.9588912,train_step,120,2,perf/logical_batch_size,136.0
1773854082.9588912,train_step,120,2,perf/logical_token_count,103081.0
1773854082.9588912,train_step,120,2,perf/physical_batches,18.0
1773854082.9588912,train_step,120,2,privacy/epsilon,6.487004247951293
1773854082.9588912,train_step,120,2,system/cuda_memory_allocated_gb,16.279295921325684
1773854082.9588912,train_step,120,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854228.6299124,train_step,130,2,train/step_loss,1.4841852043614243
1773854228.6299124,train_step,130,2,train/step_real_loss,1.1038081794977188
1773854228.6299124,train_step,130,2,train/lr,4.388129346376178e-05
1773854228.6299124,train_step,130,2,train/step_canary_loss,13.65625
1773854228.6299124,train_step,130,2,perf/step_duration_sec,14.138089621905237
1773854228.6299124,train_step,130,2,perf/samples_per_sec,9.336480637064444
1773854228.6299124,train_step,130,2,perf/tokens_per_sec,7087.591229068504
1773854228.6299124,train_step,130,2,perf/logical_batch_size,132.0
1773854228.6299124,train_step,130,2,perf/logical_token_count,100205.0
1773854228.6299124,train_step,130,2,perf/physical_batches,17.0
1773854228.6299124,train_step,130,2,privacy/epsilon,6.738888126040064
1773854228.6299124,train_step,130,2,system/cuda_memory_allocated_gb,15.509668827056885
1773854228.6299124,train_step,130,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854374.1609344,train_step,140,2,train/step_loss,1.2703148426900384
1773854374.1609344,train_step,140,2,train/step_real_loss,1.083779253065586
1773854374.1609344,train_step,140,2,train/lr,2.993053979266577e-05
1773854374.1609344,train_step,140,2,train/step_canary_loss,9.229166666666666
1773854374.1609344,train_step,140,2,perf/step_duration_sec,14.757511151023209
1773854374.1609344,train_step,140,2,perf/samples_per_sec,8.809073472459241
1773854374.1609344,train_step,140,2,perf/tokens_per_sec,6979.293388022188
1773854374.1609344,train_step,140,2,perf/logical_batch_size,130.0
1773854374.1609344,train_step,140,2,perf/logical_token_count,102997.0
1773854374.1609344,train_step,140,2,perf/physical_batches,18.0
1773854374.1609344,train_step,140,2,privacy/epsilon,6.9846017598599595
1773854374.1609344,train_step,140,2,system/cuda_memory_allocated_gb,15.124946117401123
1773854374.1609344,train_step,140,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854520.2708924,train_step,150,2,train/step_loss,1.4772591663680914
1773854520.2708924,train_step,150,2,train/step_real_loss,1.1925464905798435
1773854520.2708924,train_step,150,2,train/lr,1.825776614411082e-05
1773854520.2708924,train_step,150,2,train/step_canary_loss,13.625
1773854520.2708924,train_step,150,2,perf/step_duration_sec,14.719724283087999
1773854520.2708924,train_step,150,2,perf/samples_per_sec,8.899623218521182
1773854520.2708924,train_step,150,2,perf/tokens_per_sec,6637.556391749427
1773854520.2708924,train_step,150,2,perf/logical_batch_size,131.0
1773854520.2708924,train_step,150,2,perf/logical_token_count,97703.0
1773854520.2708924,train_step,150,2,perf/physical_batches,17.0
1773854520.2708924,train_step,150,2,privacy/epsilon,7.225647671872536
1773854520.2708924,train_step,150,2,system/cuda_memory_allocated_gb,15.317307472229004
1773854520.2708924,train_step,150,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854538.402299,eval_step,150,2,eval/loss,1.0383169391712586
1773854538.402299,eval_step,150,2,eval/duration_sec,18.111375170992687
1773854691.1560457,train_step,160,2,train/step_loss,1.6241072825531462
1773854691.1560457,train_step,160,2,train/step_real_loss,1.152385726571083
1773854691.1560457,train_step,160,2,train/lr,9.242458032904311e-06
1773854691.1560457,train_step,160,2,train/step_canary_loss,11.687500476837158
1773854691.1560457,train_step,160,2,perf/step_duration_sec,14.878075499087572
1773854691.1560457,train_step,160,2,perf/samples_per_sec,8.93932820868912
1773854691.1560457,train_step,160,2,perf/tokens_per_sec,6843.62705420095
1773854691.1560457,train_step,160,2,perf/logical_batch_size,133.0
1773854691.1560457,train_step,160,2,perf/logical_token_count,101820.0
1773854691.1560457,train_step,160,2,perf/physical_batches,18.0
1773854691.1560457,train_step,160,2,privacy/epsilon,7.457279858735506
1773854691.1560457,train_step,160,2,system/cuda_memory_allocated_gb,15.702028751373291
1773854691.1560457,train_step,160,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854837.3778715,train_step,170,2,train/step_loss,1.382974067717108
1773854837.3778715,train_step,170,2,train/step_real_loss,1.0921453274786472
1773854837.3778715,train_step,170,2,train/lr,3.1777059397436692e-06
1773854837.3778715,train_step,170,2,train/step_canary_loss,13.791666984558105
1773854837.3778715,train_step,170,2,perf/step_duration_sec,14.022617863025516
1773854837.3778715,train_step,170,2,perf/samples_per_sec,9.342050199158423
1773854837.3778715,train_step,170,2,perf/tokens_per_sec,7510.50916660128
1773854837.3778715,train_step,170,2,perf/logical_batch_size,131.0
1773854837.3778715,train_step,170,2,perf/logical_token_count,105317.0
1773854837.3778715,train_step,170,2,perf/physical_batches,17.0
1773854837.3778715,train_step,170,2,privacy/epsilon,7.687885302283429
1773854837.3778715,train_step,170,2,system/cuda_memory_allocated_gb,15.317307472229004
1773854837.3778715,train_step,170,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773854982.5852828,train_step,180,2,train/step_loss,1.7006171685677987
1773854982.5852828,train_step,180,2,train/step_real_loss,1.1432290449738503
1773854982.5852828,train_step,180,2,train/lr,2.606768482050215e-07
1773854982.5852828,train_step,180,2,train/step_canary_loss,11.892857142857142
1773854982.5852828,train_step,180,2,perf/step_duration_sec,14.94509603898041
1773854982.5852828,train_step,180,2,perf/samples_per_sec,8.966151816655827
1773854982.5852828,train_step,180,2,perf/tokens_per_sec,6624.1795798291805
1773854982.5852828,train_step,180,2,perf/logical_batch_size,134.0
1773854982.5852828,train_step,180,2,perf/logical_token_count,98999.0
1773854982.5852828,train_step,180,2,perf/physical_batches,18.0
1773854982.5852828,train_step,180,2,privacy/epsilon,7.908478086773533
1773854982.5852828,train_step,180,2,system/cuda_memory_allocated_gb,15.894390106201172
1773854982.5852828,train_step,180,2,system/cuda_max_memory_allocated_gb,54.06288194656372
1773855055.0742006,train_epoch,184,2,train/epoch_loss,1.7304604712060176
1773855055.0742006,train_epoch,184,2,train/epoch_real_loss,1.1275523616137826
1773855055.0742006,train_epoch,184,2,train/epoch_canary_loss,13.386261670760845
1773855055.0742006,train_epoch,184,2,perf/epoch_duration_sec,1389.9474938560743
1773855055.0742006,train_epoch,184,2,perf/epoch_samples_per_sec,35.36968138531038
1773855055.0742006,train_epoch,184,2,perf/epoch_tokens_per_sec,26840.30811588557
1773855055.0742006,train_epoch,184,2,perf/epoch_samples,49162.0
1773855055.0742006,train_epoch,184,2,perf/epoch_tokens,37306619.0
1773855055.0742006,train_epoch,184,2,system/cuda_epoch_peak_memory_gb,54.06288194656372
1773855055.0742006,train_epoch,184,2,eval/loss,1.0382178419983232
1773855055.0742006,train_epoch,184,2,eval/duration_sec,19.597126913955435
1773855055.0742006,train_epoch,184,2,privacy/epsilon,7.99669228530272
1773855064.748227,audit_epoch,184,2,audit/delta,1e-05
1773855064.748227,audit_epoch,184,2,audit/num_canaries,500.0
1773855064.748227,audit_epoch,184,2,audit/num_members,250.0
1773855064.748227,audit_epoch,184,2,audit/paper_guess_fraction,0.2
1773855064.748227,audit_epoch,184,2,audit/paper_guess_steps,20.0
1773855064.748227,audit_epoch,184,2,audit/loss/auc,0.562848
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon/0.05,0.42648759204894304
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon/0.01,0.16005427204072475
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon_details/0.05/epsilon,0.42648759204894304
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon_details/0.05/num_guesses,30.0
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon_details/0.05/correct_guesses,23.0
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon_details/0.01/epsilon,0.16005427204072475
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon_details/0.01/num_guesses,30.0
1773855064.748227,audit_epoch,184,2,audit/loss/empirical_epsilon_details/0.01/correct_guesses,23.0
1773855064.748227,audit_epoch,184,2,audit/embedding/auc,0.50796
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon/0.05,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon/0.01,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon_details/0.05/epsilon,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon_details/0.05/num_guesses,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon_details/0.05/correct_guesses,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon_details/0.01/epsilon,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon_details/0.01/num_guesses,0.0
1773855064.748227,audit_epoch,184,2,audit/embedding/empirical_epsilon_details/0.01/correct_guesses,0.0
1773855064.748227,audit_epoch,184,2,perf/audit_duration_sec,7.245345794828609
1773855072.9264297,audit_final,184,2,audit/delta,1e-05
1773855072.9264297,audit_final,184,2,audit/num_canaries,500.0
1773855072.9264297,audit_final,184,2,audit/num_members,250.0
1773855072.9264297,audit_final,184,2,audit/paper_guess_fraction,0.2
1773855072.9264297,audit_final,184,2,audit/paper_guess_steps,20.0
1773855072.9264297,audit_final,184,2,audit/loss/auc,0.563808
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon/0.05,0.42648759204894304
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon/0.01,0.16005427204072475
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon_details/0.05/epsilon,0.42648759204894304
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon_details/0.05/num_guesses,30.0
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon_details/0.05/correct_guesses,23.0
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon_details/0.01/epsilon,0.16005427204072475
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon_details/0.01/num_guesses,30.0
1773855072.9264297,audit_final,184,2,audit/loss/empirical_epsilon_details/0.01/correct_guesses,23.0
1773855072.9264297,audit_final,184,2,audit/embedding/auc,0.50796
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon/0.05,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon/0.01,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon_details/0.05/epsilon,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon_details/0.05/num_guesses,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon_details/0.05/correct_guesses,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon_details/0.01/epsilon,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon_details/0.01/num_guesses,0.0
1773855072.9264297,audit_final,184,2,audit/embedding/empirical_epsilon_details/0.01/correct_guesses,0.0
1773855073.4648237,energy_final,184,,energy/codecarbon/duration,2940.2213213280775
1773855073.4648237,energy_final,184,,energy/codecarbon/emissions,0.04336494635667721
1773855073.4648237,energy_final,184,,energy/codecarbon/emissions_rate,1.4748871468318434e-05
1773855073.4648237,energy_final,184,,energy/codecarbon/cpu_power,82.44163995833674
1773855073.4648237,energy_final,184,,energy/codecarbon/gpu_power,1409.9170137162785
1773855073.4648237,energy_final,184,,energy/codecarbon/ram_power,38.0
1773855073.4648237,energy_final,184,,energy/codecarbon/cpu_energy,0.06497743055379518
1773855073.4648237,energy_final,184,,energy/codecarbon/gpu_energy,1.1496539994446735
1773855073.4648237,energy_final,184,,energy/codecarbon/ram_energy,0.029949873467856485
1773855073.4648237,energy_final,184,,energy/codecarbon/energy_consumed,1.2445813034663262
1773855073.4648237,energy_final,184,,energy/codecarbon/water_consumed,0.0
1773855073.4648237,energy_final,184,,energy/codecarbon/cpu_count,8.0
1773855073.4648237,energy_final,184,,energy/codecarbon/gpu_count,5.0
1773855073.4648237,energy_final,184,,energy/codecarbon/longitude,8.212
1773855073.4648237,energy_final,184,,energy/codecarbon/latitude,47.4843
1773855073.4648237,energy_final,184,,energy/codecarbon/ram_total_size,128.0
1773855073.4648237,energy_final,184,,energy/codecarbon/cpu_utilization_percent,12.307671326287071
1773855073.4648237,energy_final,184,,energy/codecarbon/gpu_utilization_percent,66.43163995908625
1773855073.4648237,energy_final,184,,energy/codecarbon/ram_utilization_percent,8.260415956358626
1773855073.4648237,energy_final,184,,energy/codecarbon/ram_used_gb,166.47142205691932
1773855073.4648237,energy_final,184,,energy/codecarbon/pue,1.0
1773855073.4648237,energy_final,184,,energy/codecarbon/wue,0.0