sparsetrace commited on
Commit
8ea44e3
·
verified ·
1 Parent(s): 8f29853

checkpoint: promote transactional candidate: step 80000

Browse files
Files changed (4) hide show
  1. ema.safetensors +1 -1
  2. model.safetensors +1 -1
  3. optim.safetensors +1 -1
  4. state.json +257 -228
ema.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8b8002445e6d87a3c5fe6e64263bc1b820638389018b717bd1d87a388514ddf4
3
  size 1945760448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:94ced5bd346339374a730d0452456afecd8f5976c78ed9d9a0eff0edcc736897
3
  size 1945760448
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d084dabba86c6c1d5525fe4281e4dfb71b3bebfdc17eb66d2f2d820138f791a1
3
  size 2046527448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:558aea0e4e2c8f49884be5d2cd8954a643129b2bf503382573acaeac6759cf20
3
  size 2046527448
optim.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:771e9435cff2cf59472bf759dde9ffd2a1a80406f6bd4c8610cd606461e4ca48
3
  size 3891530432
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:862af4ab309bb1e2c441d424da06ca7393a5e4e9e28b28fe540d5a15ade4fbea
3
  size 3891530432
state.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "step": 70000,
3
  "run_history": [
4
  {
5
  "start_step": 0,
@@ -94,216 +94,245 @@
94
  "seed_offset": 0,
95
  "target_steps": 400000
96
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  }
98
  ],
99
  "guard_state": {
100
  "version": 3,
101
- "live_grad_ema": 0.10093997473621566,
102
- "grad_ema": 0.10093997473621566,
103
- "grad_reference": 0.13336485530715764,
104
- "grad_p90_reference": 0.2482941896375334,
105
  "spikes_total": 0,
106
  "recent_losses": [
107
- 0.8025999069213867,
108
- 0.7812247276306152,
109
- 0.7802746295928955,
110
- 0.7988201379776001,
111
- 0.802432656288147,
112
- 0.793769121170044,
113
- 0.8051333427429199,
114
- 0.8035447597503662,
115
- 0.7800968885421753,
116
- 0.7837401628494263,
117
- 0.8129797577857971,
118
- 0.8023052215576172,
119
- 0.7940864562988281,
120
- 0.7970350980758667,
121
- 0.7882712483406067,
122
- 0.7862706184387207,
123
- 0.781178891658783,
124
- 0.7873877882957458,
125
- 0.7874085903167725,
126
- 0.8068331480026245,
127
- 0.7904229164123535,
128
- 0.8128153085708618,
129
- 0.7917333245277405,
130
- 0.7896208167076111,
131
- 0.8000121116638184,
132
- 0.8042793273925781,
133
- 0.7994593381881714,
134
- 0.7975194454193115,
135
- 0.8028004169464111,
136
- 0.8031681776046753,
137
- 0.7790490388870239,
138
- 0.8120772242546082,
139
- 0.8106952905654907,
140
- 0.802364706993103,
141
- 0.7977872490882874,
142
- 0.7935895323753357,
143
- 0.7773255109786987,
144
- 0.7832024097442627,
145
- 0.7952970266342163,
146
- 0.8125020265579224,
147
- 0.7925475835800171,
148
- 0.8042447566986084,
149
- 0.7662420272827148,
150
- 0.7848743200302124,
151
- 0.793734073638916,
152
- 0.7999864220619202,
153
- 0.7991811037063599,
154
- 0.7887350916862488,
155
- 0.7967043519020081,
156
- 0.7802110910415649,
157
- 0.7817031741142273,
158
- 0.8231815099716187,
159
- 0.7956117391586304,
160
- 0.7900019884109497,
161
- 0.7889696359634399,
162
- 0.7955681681632996,
163
- 0.7945159673690796,
164
- 0.7760295867919922,
165
- 0.7827702760696411,
166
- 0.7972118854522705,
167
- 0.7901098728179932,
168
- 0.8013142943382263,
169
- 0.7777154445648193,
170
- 0.7857697010040283,
171
- 0.7908087968826294,
172
- 0.8028693795204163,
173
- 0.7997928857803345,
174
- 0.7858878970146179,
175
- 0.7779598236083984,
176
- 0.8028081655502319,
177
- 0.7757434844970703,
178
- 0.8103242516517639,
179
- 0.7952283620834351,
180
- 0.7894253730773926,
181
- 0.8221614360809326,
182
- 0.7723782658576965,
183
- 0.792094886302948,
184
- 0.781030535697937,
185
- 0.7819499969482422,
186
- 0.7943369150161743,
187
- 0.8123618364334106,
188
- 0.786348283290863,
189
- 0.8006545901298523,
190
- 0.8004565238952637,
191
- 0.7940046787261963,
192
- 0.7931481003761292,
193
- 0.7961478233337402,
194
- 0.799895167350769,
195
- 0.7882716655731201,
196
- 0.7782301902770996,
197
- 0.7854876518249512,
198
- 0.7882372140884399,
199
- 0.8108580708503723,
200
- 0.7851492762565613,
201
- 0.7768666744232178,
202
- 0.7923758625984192,
203
- 0.8191432952880859,
204
- 0.7937012910842896,
205
- 0.7980853319168091,
206
- 0.7889020442962646,
207
- 0.7968310117721558,
208
- 0.7883217334747314,
209
- 0.7871203422546387,
210
- 0.8154696226119995,
211
- 0.7977403998374939,
212
- 0.7927383184432983,
213
- 0.7958823442459106,
214
- 0.8009363412857056,
215
- 0.8107385635375977,
216
- 0.7928140163421631,
217
- 0.7953475713729858,
218
- 0.7903873324394226,
219
- 0.7850275039672852,
220
- 0.7881691455841064,
221
- 0.8031665086746216,
222
- 0.8081239461898804,
223
- 0.7937340140342712,
224
- 0.8086464405059814,
225
- 0.7959200739860535,
226
- 0.7835697531700134,
227
- 0.7958613634109497,
228
- 0.8060988187789917,
229
- 0.8173725008964539,
230
- 0.781183123588562,
231
- 0.799437403678894,
232
- 0.816066324710846,
233
- 0.7855912446975708,
234
- 0.7923034429550171,
235
- 0.7746668457984924,
236
- 0.7907034158706665,
237
- 0.7777390480041504,
238
- 0.7866688966751099,
239
- 0.802391529083252,
240
- 0.7943458557128906,
241
- 0.8009390830993652,
242
- 0.8039968013763428,
243
- 0.7872947454452515,
244
- 0.7985210418701172,
245
- 0.7941929697990417,
246
- 0.7949443459510803,
247
- 0.788948655128479,
248
- 0.7913790941238403,
249
- 0.7709335088729858,
250
- 0.7976598143577576,
251
- 0.7900689244270325,
252
- 0.7737479209899902,
253
- 0.791588306427002,
254
- 0.7892031669616699,
255
- 0.7927761673927307,
256
- 0.7997742891311646,
257
- 0.7964314222335815,
258
- 0.7844709157943726,
259
- 0.7812561988830566,
260
- 0.7896272540092468,
261
- 0.7948977947235107,
262
- 0.8038414716720581,
263
- 0.79840087890625,
264
- 0.804000973701477,
265
- 0.7764262557029724,
266
- 0.782539963722229,
267
- 0.7756984233856201,
268
- 0.7987139225006104,
269
- 0.8025899529457092,
270
- 0.7845830917358398,
271
- 0.786514163017273,
272
- 0.7901351451873779,
273
- 0.8009799718856812,
274
- 0.779059648513794,
275
- 0.7784007787704468,
276
- 0.7951036691665649,
277
- 0.7936373949050903,
278
- 0.7877393960952759,
279
- 0.7921167612075806,
280
- 0.7997759580612183,
281
- 0.7820404767990112,
282
- 0.7858753800392151,
283
- 0.8241041898727417,
284
- 0.7900389432907104,
285
- 0.7875664234161377,
286
- 0.7704766392707825,
287
- 0.7841053009033203,
288
- 0.7821608185768127,
289
- 0.7917798757553101,
290
- 0.8030577898025513,
291
- 0.7861316204071045,
292
- 0.7993047833442688,
293
- 0.7967855930328369,
294
- 0.7997536063194275,
295
- 0.786116361618042,
296
- 0.8042708039283752,
297
- 0.7887070775032043,
298
- 0.7841724753379822,
299
- 0.772618293762207,
300
- 0.7918744087219238,
301
- 0.8047670722007751,
302
- 0.8008345365524292,
303
- 0.775162935256958,
304
- 0.8022024631500244,
305
- 0.799297571182251,
306
- 0.7904259562492371
307
  ],
308
  "loss_window": 200,
309
  "grad_ema_decay": 0.99,
@@ -336,45 +365,45 @@
336
  },
337
  "committed_scale": 0.125,
338
  "attempt_factor": 1.0,
339
- "fast_loss": 0.7933706989574026,
340
- "slow_loss": 0.7941704669861516,
341
- "best_loss": 0.7924602755904198,
342
  "reference": {
343
- "loss": 0.7944041414260864,
344
- "grad_median": 0.13336485530715764,
345
- "grad_p90": 0.2482941896375334,
346
- "step": 70000,
347
- "promotions": 2
348
  },
349
  "last_metrics": {
350
- "loss": 0.7932941544055939,
351
- "grad_median": 0.10018549859523773,
352
- "grad_p90": 0.10819339230656624,
353
  "skip_rate": 0.0,
354
  "relative_spike_rate": 0.0
355
  },
356
- "last_loss_ratio": 0.9982540382098282,
357
- "last_grad_ratio": 0.7072265612273279,
358
- "last_grad_p90_ratio": 0.38187782593981767,
359
  "healthy_windows": 0,
360
  "warning_windows": 0,
361
  "retry_windows": 0,
362
- "windows_seen": 50,
363
  "retry_count": 0
364
  },
365
- "best_window": 0.7924602755904198,
366
  "blown_windows": 0
367
  },
368
  "transaction": {
369
  "status": "committed",
370
- "committed_at": "2026-07-21T04:47:40.398302+00:00",
371
  "attempt": 0,
372
  "health_reference": {
373
- "loss": 0.7944041414260864,
374
- "grad_median": 0.13336485530715764,
375
- "grad_p90": 0.2482941896375334,
376
- "step": 70000,
377
- "promotions": 2
378
  }
379
  },
380
  "config": {
@@ -417,7 +446,7 @@
417
  },
418
  "optim_param_groups": [
419
  {
420
- "lr": 1.1671124007623334e-05,
421
  "betas": [
422
  0.9,
423
  0.999
 
1
  {
2
+ "step": 80000,
3
  "run_history": [
4
  {
5
  "start_step": 0,
 
94
  "seed_offset": 0,
95
  "target_steps": 400000
96
  }
97
+ },
98
+ {
99
+ "start_step": 70000,
100
+ "end_step": 80000,
101
+ "seconds": 2527.8,
102
+ "world": 2,
103
+ "objective": "flow",
104
+ "completed": false,
105
+ "finished_at": "2026-07-21T05:29:48.245697+00:00",
106
+ "promotion_reason": "periodic healthy candidate",
107
+ "effective": {
108
+ "attempt": 0,
109
+ "precision": "tf32",
110
+ "lr_policy": "adaptive",
111
+ "lr_base": 0.0001,
112
+ "lr_start": null,
113
+ "lr_end": 1.1425746560775714e-05,
114
+ "lr_min": 1e-05,
115
+ "lr_hard_min": 1e-06,
116
+ "lr_scale": 0.125,
117
+ "weight_decay": 0.1,
118
+ "wd_ada": 0.0,
119
+ "clip": 1.0,
120
+ "spike_skip": 4.0,
121
+ "grad_ceiling": 25.0,
122
+ "steps_skipped": 0,
123
+ "seed_offset": 0,
124
+ "target_steps": 400000
125
+ }
126
  }
127
  ],
128
  "guard_state": {
129
  "version": 3,
130
+ "live_grad_ema": 0.10541458150407955,
131
+ "grad_ema": 0.10541458150407955,
132
+ "grad_reference": 0.12761892899498606,
133
+ "grad_p90_reference": 0.22177981039137173,
134
  "spikes_total": 0,
135
  "recent_losses": [
136
+ 0.7958217263221741,
137
+ 0.7977699041366577,
138
+ 0.77901291847229,
139
+ 0.770370602607727,
140
+ 0.797216534614563,
141
+ 0.7807406187057495,
142
+ 0.8035680651664734,
143
+ 0.7891042232513428,
144
+ 0.7964577078819275,
145
+ 0.7968965768814087,
146
+ 0.7889297008514404,
147
+ 0.7768633365631104,
148
+ 0.7984722852706909,
149
+ 0.7791399955749512,
150
+ 0.7925962209701538,
151
+ 0.7921831607818604,
152
+ 0.780846357345581,
153
+ 0.8034866452217102,
154
+ 0.7993617057800293,
155
+ 0.8049355745315552,
156
+ 0.8026502728462219,
157
+ 0.8026549816131592,
158
+ 0.8041970729827881,
159
+ 0.8032207489013672,
160
+ 0.7939211130142212,
161
+ 0.7963867783546448,
162
+ 0.7903079986572266,
163
+ 0.8019658327102661,
164
+ 0.7911746501922607,
165
+ 0.812740683555603,
166
+ 0.7941210269927979,
167
+ 0.800366997718811,
168
+ 0.7813549041748047,
169
+ 0.7876204252243042,
170
+ 0.8110461235046387,
171
+ 0.778113603591919,
172
+ 0.7779107093811035,
173
+ 0.7833624482154846,
174
+ 0.791386604309082,
175
+ 0.7745489478111267,
176
+ 0.7929266691207886,
177
+ 0.7926019430160522,
178
+ 0.7821005582809448,
179
+ 0.7942101955413818,
180
+ 0.7884989976882935,
181
+ 0.7952886819839478,
182
+ 0.7980111837387085,
183
+ 0.791961669921875,
184
+ 0.7800871133804321,
185
+ 0.7865033745765686,
186
+ 0.7923028469085693,
187
+ 0.7920430302619934,
188
+ 0.7883865833282471,
189
+ 0.7861016988754272,
190
+ 0.7989460229873657,
191
+ 0.8002275824546814,
192
+ 0.8023224472999573,
193
+ 0.8013152480125427,
194
+ 0.7967742681503296,
195
+ 0.7968146800994873,
196
+ 0.8060688972473145,
197
+ 0.7988928556442261,
198
+ 0.7862999439239502,
199
+ 0.7854998111724854,
200
+ 0.7765175104141235,
201
+ 0.7802256345748901,
202
+ 0.7915911078453064,
203
+ 0.7755116820335388,
204
+ 0.8048877716064453,
205
+ 0.8124485015869141,
206
+ 0.7985507845878601,
207
+ 0.7857109308242798,
208
+ 0.8062797784805298,
209
+ 0.778174877166748,
210
+ 0.7864381670951843,
211
+ 0.8261948227882385,
212
+ 0.7813535928726196,
213
+ 0.7641370296478271,
214
+ 0.7969774603843689,
215
+ 0.7905817627906799,
216
+ 0.7884480953216553,
217
+ 0.7899702191352844,
218
+ 0.7861962914466858,
219
+ 0.7916526794433594,
220
+ 0.7881284952163696,
221
+ 0.8023510575294495,
222
+ 0.7921522855758667,
223
+ 0.8151097893714905,
224
+ 0.8002135753631592,
225
+ 0.771194577217102,
226
+ 0.8068884015083313,
227
+ 0.7962524890899658,
228
+ 0.78901207447052,
229
+ 0.7853200435638428,
230
+ 0.7994428873062134,
231
+ 0.7993086576461792,
232
+ 0.7978327870368958,
233
+ 0.7830636501312256,
234
+ 0.7861357927322388,
235
+ 0.7813628911972046,
236
+ 0.8038924932479858,
237
+ 0.7962657809257507,
238
+ 0.7939842343330383,
239
+ 0.7997649908065796,
240
+ 0.7832349538803101,
241
+ 0.802859902381897,
242
+ 0.7959683537483215,
243
+ 0.7981163263320923,
244
+ 0.8031013011932373,
245
+ 0.7908291816711426,
246
+ 0.7975724935531616,
247
+ 0.8222005367279053,
248
+ 0.7992904186248779,
249
+ 0.8207582235336304,
250
+ 0.7905919551849365,
251
+ 0.8132926225662231,
252
+ 0.8019974231719971,
253
+ 0.7903037071228027,
254
+ 0.825846791267395,
255
+ 0.8016583323478699,
256
+ 0.7920042872428894,
257
+ 0.7943241596221924,
258
+ 0.8049557209014893,
259
+ 0.7738712430000305,
260
+ 0.7940982580184937,
261
+ 0.8038930892944336,
262
+ 0.7933866381645203,
263
+ 0.7818841934204102,
264
+ 0.7949215173721313,
265
+ 0.8037704229354858,
266
+ 0.7869149446487427,
267
+ 0.8071405291557312,
268
+ 0.7996875047683716,
269
+ 0.7969752550125122,
270
+ 0.8010575175285339,
271
+ 0.8031644225120544,
272
+ 0.790553629398346,
273
+ 0.7809879779815674,
274
+ 0.7843597531318665,
275
+ 0.7957068681716919,
276
+ 0.8026432991027832,
277
+ 0.7863646745681763,
278
+ 0.8028186559677124,
279
+ 0.807106077671051,
280
+ 0.7928950786590576,
281
+ 0.8013726472854614,
282
+ 0.7949384450912476,
283
+ 0.7777584791183472,
284
+ 0.7918338775634766,
285
+ 0.8006141781806946,
286
+ 0.8103543519973755,
287
+ 0.7856409549713135,
288
+ 0.8001909255981445,
289
+ 0.8144172430038452,
290
+ 0.7939871549606323,
291
+ 0.7718457579612732,
292
+ 0.7882776260375977,
293
+ 0.7858873009681702,
294
+ 0.7827022075653076,
295
+ 0.7766430377960205,
296
+ 0.7870970964431763,
297
+ 0.7912852764129639,
298
+ 0.8013837933540344,
299
+ 0.7906249165534973,
300
+ 0.7857555150985718,
301
+ 0.795992910861969,
302
+ 0.7838965058326721,
303
+ 0.8030397891998291,
304
+ 0.7962979078292847,
305
+ 0.8199806809425354,
306
+ 0.8019179105758667,
307
+ 0.8109489679336548,
308
+ 0.787373423576355,
309
+ 0.7910624146461487,
310
+ 0.8004148006439209,
311
+ 0.8061037063598633,
312
+ 0.7987639904022217,
313
+ 0.7971713542938232,
314
+ 0.7790286540985107,
315
+ 0.772291898727417,
316
+ 0.7975274324417114,
317
+ 0.7954764366149902,
318
+ 0.7991613149642944,
319
+ 0.7771415710449219,
320
+ 0.8288599252700806,
321
+ 0.7953532934188843,
322
+ 0.8029688596725464,
323
+ 0.7859041690826416,
324
+ 0.7958877682685852,
325
+ 0.7970870137214661,
326
+ 0.7974164485931396,
327
+ 0.795319139957428,
328
+ 0.7888880372047424,
329
+ 0.8042818307876587,
330
+ 0.8083038926124573,
331
+ 0.7833030223846436,
332
+ 0.798464298248291,
333
+ 0.7868218421936035,
334
+ 0.8082107901573181,
335
+ 0.795568585395813
336
  ],
337
  "loss_window": 200,
338
  "grad_ema_decay": 0.99,
 
365
  },
366
  "committed_scale": 0.125,
367
  "attempt_factor": 1.0,
368
+ "fast_loss": 0.7929503200555064,
369
+ "slow_loss": 0.7933888351661564,
370
+ "best_loss": 0.7923494011104905,
371
  "reference": {
372
+ "loss": 0.7943918004989624,
373
+ "grad_median": 0.12761892899498606,
374
+ "grad_p90": 0.22177981039137173,
375
+ "step": 80000,
376
+ "promotions": 3
377
  },
378
  "last_metrics": {
379
+ "loss": 0.7943424367904663,
380
+ "grad_median": 0.10463522374629974,
381
+ "grad_p90": 0.11572229340672494,
382
  "skip_rate": 0.0,
383
  "relative_spike_rate": 0.0
384
  },
385
+ "last_loss_ratio": 0.9999223258887985,
386
+ "last_grad_ratio": 0.7845786920797867,
387
+ "last_grad_p90_ratio": 0.4660692768351103,
388
  "healthy_windows": 0,
389
  "warning_windows": 0,
390
  "retry_windows": 0,
391
+ "windows_seen": 100,
392
  "retry_count": 0
393
  },
394
+ "best_window": 0.7923494011104905,
395
  "blown_windows": 0
396
  },
397
  "transaction": {
398
  "status": "committed",
399
+ "committed_at": "2026-07-21T05:29:48.245770+00:00",
400
  "attempt": 0,
401
  "health_reference": {
402
+ "loss": 0.7943918004989624,
403
+ "grad_median": 0.12761892899498606,
404
+ "grad_p90": 0.22177981039137173,
405
+ "step": 80000,
406
+ "promotions": 3
407
  }
408
  },
409
  "config": {
 
446
  },
447
  "optim_param_groups": [
448
  {
449
+ "lr": 1.1425746560775714e-05,
450
  "betas": [
451
  0.9,
452
  0.999