sparsetrace commited on
Commit
eb3b86e
·
verified ·
1 Parent(s): a0a6fa7

checkpoint: promote transactional candidate: step 312500

Browse files
Files changed (4) hide show
  1. ema.safetensors +1 -1
  2. model.safetensors +1 -1
  3. optim.safetensors +1 -1
  4. state.json +258 -229
ema.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d4a48178c84975737eab69c358ecd5b6214d0d2b83fc43ea06d6ce479980434f
3
  size 2046527448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fb8dd7fad365f56eb99f41f41a66c75601f2cc0dd42d617849e9d860c3b60586
3
  size 2046527448
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:403c6bfd9c7431a8cf2b535fa35c20fa79075ab493181f0041edb2f3d312c0c4
3
  size 2046527448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b851120f115d275cd69ff79be8601616251a38ba07635bcd06a8478613959363
3
  size 2046527448
optim.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3f725b8effab6de9b813a420c4f8b0dec5da0094104b37d4858db3821fc812bb
3
  size 4093066344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b55f942ddd9db4fca09b58ff94a23e8f8be75202a6f3282572819a0776c39a18
3
  size 4093066344
state.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "step": 310000,
3
  "run_history": [
4
  {
5
  "start_step": 0,
@@ -236,216 +236,245 @@
236
  "seed_offset": 0,
237
  "target_steps": 400000
238
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
239
  }
240
  ],
241
  "guard_state": {
242
  "version": 3,
243
- "live_grad_ema": 325.1300853243034,
244
- "grad_ema": 325.1300853243034,
245
- "grad_reference": 115.12889026783117,
246
- "grad_p90_reference": 288.0010243012873,
247
- "spikes_total": 2769,
248
  "recent_losses": [
249
- 0.7815734148025513,
250
- 0.7695204615592957,
251
- 0.778976559638977,
252
- 0.7780275940895081,
253
- 0.7747119069099426,
254
- 0.7762272357940674,
255
- 0.7832270860671997,
256
- 0.768639087677002,
257
- 0.7717515230178833,
258
- 0.7665273547172546,
259
- 0.7817960381507874,
260
- 0.7678589820861816,
261
- 0.7664707899093628,
262
- 0.7537945508956909,
263
- 0.761132001876831,
264
- 0.7611212730407715,
265
- 0.7704578638076782,
266
- 0.7670674920082092,
267
- 0.7762092351913452,
268
- 0.7584229707717896,
269
- 0.7806477546691895,
270
- 0.7697452306747437,
271
- 0.7449471950531006,
272
- 0.7654528617858887,
273
- 0.7666465640068054,
274
- 0.7732318639755249,
275
- 0.7813650369644165,
276
- 0.7899041175842285,
277
- 0.7653136253356934,
278
- 0.7779978513717651,
279
- 0.7851771116256714,
280
- 0.7747712731361389,
281
- 0.7766869068145752,
282
- 0.7424110770225525,
283
- 0.7727780342102051,
284
- 0.7629849910736084,
285
- 0.75688236951828,
286
- 0.7461458444595337,
287
- 0.7494951486587524,
288
- 0.774730920791626,
289
- 0.7789695262908936,
290
- 0.7730566263198853,
291
- 0.7505097389221191,
292
- 0.7506208419799805,
293
- 0.77761310338974,
294
- 0.7825862169265747,
295
- 0.7645778656005859,
296
- 0.7712517380714417,
297
- 0.763931393623352,
298
- 0.7642889022827148,
299
- 0.7814410328865051,
300
- 0.7674883604049683,
301
- 0.7693274021148682,
302
- 0.7734235525131226,
303
- 0.770982027053833,
304
- 0.7674297094345093,
305
- 0.7798652648925781,
306
- 0.7835193872451782,
307
- 0.7661323547363281,
308
- 0.7803771495819092,
309
- 0.7983541488647461,
310
- 0.7793653011322021,
311
- 0.7595428228378296,
312
- 0.781514585018158,
313
- 0.7622340321540833,
314
- 0.7566722631454468,
315
- 0.7713912725448608,
316
- 0.767433762550354,
317
- 0.7697087526321411,
318
- 0.7599937319755554,
319
- 0.7760474681854248,
320
- 0.7778599262237549,
321
- 0.7638007402420044,
322
- 0.7586327195167542,
323
- 0.7810822129249573,
324
- 0.7687883377075195,
325
- 0.7635555863380432,
326
- 0.7738674879074097,
327
- 0.7688883543014526,
328
- 0.7604380249977112,
329
- 0.7603793144226074,
330
- 0.7860535383224487,
331
- 0.7781825065612793,
332
- 0.770027756690979,
333
- 0.7694627046585083,
334
- 0.7674841284751892,
335
- 0.7658978700637817,
336
- 0.7704386711120605,
337
- 0.7583168745040894,
338
- 0.7600569725036621,
339
- 0.7677663564682007,
340
- 0.7534502148628235,
341
- 0.7551587820053101,
342
- 0.7778497338294983,
343
- 0.7743889093399048,
344
- 0.7951738834381104,
345
- 0.7660666704177856,
346
- 0.7643990516662598,
347
- 0.7628085017204285,
348
- 0.7777034044265747,
349
- 0.7555203437805176,
350
- 0.7715059518814087,
351
- 0.7733153104782104,
352
- 0.7652976512908936,
353
- 0.7560437917709351,
354
- 0.782185435295105,
355
- 0.7573839426040649,
356
- 0.7793244123458862,
357
- 0.7846203446388245,
358
- 0.7904572486877441,
359
- 0.7643678188323975,
360
- 0.7719947099685669,
361
- 0.7650443315505981,
362
- 0.7789854407310486,
363
- 0.7543730735778809,
364
- 0.7731004357337952,
365
- 0.7709360122680664,
366
- 0.7741367220878601,
367
- 0.7637652158737183,
368
- 0.7722082138061523,
369
- 0.7498772144317627,
370
- 0.7644094824790955,
371
- 0.7729204297065735,
372
- 0.778230607509613,
373
- 0.7575691342353821,
374
- 0.7752888798713684,
375
- 0.7770575881004333,
376
- 0.7610445022583008,
377
- 0.765170693397522,
378
- 0.7697356343269348,
379
- 0.7870737314224243,
380
- 0.7810636758804321,
381
- 0.7699251770973206,
382
- 0.7615000605583191,
383
- 0.7654602527618408,
384
- 0.7766352891921997,
385
- 0.7630747556686401,
386
- 0.7755551338195801,
387
- 0.7592688798904419,
388
- 0.7785106301307678,
389
- 0.7797162532806396,
390
- 0.7840276956558228,
391
- 0.7622792720794678,
392
- 0.7794853448867798,
393
- 0.7840263843536377,
394
- 0.7494792938232422,
395
- 0.7739502191543579,
396
- 0.7757754325866699,
397
- 0.7814549207687378,
398
- 0.7689940333366394,
399
- 0.7587122321128845,
400
- 0.7651452422142029,
401
- 0.7657533288002014,
402
- 0.76514732837677,
403
- 0.7590616345405579,
404
- 0.7874753475189209,
405
- 0.7683334946632385,
406
- 0.7758129835128784,
407
- 0.7612481117248535,
408
- 0.751968502998352,
409
- 0.7627632021903992,
410
- 0.7662752866744995,
411
- 0.770512580871582,
412
- 0.7587728500366211,
413
- 0.7765088081359863,
414
- 0.7663204669952393,
415
- 0.7385183572769165,
416
- 0.7527186870574951,
417
- 0.7783799171447754,
418
- 0.7729552388191223,
419
- 0.7662674188613892,
420
- 0.775795578956604,
421
- 0.7689927816390991,
422
- 0.7808484435081482,
423
- 0.7735921740531921,
424
- 0.7781112790107727,
425
- 0.7749870419502258,
426
- 0.790752649307251,
427
- 0.7638624310493469,
428
- 0.7779871225357056,
429
- 0.7725045680999756,
430
- 0.7638426423072815,
431
- 0.7668968439102173,
432
- 0.7770760655403137,
433
- 0.7824015021324158,
434
- 0.7721044421195984,
435
- 0.7699593305587769,
436
- 0.7670173645019531,
437
- 0.7575844526290894,
438
- 0.7936911582946777,
439
- 0.7692011594772339,
440
- 0.7752118110656738,
441
- 0.7845311164855957,
442
- 0.7557656764984131,
443
- 0.7653970718383789,
444
- 0.7774176001548767,
445
- 0.7762206196784973,
446
- 0.7702947854995728,
447
- 0.7780052423477173,
448
- 0.7719589471817017
449
  ],
450
  "loss_window": 200,
451
  "grad_ema_decay": 0.99,
@@ -478,30 +507,30 @@
478
  },
479
  "committed_scale": 0.28176768087105175,
480
  "attempt_factor": 1.0,
481
- "fast_loss": 0.7697179931952205,
482
- "slow_loss": 0.7698657912888203,
483
  "best_loss": 0.7673007962107659,
484
  "reference": {
485
- "loss": 0.7708657577830316,
486
- "grad_median": 115.12889026783117,
487
- "grad_p90": 288.0010243012873,
488
- "step": 310000,
489
- "promotions": 6
490
  },
491
  "last_metrics": {
492
- "loss": 0.7700112435221672,
493
- "grad_median": 245.44883728027344,
494
- "grad_p90": 630.5565490722656,
495
- "skip_rate": 0.04,
496
- "relative_spike_rate": 0.04
497
  },
498
- "last_loss_ratio": 0.9986147433779303,
499
- "last_grad_ratio": 2.9733748945133294,
500
- "last_grad_p90_ratio": 3.115980782700292,
501
  "healthy_windows": 0,
502
  "warning_windows": 0,
503
  "retry_windows": 0,
504
- "windows_seen": 200,
505
  "retry_count": 0
506
  },
507
  "best_window": 0.7673007962107659,
@@ -509,14 +538,14 @@
509
  },
510
  "transaction": {
511
  "status": "committed",
512
- "committed_at": "2026-07-21T07:32:05.358209+00:00",
513
  "attempt": 0,
514
  "health_reference": {
515
- "loss": 0.7708657577830316,
516
- "grad_median": 115.12889026783117,
517
- "grad_p90": 288.0010243012873,
518
- "step": 310000,
519
- "promotions": 6
520
  }
521
  },
522
  "config": {
@@ -559,7 +588,7 @@
559
  },
560
  "optim_param_groups": [
561
  {
562
- "lr": 5.8556849782927245e-06,
563
  "betas": [
564
  0.9,
565
  0.999
 
1
  {
2
+ "step": 312500,
3
  "run_history": [
4
  {
5
  "start_step": 0,
 
236
  "seed_offset": 0,
237
  "target_steps": 400000
238
  }
239
+ },
240
+ {
241
+ "start_step": 310000,
242
+ "end_step": 312500,
243
+ "seconds": 1129.4,
244
+ "world": 1,
245
+ "objective": "flow",
246
+ "completed": false,
247
+ "finished_at": "2026-07-21T07:50:54.720571+00:00",
248
+ "promotion_reason": "run final",
249
+ "effective": {
250
+ "attempt": 0,
251
+ "precision": "tf32",
252
+ "lr_policy": "adaptive",
253
+ "lr_base": 0.0001,
254
+ "lr_start": null,
255
+ "lr_end": 5.695864301224254e-06,
256
+ "lr_min": 1e-05,
257
+ "lr_hard_min": 1e-06,
258
+ "lr_scale": 0.28176768087105175,
259
+ "weight_decay": 0.1,
260
+ "wd_ada": 0.05,
261
+ "clip": 1.0,
262
+ "spike_skip": 4.0,
263
+ "grad_ceiling": 0.0,
264
+ "steps_skipped": 54,
265
+ "seed_offset": 0,
266
+ "target_steps": 400000
267
+ }
268
  }
269
  ],
270
  "guard_state": {
271
  "version": 3,
272
+ "live_grad_ema": 308.99704508964726,
273
+ "grad_ema": 308.99704508964726,
274
+ "grad_reference": 142.2118448192454,
275
+ "grad_p90_reference": 370.5497952711079,
276
+ "spikes_total": 2823,
277
  "recent_losses": [
278
+ 0.7719821929931641,
279
+ 0.7697741985321045,
280
+ 0.7527035474777222,
281
+ 0.7771854996681213,
282
+ 0.7851243019104004,
283
+ 0.765992283821106,
284
+ 0.7532682418823242,
285
+ 0.7758858799934387,
286
+ 0.7573052644729614,
287
+ 0.7578380107879639,
288
+ 0.7734838724136353,
289
+ 0.7732190489768982,
290
+ 0.7713576555252075,
291
+ 0.7805781960487366,
292
+ 0.7534257173538208,
293
+ 0.7518965005874634,
294
+ 0.7609991431236267,
295
+ 0.7556645274162292,
296
+ 0.7631251811981201,
297
+ 0.7710431814193726,
298
+ 0.7736701369285583,
299
+ 0.7626610994338989,
300
+ 0.7739626169204712,
301
+ 0.7643856406211853,
302
+ 0.7782695293426514,
303
+ 0.7730320692062378,
304
+ 0.7630584239959717,
305
+ 0.7592689990997314,
306
+ 0.7653300762176514,
307
+ 0.7853460907936096,
308
+ 0.7841410636901855,
309
+ 0.7883813381195068,
310
+ 0.766587495803833,
311
+ 0.7534781098365784,
312
+ 0.7733400464057922,
313
+ 0.7636327743530273,
314
+ 0.7584853768348694,
315
+ 0.7626829147338867,
316
+ 0.7874799966812134,
317
+ 0.7721030712127686,
318
+ 0.7612360119819641,
319
+ 0.7758907675743103,
320
+ 0.7593536376953125,
321
+ 0.7643862962722778,
322
+ 0.7652077674865723,
323
+ 0.7705347537994385,
324
+ 0.7704811096191406,
325
+ 0.7616401314735413,
326
+ 0.7399537563323975,
327
+ 0.7572300434112549,
328
+ 0.7714760303497314,
329
+ 0.7640082836151123,
330
+ 0.7647771835327148,
331
+ 0.7759393453598022,
332
+ 0.757431149482727,
333
+ 0.7712739109992981,
334
+ 0.7629599571228027,
335
+ 0.753789484500885,
336
+ 0.7572435736656189,
337
+ 0.7852658033370972,
338
+ 0.7666440010070801,
339
+ 0.7855952978134155,
340
+ 0.7570184469223022,
341
+ 0.7461519241333008,
342
+ 0.7635331749916077,
343
+ 0.7577164173126221,
344
+ 0.7657725811004639,
345
+ 0.7879844903945923,
346
+ 0.7893444299697876,
347
+ 0.77923583984375,
348
+ 0.7486773729324341,
349
+ 0.7676677703857422,
350
+ 0.7955640554428101,
351
+ 0.765953779220581,
352
+ 0.7643669247627258,
353
+ 0.7610448598861694,
354
+ 0.7679208517074585,
355
+ 0.7719264626502991,
356
+ 0.7525736093521118,
357
+ 0.7713336944580078,
358
+ 0.7604453563690186,
359
+ 0.7624722719192505,
360
+ 0.7447837591171265,
361
+ 0.7688546180725098,
362
+ 0.7643270492553711,
363
+ 0.7845481634140015,
364
+ 0.764289379119873,
365
+ 0.7720410823822021,
366
+ 0.767133891582489,
367
+ 0.7744083404541016,
368
+ 0.767828643321991,
369
+ 0.7684509754180908,
370
+ 0.7588315606117249,
371
+ 0.7707924246788025,
372
+ 0.7752561569213867,
373
+ 0.7805676460266113,
374
+ 0.7628724575042725,
375
+ 0.758941650390625,
376
+ 0.7836504578590393,
377
+ 0.7684669494628906,
378
+ 0.7749240398406982,
379
+ 0.7666053771972656,
380
+ 0.7745267152786255,
381
+ 0.7582672238349915,
382
+ 0.7658072113990784,
383
+ 0.7932355403900146,
384
+ 0.7827215790748596,
385
+ 0.7878477573394775,
386
+ 0.7688950896263123,
387
+ 0.7834637761116028,
388
+ 0.7753982543945312,
389
+ 0.7901803255081177,
390
+ 0.760439395904541,
391
+ 0.7713825106620789,
392
+ 0.7867081761360168,
393
+ 0.764076828956604,
394
+ 0.7671103477478027,
395
+ 0.7835704684257507,
396
+ 0.7785870432853699,
397
+ 0.7624772787094116,
398
+ 0.788772463798523,
399
+ 0.777624785900116,
400
+ 0.7678309082984924,
401
+ 0.7543090581893921,
402
+ 0.7675315737724304,
403
+ 0.7490672469139099,
404
+ 0.7633370161056519,
405
+ 0.7582913637161255,
406
+ 0.770655632019043,
407
+ 0.781661868095398,
408
+ 0.772572934627533,
409
+ 0.7664582133293152,
410
+ 0.776885449886322,
411
+ 0.7718193531036377,
412
+ 0.7796340584754944,
413
+ 0.7836461067199707,
414
+ 0.7818602323532104,
415
+ 0.7569148540496826,
416
+ 0.7662522792816162,
417
+ 0.7669554948806763,
418
+ 0.7520023584365845,
419
+ 0.7738852500915527,
420
+ 0.7717455625534058,
421
+ 0.7852076292037964,
422
+ 0.7742418646812439,
423
+ 0.7834006547927856,
424
+ 0.7739925384521484,
425
+ 0.7954076528549194,
426
+ 0.7878352403640747,
427
+ 0.7691454291343689,
428
+ 0.7759157419204712,
429
+ 0.7588131427764893,
430
+ 0.7619526386260986,
431
+ 0.7770611643791199,
432
+ 0.7496113777160645,
433
+ 0.7925460338592529,
434
+ 0.7835298776626587,
435
+ 0.7803013324737549,
436
+ 0.7734143137931824,
437
+ 0.786319375038147,
438
+ 0.7853766679763794,
439
+ 0.7637833952903748,
440
+ 0.7584264874458313,
441
+ 0.7749159336090088,
442
+ 0.7627032995223999,
443
+ 0.7783739566802979,
444
+ 0.7706066370010376,
445
+ 0.7838884592056274,
446
+ 0.7637871503829956,
447
+ 0.7712488770484924,
448
+ 0.799232006072998,
449
+ 0.7673161029815674,
450
+ 0.7745071649551392,
451
+ 0.7644798159599304,
452
+ 0.7690597772598267,
453
+ 0.770691990852356,
454
+ 0.7542893290519714,
455
+ 0.7870317697525024,
456
+ 0.7730977535247803,
457
+ 0.7511366009712219,
458
+ 0.7774863243103027,
459
+ 0.7666788101196289,
460
+ 0.7882211804389954,
461
+ 0.7758758068084717,
462
+ 0.7539352178573608,
463
+ 0.7599439024925232,
464
+ 0.7537254691123962,
465
+ 0.764796793460846,
466
+ 0.7842488288879395,
467
+ 0.7541405558586121,
468
+ 0.7699174880981445,
469
+ 0.7735539674758911,
470
+ 0.7673854827880859,
471
+ 0.7714779376983643,
472
+ 0.7695835828781128,
473
+ 0.781039834022522,
474
+ 0.7578097581863403,
475
+ 0.7605041265487671,
476
+ 0.7799188494682312,
477
+ 0.7886980772018433
478
  ],
479
  "loss_window": 200,
480
  "grad_ema_decay": 0.99,
 
507
  },
508
  "committed_scale": 0.28176768087105175,
509
  "attempt_factor": 1.0,
510
+ "fast_loss": 0.76928074066719,
511
+ "slow_loss": 0.7697922280915285,
512
  "best_loss": 0.7673007962107659,
513
  "reference": {
514
+ "loss": 0.7704072610297014,
515
+ "grad_median": 142.2118448192454,
516
+ "grad_p90": 370.5497952711079,
517
+ "step": 312500,
518
+ "promotions": 7
519
  },
520
  "last_metrics": {
521
+ "loss": 0.7685732740163803,
522
+ "grad_median": 250.54366302490234,
523
+ "grad_p90": 700.7448791503905,
524
+ "skip_rate": 0.02,
525
+ "relative_spike_rate": 0.02
526
  },
527
+ "last_loss_ratio": 0.9970260921003362,
528
+ "last_grad_ratio": 2.1762014941866266,
529
+ "last_grad_p90_ratio": 2.433133287808443,
530
  "healthy_windows": 0,
531
  "warning_windows": 0,
532
  "retry_windows": 0,
533
+ "windows_seen": 212,
534
  "retry_count": 0
535
  },
536
  "best_window": 0.7673007962107659,
 
538
  },
539
  "transaction": {
540
  "status": "committed",
541
+ "committed_at": "2026-07-21T07:50:54.720630+00:00",
542
  "attempt": 0,
543
  "health_reference": {
544
+ "loss": 0.7704072610297014,
545
+ "grad_median": 142.2118448192454,
546
+ "grad_p90": 370.5497952711079,
547
+ "step": 312500,
548
+ "promotions": 7
549
  }
550
  },
551
  "config": {
 
588
  },
589
  "optim_param_groups": [
590
  {
591
+ "lr": 5.695864301224254e-06,
592
  "betas": [
593
  0.9,
594
  0.999