sparsetrace commited on
Commit
e6eb619
·
verified ·
1 Parent(s): d41caf0

checkpoint: promote transactional candidate: step 430000

Browse files
Files changed (4) hide show
  1. ema.safetensors +1 -1
  2. model.safetensors +1 -1
  3. optim.safetensors +1 -1
  4. state.json +256 -227
ema.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:fc39af9b706889f28a7768b36fdf8216d2e59c1fc267a31120a353a1c19088a1
3
  size 1945760448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:951a543ad0bc1344c659726f9b3498d8c6255481bc010fcdd03dc0b8235126be
3
  size 1945760448
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9c3f748b31dcb4aa3146c87a97fe086048e6fc44da8d0f180315888dfbe4cd1e
3
  size 2046527448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce395e124e6b2f2d7caefb9feeb007c694b78964dcfd3de9528836e0e92c307c
3
  size 2046527448
optim.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:af4cc8486f422bb214b589b8d7d62ab6cd01be23f617b4f9a3884f9567380f21
3
  size 3891530432
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1bc7191dd831e51e0e9af0a35d17a3e575a9f4cc1eeca62cec96941e552bbb47
3
  size 3891530432
state.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "step": 420000,
3
  "run_history": [
4
  {
5
  "start_step": 0,
@@ -1225,216 +1225,245 @@
1225
  "seed_offset": 0,
1226
  "target_steps": 500000
1227
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1228
  }
1229
  ],
1230
  "guard_state": {
1231
  "version": 3,
1232
- "live_grad_ema": 0.1271673811585368,
1233
- "grad_ema": 0.1271673811585368,
1234
- "grad_reference": 0.12107273116813544,
1235
- "grad_p90_reference": 0.14080833682407548,
1236
  "spikes_total": 0,
1237
  "recent_losses": [
1238
- 0.7844154834747314,
1239
- 0.7900657057762146,
1240
- 0.7914424538612366,
1241
- 0.7985655665397644,
1242
- 0.7798577547073364,
1243
- 0.7793698310852051,
1244
- 0.7642449140548706,
1245
- 0.7781345844268799,
1246
- 0.7540674209594727,
1247
- 0.7906212210655212,
1248
- 0.776996374130249,
1249
- 0.7932469844818115,
1250
- 0.7828843593597412,
1251
- 0.7794705033302307,
1252
- 0.7651668190956116,
1253
- 0.7872708439826965,
1254
- 0.7851995229721069,
1255
- 0.7752612233161926,
1256
- 0.7669999003410339,
1257
- 0.769835352897644,
1258
- 0.7654687762260437,
1259
- 0.7688301801681519,
1260
- 0.7769685983657837,
1261
- 0.7880840301513672,
1262
- 0.7697796821594238,
1263
- 0.7759051322937012,
1264
- 0.7590314149856567,
1265
- 0.7791679501533508,
1266
- 0.767042875289917,
1267
- 0.7784203886985779,
1268
- 0.7707099914550781,
1269
- 0.7666407823562622,
1270
- 0.7945837378501892,
1271
- 0.8182512521743774,
1272
- 0.7848827242851257,
1273
- 0.7814526557922363,
1274
- 0.76530921459198,
1275
- 0.7784772515296936,
1276
- 0.7719040513038635,
1277
- 0.774870753288269,
1278
- 0.775664210319519,
1279
- 0.758037269115448,
1280
- 0.7790703773498535,
1281
- 0.7979621291160583,
1282
- 0.7729102373123169,
1283
- 0.7845079898834229,
1284
- 0.7807520627975464,
1285
- 0.7817583084106445,
1286
- 0.778288722038269,
1287
- 0.7842386960983276,
1288
- 0.802446722984314,
1289
- 0.7703600525856018,
1290
- 0.7621427774429321,
1291
- 0.765002965927124,
1292
- 0.7718648910522461,
1293
- 0.7691908478736877,
1294
- 0.7762772440910339,
1295
- 0.7743708491325378,
1296
- 0.7789310216903687,
1297
- 0.7826679944992065,
1298
- 0.7858250737190247,
1299
- 0.7893880605697632,
1300
- 0.7915815711021423,
1301
- 0.7957233786582947,
1302
- 0.776009202003479,
1303
- 0.7697072625160217,
1304
- 0.7794809341430664,
1305
- 0.7690744400024414,
1306
- 0.7933147549629211,
1307
- 0.7875101566314697,
1308
- 0.7703601121902466,
1309
- 0.7791745662689209,
1310
- 0.776046633720398,
1311
- 0.7843154668807983,
1312
- 0.7699857950210571,
1313
- 0.7876410484313965,
1314
- 0.7844052314758301,
1315
- 0.7727444171905518,
1316
- 0.7697446346282959,
1317
- 0.7852394580841064,
1318
- 0.7735921144485474,
1319
- 0.7988641858100891,
1320
- 0.7888714075088501,
1321
- 0.7797876596450806,
1322
- 0.7658687829971313,
1323
- 0.8036059141159058,
1324
- 0.767023503780365,
1325
- 0.7892134189605713,
1326
- 0.7958495616912842,
1327
- 0.7776927947998047,
1328
- 0.7807880640029907,
1329
- 0.802375078201294,
1330
- 0.7825500965118408,
1331
- 0.7898280620574951,
1332
- 0.7823948860168457,
1333
- 0.7720092535018921,
1334
- 0.7940360903739929,
1335
- 0.784995436668396,
1336
- 0.778644859790802,
1337
- 0.76335608959198,
1338
- 0.7777822017669678,
1339
- 0.7763221859931946,
1340
- 0.7946935296058655,
1341
- 0.7920470833778381,
1342
- 0.7763581275939941,
1343
- 0.7685378789901733,
1344
- 0.7711445093154907,
1345
- 0.7948620915412903,
1346
- 0.7944684028625488,
1347
- 0.799842894077301,
1348
- 0.7865338325500488,
1349
- 0.7936634421348572,
1350
- 0.7670670747756958,
1351
- 0.7752442359924316,
1352
- 0.7845955491065979,
1353
- 0.7776526212692261,
1354
- 0.7983258962631226,
1355
- 0.7717570662498474,
1356
- 0.7747572660446167,
1357
- 0.7642585635185242,
1358
- 0.7896884083747864,
1359
- 0.7776357531547546,
1360
- 0.7798128128051758,
1361
- 0.7714104652404785,
1362
- 0.7865991592407227,
1363
- 0.7701637744903564,
1364
- 0.7653399705886841,
1365
- 0.7977522015571594,
1366
- 0.7751191854476929,
1367
- 0.7872921228408813,
1368
- 0.7791725397109985,
1369
- 0.7550270557403564,
1370
- 0.7849012017250061,
1371
- 0.7677953243255615,
1372
- 0.7892708778381348,
1373
- 0.7703138589859009,
1374
- 0.8015283346176147,
1375
- 0.7944259643554688,
1376
- 0.7646466493606567,
1377
- 0.7645500898361206,
1378
- 0.7886549234390259,
1379
- 0.771674394607544,
1380
- 0.7790502309799194,
1381
- 0.7763421535491943,
1382
- 0.7840163707733154,
1383
- 0.7677236795425415,
1384
- 0.7786950469017029,
1385
- 0.7757492065429688,
1386
- 0.7704688310623169,
1387
- 0.7793031930923462,
1388
- 0.7852842807769775,
1389
- 0.7698364853858948,
1390
- 0.806840181350708,
1391
- 0.8011410236358643,
1392
- 0.7919570207595825,
1393
- 0.784784734249115,
1394
- 0.785818874835968,
1395
- 0.805206298828125,
1396
- 0.774101197719574,
1397
- 0.7767819166183472,
1398
- 0.7690863609313965,
1399
- 0.7609566450119019,
1400
- 0.7919425964355469,
1401
- 0.7965169548988342,
1402
- 0.7690175175666809,
1403
- 0.7728021144866943,
1404
- 0.7781683206558228,
1405
- 0.7662808895111084,
1406
- 0.7756146788597107,
1407
- 0.786636233329773,
1408
- 0.7565346956253052,
1409
- 0.781264066696167,
1410
- 0.7882192134857178,
1411
- 0.7956154346466064,
1412
- 0.777544379234314,
1413
- 0.7544653415679932,
1414
- 0.7728642821311951,
1415
- 0.7669949531555176,
1416
- 0.8019954562187195,
1417
- 0.7884479761123657,
1418
- 0.7655860185623169,
1419
- 0.764771044254303,
1420
- 0.7768639922142029,
1421
- 0.7744399309158325,
1422
- 0.7772291302680969,
1423
- 0.7890379428863525,
1424
- 0.7715737223625183,
1425
- 0.7742265462875366,
1426
- 0.7782158851623535,
1427
- 0.7759013175964355,
1428
- 0.7772053480148315,
1429
- 0.7741382122039795,
1430
- 0.7801560163497925,
1431
- 0.770395040512085,
1432
- 0.7899880409240723,
1433
- 0.7938807606697083,
1434
- 0.7932580709457397,
1435
- 0.7889757752418518,
1436
- 0.7987644076347351,
1437
- 0.7759979963302612
1438
  ],
1439
  "loss_window": 200,
1440
  "grad_ema_decay": 0.99,
@@ -1467,45 +1496,45 @@
1467
  },
1468
  "committed_scale": 1.0,
1469
  "attempt_factor": 1.0,
1470
- "fast_loss": 0.7795399289007362,
1471
- "slow_loss": 0.7800311677507622,
1472
- "best_loss": 0.7791871686748875,
1473
  "reference": {
1474
- "loss": 0.7806124003529549,
1475
- "grad_median": 0.12107273116813544,
1476
- "grad_p90": 0.14080833682407548,
1477
- "step": 420000,
1478
- "promotions": 3
1479
  },
1480
  "last_metrics": {
1481
- "loss": 0.779856633245945,
1482
- "grad_median": 0.12045571580529213,
1483
- "grad_p90": 0.1426164537668228,
1484
  "skip_rate": 0.0,
1485
  "relative_spike_rate": 0.0
1486
  },
1487
- "last_loss_ratio": 0.9987900777921493,
1488
- "last_grad_ratio": 0.993637809242737,
1489
- "last_grad_p90_ratio": 1.0161029184741899,
1490
  "healthy_windows": 0,
1491
  "warning_windows": 0,
1492
  "retry_windows": 0,
1493
- "windows_seen": 100,
1494
  "retry_count": 0
1495
  },
1496
- "best_window": 0.7791871686748875,
1497
  "blown_windows": 0
1498
  },
1499
  "transaction": {
1500
  "status": "committed",
1501
- "committed_at": "2026-07-23T17:30:36.297938+00:00",
1502
  "attempt": 0,
1503
  "health_reference": {
1504
- "loss": 0.7806124003529549,
1505
- "grad_median": 0.12107273116813544,
1506
- "grad_p90": 0.14080833682407548,
1507
- "step": 420000,
1508
- "promotions": 3
1509
  }
1510
  },
1511
  "config": {
 
1
  {
2
+ "step": 430000,
3
  "run_history": [
4
  {
5
  "start_step": 0,
 
1225
  "seed_offset": 0,
1226
  "target_steps": 500000
1227
  }
1228
+ },
1229
+ {
1230
+ "start_step": 420000,
1231
+ "end_step": 430000,
1232
+ "seconds": 4482.4,
1233
+ "world": 1,
1234
+ "objective": "flow",
1235
+ "completed": false,
1236
+ "finished_at": "2026-07-23T18:45:18.663356+00:00",
1237
+ "promotion_reason": "periodic healthy candidate",
1238
+ "effective": {
1239
+ "attempt": 0,
1240
+ "precision": "tf32",
1241
+ "lr_policy": "constant",
1242
+ "lr_base": 0.0001,
1243
+ "lr_start": null,
1244
+ "lr_end": 0.0001,
1245
+ "lr_min": 1e-05,
1246
+ "lr_hard_min": 1e-06,
1247
+ "lr_scale": 1.0,
1248
+ "weight_decay": 0.1,
1249
+ "wd_ada": 0.0,
1250
+ "clip": 1.0,
1251
+ "spike_skip": 4.0,
1252
+ "grad_ceiling": 25.0,
1253
+ "steps_skipped": 0,
1254
+ "seed_offset": 0,
1255
+ "target_steps": 500000
1256
+ }
1257
  }
1258
  ],
1259
  "guard_state": {
1260
  "version": 3,
1261
+ "live_grad_ema": 0.1273210148786569,
1262
+ "grad_ema": 0.1273210148786569,
1263
+ "grad_reference": 0.12170863464559273,
1264
+ "grad_p90_reference": 0.14147012085906355,
1265
  "spikes_total": 0,
1266
  "recent_losses": [
1267
+ 0.7611697912216187,
1268
+ 0.7859362959861755,
1269
+ 0.7639317512512207,
1270
+ 0.7797172665596008,
1271
+ 0.7679272294044495,
1272
+ 0.7858853936195374,
1273
+ 0.7774540185928345,
1274
+ 0.7651809453964233,
1275
+ 0.7625945806503296,
1276
+ 0.7641668319702148,
1277
+ 0.7863965034484863,
1278
+ 0.7869222164154053,
1279
+ 0.7565844655036926,
1280
+ 0.7902112007141113,
1281
+ 0.7978968620300293,
1282
+ 0.7748574018478394,
1283
+ 0.7873219847679138,
1284
+ 0.7655407786369324,
1285
+ 0.7606543898582458,
1286
+ 0.7847299575805664,
1287
+ 0.773524820804596,
1288
+ 0.7716867923736572,
1289
+ 0.7879720330238342,
1290
+ 0.7851314544677734,
1291
+ 0.7827275991439819,
1292
+ 0.7746939659118652,
1293
+ 0.7724999785423279,
1294
+ 0.7801138162612915,
1295
+ 0.790901243686676,
1296
+ 0.7884348034858704,
1297
+ 0.7803795337677002,
1298
+ 0.7776947617530823,
1299
+ 0.7581580281257629,
1300
+ 0.7896554470062256,
1301
+ 0.7804880142211914,
1302
+ 0.7620750069618225,
1303
+ 0.7756158113479614,
1304
+ 0.7674772143363953,
1305
+ 0.7809589505195618,
1306
+ 0.7598493099212646,
1307
+ 0.7973071336746216,
1308
+ 0.7575341463088989,
1309
+ 0.7792950868606567,
1310
+ 0.756394624710083,
1311
+ 0.8042755126953125,
1312
+ 0.7894731760025024,
1313
+ 0.7734087705612183,
1314
+ 0.7713190317153931,
1315
+ 0.7733933925628662,
1316
+ 0.7869383096694946,
1317
+ 0.7916207313537598,
1318
+ 0.7824002504348755,
1319
+ 0.7731986045837402,
1320
+ 0.7848385572433472,
1321
+ 0.769917368888855,
1322
+ 0.7761365175247192,
1323
+ 0.7878182530403137,
1324
+ 0.7651918530464172,
1325
+ 0.7634813189506531,
1326
+ 0.7789775729179382,
1327
+ 0.7592380046844482,
1328
+ 0.7928231358528137,
1329
+ 0.7714013457298279,
1330
+ 0.7689073085784912,
1331
+ 0.7758734226226807,
1332
+ 0.7779701948165894,
1333
+ 0.7809176445007324,
1334
+ 0.781708836555481,
1335
+ 0.7909632921218872,
1336
+ 0.7622993588447571,
1337
+ 0.7866973280906677,
1338
+ 0.7720290422439575,
1339
+ 0.7733672857284546,
1340
+ 0.7902675867080688,
1341
+ 0.7678062915802002,
1342
+ 0.7767971754074097,
1343
+ 0.7744797468185425,
1344
+ 0.792242169380188,
1345
+ 0.7812575101852417,
1346
+ 0.7901678085327148,
1347
+ 0.7877702713012695,
1348
+ 0.775005042552948,
1349
+ 0.7859253883361816,
1350
+ 0.7835381031036377,
1351
+ 0.7937123775482178,
1352
+ 0.7808881402015686,
1353
+ 0.782737135887146,
1354
+ 0.8077836036682129,
1355
+ 0.7705615162849426,
1356
+ 0.7792931199073792,
1357
+ 0.7741867303848267,
1358
+ 0.758423388004303,
1359
+ 0.7733595371246338,
1360
+ 0.7719141840934753,
1361
+ 0.7665902376174927,
1362
+ 0.7745699882507324,
1363
+ 0.7793188095092773,
1364
+ 0.776689887046814,
1365
+ 0.7783402800559998,
1366
+ 0.7793123722076416,
1367
+ 0.748462438583374,
1368
+ 0.7847517728805542,
1369
+ 0.778985857963562,
1370
+ 0.765911340713501,
1371
+ 0.8044880628585815,
1372
+ 0.7837658524513245,
1373
+ 0.7788437604904175,
1374
+ 0.7672808170318604,
1375
+ 0.7926174998283386,
1376
+ 0.7772773504257202,
1377
+ 0.7955619692802429,
1378
+ 0.7905150651931763,
1379
+ 0.7708892822265625,
1380
+ 0.8017271757125854,
1381
+ 0.7916097640991211,
1382
+ 0.7813031077384949,
1383
+ 0.768671989440918,
1384
+ 0.7688347101211548,
1385
+ 0.7929792404174805,
1386
+ 0.7643711566925049,
1387
+ 0.7710295915603638,
1388
+ 0.7956861257553101,
1389
+ 0.775759220123291,
1390
+ 0.7851706743240356,
1391
+ 0.7763235569000244,
1392
+ 0.7840069532394409,
1393
+ 0.7891350984573364,
1394
+ 0.7790133953094482,
1395
+ 0.7865608930587769,
1396
+ 0.7712998390197754,
1397
+ 0.7953822612762451,
1398
+ 0.7522621750831604,
1399
+ 0.7789600491523743,
1400
+ 0.7822747230529785,
1401
+ 0.7683793902397156,
1402
+ 0.7780958414077759,
1403
+ 0.7778472304344177,
1404
+ 0.7890318632125854,
1405
+ 0.7912524342536926,
1406
+ 0.7955080270767212,
1407
+ 0.7843313217163086,
1408
+ 0.7704141736030579,
1409
+ 0.8011472225189209,
1410
+ 0.7723038196563721,
1411
+ 0.7629398107528687,
1412
+ 0.7943845987319946,
1413
+ 0.7824939489364624,
1414
+ 0.7986029386520386,
1415
+ 0.7799835205078125,
1416
+ 0.7753137350082397,
1417
+ 0.7764695882797241,
1418
+ 0.7789475917816162,
1419
+ 0.7772172093391418,
1420
+ 0.7792038917541504,
1421
+ 0.7663732171058655,
1422
+ 0.7891214489936829,
1423
+ 0.7658422589302063,
1424
+ 0.76682448387146,
1425
+ 0.7720687985420227,
1426
+ 0.7804040908813477,
1427
+ 0.7830777168273926,
1428
+ 0.7594829797744751,
1429
+ 0.7839692831039429,
1430
+ 0.7788735628128052,
1431
+ 0.7754986882209778,
1432
+ 0.7855756282806396,
1433
+ 0.7806379199028015,
1434
+ 0.7652832865715027,
1435
+ 0.7549910545349121,
1436
+ 0.777277946472168,
1437
+ 0.7977598905563354,
1438
+ 0.7900921106338501,
1439
+ 0.7777581810951233,
1440
+ 0.7619286179542542,
1441
+ 0.79695063829422,
1442
+ 0.7639628052711487,
1443
+ 0.7746571898460388,
1444
+ 0.7828299403190613,
1445
+ 0.7545140385627747,
1446
+ 0.7856276035308838,
1447
+ 0.7859327793121338,
1448
+ 0.7729083299636841,
1449
+ 0.7775464057922363,
1450
+ 0.7831757068634033,
1451
+ 0.7666918039321899,
1452
+ 0.7919015288352966,
1453
+ 0.7977434396743774,
1454
+ 0.7780756950378418,
1455
+ 0.7900899052619934,
1456
+ 0.7669292092323303,
1457
+ 0.7722169756889343,
1458
+ 0.7852177619934082,
1459
+ 0.7992111444473267,
1460
+ 0.7724581956863403,
1461
+ 0.785639762878418,
1462
+ 0.759498655796051,
1463
+ 0.7778706550598145,
1464
+ 0.7979644536972046,
1465
+ 0.7809131741523743,
1466
+ 0.7950116395950317
1467
  ],
1468
  "loss_window": 200,
1469
  "grad_ema_decay": 0.99,
 
1496
  },
1497
  "committed_scale": 1.0,
1498
  "attempt_factor": 1.0,
1499
+ "fast_loss": 0.7788921701746249,
1500
+ "slow_loss": 0.7794827720793691,
1501
+ "best_loss": 0.778826138998272,
1502
  "reference": {
1503
+ "loss": 0.7802190550804138,
1504
+ "grad_median": 0.12170863464559273,
1505
+ "grad_p90": 0.14147012085906355,
1506
+ "step": 430000,
1507
+ "promotions": 4
1508
  },
1509
  "last_metrics": {
1510
+ "loss": 0.7786456739902496,
1511
+ "grad_median": 0.12425224855542183,
1512
+ "grad_p90": 0.14411725699901579,
1513
  "skip_rate": 0.0,
1514
  "relative_spike_rate": 0.0
1515
  },
1516
+ "last_loss_ratio": 0.9974805340501689,
1517
+ "last_grad_ratio": 1.026261218001855,
1518
+ "last_grad_p90_ratio": 1.023499462102691,
1519
  "healthy_windows": 0,
1520
  "warning_windows": 0,
1521
  "retry_windows": 0,
1522
+ "windows_seen": 150,
1523
  "retry_count": 0
1524
  },
1525
+ "best_window": 0.778826138998272,
1526
  "blown_windows": 0
1527
  },
1528
  "transaction": {
1529
  "status": "committed",
1530
+ "committed_at": "2026-07-23T18:45:18.663474+00:00",
1531
  "attempt": 0,
1532
  "health_reference": {
1533
+ "loss": 0.7802190550804138,
1534
+ "grad_median": 0.12170863464559273,
1535
+ "grad_p90": 0.14147012085906355,
1536
+ "step": 430000,
1537
+ "promotions": 4
1538
  }
1539
  },
1540
  "config": {