sparsetrace commited on
Commit
d7dfcf7
·
verified ·
1 Parent(s): c8e31a6

checkpoint: promote transactional candidate: step 380000

Browse files
Files changed (4) hide show
  1. ema.safetensors +1 -1
  2. model.safetensors +1 -1
  3. optim.safetensors +1 -1
  4. state.json +254 -225
ema.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0cc404d03f51297c17babafadab8c1f626e3ca9b54d842f434ac38c5244c9160
3
  size 1945760448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e08e86196419e22ddeeb158eb97466da33e49255be77fb3abd31067b5bb84ca6
3
  size 1945760448
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e2bd2a634d1755dbe2b9665666cdc535bf49a8592c9ddfc7096a8cdbc84d31f9
3
  size 2046527448
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b5b33e02d4d5030f582191e1c47a72bf73c629ff14c86575300bb75a9a429d6
3
  size 2046527448
optim.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4936eff2928c90f73897387454e907fb3a6cb8630367fdece431fb3e7d68270a
3
  size 3891530432
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab3ed91a1b4be1107a7ab1cc4bf65e03eacdfaaa6eb4d7f13c5398246d34cf52
3
  size 3891530432
state.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "step": 370000,
3
  "run_history": [
4
  {
5
  "start_step": 0,
@@ -1080,216 +1080,245 @@
1080
  "seed_offset": 0,
1081
  "target_steps": 400000
1082
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1083
  }
1084
  ],
1085
  "guard_state": {
1086
  "version": 3,
1087
- "live_grad_ema": 0.12521205148990533,
1088
- "grad_ema": 0.12521205148990533,
1089
- "grad_reference": 0.11887033767056938,
1090
- "grad_p90_reference": 0.13989912932343368,
1091
  "spikes_total": 0,
1092
  "recent_losses": [
1093
- 0.7978678941726685,
1094
- 0.7751516103744507,
1095
- 0.7855542898178101,
1096
- 0.7929452061653137,
1097
- 0.7977163791656494,
1098
- 0.8016595840454102,
1099
- 0.7931407690048218,
1100
- 0.7856813073158264,
1101
- 0.7782225608825684,
1102
- 0.7713536024093628,
1103
- 0.7797688245773315,
1104
- 0.7937817573547363,
1105
- 0.7918097376823425,
1106
- 0.8009408712387085,
1107
- 0.780621349811554,
1108
- 0.7973782420158386,
1109
- 0.7865002155303955,
1110
- 0.7804096937179565,
1111
- 0.7874341607093811,
1112
- 0.768108606338501,
1113
- 0.7781695127487183,
1114
- 0.7720744609832764,
1115
- 0.7748705744743347,
1116
- 0.7903978228569031,
1117
- 0.7955414056777954,
1118
- 0.7895504832267761,
1119
- 0.7955923080444336,
1120
- 0.7797942161560059,
1121
- 0.8003368377685547,
1122
- 0.7947072386741638,
1123
- 0.7822599411010742,
1124
- 0.8023264408111572,
1125
- 0.7790060043334961,
1126
- 0.779749870300293,
1127
- 0.769734263420105,
1128
- 0.7868667840957642,
1129
- 0.7710761427879333,
1130
- 0.774529218673706,
1131
- 0.7693489789962769,
1132
- 0.7976977825164795,
1133
- 0.7608304023742676,
1134
- 0.7721688747406006,
1135
- 0.7810112237930298,
1136
- 0.790211021900177,
1137
- 0.7798612117767334,
1138
- 0.7930037975311279,
1139
- 0.7683382034301758,
1140
- 0.7718660831451416,
1141
- 0.7883671522140503,
1142
- 0.7839291095733643,
1143
- 0.7700468301773071,
1144
- 0.7842354774475098,
1145
- 0.7720392346382141,
1146
- 0.7676264643669128,
1147
- 0.7869400382041931,
1148
- 0.7878783941268921,
1149
- 0.7853030562400818,
1150
- 0.7831085920333862,
1151
- 0.7681159973144531,
1152
- 0.8027441501617432,
1153
- 0.7716056108474731,
1154
- 0.7608656883239746,
1155
- 0.7842078804969788,
1156
- 0.7720915079116821,
1157
- 0.7835590243339539,
1158
- 0.7762640714645386,
1159
- 0.7661155462265015,
1160
- 0.7638898491859436,
1161
- 0.7642673254013062,
1162
- 0.7968372702598572,
1163
- 0.7687630653381348,
1164
- 0.7989886999130249,
1165
- 0.8001572489738464,
1166
- 0.7925434112548828,
1167
- 0.7850891947746277,
1168
- 0.7865904569625854,
1169
- 0.7750698924064636,
1170
- 0.7881876230239868,
1171
- 0.7665663361549377,
1172
- 0.7760233283042908,
1173
- 0.7843261361122131,
1174
- 0.7858600616455078,
1175
- 0.783657431602478,
1176
- 0.797662615776062,
1177
- 0.773908257484436,
1178
- 0.8006343245506287,
1179
- 0.7975600361824036,
1180
- 0.7721484899520874,
1181
- 0.7774638533592224,
1182
- 0.7803500890731812,
1183
- 0.7840143442153931,
1184
- 0.779275119304657,
1185
- 0.7676138877868652,
1186
- 0.7779205441474915,
1187
- 0.7833095788955688,
1188
- 0.7984818816184998,
1189
- 0.7766016721725464,
1190
- 0.7921992540359497,
1191
- 0.7730411291122437,
1192
- 0.7805932760238647,
1193
- 0.7879430055618286,
1194
- 0.7681646943092346,
1195
- 0.7747885584831238,
1196
- 0.7853114008903503,
1197
- 0.783561110496521,
1198
- 0.7732912302017212,
1199
- 0.7877262830734253,
1200
- 0.7865772247314453,
1201
- 0.7732062339782715,
1202
- 0.7771124839782715,
1203
- 0.7986534833908081,
1204
- 0.7964777946472168,
1205
- 0.7843698263168335,
1206
- 0.7879964113235474,
1207
- 0.7894013524055481,
1208
- 0.788426399230957,
1209
- 0.7863854169845581,
1210
- 0.7783207893371582,
1211
- 0.7711477279663086,
1212
- 0.7906065583229065,
1213
- 0.7997589111328125,
1214
- 0.792722225189209,
1215
- 0.7722136974334717,
1216
- 0.7796634435653687,
1217
- 0.7661240696907043,
1218
- 0.7794838547706604,
1219
- 0.7811031341552734,
1220
- 0.7752534747123718,
1221
- 0.7910878658294678,
1222
- 0.7933632135391235,
1223
- 0.7769409418106079,
1224
- 0.8001601696014404,
1225
- 0.7801427245140076,
1226
- 0.7891408801078796,
1227
- 0.7812725901603699,
1228
- 0.8083652257919312,
1229
- 0.7628598213195801,
1230
- 0.7905555963516235,
1231
- 0.7802824378013611,
1232
- 0.7886943221092224,
1233
- 0.7859426736831665,
1234
- 0.7777540683746338,
1235
- 0.7939848899841309,
1236
- 0.7756738662719727,
1237
- 0.7704358100891113,
1238
- 0.7792330980300903,
1239
- 0.7740166783332825,
1240
- 0.773828387260437,
1241
- 0.782011091709137,
1242
- 0.7961677312850952,
1243
- 0.7744059562683105,
1244
- 0.8045989274978638,
1245
- 0.7884491682052612,
1246
- 0.7689552307128906,
1247
- 0.7803974151611328,
1248
- 0.768400251865387,
1249
- 0.7757793664932251,
1250
- 0.7915592193603516,
1251
- 0.7801134586334229,
1252
- 0.767184853553772,
1253
- 0.7686281204223633,
1254
- 0.7917814254760742,
1255
- 0.7928374409675598,
1256
- 0.7809380292892456,
1257
- 0.775888204574585,
1258
- 0.7773340344429016,
1259
- 0.8050969839096069,
1260
- 0.7807714939117432,
1261
- 0.7794414758682251,
1262
- 0.7935159802436829,
1263
- 0.7818305492401123,
1264
- 0.7673047780990601,
1265
- 0.7759720087051392,
1266
- 0.7916602492332458,
1267
- 0.8164181709289551,
1268
- 0.7827853560447693,
1269
- 0.8029986619949341,
1270
- 0.7899382710456848,
1271
- 0.7876236438751221,
1272
- 0.7783252000808716,
1273
- 0.7788745164871216,
1274
- 0.7808921337127686,
1275
- 0.804801344871521,
1276
- 0.7945502996444702,
1277
- 0.7755527496337891,
1278
- 0.8046509027481079,
1279
- 0.7876801490783691,
1280
- 0.7907925844192505,
1281
- 0.78558349609375,
1282
- 0.7998905181884766,
1283
- 0.7946356534957886,
1284
- 0.7978602647781372,
1285
- 0.7723987102508545,
1286
- 0.7749336957931519,
1287
- 0.7792190909385681,
1288
- 0.7843501567840576,
1289
- 0.7864072322845459,
1290
- 0.7880149483680725,
1291
- 0.7962101101875305,
1292
- 0.7930788993835449
1293
  ],
1294
  "loss_window": 200,
1295
  "grad_ema_decay": 0.99,
@@ -1322,30 +1351,30 @@
1322
  },
1323
  "committed_scale": 1.0,
1324
  "attempt_factor": 1.0,
1325
- "fast_loss": 0.7837044753459748,
1326
- "slow_loss": 0.7846865615314718,
1327
  "best_loss": 0.7812624632790134,
1328
  "reference": {
1329
- "loss": 0.7857006221151352,
1330
- "grad_median": 0.11887033767056938,
1331
- "grad_p90": 0.13989912932343368,
1332
- "step": 370000,
1333
- "promotions": 4
1334
  },
1335
  "last_metrics": {
1336
- "loss": 0.7833779042959214,
1337
- "grad_median": 0.11908036470413208,
1338
- "grad_p90": 0.1395524427294731,
1339
  "skip_rate": 0.0,
1340
  "relative_spike_rate": 0.0
1341
  },
1342
- "last_loss_ratio": 0.9963074318717484,
1343
- "last_grad_ratio": 1.0022095487683782,
1344
- "last_grad_p90_ratio": 0.9969042700085157,
1345
  "healthy_windows": 0,
1346
  "warning_windows": 0,
1347
  "retry_windows": 0,
1348
- "windows_seen": 127,
1349
  "retry_count": 0
1350
  },
1351
  "best_window": 0.7812624632790134,
@@ -1353,14 +1382,14 @@
1353
  },
1354
  "transaction": {
1355
  "status": "committed",
1356
- "committed_at": "2026-07-22T22:45:13.774561+00:00",
1357
  "attempt": 0,
1358
  "health_reference": {
1359
- "loss": 0.7857006221151352,
1360
- "grad_median": 0.11887033767056938,
1361
- "grad_p90": 0.13989912932343368,
1362
- "step": 370000,
1363
- "promotions": 4
1364
  }
1365
  },
1366
  "config": {
 
1
  {
2
+ "step": 380000,
3
  "run_history": [
4
  {
5
  "start_step": 0,
 
1080
  "seed_offset": 0,
1081
  "target_steps": 400000
1082
  }
1083
+ },
1084
+ {
1085
+ "start_step": 370000,
1086
+ "end_step": 380000,
1087
+ "seconds": 4589.9,
1088
+ "world": 1,
1089
+ "objective": "flow",
1090
+ "completed": false,
1091
+ "finished_at": "2026-07-23T00:39:13.925275+00:00",
1092
+ "promotion_reason": "periodic healthy candidate",
1093
+ "effective": {
1094
+ "attempt": 0,
1095
+ "precision": "tf32",
1096
+ "lr_policy": "constant",
1097
+ "lr_base": 0.0001,
1098
+ "lr_start": 0.0001,
1099
+ "lr_end": 0.0001,
1100
+ "lr_min": 1e-05,
1101
+ "lr_hard_min": 1e-06,
1102
+ "lr_scale": 1.0,
1103
+ "weight_decay": 0.1,
1104
+ "wd_ada": 0.0,
1105
+ "clip": 1.0,
1106
+ "spike_skip": 4.0,
1107
+ "grad_ceiling": 25.0,
1108
+ "steps_skipped": 0,
1109
+ "seed_offset": 0,
1110
+ "target_steps": 400000
1111
+ }
1112
  }
1113
  ],
1114
  "guard_state": {
1115
  "version": 3,
1116
+ "live_grad_ema": 0.12047874666031405,
1117
+ "grad_ema": 0.12047874666031405,
1118
+ "grad_reference": 0.12339134257551497,
1119
+ "grad_p90_reference": 0.13971604220252898,
1120
  "spikes_total": 0,
1121
  "recent_losses": [
1122
+ 0.7931565046310425,
1123
+ 0.7827502489089966,
1124
+ 0.7755047082901001,
1125
+ 0.7518191337585449,
1126
+ 0.7874758243560791,
1127
+ 0.8052571415901184,
1128
+ 0.7914084792137146,
1129
+ 0.7584506869316101,
1130
+ 0.7790533304214478,
1131
+ 0.7826451659202576,
1132
+ 0.794867217540741,
1133
+ 0.798413872718811,
1134
+ 0.7825874090194702,
1135
+ 0.7727178931236267,
1136
+ 0.7912687063217163,
1137
+ 0.7785184383392334,
1138
+ 0.797482967376709,
1139
+ 0.7782487869262695,
1140
+ 0.7757858633995056,
1141
+ 0.78122878074646,
1142
+ 0.7822624444961548,
1143
+ 0.7690896987915039,
1144
+ 0.7828480005264282,
1145
+ 0.7897740602493286,
1146
+ 0.7736449241638184,
1147
+ 0.7746298313140869,
1148
+ 0.7822747230529785,
1149
+ 0.783113956451416,
1150
+ 0.7789171934127808,
1151
+ 0.7683112621307373,
1152
+ 0.7669581174850464,
1153
+ 0.7692807912826538,
1154
+ 0.7796012163162231,
1155
+ 0.7673876285552979,
1156
+ 0.7916824221611023,
1157
+ 0.7738845944404602,
1158
+ 0.786881685256958,
1159
+ 0.7967593669891357,
1160
+ 0.7723098993301392,
1161
+ 0.7874491214752197,
1162
+ 0.7888399362564087,
1163
+ 0.7785971760749817,
1164
+ 0.7811653017997742,
1165
+ 0.7640054225921631,
1166
+ 0.7747929096221924,
1167
+ 0.7835151553153992,
1168
+ 0.7730658054351807,
1169
+ 0.7636295557022095,
1170
+ 0.7770801186561584,
1171
+ 0.7802595496177673,
1172
+ 0.7856400609016418,
1173
+ 0.7910728454589844,
1174
+ 0.7913957834243774,
1175
+ 0.773612380027771,
1176
+ 0.7665975689888,
1177
+ 0.7795747518539429,
1178
+ 0.7999171614646912,
1179
+ 0.7754309177398682,
1180
+ 0.7804783582687378,
1181
+ 0.7698723077774048,
1182
+ 0.7806293368339539,
1183
+ 0.786333441734314,
1184
+ 0.7758967280387878,
1185
+ 0.7785554528236389,
1186
+ 0.7884644269943237,
1187
+ 0.7789888978004456,
1188
+ 0.7641298770904541,
1189
+ 0.8010116815567017,
1190
+ 0.7582226991653442,
1191
+ 0.7559333443641663,
1192
+ 0.7906558513641357,
1193
+ 0.7694743871688843,
1194
+ 0.7894136905670166,
1195
+ 0.7954419851303101,
1196
+ 0.75834059715271,
1197
+ 0.762528121471405,
1198
+ 0.7948977947235107,
1199
+ 0.7769706845283508,
1200
+ 0.7807410359382629,
1201
+ 0.7833594083786011,
1202
+ 0.7690268754959106,
1203
+ 0.796924352645874,
1204
+ 0.7924069166183472,
1205
+ 0.7706602811813354,
1206
+ 0.7738072872161865,
1207
+ 0.7978936433792114,
1208
+ 0.788293182849884,
1209
+ 0.7758481502532959,
1210
+ 0.776520311832428,
1211
+ 0.7920444011688232,
1212
+ 0.7684814929962158,
1213
+ 0.798369288444519,
1214
+ 0.8058817386627197,
1215
+ 0.8061990737915039,
1216
+ 0.769462525844574,
1217
+ 0.7877870202064514,
1218
+ 0.788791298866272,
1219
+ 0.7703174352645874,
1220
+ 0.7726359963417053,
1221
+ 0.7893047332763672,
1222
+ 0.7839610576629639,
1223
+ 0.7676035761833191,
1224
+ 0.7873141765594482,
1225
+ 0.7862273454666138,
1226
+ 0.770391583442688,
1227
+ 0.7806901931762695,
1228
+ 0.7804044485092163,
1229
+ 0.7603528499603271,
1230
+ 0.7910115718841553,
1231
+ 0.7985414266586304,
1232
+ 0.7839707136154175,
1233
+ 0.7956773042678833,
1234
+ 0.7961212396621704,
1235
+ 0.784096360206604,
1236
+ 0.7728941440582275,
1237
+ 0.795786440372467,
1238
+ 0.7580296993255615,
1239
+ 0.7708978652954102,
1240
+ 0.7873404026031494,
1241
+ 0.795462965965271,
1242
+ 0.7994289398193359,
1243
+ 0.7719279527664185,
1244
+ 0.7732014656066895,
1245
+ 0.7951341867446899,
1246
+ 0.7899830341339111,
1247
+ 0.769656777381897,
1248
+ 0.7683723568916321,
1249
+ 0.7922228574752808,
1250
+ 0.7827354073524475,
1251
+ 0.7863415479660034,
1252
+ 0.7483392357826233,
1253
+ 0.785744309425354,
1254
+ 0.7882087826728821,
1255
+ 0.7632929086685181,
1256
+ 0.7827844023704529,
1257
+ 0.780449628829956,
1258
+ 0.8171622157096863,
1259
+ 0.8015577793121338,
1260
+ 0.7797446250915527,
1261
+ 0.7879010438919067,
1262
+ 0.7707571387290955,
1263
+ 0.782752275466919,
1264
+ 0.7819050550460815,
1265
+ 0.7698577642440796,
1266
+ 0.783696174621582,
1267
+ 0.7831618785858154,
1268
+ 0.7665432691574097,
1269
+ 0.7908247709274292,
1270
+ 0.7932536005973816,
1271
+ 0.7755106091499329,
1272
+ 0.7739017009735107,
1273
+ 0.785597026348114,
1274
+ 0.7896386384963989,
1275
+ 0.7833244800567627,
1276
+ 0.7808831930160522,
1277
+ 0.7832551002502441,
1278
+ 0.7922441959381104,
1279
+ 0.7913550734519958,
1280
+ 0.7831692695617676,
1281
+ 0.7929463982582092,
1282
+ 0.7936990261077881,
1283
+ 0.7734542489051819,
1284
+ 0.7818700075149536,
1285
+ 0.7912338972091675,
1286
+ 0.7859599590301514,
1287
+ 0.783903956413269,
1288
+ 0.7943370342254639,
1289
+ 0.768065333366394,
1290
+ 0.8002237677574158,
1291
+ 0.7743337154388428,
1292
+ 0.79447340965271,
1293
+ 0.7826780676841736,
1294
+ 0.7769286632537842,
1295
+ 0.7911287546157837,
1296
+ 0.7813004851341248,
1297
+ 0.794337809085846,
1298
+ 0.7870470285415649,
1299
+ 0.7929540872573853,
1300
+ 0.7921295166015625,
1301
+ 0.7875479459762573,
1302
+ 0.7707194089889526,
1303
+ 0.7964199781417847,
1304
+ 0.7824279069900513,
1305
+ 0.7839290499687195,
1306
+ 0.7728868126869202,
1307
+ 0.7746325135231018,
1308
+ 0.7962530255317688,
1309
+ 0.7792419195175171,
1310
+ 0.7559351921081543,
1311
+ 0.7612786293029785,
1312
+ 0.7787858247756958,
1313
+ 0.7943848967552185,
1314
+ 0.7864570021629333,
1315
+ 0.7771327495574951,
1316
+ 0.7811050415039062,
1317
+ 0.7812439203262329,
1318
+ 0.7885185480117798,
1319
+ 0.783769965171814,
1320
+ 0.7689976692199707,
1321
+ 0.7745211720466614
1322
  ],
1323
  "loss_window": 200,
1324
  "grad_ema_decay": 0.99,
 
1351
  },
1352
  "committed_scale": 1.0,
1353
  "attempt_factor": 1.0,
1354
+ "fast_loss": 0.7825673768084968,
1355
+ "slow_loss": 0.7831768108295837,
1356
  "best_loss": 0.7812624632790134,
1357
  "reference": {
1358
+ "loss": 0.7832791751623154,
1359
+ "grad_median": 0.12339134257551497,
1360
+ "grad_p90": 0.13971604220252898,
1361
+ "step": 380000,
1362
+ "promotions": 2
1363
  },
1364
  "last_metrics": {
1365
+ "loss": 0.7817834049463273,
1366
+ "grad_median": 0.11610850691795349,
1367
+ "grad_p90": 0.1389836937189102,
1368
  "skip_rate": 0.0,
1369
  "relative_spike_rate": 0.0
1370
  },
1371
+ "last_loss_ratio": 0.9976141066401499,
1372
+ "last_grad_ratio": 0.9272949810850614,
1373
+ "last_grad_p90_ratio": 0.9934564596009238,
1374
  "healthy_windows": 0,
1375
  "warning_windows": 0,
1376
  "retry_windows": 0,
1377
+ "windows_seen": 50,
1378
  "retry_count": 0
1379
  },
1380
  "best_window": 0.7812624632790134,
 
1382
  },
1383
  "transaction": {
1384
  "status": "committed",
1385
+ "committed_at": "2026-07-23T00:39:13.928456+00:00",
1386
  "attempt": 0,
1387
  "health_reference": {
1388
+ "loss": 0.7832791751623154,
1389
+ "grad_median": 0.12339134257551497,
1390
+ "grad_p90": 0.13971604220252898,
1391
+ "step": 380000,
1392
+ "promotions": 2
1393
  }
1394
  },
1395
  "config": {