ditflex-L2-flow / state.json
sparsetrace's picture
checkpoint: promote transactional candidate: step 340000
8a5823e verified
Raw
History Blame Contribute Delete
25.1 kB
{
"step": 340000,
"run_history": [
{
"start_step": 0,
"end_step": 93500,
"seconds": 14451.7,
"world": 2,
"objective": "flow",
"completed": true,
"finished_at": "2026-07-19T22:25:14.507322+00:00"
},
{
"start_step": 93500,
"end_step": 192500,
"seconds": 14410.2,
"world": 2,
"objective": "flow",
"completed": true,
"finished_at": "2026-07-20T04:05:19.860941+00:00"
},
{
"start_step": 192500,
"end_step": 210000,
"seconds": 3073.8,
"world": 2,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-20T05:38:26.749589+00:00"
},
{
"start_step": 210000,
"end_step": 240000,
"seconds": 3419.9,
"world": 2,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-20T06:48:12.738761+00:00"
},
{
"start_step": 240000,
"end_step": 250000,
"seconds": 1781.7,
"world": 2,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-20T14:46:34.505065+00:00"
},
{
"start_step": 250000,
"end_step": 268500,
"seconds": 3641.4,
"world": 1,
"objective": "flow",
"completed": true,
"finished_at": "2026-07-20T19:24:03.421837+00:00",
"effective": {
"lr": 9e-06,
"weight_decay": 0.1,
"clip": 0.5,
"spike_skip": 4.0,
"steps_skipped_this_run": 1524,
"steps_skipped_total": 1524,
"seed_offset": 797,
"grad_ceiling": 25.0
}
},
{
"start_step": 268500,
"end_step": 270000,
"seconds": 368.6,
"world": 1,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-20T20:26:00.017328+00:00",
"effective": {
"lr_policy": "adaptive",
"lr_base": 0.0001,
"lr_start": 9e-06,
"lr_end": 8.87226296048255e-06,
"lr_min": 1e-05,
"lr_hard_min": 1e-06,
"lr_scale": 0.28176768087105175,
"lr_backoffs_total": 0,
"weight_decay": 0.1,
"clip": 1.0,
"spike_skip": 4.0,
"grad_ceiling": 0.0,
"steps_skipped_this_run": 18,
"steps_skipped_total": 1542,
"seed_offset": 0,
"target_steps": 400000
}
},
{
"start_step": 270000,
"end_step": 275000,
"seconds": 2254.3,
"world": 1,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-21T02:12:18.577289+00:00",
"promotion_reason": "run final",
"effective": {
"attempt": 0,
"precision": "tf32",
"lr_policy": "adaptive",
"lr_base": 0.0001,
"lr_start": 8.872178050176876e-06,
"lr_end": 8.452927124607216e-06,
"lr_min": 1e-05,
"lr_hard_min": 1e-06,
"lr_scale": 0.28176768087105175,
"weight_decay": 0.1,
"wd_ada": 0.01,
"clip": 1.0,
"spike_skip": 10.0,
"grad_ceiling": 500.0,
"steps_skipped": 5,
"seed_offset": 0,
"target_steps": 400000
}
},
{
"start_step": 275000,
"end_step": 280000,
"seconds": 2309.0,
"world": 1,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-21T02:55:39.928314+00:00",
"promotion_reason": "periodic healthy candidate",
"effective": {
"attempt": 0,
"precision": "tf32",
"lr_policy": "adaptive",
"lr_base": 0.0001,
"lr_start": 8.452844322556308e-06,
"lr_end": 8.044453081539695e-06,
"lr_min": 1e-05,
"lr_hard_min": 1e-06,
"lr_scale": 0.28176768087105175,
"weight_decay": 0.1,
"wd_ada": 0.01,
"clip": 1.0,
"spike_skip": 4.0,
"grad_ceiling": 0.0,
"steps_skipped": 139,
"seed_offset": 0,
"target_steps": 400000
}
},
{
"start_step": 280000,
"end_step": 290000,
"seconds": 4446.3,
"world": 1,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-21T05:04:30.114007+00:00",
"promotion_reason": "periodic healthy candidate",
"effective": {
"attempt": 0,
"precision": "tf32",
"lr_policy": "adaptive",
"lr_base": 0.0001,
"lr_start": 8.044372515418319e-06,
"lr_end": 7.2625920040378e-06,
"lr_min": 1e-05,
"lr_hard_min": 1e-06,
"lr_scale": 0.28176768087105175,
"weight_decay": 0.1,
"wd_ada": 0.05,
"clip": 1.0,
"spike_skip": 4.0,
"grad_ceiling": 0.0,
"steps_skipped": 179,
"seed_offset": 0,
"target_steps": 400000
}
},
{
"start_step": 290000,
"end_step": 300000,
"seconds": 4428.6,
"world": 1,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-21T06:18:18.741141+00:00",
"promotion_reason": "periodic healthy candidate",
"effective": {
"attempt": 0,
"precision": "tf32",
"lr_policy": "adaptive",
"lr_base": 0.0001,
"lr_start": null,
"lr_end": 6.5315001615148555e-06,
"lr_min": 1e-05,
"lr_hard_min": 1e-06,
"lr_scale": 0.28176768087105175,
"weight_decay": 0.1,
"wd_ada": 0.05,
"clip": 1.0,
"spike_skip": 4.0,
"grad_ceiling": 0.0,
"steps_skipped": 290,
"seed_offset": 0,
"target_steps": 400000
}
},
{
"start_step": 300000,
"end_step": 310000,
"seconds": 4426.6,
"world": 1,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-21T07:32:05.358144+00:00",
"promotion_reason": "periodic healthy candidate",
"effective": {
"attempt": 0,
"precision": "tf32",
"lr_policy": "adaptive",
"lr_base": 0.0001,
"lr_start": null,
"lr_end": 5.8556849782927245e-06,
"lr_min": 1e-05,
"lr_hard_min": 1e-06,
"lr_scale": 0.28176768087105175,
"weight_decay": 0.1,
"wd_ada": 0.05,
"clip": 1.0,
"spike_skip": 4.0,
"grad_ceiling": 0.0,
"steps_skipped": 614,
"seed_offset": 0,
"target_steps": 400000
}
},
{
"start_step": 310000,
"end_step": 312500,
"seconds": 1129.4,
"world": 1,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-21T07:50:54.720571+00:00",
"promotion_reason": "run final",
"effective": {
"attempt": 0,
"precision": "tf32",
"lr_policy": "adaptive",
"lr_base": 0.0001,
"lr_start": null,
"lr_end": 5.695864301224254e-06,
"lr_min": 1e-05,
"lr_hard_min": 1e-06,
"lr_scale": 0.28176768087105175,
"weight_decay": 0.1,
"wd_ada": 0.05,
"clip": 1.0,
"spike_skip": 4.0,
"grad_ceiling": 0.0,
"steps_skipped": 54,
"seed_offset": 0,
"target_steps": 400000
}
},
{
"start_step": 312500,
"end_step": 320000,
"seconds": 3445.6,
"world": 1,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-21T15:45:13.044752+00:00",
"promotion_reason": "periodic healthy candidate",
"effective": {
"attempt": 0,
"precision": "tf32",
"lr_policy": "adaptive",
"lr_base": 0.0001,
"lr_start": 5.695801124918405e-06,
"lr_end": 5.239313079707324e-06,
"lr_min": 1e-05,
"lr_hard_min": 1e-06,
"lr_scale": 0.28176768087105175,
"weight_decay": 0.1,
"wd_ada": 0.05,
"clip": 1.0,
"spike_skip": 4.0,
"grad_ceiling": 0.0,
"steps_skipped": 77,
"seed_offset": 0,
"target_steps": 400000
}
},
{
"start_step": 320000,
"end_step": 330000,
"seconds": 4516.6,
"world": 1,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-21T17:00:29.615182+00:00",
"promotion_reason": "periodic healthy candidate",
"effective": {
"attempt": 0,
"precision": "tf32",
"lr_policy": "adaptive",
"lr_base": 0.0001,
"lr_start": null,
"lr_end": 4.686184603477889e-06,
"lr_min": 1e-05,
"lr_hard_min": 1e-06,
"lr_scale": 0.28176768087105175,
"weight_decay": 0.1,
"wd_ada": 0.05,
"clip": 1.0,
"spike_skip": 4.0,
"grad_ceiling": 0.0,
"steps_skipped": 171,
"seed_offset": 0,
"target_steps": 400000
}
},
{
"start_step": 330000,
"end_step": 337500,
"seconds": 3341.5,
"world": 1,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-21T18:49:32.212552+00:00",
"promotion_reason": "run final",
"effective": {
"attempt": 1,
"precision": "tf32",
"lr_policy": "adaptive",
"lr_base": 0.0001,
"lr_start": 2.3430662850837115e-06,
"lr_end": 2.157454235387407e-06,
"lr_min": 1e-05,
"lr_hard_min": 1e-06,
"lr_scale": 0.14088384043552588,
"weight_decay": 0.1,
"wd_ada": 0.05,
"clip": 1.0,
"spike_skip": 4.0,
"grad_ceiling": 0.0,
"steps_skipped": 48,
"seed_offset": 1000003,
"target_steps": 400000
}
},
{
"start_step": 337500,
"end_step": 340000,
"seconds": 1548.1,
"world": 1,
"objective": "flow",
"completed": false,
"finished_at": "2026-07-21T19:32:23.605941+00:00",
"promotion_reason": "periodic healthy candidate",
"effective": {
"attempt": 0,
"precision": "tf32",
"lr_policy": "adaptive",
"lr_base": 0.0001,
"lr_start": 2.157430763212134e-06,
"lr_end": 2.099854885297127e-06,
"lr_min": 1e-05,
"lr_hard_min": 1e-06,
"lr_scale": 0.14088384043552588,
"weight_decay": 0.1,
"wd_ada": 0.01,
"clip": 1.0,
"spike_skip": 4.0,
"grad_ceiling": 0.0,
"steps_skipped": 2,
"seed_offset": 0,
"target_steps": 400000
}
}
],
"guard_state": {
"version": 3,
"live_grad_ema": 1105.722598976348,
"grad_ema": 1105.722598976348,
"grad_reference": 533.3278690397134,
"grad_p90_reference": 1076.281941084452,
"spikes_total": 3121,
"recent_losses": [
0.7659851312637329,
0.7799718379974365,
0.767326831817627,
0.7740594744682312,
0.7746687531471252,
0.7693485021591187,
0.7744090557098389,
0.76472407579422,
0.7712109684944153,
0.7667823433876038,
0.7734376192092896,
0.7631382942199707,
0.7683030962944031,
0.7651751637458801,
0.7694488167762756,
0.7677610516548157,
0.7600035071372986,
0.7641050815582275,
0.758497953414917,
0.75881028175354,
0.7905396223068237,
0.7816970348358154,
0.7721761465072632,
0.7704170942306519,
0.7622644901275635,
0.785053014755249,
0.7627816200256348,
0.7816370725631714,
0.7777868509292603,
0.7586430311203003,
0.757468581199646,
0.768932580947876,
0.7764378786087036,
0.7670449018478394,
0.7591977119445801,
0.7605947256088257,
0.7780417203903198,
0.7622751593589783,
0.7680383324623108,
0.7845712304115295,
0.7854090929031372,
0.7671298980712891,
0.7803963422775269,
0.7852848768234253,
0.7887732982635498,
0.7690376043319702,
0.7525300979614258,
0.8054315447807312,
0.776677131652832,
0.7609086036682129,
0.7614132165908813,
0.7684108018875122,
0.7619225978851318,
0.7682492733001709,
0.7727726697921753,
0.7575544118881226,
0.7509193420410156,
0.7757312059402466,
0.7860416769981384,
0.7617193460464478,
0.7782135009765625,
0.7646865844726562,
0.7678159475326538,
0.7605258822441101,
0.7857308387756348,
0.7728601694107056,
0.7700196504592896,
0.7775273323059082,
0.7790310382843018,
0.7775787115097046,
0.781828761100769,
0.7681038975715637,
0.765837550163269,
0.7664794921875,
0.7541241645812988,
0.7896788120269775,
0.7635741233825684,
0.7778509855270386,
0.7496468424797058,
0.753919780254364,
0.7757487297058105,
0.7724413871765137,
0.7722828388214111,
0.7738194465637207,
0.7687792181968689,
0.7757934331893921,
0.7694065570831299,
0.7718388438224792,
0.7744388580322266,
0.764975368976593,
0.7768993973731995,
0.7564750909805298,
0.760976254940033,
0.7610824704170227,
0.7636446952819824,
0.7474631667137146,
0.7591776847839355,
0.7641832828521729,
0.776297926902771,
0.7714574337005615,
0.758858859539032,
0.7527077198028564,
0.7844610214233398,
0.7654604315757751,
0.7850476503372192,
0.7593823671340942,
0.7806195616722107,
0.7746589779853821,
0.7741937041282654,
0.771770715713501,
0.7561997175216675,
0.7938714027404785,
0.76432204246521,
0.7739542126655579,
0.7850971221923828,
0.7664803266525269,
0.7699167132377625,
0.7678929567337036,
0.7696297764778137,
0.7723866105079651,
0.7579843997955322,
0.7670145034790039,
0.7692501544952393,
0.7637323141098022,
0.763234555721283,
0.7680944204330444,
0.7569077610969543,
0.7567135095596313,
0.7794183492660522,
0.7721916437149048,
0.7629839181900024,
0.7838914394378662,
0.7697135210037231,
0.7794018387794495,
0.7710062265396118,
0.7675776481628418,
0.7608529925346375,
0.7764639854431152,
0.7648787498474121,
0.7799038887023926,
0.7621294856071472,
0.7661650776863098,
0.7731010913848877,
0.7682425379753113,
0.7695325613021851,
0.7721757888793945,
0.792959451675415,
0.7791135311126709,
0.7931289672851562,
0.7856360673904419,
0.7798048257827759,
0.7585338354110718,
0.7685825824737549,
0.7825548052787781,
0.752373993396759,
0.7753353118896484,
0.776984453201294,
0.7808583378791809,
0.7686702013015747,
0.7760231494903564,
0.7672873735427856,
0.7529267072677612,
0.776481032371521,
0.7770534753799438,
0.7646709680557251,
0.7683137059211731,
0.7689797878265381,
0.7632043361663818,
0.759833574295044,
0.7842649221420288,
0.7561384439468384,
0.7540510892868042,
0.7610975503921509,
0.7695935964584351,
0.7842625379562378,
0.7737125158309937,
0.771269679069519,
0.7800078988075256,
0.7660631537437439,
0.7720063924789429,
0.7702754735946655,
0.764051079750061,
0.7770617008209229,
0.781154990196228,
0.7817167043685913,
0.7703344225883484,
0.7748950719833374,
0.7638087868690491,
0.7834888696670532,
0.7507861852645874,
0.7671807408332825,
0.7692813277244568,
0.7647513747215271,
0.7708958983421326,
0.7879855036735535,
0.7646057605743408,
0.7754836082458496,
0.7569471597671509,
0.7684503197669983,
0.7903959155082703
],
"loss_window": 200,
"grad_ema_decay": 0.99,
"stability_controller": {
"version": 4,
"spec": {
"policy": "adaptive",
"total_steps": 400000,
"base_lr": 0.0001,
"min_lr": 1e-05,
"hard_min_lr": 1e-06,
"min_scale": 0.125,
"commit_patience_windows": 2,
"warning_patience_windows": 2,
"loss_warn_ratio": 1.015,
"loss_retry_ratio": 1.025,
"loss_emergency_ratio": 1.05,
"grad_warn_ratio": 2.0,
"grad_retry_ratio": 4.0,
"grad_emergency_ratio": 8.0,
"grad_p90_warn_ratio": 2.5,
"grad_p90_retry_ratio": 5.0,
"grad_p90_emergency_ratio": 10.0,
"skip_warn_rate": 0.3,
"skip_retry_rate": 0.4,
"skip_emergency_rate": 0.6,
"reference_decay": 0.8,
"loss_reference_max_growth": 1.01,
"grad_reference_max_growth": 1.5
},
"committed_scale": 0.14088384043552588,
"attempt_factor": 1.0,
"fast_loss": 0.7709401466758908,
"slow_loss": 0.7704296880975201,
"best_loss": 0.7673007962107659,
"reference": {
"loss": 0.7697591851380827,
"grad_median": 533.3278690397134,
"grad_p90": 1076.281941084452,
"step": 340000,
"promotions": 11
},
"last_metrics": {
"loss": 0.7707032078504562,
"grad_median": 951.15771484375,
"grad_p90": 1791.2954711914062,
"skip_rate": 0.0,
"relative_spike_rate": 0.0
},
"last_loss_ratio": 1.0015334539906842,
"last_grad_ratio": 2.217820810233963,
"last_grad_p90_ratio": 1.995808884421388,
"healthy_windows": 0,
"warning_windows": 0,
"retry_windows": 0,
"windows_seen": 348,
"retry_count": 0
},
"best_window": 0.7673007962107659,
"blown_windows": 0
},
"transaction": {
"status": "committed",
"committed_at": "2026-07-21T19:32:23.606071+00:00",
"attempt": 0,
"health_reference": {
"loss": 0.7697591851380827,
"grad_median": 533.3278690397134,
"grad_p90": 1076.281941084452,
"step": 340000,
"promotions": 11
}
},
"config": {
"model": {
"num_attention_heads": 16,
"attention_head_dim": 64,
"num_layers": 24,
"patch_size": 2,
"sample_size": 32,
"in_channels": 4,
"out_channels": 4,
"num_classes": 1000,
"qk_mode": "amap",
"dmap_alpha": 0.0,
"qk_norm": false
},
"data": {
"hub_repo": "sparsetrace/dlatentzz",
"latent_shape": [
4,
32,
32
],
"expected_total": 1281167
},
"train": {
"objective": "flow",
"global_batch": 256,
"lr": 0.0001,
"weight_decay": 0.0,
"ema_decay": 0.9999,
"label_dropout": 0.1,
"base_seed": 0,
"deadline_check_every": 500
},
"hub": {
"checkpoint_repo": "sparsetrace/ditflex-L2-flow",
"archive_every_steps": 200000,
"save_every_steps": 10000
}
},
"optim_param_groups": [
{
"lr": 2.099854885297127e-06,
"betas": [
0.9,
0.999
],
"eps": 1e-08,
"weight_decay": 0.1,
"amsgrad": false,
"maximize": false,
"foreach": null,
"capturable": false,
"differentiable": false,
"fused": null,
"decoupled_weight_decay": true,
"params": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59,
60,
61,
62,
63,
64,
65,
66,
67,
68,
69,
70,
71,
72,
73,
74,
75,
76,
77,
78,
79,
80,
81,
82,
83,
84,
85,
86,
87,
88,
89,
90,
91,
92,
93,
94,
95,
96,
97,
98,
99,
100,
101,
102,
103,
104,
105,
106,
107,
108,
109,
110,
111,
112,
113,
114,
115,
116,
117,
118,
119,
120,
121,
122,
123,
124,
125,
126,
127,
128,
129,
130,
131,
132,
133,
134,
135,
136,
137,
138,
139,
140,
141,
142,
143,
144,
145,
146,
147,
148,
149,
150,
151,
152,
153,
154,
155,
156,
157,
158,
159,
160,
161,
162,
163,
164,
165,
166,
167,
168,
169,
170,
171,
172,
173,
174,
175,
176,
177,
178,
179,
180,
181,
182,
183,
184,
185,
186,
187,
188,
189,
190,
191,
192,
193,
194,
195,
196,
197,
198,
199,
200,
201,
202,
203,
204,
205,
206,
207,
208,
209,
210,
211,
212,
213,
214,
215,
216,
217,
218,
219,
220,
221,
222,
223,
224,
225,
226,
227,
228,
229,
230,
231,
232,
233,
234,
235,
236,
237,
238,
239,
240,
241,
242,
243,
244,
245,
246,
247,
248,
249,
250,
251,
252,
253,
254,
255,
256,
257,
258,
259,
260,
261,
262,
263,
264,
265,
266,
267,
268,
269,
270,
271,
272,
273,
274,
275,
276,
277,
278,
279,
280,
281,
282,
283,
284,
285,
286,
287,
288,
289,
290,
291,
292,
293,
294,
295,
296,
297,
298,
299,
300,
301,
302,
303,
304,
305,
306,
307,
308,
309,
310,
311,
312,
313,
314,
315,
316,
317,
318,
319,
320,
321,
322,
323,
324,
325,
326,
327,
328,
329,
330,
331,
332,
333,
334,
335,
336,
337,
338,
339,
340,
341,
342,
343,
344,
345,
346,
347,
348,
349,
350,
351,
352,
353,
354,
355,
356,
357,
358,
359,
360,
361,
362,
363,
364,
365,
366,
367,
368,
369,
370,
371,
372,
373,
374,
375,
376,
377,
378,
379,
380,
381,
382,
383,
384,
385,
386,
387,
388,
389,
390,
391,
392,
393,
394,
395,
396,
397,
398,
399,
400,
401,
402,
403,
404,
405,
406,
407,
408,
409,
410,
411,
412,
413,
414,
415,
416,
417,
418,
419,
420,
421,
422,
423,
424,
425,
426,
427,
428,
429,
430,
431,
432,
433,
434,
435,
436,
437,
438,
439,
440,
441,
442,
443,
444,
445,
446,
447,
448,
449,
450,
451,
452,
453,
454,
455,
456,
457,
458,
459,
460,
461
]
}
],
"torch_version": "2.11.0+cu128"
}