{ "step": 340000, "run_history": [ { "start_step": 0, "end_step": 93500, "seconds": 14451.7, "world": 2, "objective": "flow", "completed": true, "finished_at": "2026-07-19T22:25:14.507322+00:00" }, { "start_step": 93500, "end_step": 192500, "seconds": 14410.2, "world": 2, "objective": "flow", "completed": true, "finished_at": "2026-07-20T04:05:19.860941+00:00" }, { "start_step": 192500, "end_step": 210000, "seconds": 3073.8, "world": 2, "objective": "flow", "completed": false, "finished_at": "2026-07-20T05:38:26.749589+00:00" }, { "start_step": 210000, "end_step": 240000, "seconds": 3419.9, "world": 2, "objective": "flow", "completed": false, "finished_at": "2026-07-20T06:48:12.738761+00:00" }, { "start_step": 240000, "end_step": 250000, "seconds": 1781.7, "world": 2, "objective": "flow", "completed": false, "finished_at": "2026-07-20T14:46:34.505065+00:00" }, { "start_step": 250000, "end_step": 268500, "seconds": 3641.4, "world": 1, "objective": "flow", "completed": true, "finished_at": "2026-07-20T19:24:03.421837+00:00", "effective": { "lr": 9e-06, "weight_decay": 0.1, "clip": 0.5, "spike_skip": 4.0, "steps_skipped_this_run": 1524, "steps_skipped_total": 1524, "seed_offset": 797, "grad_ceiling": 25.0 } }, { "start_step": 268500, "end_step": 270000, "seconds": 368.6, "world": 1, "objective": "flow", "completed": false, "finished_at": "2026-07-20T20:26:00.017328+00:00", "effective": { "lr_policy": "adaptive", "lr_base": 0.0001, "lr_start": 9e-06, "lr_end": 8.87226296048255e-06, "lr_min": 1e-05, "lr_hard_min": 1e-06, "lr_scale": 0.28176768087105175, "lr_backoffs_total": 0, "weight_decay": 0.1, "clip": 1.0, "spike_skip": 4.0, "grad_ceiling": 0.0, "steps_skipped_this_run": 18, "steps_skipped_total": 1542, "seed_offset": 0, "target_steps": 400000 } }, { "start_step": 270000, "end_step": 275000, "seconds": 2254.3, "world": 1, "objective": "flow", "completed": false, "finished_at": "2026-07-21T02:12:18.577289+00:00", "promotion_reason": "run final", "effective": { "attempt": 0, "precision": "tf32", "lr_policy": "adaptive", "lr_base": 0.0001, "lr_start": 8.872178050176876e-06, "lr_end": 8.452927124607216e-06, "lr_min": 1e-05, "lr_hard_min": 1e-06, "lr_scale": 0.28176768087105175, "weight_decay": 0.1, "wd_ada": 0.01, "clip": 1.0, "spike_skip": 10.0, "grad_ceiling": 500.0, "steps_skipped": 5, "seed_offset": 0, "target_steps": 400000 } }, { "start_step": 275000, "end_step": 280000, "seconds": 2309.0, "world": 1, "objective": "flow", "completed": false, "finished_at": "2026-07-21T02:55:39.928314+00:00", "promotion_reason": "periodic healthy candidate", "effective": { "attempt": 0, "precision": "tf32", "lr_policy": "adaptive", "lr_base": 0.0001, "lr_start": 8.452844322556308e-06, "lr_end": 8.044453081539695e-06, "lr_min": 1e-05, "lr_hard_min": 1e-06, "lr_scale": 0.28176768087105175, "weight_decay": 0.1, "wd_ada": 0.01, "clip": 1.0, "spike_skip": 4.0, "grad_ceiling": 0.0, "steps_skipped": 139, "seed_offset": 0, "target_steps": 400000 } }, { "start_step": 280000, "end_step": 290000, "seconds": 4446.3, "world": 1, "objective": "flow", "completed": false, "finished_at": "2026-07-21T05:04:30.114007+00:00", "promotion_reason": "periodic healthy candidate", "effective": { "attempt": 0, "precision": "tf32", "lr_policy": "adaptive", "lr_base": 0.0001, "lr_start": 8.044372515418319e-06, "lr_end": 7.2625920040378e-06, "lr_min": 1e-05, "lr_hard_min": 1e-06, "lr_scale": 0.28176768087105175, "weight_decay": 0.1, "wd_ada": 0.05, "clip": 1.0, "spike_skip": 4.0, "grad_ceiling": 0.0, "steps_skipped": 179, "seed_offset": 0, "target_steps": 400000 } }, { "start_step": 290000, "end_step": 300000, "seconds": 4428.6, "world": 1, "objective": "flow", "completed": false, "finished_at": "2026-07-21T06:18:18.741141+00:00", "promotion_reason": "periodic healthy candidate", "effective": { "attempt": 0, "precision": "tf32", "lr_policy": "adaptive", "lr_base": 0.0001, "lr_start": null, "lr_end": 6.5315001615148555e-06, "lr_min": 1e-05, "lr_hard_min": 1e-06, "lr_scale": 0.28176768087105175, "weight_decay": 0.1, "wd_ada": 0.05, "clip": 1.0, "spike_skip": 4.0, "grad_ceiling": 0.0, "steps_skipped": 290, "seed_offset": 0, "target_steps": 400000 } }, { "start_step": 300000, "end_step": 310000, "seconds": 4426.6, "world": 1, "objective": "flow", "completed": false, "finished_at": "2026-07-21T07:32:05.358144+00:00", "promotion_reason": "periodic healthy candidate", "effective": { "attempt": 0, "precision": "tf32", "lr_policy": "adaptive", "lr_base": 0.0001, "lr_start": null, "lr_end": 5.8556849782927245e-06, "lr_min": 1e-05, "lr_hard_min": 1e-06, "lr_scale": 0.28176768087105175, "weight_decay": 0.1, "wd_ada": 0.05, "clip": 1.0, "spike_skip": 4.0, "grad_ceiling": 0.0, "steps_skipped": 614, "seed_offset": 0, "target_steps": 400000 } }, { "start_step": 310000, "end_step": 312500, "seconds": 1129.4, "world": 1, "objective": "flow", "completed": false, "finished_at": "2026-07-21T07:50:54.720571+00:00", "promotion_reason": "run final", "effective": { "attempt": 0, "precision": "tf32", "lr_policy": "adaptive", "lr_base": 0.0001, "lr_start": null, "lr_end": 5.695864301224254e-06, "lr_min": 1e-05, "lr_hard_min": 1e-06, "lr_scale": 0.28176768087105175, "weight_decay": 0.1, "wd_ada": 0.05, "clip": 1.0, "spike_skip": 4.0, "grad_ceiling": 0.0, "steps_skipped": 54, "seed_offset": 0, "target_steps": 400000 } }, { "start_step": 312500, "end_step": 320000, "seconds": 3445.6, "world": 1, "objective": "flow", "completed": false, "finished_at": "2026-07-21T15:45:13.044752+00:00", "promotion_reason": "periodic healthy candidate", "effective": { "attempt": 0, "precision": "tf32", "lr_policy": "adaptive", "lr_base": 0.0001, "lr_start": 5.695801124918405e-06, "lr_end": 5.239313079707324e-06, "lr_min": 1e-05, "lr_hard_min": 1e-06, "lr_scale": 0.28176768087105175, "weight_decay": 0.1, "wd_ada": 0.05, "clip": 1.0, "spike_skip": 4.0, "grad_ceiling": 0.0, "steps_skipped": 77, "seed_offset": 0, "target_steps": 400000 } }, { "start_step": 320000, "end_step": 330000, "seconds": 4516.6, "world": 1, "objective": "flow", "completed": false, "finished_at": "2026-07-21T17:00:29.615182+00:00", "promotion_reason": "periodic healthy candidate", "effective": { "attempt": 0, "precision": "tf32", "lr_policy": "adaptive", "lr_base": 0.0001, "lr_start": null, "lr_end": 4.686184603477889e-06, "lr_min": 1e-05, "lr_hard_min": 1e-06, "lr_scale": 0.28176768087105175, "weight_decay": 0.1, "wd_ada": 0.05, "clip": 1.0, "spike_skip": 4.0, "grad_ceiling": 0.0, "steps_skipped": 171, "seed_offset": 0, "target_steps": 400000 } }, { "start_step": 330000, "end_step": 337500, "seconds": 3341.5, "world": 1, "objective": "flow", "completed": false, "finished_at": "2026-07-21T18:49:32.212552+00:00", "promotion_reason": "run final", "effective": { "attempt": 1, "precision": "tf32", "lr_policy": "adaptive", "lr_base": 0.0001, "lr_start": 2.3430662850837115e-06, "lr_end": 2.157454235387407e-06, "lr_min": 1e-05, "lr_hard_min": 1e-06, "lr_scale": 0.14088384043552588, "weight_decay": 0.1, "wd_ada": 0.05, "clip": 1.0, "spike_skip": 4.0, "grad_ceiling": 0.0, "steps_skipped": 48, "seed_offset": 1000003, "target_steps": 400000 } }, { "start_step": 337500, "end_step": 340000, "seconds": 1548.1, "world": 1, "objective": "flow", "completed": false, "finished_at": "2026-07-21T19:32:23.605941+00:00", "promotion_reason": "periodic healthy candidate", "effective": { "attempt": 0, "precision": "tf32", "lr_policy": "adaptive", "lr_base": 0.0001, "lr_start": 2.157430763212134e-06, "lr_end": 2.099854885297127e-06, "lr_min": 1e-05, "lr_hard_min": 1e-06, "lr_scale": 0.14088384043552588, "weight_decay": 0.1, "wd_ada": 0.01, "clip": 1.0, "spike_skip": 4.0, "grad_ceiling": 0.0, "steps_skipped": 2, "seed_offset": 0, "target_steps": 400000 } } ], "guard_state": { "version": 3, "live_grad_ema": 1105.722598976348, "grad_ema": 1105.722598976348, "grad_reference": 533.3278690397134, "grad_p90_reference": 1076.281941084452, "spikes_total": 3121, "recent_losses": [ 0.7659851312637329, 0.7799718379974365, 0.767326831817627, 0.7740594744682312, 0.7746687531471252, 0.7693485021591187, 0.7744090557098389, 0.76472407579422, 0.7712109684944153, 0.7667823433876038, 0.7734376192092896, 0.7631382942199707, 0.7683030962944031, 0.7651751637458801, 0.7694488167762756, 0.7677610516548157, 0.7600035071372986, 0.7641050815582275, 0.758497953414917, 0.75881028175354, 0.7905396223068237, 0.7816970348358154, 0.7721761465072632, 0.7704170942306519, 0.7622644901275635, 0.785053014755249, 0.7627816200256348, 0.7816370725631714, 0.7777868509292603, 0.7586430311203003, 0.757468581199646, 0.768932580947876, 0.7764378786087036, 0.7670449018478394, 0.7591977119445801, 0.7605947256088257, 0.7780417203903198, 0.7622751593589783, 0.7680383324623108, 0.7845712304115295, 0.7854090929031372, 0.7671298980712891, 0.7803963422775269, 0.7852848768234253, 0.7887732982635498, 0.7690376043319702, 0.7525300979614258, 0.8054315447807312, 0.776677131652832, 0.7609086036682129, 0.7614132165908813, 0.7684108018875122, 0.7619225978851318, 0.7682492733001709, 0.7727726697921753, 0.7575544118881226, 0.7509193420410156, 0.7757312059402466, 0.7860416769981384, 0.7617193460464478, 0.7782135009765625, 0.7646865844726562, 0.7678159475326538, 0.7605258822441101, 0.7857308387756348, 0.7728601694107056, 0.7700196504592896, 0.7775273323059082, 0.7790310382843018, 0.7775787115097046, 0.781828761100769, 0.7681038975715637, 0.765837550163269, 0.7664794921875, 0.7541241645812988, 0.7896788120269775, 0.7635741233825684, 0.7778509855270386, 0.7496468424797058, 0.753919780254364, 0.7757487297058105, 0.7724413871765137, 0.7722828388214111, 0.7738194465637207, 0.7687792181968689, 0.7757934331893921, 0.7694065570831299, 0.7718388438224792, 0.7744388580322266, 0.764975368976593, 0.7768993973731995, 0.7564750909805298, 0.760976254940033, 0.7610824704170227, 0.7636446952819824, 0.7474631667137146, 0.7591776847839355, 0.7641832828521729, 0.776297926902771, 0.7714574337005615, 0.758858859539032, 0.7527077198028564, 0.7844610214233398, 0.7654604315757751, 0.7850476503372192, 0.7593823671340942, 0.7806195616722107, 0.7746589779853821, 0.7741937041282654, 0.771770715713501, 0.7561997175216675, 0.7938714027404785, 0.76432204246521, 0.7739542126655579, 0.7850971221923828, 0.7664803266525269, 0.7699167132377625, 0.7678929567337036, 0.7696297764778137, 0.7723866105079651, 0.7579843997955322, 0.7670145034790039, 0.7692501544952393, 0.7637323141098022, 0.763234555721283, 0.7680944204330444, 0.7569077610969543, 0.7567135095596313, 0.7794183492660522, 0.7721916437149048, 0.7629839181900024, 0.7838914394378662, 0.7697135210037231, 0.7794018387794495, 0.7710062265396118, 0.7675776481628418, 0.7608529925346375, 0.7764639854431152, 0.7648787498474121, 0.7799038887023926, 0.7621294856071472, 0.7661650776863098, 0.7731010913848877, 0.7682425379753113, 0.7695325613021851, 0.7721757888793945, 0.792959451675415, 0.7791135311126709, 0.7931289672851562, 0.7856360673904419, 0.7798048257827759, 0.7585338354110718, 0.7685825824737549, 0.7825548052787781, 0.752373993396759, 0.7753353118896484, 0.776984453201294, 0.7808583378791809, 0.7686702013015747, 0.7760231494903564, 0.7672873735427856, 0.7529267072677612, 0.776481032371521, 0.7770534753799438, 0.7646709680557251, 0.7683137059211731, 0.7689797878265381, 0.7632043361663818, 0.759833574295044, 0.7842649221420288, 0.7561384439468384, 0.7540510892868042, 0.7610975503921509, 0.7695935964584351, 0.7842625379562378, 0.7737125158309937, 0.771269679069519, 0.7800078988075256, 0.7660631537437439, 0.7720063924789429, 0.7702754735946655, 0.764051079750061, 0.7770617008209229, 0.781154990196228, 0.7817167043685913, 0.7703344225883484, 0.7748950719833374, 0.7638087868690491, 0.7834888696670532, 0.7507861852645874, 0.7671807408332825, 0.7692813277244568, 0.7647513747215271, 0.7708958983421326, 0.7879855036735535, 0.7646057605743408, 0.7754836082458496, 0.7569471597671509, 0.7684503197669983, 0.7903959155082703 ], "loss_window": 200, "grad_ema_decay": 0.99, "stability_controller": { "version": 4, "spec": { "policy": "adaptive", "total_steps": 400000, "base_lr": 0.0001, "min_lr": 1e-05, "hard_min_lr": 1e-06, "min_scale": 0.125, "commit_patience_windows": 2, "warning_patience_windows": 2, "loss_warn_ratio": 1.015, "loss_retry_ratio": 1.025, "loss_emergency_ratio": 1.05, "grad_warn_ratio": 2.0, "grad_retry_ratio": 4.0, "grad_emergency_ratio": 8.0, "grad_p90_warn_ratio": 2.5, "grad_p90_retry_ratio": 5.0, "grad_p90_emergency_ratio": 10.0, "skip_warn_rate": 0.3, "skip_retry_rate": 0.4, "skip_emergency_rate": 0.6, "reference_decay": 0.8, "loss_reference_max_growth": 1.01, "grad_reference_max_growth": 1.5 }, "committed_scale": 0.14088384043552588, "attempt_factor": 1.0, "fast_loss": 0.7709401466758908, "slow_loss": 0.7704296880975201, "best_loss": 0.7673007962107659, "reference": { "loss": 0.7697591851380827, "grad_median": 533.3278690397134, "grad_p90": 1076.281941084452, "step": 340000, "promotions": 11 }, "last_metrics": { "loss": 0.7707032078504562, "grad_median": 951.15771484375, "grad_p90": 1791.2954711914062, "skip_rate": 0.0, "relative_spike_rate": 0.0 }, "last_loss_ratio": 1.0015334539906842, "last_grad_ratio": 2.217820810233963, "last_grad_p90_ratio": 1.995808884421388, "healthy_windows": 0, "warning_windows": 0, "retry_windows": 0, "windows_seen": 348, "retry_count": 0 }, "best_window": 0.7673007962107659, "blown_windows": 0 }, "transaction": { "status": "committed", "committed_at": "2026-07-21T19:32:23.606071+00:00", "attempt": 0, "health_reference": { "loss": 0.7697591851380827, "grad_median": 533.3278690397134, "grad_p90": 1076.281941084452, "step": 340000, "promotions": 11 } }, "config": { "model": { "num_attention_heads": 16, "attention_head_dim": 64, "num_layers": 24, "patch_size": 2, "sample_size": 32, "in_channels": 4, "out_channels": 4, "num_classes": 1000, "qk_mode": "amap", "dmap_alpha": 0.0, "qk_norm": false }, "data": { "hub_repo": "sparsetrace/dlatentzz", "latent_shape": [ 4, 32, 32 ], "expected_total": 1281167 }, "train": { "objective": "flow", "global_batch": 256, "lr": 0.0001, "weight_decay": 0.0, "ema_decay": 0.9999, "label_dropout": 0.1, "base_seed": 0, "deadline_check_every": 500 }, "hub": { "checkpoint_repo": "sparsetrace/ditflex-L2-flow", "archive_every_steps": 200000, "save_every_steps": 10000 } }, "optim_param_groups": [ { "lr": 2.099854885297127e-06, "betas": [ 0.9, 0.999 ], "eps": 1e-08, "weight_decay": 0.1, "amsgrad": false, "maximize": false, "foreach": null, "capturable": false, "differentiable": false, "fused": null, "decoupled_weight_decay": true, "params": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461 ] } ], "torch_version": "2.11.0+cu128" }