Add files using upload-large-folder tool

LTA_openwebtext_dualt/logs/owt_fully_marginal_categorical_probe_step118k_n8.log

@@ -0,0 +1,40 @@
+[forbid_endpoint_ids] n=352 first=[94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125]
+[decode] steps32_c128_mtpre_t1p0_tpow1p0_noise0_state_marginal_categorical
+[summary] {"name": "steps32_c128_mtpre_t1p0_tpow1p0_noise0_state_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.0, "temp_end": 1.0, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "state", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "pre", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 2.138821908505566, "distinct_1": 0.045166015625, "distinct_2": 0.16300097751710654, "top_token_mass": 0.4710693359375, "tokens_scored": 0, "readability_score": 2.701582987844995, "mean_chars": 1753.0, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpost_t1p0_tpow1p0_noise0_state_marginal_categorical
+[summary] {"name": "steps32_c128_mtpost_t1p0_tpow1p0_noise0_state_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.0, "temp_end": 1.0, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "state", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "post", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 1.8338167258452847, "distinct_1": 0.0372314453125, "distinct_2": 0.11742424242424243, "top_token_mass": 0.5498046875, "tokens_scored": 0, "readability_score": 2.144895081574039, "mean_chars": 1478.875, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpre_t1p0_tpow1p0_noise0_endpoint_marginal_categorical
+[summary] {"name": "steps32_c128_mtpre_t1p0_tpow1p0_noise0_endpoint_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.0, "temp_end": 1.0, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "endpoint", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "pre", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 2.138821908505566, "distinct_1": 0.045166015625, "distinct_2": 0.16300097751710654, "top_token_mass": 0.4710693359375, "tokens_scored": 0, "readability_score": 2.701582987844995, "mean_chars": 1753.0, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpost_t1p0_tpow1p0_noise0_endpoint_marginal_categorical
+[summary] {"name": "steps32_c128_mtpost_t1p0_tpow1p0_noise0_endpoint_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.0, "temp_end": 1.0, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "endpoint", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "post", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 1.8338167258452847, "distinct_1": 0.0372314453125, "distinct_2": 0.11742424242424243, "top_token_mass": 0.5498046875, "tokens_scored": 0, "readability_score": 2.144895081574039, "mean_chars": 1478.875, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpre_t1p0_tpow1p0_noise0_blend_marginal_categorical
+[summary] {"name": "steps32_c128_mtpre_t1p0_tpow1p0_noise0_blend_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.0, "temp_end": 1.0, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "blend", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "pre", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 2.138821908505566, "distinct_1": 0.045166015625, "distinct_2": 0.16300097751710654, "top_token_mass": 0.4710693359375, "tokens_scored": 0, "readability_score": 2.701582987844995, "mean_chars": 1753.0, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpost_t1p0_tpow1p0_noise0_blend_marginal_categorical
+[summary] {"name": "steps32_c128_mtpost_t1p0_tpow1p0_noise0_blend_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.0, "temp_end": 1.0, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "blend", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "post", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 1.8338167258452847, "distinct_1": 0.0372314453125, "distinct_2": 0.11742424242424243, "top_token_mass": 0.5498046875, "tokens_scored": 0, "readability_score": 2.144895081574039, "mean_chars": 1478.875, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpre_t1p1_tpow1p0_noise0_state_marginal_categorical
+[summary] {"name": "steps32_c128_mtpre_t1p1_tpow1p0_noise0_state_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.1, "temp_end": 1.1, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "state", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "pre", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 2.9428859927624273, "distinct_1": 0.0694580078125, "distinct_2": 0.30584066471163246, "top_token_mass": 0.3778076171875, "tokens_scored": 0, "readability_score": 3.9354258205976476, "mean_chars": 2241.25, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpost_t1p1_tpow1p0_noise0_state_marginal_categorical
+[summary] {"name": "steps32_c128_mtpost_t1p1_tpow1p0_noise0_state_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.1, "temp_end": 1.1, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "state", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "post", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 2.4547148052858425, "distinct_1": 0.06396484375, "distinct_2": 0.2371700879765396, "top_token_mass": 0.4598388671875, "tokens_scored": 0, "readability_score": 3.661850259272684, "mean_chars": 1802.75, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpre_t1p1_tpow1p0_noise0_endpoint_marginal_categorical
+[summary] {"name": "steps32_c128_mtpre_t1p1_tpow1p0_noise0_endpoint_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.1, "temp_end": 1.1, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "endpoint", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "pre", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 2.9428859927624273, "distinct_1": 0.0694580078125, "distinct_2": 0.30584066471163246, "top_token_mass": 0.3778076171875, "tokens_scored": 0, "readability_score": 3.9354258205976476, "mean_chars": 2241.25, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpost_t1p1_tpow1p0_noise0_endpoint_marginal_categorical
+[summary] {"name": "steps32_c128_mtpost_t1p1_tpow1p0_noise0_endpoint_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.1, "temp_end": 1.1, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "endpoint", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "post", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 2.4547148052858425, "distinct_1": 0.06396484375, "distinct_2": 0.2371700879765396, "top_token_mass": 0.4598388671875, "tokens_scored": 0, "readability_score": 3.661850259272684, "mean_chars": 1802.75, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpre_t1p1_tpow1p0_noise0_blend_marginal_categorical
+[summary] {"name": "steps32_c128_mtpre_t1p1_tpow1p0_noise0_blend_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.1, "temp_end": 1.1, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "blend", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "pre", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 2.9428859927624273, "distinct_1": 0.0694580078125, "distinct_2": 0.30584066471163246, "top_token_mass": 0.3778076171875, "tokens_scored": 0, "readability_score": 3.9354258205976476, "mean_chars": 2241.25, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpost_t1p1_tpow1p0_noise0_blend_marginal_categorical
+[summary] {"name": "steps32_c128_mtpost_t1p1_tpow1p0_noise0_blend_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.1, "temp_end": 1.1, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "blend", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "post", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 2.4547148052858425, "distinct_1": 0.06396484375, "distinct_2": 0.2371700879765396, "top_token_mass": 0.4598388671875, "tokens_scored": 0, "readability_score": 3.661850259272684, "mean_chars": 1802.75, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpre_t1p2_tpow1p0_noise0_state_marginal_categorical
+[summary] {"name": "steps32_c128_mtpre_t1p2_tpow1p0_noise0_state_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.2, "temp_end": 1.2, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "state", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "pre", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 3.9514859184602247, "distinct_1": 0.102294921875, "distinct_2": 0.49108015640273706, "top_token_mass": 0.2208251953125, "tokens_scored": 0, "readability_score": 4.272469777249244, "mean_chars": 3242.625, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpost_t1p2_tpow1p0_noise0_state_marginal_categorical
+[summary] {"name": "steps32_c128_mtpost_t1p2_tpow1p0_noise0_state_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.2, "temp_end": 1.2, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "state", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "post", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 3.735299917945654, "distinct_1": 0.1103515625, "distinct_2": 0.4671309872922776, "top_token_mass": 0.20361328125, "tokens_scored": 0, "readability_score": 4.142991364470012, "mean_chars": 2971.625, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpre_t1p2_tpow1p0_noise0_endpoint_marginal_categorical
+[summary] {"name": "steps32_c128_mtpre_t1p2_tpow1p0_noise0_endpoint_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.2, "temp_end": 1.2, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "endpoint", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "pre", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 3.9514859184602247, "distinct_1": 0.102294921875, "distinct_2": 0.49108015640273706, "top_token_mass": 0.2208251953125, "tokens_scored": 0, "readability_score": 4.272469777249244, "mean_chars": 3242.625, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpost_t1p2_tpow1p0_noise0_endpoint_marginal_categorical
+[summary] {"name": "steps32_c128_mtpost_t1p2_tpow1p0_noise0_endpoint_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.2, "temp_end": 1.2, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "endpoint", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "post", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 3.735299917945654, "distinct_1": 0.1103515625, "distinct_2": 0.4671309872922776, "top_token_mass": 0.20361328125, "tokens_scored": 0, "readability_score": 4.142991364470012, "mean_chars": 2971.625, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpre_t1p2_tpow1p0_noise0_blend_marginal_categorical
+[summary] {"name": "steps32_c128_mtpre_t1p2_tpow1p0_noise0_blend_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.2, "temp_end": 1.2, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "blend", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "pre", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 3.9514859184602247, "distinct_1": 0.102294921875, "distinct_2": 0.49108015640273706, "top_token_mass": 0.2208251953125, "tokens_scored": 0, "readability_score": 4.272469777249244, "mean_chars": 3242.625, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpost_t1p2_tpow1p0_noise0_blend_marginal_categorical
+[summary] {"name": "steps32_c128_mtpost_t1p2_tpow1p0_noise0_blend_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.2, "temp_end": 1.2, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "blend", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "post", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 3.735299917945654, "distinct_1": 0.1103515625, "distinct_2": 0.4671309872922776, "top_token_mass": 0.20361328125, "tokens_scored": 0, "readability_score": 4.142991364470012, "mean_chars": 2971.625, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpre_t1p3_tpow1p0_noise0_state_marginal_categorical
+[summary] {"name": "steps32_c128_mtpre_t1p3_tpow1p0_noise0_state_marginal_categorical", "step": 118000, "n_samples": 8, "steps": 32, "concentration_max": 128.0, "temp_start": 1.3, "temp_end": 1.3, "temp_schedule": "const", "t_power": 1.0, "eta0": 0.0, "eta_schedule": "none", "noise_conc": 1.0, "final_from": "state", "final_decode": "argmax", "final_temp": 1.0, "final_top_k": 0, "final_uncertain_threshold": 0.85, "update_rule": "marginal_categorical", "model_t_mode": "pre", "lock_bos": true, "lock_final_eos": false, "detok_genppl": NaN, "sample_entropy": 4.465642952565066, "distinct_1": 0.117919921875, "distinct_2": 0.5981182795698925, "top_token_mass": 0.0947265625, "tokens_scored": 0, "readability_score": 4.276690649955215, "mean_chars": 4014.75, "replacement_chars": 0.0}
+[decode] steps32_c128_mtpost_t1p3_tpow1p0_noise0_state_marginal_categorical

LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_ctx1024_randk_p50_path3_rand2_3_unif0_0p25_ctx1024_uniformt_temp1_randk_20260518_010217.log

@@ -0,0 +1,404 @@
+NCCL version 2.25.1+cuda12.8
+{
+  "device": "cuda:0",
+  "rank": 0,
+  "world_size": 4,
+  "samples": "owt_cached_chunks:8",
+  "vocab_size": 2664,
+  "tokenizer_vocab_size": 50257,
+  "save_dir": "runs/train8_ctx1024_randk_p50_path3_rand2_3_unif0_0p25_ctx1024_uniformt_temp1_randk_20260518_010217",
+  "batch_size": 128,
+  "grad_accum": 1,
+  "effective_batch_size": 512,
+  "global_batch_size": 512,
+  "lr_schedule": "constant_warmup",
+  "optimizer": "muon",
+  "epochs": 0.0,
+  "steps_per_epoch": 1,
+  "total_steps": 1000,
+  "warmup_steps": 10,
+  "warmup_epochs": -1.0,
+  "min_lr": 0.0,
+  "weight_decay": 0.1,
+  "output_weight_decay": -1.0,
+  "adamw_param_groups": "nanogpt",
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_eps": 1e-08,
+  "muon_impl": "legacy",
+  "muon_momentum": 0.95,
+  "muon_ns_steps": 5,
+  "muon_update_scale": 1.0,
+  "muon_nesterov": false,
+  "muon_width_scale": false,
+  "muon_grouping": "legacy_dim_ge_2",
+  "muon_param_count": 2616320,
+  "muon_adam_param_count": 8192,
+  "muon_param_names": [
+    "vocab_embed.embedding",
+    "sigma_map.net.0.weight",
+    "sigma_map.net.2.weight",
+    "blocks.0.attn_qkv.weight",
+    "blocks.0.attn_out.weight",
+    "blocks.0.mlp.0.weight",
+    "blocks.0.mlp.2.weight",
+    "blocks.0.adaLN_modulation.weight",
+    "blocks.1.attn_qkv.weight",
+    "blocks.1.attn_out.weight",
+    "blocks.1.mlp.0.weight",
+    "blocks.1.mlp.2.weight",
+    "blocks.1.adaLN_modulation.weight",
+    "blocks.2.attn_qkv.weight",
+    "blocks.2.attn_out.weight",
+    "blocks.2.mlp.0.weight",
+    "blocks.2.mlp.2.weight",
+    "blocks.2.adaLN_modulation.weight",
+    "output_layer.linear.weight",
+    "output_layer.adaLN_modulation.weight"
+  ],
+  "muon_adam_param_names": [
+    "sigma_map.net.0.bias",
+    "sigma_map.net.2.bias",
+    "blocks.0.norm1.weight",
+    "blocks.0.norm2.weight",
+    "blocks.0.mlp.0.bias",
+    "blocks.0.mlp.2.bias",
+    "blocks.0.adaLN_modulation.bias",
+    "blocks.1.norm1.weight",
+    "blocks.1.norm2.weight",
+    "blocks.1.mlp.0.bias",
+    "blocks.1.mlp.2.bias",
+    "blocks.1.adaLN_modulation.bias",
+    "blocks.2.norm1.weight",
+    "blocks.2.norm2.weight",
+    "blocks.2.mlp.0.bias",
+    "blocks.2.mlp.2.bias",
+    "blocks.2.adaLN_modulation.bias",
+    "output_layer.norm_final.weight",
+    "output_layer.adaLN_modulation.bias"
+  ],
+  "muon_effective_nesterov": false,
+  "muon_effective_width_scale": false,
+  "muon_effective_weight_decay": 0.1,
+  "muon_adam_fallback_nesterov": false,
+  "muon_adam_fallback_weight_decay": 0.1,
+  "ema_decay": 0.9999,
+  "ema_start_step": 0,
+  "model_type": "ddit",
+  "ddit_mlp_type": "gelu",
+  "elf_num_time_tokens": 4,
+  "elf_num_model_mode_tokens": 0,
+  "qk_norm": true,
+  "output_bias": false,
+  "output_init_std": -1.0,
+  "norm_type": "rmsnorm",
+  "target_loss": "hard_ce",
+  "linear_soft_target_power": 1.0,
+  "linear_soft_target_min_conf": 0.0,
+  "linear_soft_target_max_conf": 1.0,
+  "t_sampling_mode": "uniform",
+  "t_sampling_power": 1.0,
+  "t_sampling_eps": 0.0001,
+  "t_sampling_logit_mean": -1.5,
+  "t_sampling_logit_std": 0.8,
+  "dual_t": true,
+  "corrupt_t_mode": "same",
+  "corrupt_min_t": 0.0,
+  "corrupt_max_t": 1.0,
+  "prefix_block_prob": 0.0,
+  "prefix_block_len": 128,
+  "mask_ratio_floor_schedule": "none",
+  "dirichlet_endpoint_mode": "categorical_dual_t",
+  "dirichlet_semantic_t_mode": "same",
+  "dirichlet_semantic_t_value": 0.0,
+  "dirichlet_semantic_t_curve": "linear",
+  "dirichlet_semantic_t_power": 1.0,
+  "endpoint_sequence_random_prob_alpha": 0.0,
+  "categorical_wrong_from_full_vocab": true,
+  "categorical_wrong_from_batch_valid_tokens": false,
+  "categorical_wrong_basin_token_ids": "",
+  "categorical_wrong_basin_prob": 0.0,
+  "categorical_wrong_unigram_prob": 0.0,
+  "categorical_wrong_uniform_prob": 0.0,
+  "categorical_wrong_prob_floor": 0.0,
+  "categorical_wrong_corpus_unigram_path": "",
+  "categorical_wrong_corpus_unigram_alpha": 1.0,
+  "categorical_wrong_basin_shared_prob": 0.0,
+  "categorical_wrong_unigram_shared_prob": 0.0,
+  "mask_mixture_original_prob": 0.0,
+  "mask_mixture_lowk_prob": 0.0,
+  "mask_mixture_lowcorrupt_prob": 0.0,
+  "mask_mixture_block_prob": 0.0,
+  "mask_mixture_all_prob": 1.0,
+  "mask_mixture_lowk_clean_tokens": "0",
+  "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_block_tokens": "64,128",
+  "simplex_bridge_sampler": "dirichlet",
+  "logistic_normal_sigma_min": 0.1,
+  "logistic_normal_sigma_max": 1.0,
+  "logistic_normal_tau_min": 1.0,
+  "logistic_normal_tau_max": 1.0,
+  "torch_compile": false,
+  "compile_mode": "max-autotune",
+  "state_format": "prob",
+  "meanflow_weight": 0.0,
+  "rollout_train_prob": 0.5,
+  "rollout_train_steps": 3,
+  "rollout_train_steps_min": 2,
+  "rollout_train_infer_steps": 1,
+  "rollout_train_time_mode": "sampled_path",
+  "rollout_train_s_dist": "uniform",
+  "rollout_train_s_min_frac": 0.0,
+  "rollout_train_s_max_frac": 0.25,
+  "rollout_train_s_beta_alpha": 2.0,
+  "rollout_train_s_beta_beta": 6.0,
+  "rollout_train_temp": 1.0,
+  "rollout_train_max_gamma": 1.0,
+  "rollout_train_corrupt_only": true,
+  "rollout_train_samplewise": true,
+  "rollout_train_compute_always": false,
+  "rollout_train_sync_t": true,
+  "bridge_noise_init": "logistic_normal",
+  "noise_sigma": -1.0,
+  "allow_tf32": true,
+  "activation_checkpointing": false,
+  "activation_checkpoint_interval": 1,
+  "activation_checkpoint_scope": "block",
+  "ddp_static_graph": false,
+  "ddp_gradient_as_bucket_view": true,
+  "blocking_data_transfer": false,
+  "dataloader_prefetch_factor": 4,
+  "full_train_stats": false,
+  "tokenized_hf": false,
+  "tokenized_pad_token": "pad",
+  "elf_conditional_hf": false,
+  "record_pad_truncate": false,
+  "record_add_eos": false,
+  "record_add_special_tokens": false,
+  "record_pad_token": "pad",
+  "record_shuffle_buffer": 10000,
+  "wrap": true,
+  "wrap_mode": "stream",
+  "wrap_record_buffer_size": 200,
+  "owt_cached_chunks": true,
+  "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit",
+  "owt_chunk_cache_rebuild": false,
+  "owt_chunk_cache_write_batch": 4096,
+  "owt_exact_repeat_per_chunk": 64,
+  "online_chunk_shuffle": false,
+  "online_chunk_shuffle_buffer": 10000,
+  "openwebtext_split": "train_minus_100k",
+  "detokenizer": "auto",
+  "resolved_detokenizer": null,
+  "num_workers": 0,
+  "latest_every": 1000,
+  "resume_path": ""
+}
+step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=22.1s lr=2.000000e-03 loss=7.4577 loss_recon=7.4577 loss_meanflow=0.0000 mean_model_t=0.4971 mean_corrupt_t=0.4971 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5052 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3139 corrupt_frac=1.0000 acc_corrupt=0.3139 loss_corrupt=7.4577 wrong_frac=0.5028 init_acc_corrupt=0.4627 acc_corrupt_t_0p0_0p2=0.0440 corrupt_frac_t_0p0_0p2=0.2071 acc_corrupt_t_0p2_0p4=0.1523 corrupt_frac_t_0p2_0p4=0.1988 acc_corrupt_t_0p4_0p6=0.3124 corrupt_frac_t_0p4_0p6=0.1976 acc_corrupt_t_0p6_0p8=0.4627 corrupt_frac_t_0p6_0p8=0.1951 acc_corrupt_t_0p8_1p0=0.6082 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=1.0709 out_g_norm=0.9530 loss_all=6.9234 init_gold_top10=0.4680 init_gold_top100=0.5774 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.4067 init_acc_rollout_kept=0.4381 logit_acc_rollout_applied=0.2671 logit_acc_rollout_kept=0.2807
+step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=21.1s lr=2.000000e-03 loss=6.0057 loss_recon=6.0057 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5026 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3125 corrupt_frac=1.0000 acc_corrupt=0.3125 loss_corrupt=6.0057 wrong_frac=0.4984 init_acc_corrupt=0.4680 acc_corrupt_t_0p0_0p2=0.0494 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.1494 corrupt_frac_t_0p2_0p4=0.1979 acc_corrupt_t_0p4_0p6=0.3107 corrupt_frac_t_0p4_0p6=0.1994 acc_corrupt_t_0p6_0p8=0.4530 corrupt_frac_t_0p6_0p8=0.2007 acc_corrupt_t_0p8_1p0=0.5961 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=3.4215 out_g_norm=1.2538 loss_all=5.2425 init_gold_top10=0.5104 init_gold_top100=0.6209 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.4934 init_acc_rollout_kept=0.4670 logit_acc_rollout_applied=0.3498 logit_acc_rollout_kept=0.3434
+step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=21.2s lr=2.000000e-03 loss=4.9295 loss_recon=4.9295 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5045 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3538 corrupt_frac=1.0000 acc_corrupt=0.3538 loss_corrupt=4.9295 wrong_frac=0.4985 init_acc_corrupt=0.4693 acc_corrupt_t_0p0_0p2=0.0519 corrupt_frac_t_0p0_0p2=0.1947 acc_corrupt_t_0p2_0p4=0.1777 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.3483 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=0.5083 corrupt_frac_t_0p6_0p8=0.2045 acc_corrupt_t_0p8_1p0=0.6758 corrupt_frac_t_0p8_1p0=0.1988 out_w_norm=5.5236 out_g_norm=0.5180 loss_all=4.5734 init_gold_top10=0.5207 init_gold_top100=0.6348 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.4713 init_acc_rollout_kept=0.4756 logit_acc_rollout_applied=0.3741 logit_acc_rollout_kept=0.3869
+step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=21.1s lr=2.000000e-03 loss=4.2910 loss_recon=4.2910 loss_meanflow=0.0000 mean_model_t=0.4986 mean_corrupt_t=0.4986 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4976 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4064 corrupt_frac=1.0000 acc_corrupt=0.4064 loss_corrupt=4.2910 wrong_frac=0.5016 init_acc_corrupt=0.4661 acc_corrupt_t_0p0_0p2=0.0554 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.2003 corrupt_frac_t_0p2_0p4=0.2001 acc_corrupt_t_0p4_0p6=0.4018 corrupt_frac_t_0p4_0p6=0.2023 acc_corrupt_t_0p6_0p8=0.5897 corrupt_frac_t_0p6_0p8=0.1992 acc_corrupt_t_0p8_1p0=0.7905 corrupt_frac_t_0p8_1p0=0.1979 out_w_norm=7.1059 out_g_norm=0.2729 loss_all=4.1041 init_gold_top10=0.4976 init_gold_top100=0.6342 rollout_applied_pos_frac=0.5469 init_acc_rollout_applied=0.4729 init_acc_rollout_kept=0.4091 logit_acc_rollout_applied=0.4558 logit_acc_rollout_kept=0.4033
+step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=21.1s lr=2.000000e-03 loss=3.6585 loss_recon=3.6585 loss_meanflow=0.0000 mean_model_t=0.4977 mean_corrupt_t=0.4977 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4981 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4771 corrupt_frac=1.0000 acc_corrupt=0.4771 loss_corrupt=3.6585 wrong_frac=0.5023 init_acc_corrupt=0.4655 acc_corrupt_t_0p0_0p2=0.0565 corrupt_frac_t_0p0_0p2=0.2059 acc_corrupt_t_0p2_0p4=0.2322 corrupt_frac_t_0p2_0p4=0.1985 acc_corrupt_t_0p4_0p6=0.4947 corrupt_frac_t_0p4_0p6=0.1969 acc_corrupt_t_0p6_0p8=0.7063 corrupt_frac_t_0p6_0p8=0.1927 acc_corrupt_t_0p8_1p0=0.9024 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=8.4366 out_g_norm=0.2530 loss_all=3.4213 init_gold_top10=0.5026 init_gold_top100=0.6556 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.4297 init_acc_rollout_kept=0.4792 logit_acc_rollout_applied=0.4470 logit_acc_rollout_kept=0.5017
+step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=21.2s lr=2.000000e-03 loss=3.1426 loss_recon=3.1426 loss_meanflow=0.0000 mean_model_t=0.5012 mean_corrupt_t=0.5012 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5030 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4931 corrupt_frac=1.0000 acc_corrupt=0.4931 loss_corrupt=3.1426 wrong_frac=0.4987 init_acc_corrupt=0.4696 acc_corrupt_t_0p0_0p2=0.0591 corrupt_frac_t_0p0_0p2=0.1972 acc_corrupt_t_0p2_0p4=0.2633 corrupt_frac_t_0p2_0p4=0.2024 acc_corrupt_t_0p4_0p6=0.5212 corrupt_frac_t_0p4_0p6=0.1990 acc_corrupt_t_0p6_0p8=0.7135 corrupt_frac_t_0p6_0p8=0.2017 acc_corrupt_t_0p8_1p0=0.9043 corrupt_frac_t_0p8_1p0=0.1997 out_w_norm=9.7029 out_g_norm=0.2761 loss_all=2.5311 init_gold_top10=0.5979 init_gold_top100=0.7268 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.5201 init_acc_rollout_kept=0.5812 logit_acc_rollout_applied=0.5417 logit_acc_rollout_kept=0.6050
+step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=21.0s lr=2.000000e-03 loss=2.8141 loss_recon=2.8141 loss_meanflow=0.0000 mean_model_t=0.5008 mean_corrupt_t=0.5008 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4924 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4998 corrupt_frac=1.0000 acc_corrupt=0.4998 loss_corrupt=2.8141 wrong_frac=0.4994 init_acc_corrupt=0.4693 acc_corrupt_t_0p0_0p2=0.0609 corrupt_frac_t_0p0_0p2=0.2030 acc_corrupt_t_0p2_0p4=0.2757 corrupt_frac_t_0p2_0p4=0.1915 acc_corrupt_t_0p4_0p6=0.5280 corrupt_frac_t_0p4_0p6=0.2023 acc_corrupt_t_0p6_0p8=0.7194 corrupt_frac_t_0p6_0p8=0.2001 acc_corrupt_t_0p8_1p0=0.9050 corrupt_frac_t_0p8_1p0=0.2032 out_w_norm=10.7186 out_g_norm=0.3014 loss_all=2.6805 init_gold_top10=0.5397 init_gold_top100=0.7218 rollout_applied_pos_frac=0.5000 init_acc_rollout_applied=0.4357 init_acc_rollout_kept=0.5183 logit_acc_rollout_applied=0.4688 logit_acc_rollout_kept=0.5495
+step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=21.1s lr=2.000000e-03 loss=2.3910 loss_recon=2.3910 loss_meanflow=0.0000 mean_model_t=0.4962 mean_corrupt_t=0.4962 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4930 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5223 corrupt_frac=1.0000 acc_corrupt=0.5223 loss_corrupt=2.3910 wrong_frac=0.5037 init_acc_corrupt=0.4653 acc_corrupt_t_0p0_0p2=0.0621 corrupt_frac_t_0p0_0p2=0.2026 acc_corrupt_t_0p2_0p4=0.3034 corrupt_frac_t_0p2_0p4=0.2020 acc_corrupt_t_0p4_0p6=0.5729 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=0.7638 corrupt_frac_t_0p6_0p8=0.1965 acc_corrupt_t_0p8_1p0=0.9245 corrupt_frac_t_0p8_1p0=0.1984 out_w_norm=11.2253 out_g_norm=0.3916 loss_all=2.2693 init_gold_top10=0.5604 init_gold_top100=0.7066 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.5007 init_acc_rollout_kept=0.4082 logit_acc_rollout_applied=0.5893 logit_acc_rollout_kept=0.4892
+step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=21.2s lr=2.000000e-03 loss=1.8221 loss_recon=1.8221 loss_meanflow=0.0000 mean_model_t=0.5040 mean_corrupt_t=0.5040 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5050 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6121 corrupt_frac=1.0000 acc_corrupt=0.6121 loss_corrupt=1.8221 wrong_frac=0.4960 init_acc_corrupt=0.4781 acc_corrupt_t_0p0_0p2=0.0648 corrupt_frac_t_0p0_0p2=0.1931 acc_corrupt_t_0p2_0p4=0.3866 corrupt_frac_t_0p2_0p4=0.2013 acc_corrupt_t_0p4_0p6=0.7303 corrupt_frac_t_0p4_0p6=0.2019 acc_corrupt_t_0p6_0p8=0.8832 corrupt_frac_t_0p6_0p8=0.1981 acc_corrupt_t_0p8_1p0=0.9695 corrupt_frac_t_0p8_1p0=0.2056 out_w_norm=11.6309 out_g_norm=0.4861 loss_all=1.5186 init_gold_top10=0.6299 init_gold_top100=0.7419 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.4838 init_acc_rollout_kept=0.5220 logit_acc_rollout_applied=0.6371 logit_acc_rollout_kept=0.6940
+step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=21.2s lr=2.000000e-03 loss=1.4473 loss_recon=1.4473 loss_meanflow=0.0000 mean_model_t=0.5008 mean_corrupt_t=0.5008 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4999 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6901 corrupt_frac=1.0000 acc_corrupt=0.6901 loss_corrupt=1.4473 wrong_frac=0.4993 init_acc_corrupt=0.4866 acc_corrupt_t_0p0_0p2=0.0746 corrupt_frac_t_0p0_0p2=0.2034 acc_corrupt_t_0p2_0p4=0.5408 corrupt_frac_t_0p2_0p4=0.1967 acc_corrupt_t_0p4_0p6=0.8814 corrupt_frac_t_0p4_0p6=0.2010 acc_corrupt_t_0p6_0p8=0.9673 corrupt_frac_t_0p6_0p8=0.1933 acc_corrupt_t_0p8_1p0=0.9943 corrupt_frac_t_0p8_1p0=0.2055 out_w_norm=11.9531 out_g_norm=0.5594 loss_all=1.4882 init_gold_top10=0.6393 init_gold_top100=0.7602 rollout_applied_pos_frac=0.5547 init_acc_rollout_applied=0.5451 init_acc_rollout_kept=0.4294 logit_acc_rollout_applied=0.7345 logit_acc_rollout_kept=0.6450
+NCCL version 2.25.1+cuda12.8
+resumed_from=runs/train8_ctx1024_randk_p50_path3_rand2_3_unif0_0p25_ctx1024_uniformt_temp1_randk_20260518_010217/latest.pt start_step=1001
+{
+  "device": "cuda:0",
+  "rank": 0,
+  "world_size": 4,
+  "samples": "owt_cached_chunks:8",
+  "vocab_size": 2664,
+  "tokenizer_vocab_size": 50257,
+  "save_dir": "runs/train8_ctx1024_randk_p50_path3_rand2_3_unif0_0p25_ctx1024_uniformt_temp1_randk_20260518_010217",
+  "batch_size": 128,
+  "grad_accum": 1,
+  "effective_batch_size": 512,
+  "global_batch_size": 512,
+  "lr_schedule": "constant_warmup",
+  "optimizer": "muon",
+  "epochs": 0.0,
+  "steps_per_epoch": 1,
+  "total_steps": 2000,
+  "warmup_steps": 10,
+  "warmup_epochs": -1.0,
+  "min_lr": 0.0,
+  "weight_decay": 0.1,
+  "output_weight_decay": -1.0,
+  "adamw_param_groups": "nanogpt",
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_eps": 1e-08,
+  "muon_impl": "legacy",
+  "muon_momentum": 0.95,
+  "muon_ns_steps": 5,
+  "muon_update_scale": 1.0,
+  "muon_nesterov": false,
+  "muon_width_scale": false,
+  "muon_grouping": "legacy_dim_ge_2",
+  "muon_param_count": 2616320,
+  "muon_adam_param_count": 8192,
+  "muon_param_names": [
+    "vocab_embed.embedding",
+    "sigma_map.net.0.weight",
+    "sigma_map.net.2.weight",
+    "blocks.0.attn_qkv.weight",
+    "blocks.0.attn_out.weight",
+    "blocks.0.mlp.0.weight",
+    "blocks.0.mlp.2.weight",
+    "blocks.0.adaLN_modulation.weight",
+    "blocks.1.attn_qkv.weight",
+    "blocks.1.attn_out.weight",
+    "blocks.1.mlp.0.weight",
+    "blocks.1.mlp.2.weight",
+    "blocks.1.adaLN_modulation.weight",
+    "blocks.2.attn_qkv.weight",
+    "blocks.2.attn_out.weight",
+    "blocks.2.mlp.0.weight",
+    "blocks.2.mlp.2.weight",
+    "blocks.2.adaLN_modulation.weight",
+    "output_layer.linear.weight",
+    "output_layer.adaLN_modulation.weight"
+  ],
+  "muon_adam_param_names": [
+    "sigma_map.net.0.bias",
+    "sigma_map.net.2.bias",
+    "blocks.0.norm1.weight",
+    "blocks.0.norm2.weight",
+    "blocks.0.mlp.0.bias",
+    "blocks.0.mlp.2.bias",
+    "blocks.0.adaLN_modulation.bias",
+    "blocks.1.norm1.weight",
+    "blocks.1.norm2.weight",
+    "blocks.1.mlp.0.bias",
+    "blocks.1.mlp.2.bias",
+    "blocks.1.adaLN_modulation.bias",
+    "blocks.2.norm1.weight",
+    "blocks.2.norm2.weight",
+    "blocks.2.mlp.0.bias",
+    "blocks.2.mlp.2.bias",
+    "blocks.2.adaLN_modulation.bias",
+    "output_layer.norm_final.weight",
+    "output_layer.adaLN_modulation.bias"
+  ],
+  "muon_effective_nesterov": false,
+  "muon_effective_width_scale": false,
+  "muon_effective_weight_decay": 0.1,
+  "muon_adam_fallback_nesterov": false,
+  "muon_adam_fallback_weight_decay": 0.1,
+  "ema_decay": 0.9999,
+  "ema_start_step": 0,
+  "model_type": "ddit",
+  "ddit_mlp_type": "gelu",
+  "elf_num_time_tokens": 4,
+  "elf_num_model_mode_tokens": 0,
+  "qk_norm": true,
+  "output_bias": false,
+  "output_init_std": -1.0,
+  "norm_type": "rmsnorm",
+  "target_loss": "hard_ce",
+  "linear_soft_target_power": 1.0,
+  "linear_soft_target_min_conf": 0.0,
+  "linear_soft_target_max_conf": 1.0,
+  "t_sampling_mode": "uniform",
+  "t_sampling_power": 1.0,
+  "t_sampling_eps": 0.0001,
+  "t_sampling_logit_mean": -1.5,
+  "t_sampling_logit_std": 0.8,
+  "dual_t": true,
+  "corrupt_t_mode": "same",
+  "corrupt_min_t": 0.0,
+  "corrupt_max_t": 1.0,
+  "prefix_block_prob": 0.0,
+  "prefix_block_len": 128,
+  "mask_ratio_floor_schedule": "none",
+  "dirichlet_endpoint_mode": "categorical_dual_t",
+  "dirichlet_semantic_t_mode": "same",
+  "dirichlet_semantic_t_value": 0.0,
+  "dirichlet_semantic_t_curve": "linear",
+  "dirichlet_semantic_t_power": 1.0,
+  "endpoint_sequence_random_prob_alpha": 0.0,
+  "categorical_wrong_from_full_vocab": true,
+  "categorical_wrong_from_batch_valid_tokens": false,
+  "categorical_wrong_basin_token_ids": "",
+  "categorical_wrong_basin_prob": 0.0,
+  "categorical_wrong_unigram_prob": 0.0,
+  "categorical_wrong_uniform_prob": 0.0,
+  "categorical_wrong_prob_floor": 0.0,
+  "categorical_wrong_corpus_unigram_path": "",
+  "categorical_wrong_corpus_unigram_alpha": 1.0,
+  "categorical_wrong_basin_shared_prob": 0.0,
+  "categorical_wrong_unigram_shared_prob": 0.0,
+  "mask_mixture_original_prob": 0.0,
+  "mask_mixture_lowk_prob": 0.0,
+  "mask_mixture_lowcorrupt_prob": 0.0,
+  "mask_mixture_block_prob": 0.0,
+  "mask_mixture_all_prob": 1.0,
+  "mask_mixture_lowk_clean_tokens": "0",
+  "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_block_tokens": "64,128",
+  "simplex_bridge_sampler": "dirichlet",
+  "logistic_normal_sigma_min": 0.1,
+  "logistic_normal_sigma_max": 1.0,
+  "logistic_normal_tau_min": 1.0,
+  "logistic_normal_tau_max": 1.0,
+  "torch_compile": false,
+  "compile_mode": "max-autotune",
+  "state_format": "prob",
+  "meanflow_weight": 0.0,
+  "rollout_train_prob": 0.5,
+  "rollout_train_steps": 3,
+  "rollout_train_steps_min": 2,
+  "rollout_train_infer_steps": 1,
+  "rollout_train_time_mode": "sampled_path",
+  "rollout_train_s_dist": "uniform",
+  "rollout_train_s_min_frac": 0.0,
+  "rollout_train_s_max_frac": 0.25,
+  "rollout_train_s_beta_alpha": 2.0,
+  "rollout_train_s_beta_beta": 6.0,
+  "rollout_train_temp": 1.0,
+  "rollout_train_max_gamma": 1.0,
+  "rollout_train_corrupt_only": true,
+  "rollout_train_samplewise": true,
+  "rollout_train_compute_always": false,
+  "rollout_train_sync_t": true,
+  "bridge_noise_init": "logistic_normal",
+  "noise_sigma": -1.0,
+  "allow_tf32": true,
+  "activation_checkpointing": false,
+  "activation_checkpoint_interval": 1,
+  "activation_checkpoint_scope": "block",
+  "ddp_static_graph": false,
+  "ddp_gradient_as_bucket_view": true,
+  "blocking_data_transfer": false,
+  "dataloader_prefetch_factor": 4,
+  "full_train_stats": false,
+  "tokenized_hf": false,
+  "tokenized_pad_token": "pad",
+  "elf_conditional_hf": false,
+  "record_pad_truncate": false,
+  "record_add_eos": false,
+  "record_add_special_tokens": false,
+  "record_pad_token": "pad",
+  "record_shuffle_buffer": 10000,
+  "wrap": true,
+  "wrap_mode": "stream",
+  "wrap_record_buffer_size": 200,
+  "owt_cached_chunks": true,
+  "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_compact_overfit",
+  "owt_chunk_cache_rebuild": false,
+  "owt_chunk_cache_write_batch": 4096,
+  "owt_exact_repeat_per_chunk": 64,
+  "online_chunk_shuffle": false,
+  "online_chunk_shuffle_buffer": 10000,
+  "openwebtext_split": "train_minus_100k",
+  "detokenizer": "auto",
+  "resolved_detokenizer": null,
+  "num_workers": 0,
+  "latest_every": 1000,
+  "resume_path": "runs/train8_ctx1024_randk_p50_path3_rand2_3_unif0_0p25_ctx1024_uniformt_temp1_randk_20260518_010217/latest.pt"
+}
+step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=1100 elapsed=21.9s lr=2.000000e-03 loss=1.1732 loss_recon=1.1732 loss_meanflow=0.0000 mean_model_t=0.4971 mean_corrupt_t=0.4971 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5052 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7453 corrupt_frac=1.0000 acc_corrupt=0.7453 loss_corrupt=1.1732 wrong_frac=0.5028 init_acc_corrupt=0.4994 acc_corrupt_t_0p0_0p2=0.1029 corrupt_frac_t_0p0_0p2=0.2071 acc_corrupt_t_0p2_0p4=0.7067 corrupt_frac_t_0p2_0p4=0.1988 acc_corrupt_t_0p4_0p6=0.9560 corrupt_frac_t_0p4_0p6=0.1976 acc_corrupt_t_0p6_0p8=0.9915 corrupt_frac_t_0p6_0p8=0.1951 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=12.1903 out_g_norm=0.5876 loss_all=0.9405 init_gold_top10=0.6469 init_gold_top100=0.7430 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.5473 init_acc_rollout_kept=0.4381 logit_acc_rollout_applied=0.7878 logit_acc_rollout_kept=0.7687

LTA_openwebtext_dualt/logs/softendpoint_mn_pilot_4gpu/train8_n1024_hard_ce_bridge_20260517_train8_overfit.log

@@ -0,0 +1,316 @@
+NCCL version 2.25.1+cuda12.8
+{
+  "device": "cuda:0",
+  "rank": 0,
+  "world_size": 4,
+  "samples": "owt_cached_chunks:8",
+  "vocab_size": 50257,
+  "tokenizer_vocab_size": 50257,
+  "save_dir": "runs/train8_n1024_hard_ce_bridge_20260517_train8_overfit",
+  "batch_size": 1,
+  "grad_accum": 1,
+  "effective_batch_size": 4,
+  "global_batch_size": 4,
+  "lr_schedule": "constant_warmup",
+  "optimizer": "muon",
+  "epochs": 0.0,
+  "steps_per_epoch": 2,
+  "total_steps": 1000,
+  "warmup_steps": 20,
+  "warmup_epochs": -1.0,
+  "min_lr": 0.0,
+  "weight_decay": 0.1,
+  "output_weight_decay": -1.0,
+  "adamw_param_groups": "nanogpt",
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_eps": 1e-08,
+  "muon_impl": "legacy",
+  "muon_momentum": 0.95,
+  "muon_ns_steps": 5,
+  "muon_update_scale": 1.0,
+  "muon_nesterov": false,
+  "muon_width_scale": false,
+  "muon_grouping": "legacy_dim_ge_2",
+  "muon_param_count": 169453056,
+  "muon_adam_param_count": 122368,
+  "muon_param_names": [
+    "vocab_embed.embedding",
+    "sigma_map.net.0.weight",
+    "sigma_map.net.2.weight",
+    "blocks.0.attn_qkv.weight",
+    "blocks.0.attn_out.weight",
+    "blocks.0.mlp.0.weight",
+    "blocks.0.mlp.2.weight",
+    "blocks.0.adaLN_modulation.weight",
+    "blocks.1.attn_qkv.weight",
+    "blocks.1.attn_out.weight",
+    "blocks.1.mlp.0.weight",
+    "blocks.1.mlp.2.weight",
+    "blocks.1.adaLN_modulation.weight",
+    "blocks.2.attn_qkv.weight",
+    "blocks.2.attn_out.weight",
+    "blocks.2.mlp.0.weight",
+    "blocks.2.mlp.2.weight",
+    "blocks.2.adaLN_modulation.weight",
+    "blocks.3.attn_qkv.weight",
+    "blocks.3.attn_out.weight",
+    "blocks.3.mlp.0.weight",
+    "blocks.3.mlp.2.weight",
+    "blocks.3.adaLN_modulation.weight",
+    "blocks.4.attn_qkv.weight",
+    "blocks.4.attn_out.weight",
+    "blocks.4.mlp.0.weight",
+    "blocks.4.mlp.2.weight",
+    "blocks.4.adaLN_modulation.weight",
+    "blocks.5.attn_qkv.weight",
+    "blocks.5.attn_out.weight",
+    "blocks.5.mlp.0.weight",
+    "blocks.5.mlp.2.weight",
+    "blocks.5.adaLN_modulation.weight",
+    "blocks.6.attn_qkv.weight",
+    "blocks.6.attn_out.weight",
+    "blocks.6.mlp.0.weight",
+    "blocks.6.mlp.2.weight",
+    "blocks.6.adaLN_modulation.weight",
+    "blocks.7.attn_qkv.weight",
+    "blocks.7.attn_out.weight",
+    "blocks.7.mlp.0.weight",
+    "blocks.7.mlp.2.weight",
+    "blocks.7.adaLN_modulation.weight",
+    "blocks.8.attn_qkv.weight",
+    "blocks.8.attn_out.weight",
+    "blocks.8.mlp.0.weight",
+    "blocks.8.mlp.2.weight",
+    "blocks.8.adaLN_modulation.weight",
+    "blocks.9.attn_qkv.weight",
+    "blocks.9.attn_out.weight",
+    "blocks.9.mlp.0.weight",
+    "blocks.9.mlp.2.weight",
+    "blocks.9.adaLN_modulation.weight",
+    "blocks.10.attn_qkv.weight",
+    "blocks.10.attn_out.weight",
+    "blocks.10.mlp.0.weight",
+    "blocks.10.mlp.2.weight",
+    "blocks.10.adaLN_modulation.weight",
+    "blocks.11.attn_qkv.weight",
+    "blocks.11.attn_out.weight",
+    "blocks.11.mlp.0.weight",
+    "blocks.11.mlp.2.weight",
+    "blocks.11.adaLN_modulation.weight",
+    "output_layer.linear.weight",
+    "output_layer.adaLN_modulation.weight"
+  ],
+  "muon_adam_param_names": [
+    "sigma_map.net.0.bias",
+    "sigma_map.net.2.bias",
+    "blocks.0.norm1.weight",
+    "blocks.0.norm2.weight",
+    "blocks.0.mlp.0.bias",
+    "blocks.0.mlp.2.bias",
+    "blocks.0.adaLN_modulation.bias",
+    "blocks.1.norm1.weight",
+    "blocks.1.norm2.weight",
+    "blocks.1.mlp.0.bias",
+    "blocks.1.mlp.2.bias",
+    "blocks.1.adaLN_modulation.bias",
+    "blocks.2.norm1.weight",
+    "blocks.2.norm2.weight",
+    "blocks.2.mlp.0.bias",
+    "blocks.2.mlp.2.bias",
+    "blocks.2.adaLN_modulation.bias",
+    "blocks.3.norm1.weight",
+    "blocks.3.norm2.weight",
+    "blocks.3.mlp.0.bias",
+    "blocks.3.mlp.2.bias",
+    "blocks.3.adaLN_modulation.bias",
+    "blocks.4.norm1.weight",
+    "blocks.4.norm2.weight",
+    "blocks.4.mlp.0.bias",
+    "blocks.4.mlp.2.bias",
+    "blocks.4.adaLN_modulation.bias",
+    "blocks.5.norm1.weight",
+    "blocks.5.norm2.weight",
+    "blocks.5.mlp.0.bias",
+    "blocks.5.mlp.2.bias",
+    "blocks.5.adaLN_modulation.bias",
+    "blocks.6.norm1.weight",
+    "blocks.6.norm2.weight",
+    "blocks.6.mlp.0.bias",
+    "blocks.6.mlp.2.bias",
+    "blocks.6.adaLN_modulation.bias",
+    "blocks.7.norm1.weight",
+    "blocks.7.norm2.weight",
+    "blocks.7.mlp.0.bias",
+    "blocks.7.mlp.2.bias",
+    "blocks.7.adaLN_modulation.bias",
+    "blocks.8.norm1.weight",
+    "blocks.8.norm2.weight",
+    "blocks.8.mlp.0.bias",
+    "blocks.8.mlp.2.bias",
+    "blocks.8.adaLN_modulation.bias",
+    "blocks.9.norm1.weight",
+    "blocks.9.norm2.weight",
+    "blocks.9.mlp.0.bias",
+    "blocks.9.mlp.2.bias",
+    "blocks.9.adaLN_modulation.bias",
+    "blocks.10.norm1.weight",
+    "blocks.10.norm2.weight",
+    "blocks.10.mlp.0.bias",
+    "blocks.10.mlp.2.bias",
+    "blocks.10.adaLN_modulation.bias",
+    "blocks.11.norm1.weight",
+    "blocks.11.norm2.weight",
+    "blocks.11.mlp.0.bias",
+    "blocks.11.mlp.2.bias",
+    "blocks.11.adaLN_modulation.bias",
+    "output_layer.norm_final.weight",
+    "output_layer.adaLN_modulation.bias"
+  ],
+  "muon_effective_nesterov": false,
+  "muon_effective_width_scale": false,
+  "muon_effective_weight_decay": 0.1,
+  "muon_adam_fallback_nesterov": false,
+  "muon_adam_fallback_weight_decay": 0.1,
+  "ema_decay": 0.9999,
+  "ema_start_step": 0,
+  "model_type": "ddit",
+  "elf_num_time_tokens": 4,
+  "elf_num_model_mode_tokens": 0,
+  "qk_norm": true,
+  "output_bias": false,
+  "output_init_std": -1.0,
+  "norm_type": "rmsnorm",
+  "target_loss": "hard_ce",
+  "linear_soft_target_power": 1.0,
+  "linear_soft_target_min_conf": 0.0,
+  "linear_soft_target_max_conf": 1.0,
+  "t_sampling_mode": "logit_normal",
+  "t_sampling_power": 1.0,
+  "t_sampling_eps": 0.0001,
+  "t_sampling_logit_mean": -1.5,
+  "t_sampling_logit_std": 0.8,
+  "dual_t": true,
+  "corrupt_t_mode": "same",
+  "corrupt_min_t": 0.0,
+  "corrupt_max_t": 1.0,
+  "prefix_block_prob": 0.0,
+  "prefix_block_len": 128,
+  "mask_ratio_floor_schedule": "none",
+  "dirichlet_endpoint_mode": "categorical_dual_t",
+  "dirichlet_semantic_t_mode": "same",
+  "dirichlet_semantic_t_value": 0.0,
+  "dirichlet_semantic_t_curve": "linear",
+  "dirichlet_semantic_t_power": 1.0,
+  "endpoint_sequence_random_prob_alpha": 0.0,
+  "categorical_wrong_from_full_vocab": true,
+  "categorical_wrong_from_batch_valid_tokens": false,
+  "categorical_wrong_basin_token_ids": "",
+  "categorical_wrong_basin_prob": 0.0,
+  "categorical_wrong_unigram_prob": 0.0,
+  "categorical_wrong_uniform_prob": 0.0,
+  "categorical_wrong_corpus_unigram_path": "",
+  "categorical_wrong_corpus_unigram_alpha": 1.0,
+  "categorical_wrong_basin_shared_prob": 0.0,
+  "categorical_wrong_unigram_shared_prob": 0.0,
+  "mask_mixture_original_prob": 0.0,
+  "mask_mixture_lowk_prob": 1.0,
+  "mask_mixture_lowcorrupt_prob": 0.0,
+  "mask_mixture_block_prob": 0.0,
+  "mask_mixture_all_prob": 0.0,
+  "mask_mixture_lowk_clean_tokens": "64,128,256",
+  "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_block_tokens": "64,128",
+  "simplex_bridge_sampler": "dirichlet",
+  "logistic_normal_sigma_min": 0.18,
+  "logistic_normal_sigma_max": 2.2,
+  "logistic_normal_tau_min": 0.65,
+  "logistic_normal_tau_max": 1.15,
+  "torch_compile": false,
+  "compile_mode": "max-autotune",
+  "state_format": "prob",
+  "meanflow_weight": 0.0,
+  "rollout_train_prob": 0.0,
+  "rollout_train_steps": 1,
+  "rollout_train_infer_steps": 64,
+  "rollout_train_temp": 1.45,
+  "rollout_train_max_gamma": 1.0,
+  "rollout_train_corrupt_only": true,
+  "rollout_train_samplewise": false,
+  "rollout_train_compute_always": false,
+  "bridge_noise_init": "logistic_normal",
+  "noise_sigma": -1.0,
+  "allow_tf32": true,
+  "activation_checkpointing": false,
+  "activation_checkpoint_interval": 1,
+  "activation_checkpoint_scope": "block",
+  "ddp_static_graph": false,
+  "ddp_gradient_as_bucket_view": true,
+  "blocking_data_transfer": false,
+  "dataloader_prefetch_factor": 4,
+  "full_train_stats": false,
+  "tokenized_hf": false,
+  "tokenized_pad_token": "pad",
+  "elf_conditional_hf": false,
+  "record_pad_truncate": false,
+  "record_add_eos": false,
+  "record_add_special_tokens": false,
+  "record_pad_token": "pad",
+  "record_shuffle_buffer": 10000,
+  "wrap": true,
+  "wrap_mode": "stream",
+  "wrap_record_buffer_size": 200,
+  "owt_cached_chunks": true,
+  "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train8_overfit",
+  "owt_chunk_cache_rebuild": false,
+  "owt_chunk_cache_write_batch": 4096,
+  "owt_exact_repeat_per_chunk": 0,
+  "online_chunk_shuffle": false,
+  "online_chunk_shuffle_buffer": 10000,
+  "openwebtext_split": "train_minus_100k",
+  "detokenizer": "auto",
+  "resolved_detokenizer": null,
+  "num_workers": 0,
+  "latest_every": 50,
+  "resume_path": ""
+}
+step=25 epoch=13/500 epoch_step=1/2 micro_steps=25 elapsed=3.9s lr=2.000000e-03 loss=10.7864 loss_recon=10.7864 loss_meanflow=0.0000 mean_model_t=0.2229 mean_corrupt_t=0.2229 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1332 corrupt_frac=0.8125 acc_corrupt=0.1000 loss_corrupt=10.7864 wrong_frac=0.7737 init_acc_corrupt=0.1321 acc_corrupt_t_0p0_0p2=0.0574 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=0.2483 out_g_norm=1.5841 acc_corrupt_t_0p2_0p4=0.1211 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.2533 corrupt_frac_t_0p4_0p6=1.0000 loss_all=10.6718 init_gold_top10=0.3646 init_gold_top100=0.3646
+step=50 epoch=25/500 epoch_step=2/2 micro_steps=50 elapsed=3.2s lr=2.000000e-03 loss=10.6536 loss_recon=10.6536 loss_meanflow=0.0000 mean_model_t=0.2418 mean_corrupt_t=0.2418 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1506 corrupt_frac=0.8675 acc_corrupt=0.1229 loss_corrupt=10.6536 wrong_frac=0.7587 init_acc_corrupt=0.1501 acc_corrupt_t_0p0_0p2=0.0635 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=1.1359 out_g_norm=2.1812 acc_corrupt_t_0p2_0p4=0.1465 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.2672 corrupt_frac_t_0p4_0p6=1.0000 loss_all=10.6088 init_gold_top10=0.1135 init_gold_top100=0.2458
+step=75 epoch=38/500 epoch_step=1/2 micro_steps=75 elapsed=7.1s lr=2.000000e-03 loss=10.4160 loss_recon=10.4160 loss_meanflow=0.0000 mean_model_t=0.1995 mean_corrupt_t=0.1995 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1303 corrupt_frac=0.8675 acc_corrupt=0.1041 loss_corrupt=10.4160 wrong_frac=0.7995 init_acc_corrupt=0.1001 acc_corrupt_t_0p2_0p4=0.1574 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=1.7525 out_g_norm=2.6698 acc_corrupt_t_0p0_0p2=0.0612 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.2458 corrupt_frac_t_0p4_0p6=1.0000 loss_all=9.9453 init_gold_top10=0.3073 init_gold_top100=0.3292
+step=100 epoch=50/500 epoch_step=2/2 micro_steps=100 elapsed=3.2s lr=2.000000e-03 loss=10.0502 loss_recon=10.0502 loss_meanflow=0.0000 mean_model_t=0.2487 mean_corrupt_t=0.2487 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1689 corrupt_frac=0.8425 acc_corrupt=0.1328 loss_corrupt=10.0502 wrong_frac=0.7557 init_acc_corrupt=0.1566 acc_corrupt_t_0p0_0p2=0.0632 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.2511 out_g_norm=2.9631 acc_corrupt_t_0p2_0p4=0.1522 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.2891 corrupt_frac_t_0p4_0p6=1.0000 loss_all=9.5690 init_gold_top10=0.3372 init_gold_top100=0.3411
+step=125 epoch=63/500 epoch_step=1/2 micro_steps=125 elapsed=7.3s lr=2.000000e-03 loss=9.8011 loss_recon=9.8011 loss_meanflow=0.0000 mean_model_t=0.2121 mean_corrupt_t=0.2121 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1487 corrupt_frac=0.8450 acc_corrupt=0.1153 loss_corrupt=9.8011 wrong_frac=0.7876 init_acc_corrupt=0.1138 acc_corrupt_t_0p0_0p2=0.0628 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=2.7598 out_g_norm=3.4495 acc_corrupt_t_0p2_0p4=0.1326 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.3001 corrupt_frac_t_0p4_0p6=1.0000 loss_all=9.5979 init_gold_top10=0.1384 init_gold_top100=0.2511
+step=150 epoch=75/500 epoch_step=2/2 micro_steps=150 elapsed=3.2s lr=2.000000e-03 loss=9.3065 loss_recon=9.3065 loss_meanflow=0.0000 mean_model_t=0.2285 mean_corrupt_t=0.2285 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1630 corrupt_frac=0.8525 acc_corrupt=0.1323 loss_corrupt=9.3065 wrong_frac=0.7725 init_acc_corrupt=0.1303 acc_corrupt_t_0p2_0p4=0.1598 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=3.2724 out_g_norm=3.7085 acc_corrupt_t_0p4_0p6=0.3238 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p0_0p2=0.0682 corrupt_frac_t_0p0_0p2=1.0000 loss_all=8.2090 init_gold_top10=0.2612 init_gold_top100=0.2991
+step=175 epoch=88/500 epoch_step=1/2 micro_steps=175 elapsed=7.2s lr=2.000000e-03 loss=9.0808 loss_recon=9.0808 loss_meanflow=0.0000 mean_model_t=0.2193 mean_corrupt_t=0.2193 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1660 corrupt_frac=0.8375 acc_corrupt=0.1244 loss_corrupt=9.0808 wrong_frac=0.7870 init_acc_corrupt=0.1225 acc_corrupt_t_0p4_0p6=0.3179 corrupt_frac_t_0p4_0p6=1.0000 out_w_norm=3.7787 out_g_norm=3.8401 acc_corrupt_t_0p2_0p4=0.1766 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p0_0p2=0.0676 corrupt_frac_t_0p0_0p2=1.0000 loss_all=7.1403 init_gold_top10=0.3661 init_gold_top100=0.3694
+step=200 epoch=100/500 epoch_step=2/2 micro_steps=200 elapsed=3.2s lr=2.000000e-03 loss=8.9196 loss_recon=8.9196 loss_meanflow=0.0000 mean_model_t=0.2000 mean_corrupt_t=0.2000 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1503 corrupt_frac=0.8400 acc_corrupt=0.1162 loss_corrupt=8.9196 wrong_frac=0.7967 init_acc_corrupt=0.1161 acc_corrupt_t_0p0_0p2=0.0644 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.2129 out_g_norm=3.8417 acc_corrupt_t_0p2_0p4=0.1556 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.2713 corrupt_frac_t_0p4_0p6=1.0000 loss_all=9.1738 init_gold_top10=0.0104 init_gold_top100=0.1367
+step=225 epoch=113/500 epoch_step=1/2 micro_steps=225 elapsed=6.5s lr=2.000000e-03 loss=8.5644 loss_recon=8.5644 loss_meanflow=0.0000 mean_model_t=0.2059 mean_corrupt_t=0.2059 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1668 corrupt_frac=0.8475 acc_corrupt=0.1290 loss_corrupt=8.5644 wrong_frac=0.7966 init_acc_corrupt=0.1126 acc_corrupt_t_0p0_0p2=0.0478 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=4.5749 out_g_norm=3.5982 acc_corrupt_t_0p2_0p4=0.1699 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.3385 corrupt_frac_t_0p4_0p6=1.0000 loss_all=7.4746 init_gold_top10=0.2122 init_gold_top100=0.3138
+step=250 epoch=125/500 epoch_step=2/2 micro_steps=250 elapsed=3.2s lr=2.000000e-03 loss=8.4566 loss_recon=8.4566 loss_meanflow=0.0000 mean_model_t=0.1793 mean_corrupt_t=0.1793 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1480 corrupt_frac=0.8500 acc_corrupt=0.1107 loss_corrupt=8.4566 wrong_frac=0.8201 init_acc_corrupt=0.0854 acc_corrupt_t_0p2_0p4=0.1787 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=4.8587 out_g_norm=3.4425 acc_corrupt_t_0p0_0p2=0.0583 corrupt_frac_t_0p0_0p2=1.0000 loss_all=8.5487 init_gold_top10=0.0837 init_gold_top100=0.1987
+step=275 epoch=138/500 epoch_step=1/2 micro_steps=275 elapsed=7.4s lr=2.000000e-03 loss=8.2703 loss_recon=8.2703 loss_meanflow=0.0000 mean_model_t=0.2038 mean_corrupt_t=0.2038 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1492 corrupt_frac=0.8600 acc_corrupt=0.1137 loss_corrupt=8.2703 wrong_frac=0.8049 init_acc_corrupt=0.0914 acc_corrupt_t_0p0_0p2=0.0710 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.0798 out_g_norm=3.1796 acc_corrupt_t_0p2_0p4=0.1635 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=0.3464 corrupt_frac_t_0p6_0p8=1.0000 loss_all=8.2362 init_gold_top10=0.0182 init_gold_top100=0.1224
+step=300 epoch=150/500 epoch_step=2/2 micro_steps=300 elapsed=3.2s lr=2.000000e-03 loss=8.1003 loss_recon=8.1003 loss_meanflow=0.0000 mean_model_t=0.1754 mean_corrupt_t=0.1754 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1366 corrupt_frac=0.8725 acc_corrupt=0.1104 loss_corrupt=8.1003 wrong_frac=0.8226 init_acc_corrupt=0.0753 acc_corrupt_t_0p2_0p4=0.1806 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.2769 out_g_norm=2.9771 acc_corrupt_t_0p0_0p2=0.0692 corrupt_frac_t_0p0_0p2=1.0000 loss_all=6.8203 init_gold_top10=0.2604 init_gold_top100=0.3187
+step=325 epoch=163/500 epoch_step=1/2 micro_steps=325 elapsed=7.2s lr=2.000000e-03 loss=7.5749 loss_recon=7.5749 loss_meanflow=0.0000 mean_model_t=0.2307 mean_corrupt_t=0.2307 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1913 corrupt_frac=0.8425 acc_corrupt=0.1466 loss_corrupt=7.5749 wrong_frac=0.7709 init_acc_corrupt=0.1407 acc_corrupt_t_0p2_0p4=0.1861 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.4730 out_g_norm=2.6582 acc_corrupt_t_0p0_0p2=0.0727 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.3095 corrupt_frac_t_0p4_0p6=1.0000 loss_all=7.2629 init_gold_top10=0.1471 init_gold_top100=0.2539
+step=350 epoch=175/500 epoch_step=2/2 micro_steps=350 elapsed=3.2s lr=2.000000e-03 loss=7.7026 loss_recon=7.7026 loss_meanflow=0.0000 mean_model_t=0.2000 mean_corrupt_t=0.2000 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1591 corrupt_frac=0.8525 acc_corrupt=0.1239 loss_corrupt=7.7026 wrong_frac=0.7984 init_acc_corrupt=0.1058 acc_corrupt_t_0p2_0p4=0.1731 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=5.6895 out_g_norm=2.5375 acc_corrupt_t_0p0_0p2=0.0568 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.3149 corrupt_frac_t_0p4_0p6=1.0000 loss_all=7.8661 init_gold_top10=0.0716 init_gold_top100=0.1667
+step=375 epoch=188/500 epoch_step=1/2 micro_steps=375 elapsed=6.0s lr=2.000000e-03 loss=7.5492 loss_recon=7.5492 loss_meanflow=0.0000 mean_model_t=0.1848 mean_corrupt_t=0.1848 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1554 corrupt_frac=0.8275 acc_corrupt=0.1158 loss_corrupt=7.5492 wrong_frac=0.8215 init_acc_corrupt=0.0898 acc_corrupt_t_0p0_0p2=0.0650 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=5.8912 out_g_norm=2.7020 acc_corrupt_t_0p2_0p4=0.1840 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.3259 corrupt_frac_t_0p4_0p6=1.0000 loss_all=6.7869 init_gold_top10=0.2943 init_gold_top100=0.3060
+step=400 epoch=200/500 epoch_step=2/2 micro_steps=400 elapsed=3.2s lr=2.000000e-03 loss=6.8606 loss_recon=6.8606 loss_meanflow=0.0000 mean_model_t=0.2432 mean_corrupt_t=0.2432 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2130 corrupt_frac=0.8475 acc_corrupt=0.1721 loss_corrupt=6.8606 wrong_frac=0.7520 init_acc_corrupt=0.1677 acc_corrupt_t_0p4_0p6=0.4051 corrupt_frac_t_0p4_0p6=1.0000 out_w_norm=6.0772 out_g_norm=2.5310 acc_corrupt_t_0p0_0p2=0.0706 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p2_0p4=0.2320 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=0.4927 corrupt_frac_t_0p6_0p8=1.0000 loss_all=7.3142 init_gold_top10=0.1354 init_gold_top100=0.2435
+step=425 epoch=213/500 epoch_step=1/2 micro_steps=425 elapsed=6.6s lr=2.000000e-03 loss=6.8943 loss_recon=6.8943 loss_meanflow=0.0000 mean_model_t=0.2254 mean_corrupt_t=0.2254 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2041 corrupt_frac=0.8475 acc_corrupt=0.1616 loss_corrupt=6.8943 wrong_frac=0.7710 init_acc_corrupt=0.1320 acc_corrupt_t_0p0_0p2=0.0746 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.2709 out_g_norm=2.3976 acc_corrupt_t_0p2_0p4=0.2291 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.4657 corrupt_frac_t_0p4_0p6=1.0000 loss_all=8.1770 init_gold_top10=0.1339 init_gold_top100=0.2333
+step=450 epoch=225/500 epoch_step=2/2 micro_steps=450 elapsed=3.2s lr=2.000000e-03 loss=6.5882 loss_recon=6.5882 loss_meanflow=0.0000 mean_model_t=0.2573 mean_corrupt_t=0.2573 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2300 corrupt_frac=0.8425 acc_corrupt=0.1835 loss_corrupt=6.5882 wrong_frac=0.7421 init_acc_corrupt=0.1684 acc_corrupt_t_0p2_0p4=0.1989 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.4616 out_g_norm=2.4658 acc_corrupt_t_0p0_0p2=0.0797 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.4002 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=0.4933 corrupt_frac_t_0p6_0p8=1.0000 loss_all=6.0882 init_gold_top10=0.2148 init_gold_top100=0.3008
+step=475 epoch=238/500 epoch_step=1/2 micro_steps=475 elapsed=7.2s lr=2.000000e-03 loss=6.6024 loss_recon=6.6024 loss_meanflow=0.0000 mean_model_t=0.2275 mean_corrupt_t=0.2275 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2090 corrupt_frac=0.8325 acc_corrupt=0.1627 loss_corrupt=6.6024 wrong_frac=0.7694 init_acc_corrupt=0.1501 acc_corrupt_t_0p0_0p2=0.0717 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=6.6827 out_g_norm=2.4195 acc_corrupt_t_0p2_0p4=0.2184 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.3401 corrupt_frac_t_0p4_0p6=1.0000 loss_all=7.1854 init_gold_top10=0.0982 init_gold_top100=0.2254
+step=500 epoch=250/500 epoch_step=2/2 micro_steps=500 elapsed=3.2s lr=2.000000e-03 loss=6.6715 loss_recon=6.6715 loss_meanflow=0.0000 mean_model_t=0.2087 mean_corrupt_t=0.2087 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1925 corrupt_frac=0.8325 acc_corrupt=0.1487 loss_corrupt=6.6715 wrong_frac=0.7906 init_acc_corrupt=0.1262 acc_corrupt_t_0p2_0p4=0.2258 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=6.8888 out_g_norm=2.5030 acc_corrupt_t_0p0_0p2=0.0652 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.4675 corrupt_frac_t_0p4_0p6=1.0000 loss_all=5.3250 init_gold_top10=0.3604 init_gold_top100=0.3615
+step=525 epoch=263/500 epoch_step=1/2 micro_steps=525 elapsed=6.0s lr=2.000000e-03 loss=6.4744 loss_recon=6.4744 loss_meanflow=0.0000 mean_model_t=0.2178 mean_corrupt_t=0.2178 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1979 corrupt_frac=0.8500 acc_corrupt=0.1515 loss_corrupt=6.4744 wrong_frac=0.7787 init_acc_corrupt=0.1194 acc_corrupt_t_0p2_0p4=0.1977 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=7.0787 out_g_norm=2.5506 acc_corrupt_t_0p0_0p2=0.0755 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.3393 corrupt_frac_t_0p4_0p6=1.0000 loss_all=4.8922 init_gold_top10=0.4342 init_gold_top100=0.4353
+step=550 epoch=275/500 epoch_step=2/2 micro_steps=550 elapsed=3.2s lr=2.000000e-03 loss=6.2602 loss_recon=6.2602 loss_meanflow=0.0000 mean_model_t=0.2122 mean_corrupt_t=0.2122 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2027 corrupt_frac=0.8475 acc_corrupt=0.1587 loss_corrupt=6.2602 wrong_frac=0.7820 init_acc_corrupt=0.1212 acc_corrupt_t_0p0_0p2=0.0710 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=7.2750 out_g_norm=2.6923 acc_corrupt_t_0p2_0p4=0.2271 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.3438 corrupt_frac_t_0p4_0p6=1.0000 loss_all=7.6317 init_gold_top10=0.0145 init_gold_top100=0.1194
+step=575 epoch=288/500 epoch_step=1/2 micro_steps=575 elapsed=6.0s lr=2.000000e-03 loss=6.6165 loss_recon=6.6165 loss_meanflow=0.0000 mean_model_t=0.1913 mean_corrupt_t=0.1913 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1636 corrupt_frac=0.8675 acc_corrupt=0.1295 loss_corrupt=6.6165 wrong_frac=0.8059 init_acc_corrupt=0.0965 acc_corrupt_t_0p0_0p2=0.0527 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=7.4940 out_g_norm=2.8035 acc_corrupt_t_0p2_0p4=0.1950 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=0.6003 corrupt_frac_t_0p6_0p8=1.0000 loss_all=8.5299 init_gold_top10=0.0146 init_gold_top100=0.1229
+step=600 epoch=300/500 epoch_step=2/2 micro_steps=600 elapsed=3.2s lr=2.000000e-03 loss=6.3523 loss_recon=6.3523 loss_meanflow=0.0000 mean_model_t=0.2002 mean_corrupt_t=0.2002 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1777 corrupt_frac=0.8600 acc_corrupt=0.1367 loss_corrupt=6.3523 wrong_frac=0.8077 init_acc_corrupt=0.1017 acc_corrupt_t_0p2_0p4=0.1968 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=7.6925 out_g_norm=2.6466 acc_corrupt_t_0p0_0p2=0.0663 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.3737 corrupt_frac_t_0p4_0p6=1.0000 loss_all=4.8443 init_gold_top10=0.2422 init_gold_top100=0.3177
+step=625 epoch=313/500 epoch_step=1/2 micro_steps=625 elapsed=7.0s lr=2.000000e-03 loss=6.1065 loss_recon=6.1065 loss_meanflow=0.0000 mean_model_t=0.1977 mean_corrupt_t=0.1977 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1844 corrupt_frac=0.8650 acc_corrupt=0.1472 loss_corrupt=6.1065 wrong_frac=0.8011 init_acc_corrupt=0.1018 acc_corrupt_t_0p2_0p4=0.2090 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=7.8705 out_g_norm=2.7706 acc_corrupt_t_0p0_0p2=0.0697 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.4319 corrupt_frac_t_0p4_0p6=1.0000 loss_all=4.5681 init_gold_top10=0.3052 init_gold_top100=0.3052
+step=650 epoch=325/500 epoch_step=2/2 micro_steps=650 elapsed=3.2s lr=2.000000e-03 loss=5.3613 loss_recon=5.3613 loss_meanflow=0.0000 mean_model_t=0.2686 mean_corrupt_t=0.2686 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2696 corrupt_frac=0.8500 acc_corrupt=0.2165 loss_corrupt=5.3613 wrong_frac=0.7288 init_acc_corrupt=0.1849 acc_corrupt_t_0p2_0p4=0.2419 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=8.0587 out_g_norm=2.9153 acc_corrupt_t_0p0_0p2=0.0758 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.3771 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p6_0p8=0.5826 corrupt_frac_t_0p6_0p8=1.0000 loss_all=6.6134 init_gold_top10=0.0586 init_gold_top100=0.2070
+step=675 epoch=338/500 epoch_step=1/2 micro_steps=675 elapsed=6.0s lr=2.000000e-03 loss=5.6003 loss_recon=5.6003 loss_meanflow=0.0000 mean_model_t=0.2244 mean_corrupt_t=0.2244 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2217 corrupt_frac=0.8450 acc_corrupt=0.1732 loss_corrupt=5.6003 wrong_frac=0.7735 init_acc_corrupt=0.1386 acc_corrupt_t_0p2_0p4=0.2217 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=8.2271 out_g_norm=3.2682 acc_corrupt_t_0p0_0p2=0.0735 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.3936 corrupt_frac_t_0p4_0p6=1.0000 loss_all=4.4996 init_gold_top10=0.2533 init_gold_top100=0.2935
+step=700 epoch=350/500 epoch_step=2/2 micro_steps=700 elapsed=3.2s lr=2.000000e-03 loss=5.7057 loss_recon=5.7057 loss_meanflow=0.0000 mean_model_t=0.1820 mean_corrupt_t=0.1820 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1834 corrupt_frac=0.8550 acc_corrupt=0.1393 loss_corrupt=5.7057 wrong_frac=0.8183 init_acc_corrupt=0.0776 acc_corrupt_t_0p2_0p4=0.1946 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=8.4344 out_g_norm=3.0268 acc_corrupt_t_0p0_0p2=0.0733 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.3880 corrupt_frac_t_0p4_0p6=1.0000 loss_all=4.3138 init_gold_top10=0.3094 init_gold_top100=0.3146
+step=725 epoch=363/500 epoch_step=1/2 micro_steps=725 elapsed=6.0s lr=2.000000e-03 loss=5.3601 loss_recon=5.3601 loss_meanflow=0.0000 mean_model_t=0.2167 mean_corrupt_t=0.2167 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2263 corrupt_frac=0.8275 acc_corrupt=0.1640 loss_corrupt=5.3601 wrong_frac=0.7835 init_acc_corrupt=0.1159 acc_corrupt_t_0p0_0p2=0.0829 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=8.6344 out_g_norm=3.1377 acc_corrupt_t_0p2_0p4=0.2374 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.4219 corrupt_frac_t_0p4_0p6=1.0000 loss_all=4.7756 init_gold_top10=0.1979 init_gold_top100=0.2875
+step=750 epoch=375/500 epoch_step=2/2 micro_steps=750 elapsed=3.2s lr=2.000000e-03 loss=5.4931 loss_recon=5.4931 loss_meanflow=0.0000 mean_model_t=0.2249 mean_corrupt_t=0.2249 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2207 corrupt_frac=0.8575 acc_corrupt=0.1721 loss_corrupt=5.4931 wrong_frac=0.7773 init_acc_corrupt=0.1290 acc_corrupt_t_0p2_0p4=0.2201 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=8.8160 out_g_norm=3.4480 acc_corrupt_t_0p4_0p6=0.4258 corrupt_frac_t_0p4_0p6=1.0000 acc_corrupt_t_0p8_1p0=0.7790 corrupt_frac_t_0p8_1p0=1.0000 acc_corrupt_t_0p0_0p2=0.0803 corrupt_frac_t_0p0_0p2=1.0000 loss_all=4.8351 init_gold_top10=0.1654 init_gold_top100=0.2630
+step=775 epoch=388/500 epoch_step=1/2 micro_steps=775 elapsed=6.8s lr=2.000000e-03 loss=5.2346 loss_recon=5.2346 loss_meanflow=0.0000 mean_model_t=0.2047 mean_corrupt_t=0.2047 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2190 corrupt_frac=0.8525 acc_corrupt=0.1645 loss_corrupt=5.2346 wrong_frac=0.8014 init_acc_corrupt=0.1122 acc_corrupt_t_0p0_0p2=0.0713 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=8.9976 out_g_norm=3.1033 acc_corrupt_t_0p2_0p4=0.2526 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.4590 corrupt_frac_t_0p4_0p6=1.0000 loss_all=4.2737 init_gold_top10=0.2271 init_gold_top100=0.3146
+step=800 epoch=400/500 epoch_step=2/2 micro_steps=800 elapsed=3.2s lr=2.000000e-03 loss=4.9681 loss_recon=4.9681 loss_meanflow=0.0000 mean_model_t=0.2219 mean_corrupt_t=0.2219 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2404 corrupt_frac=0.8575 acc_corrupt=0.1852 loss_corrupt=4.9681 wrong_frac=0.7766 init_acc_corrupt=0.1295 acc_corrupt_t_0p4_0p6=0.4122 corrupt_frac_t_0p4_0p6=1.0000 out_w_norm=9.1570 out_g_norm=3.3703 acc_corrupt_t_0p0_0p2=0.0803 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p2_0p4=0.2411 corrupt_frac_t_0p2_0p4=1.0000 loss_all=5.4143 init_gold_top10=0.1094 init_gold_top100=0.2176
+step=825 epoch=413/500 epoch_step=1/2 micro_steps=825 elapsed=6.0s lr=2.000000e-03 loss=5.3848 loss_recon=5.3848 loss_meanflow=0.0000 mean_model_t=0.1673 mean_corrupt_t=0.1673 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1622 corrupt_frac=0.8700 acc_corrupt=0.1154 loss_corrupt=5.3848 wrong_frac=0.8311 init_acc_corrupt=0.0565 acc_corrupt_t_0p0_0p2=0.0731 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=9.3244 out_g_norm=3.7546 acc_corrupt_t_0p2_0p4=0.1967 corrupt_frac_t_0p2_0p4=1.0000 loss_all=7.0083 init_gold_top10=0.0188 init_gold_top100=0.1240
+step=850 epoch=425/500 epoch_step=2/2 micro_steps=850 elapsed=3.2s lr=2.000000e-03 loss=4.8082 loss_recon=4.8082 loss_meanflow=0.0000 mean_model_t=0.2215 mean_corrupt_t=0.2215 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2529 corrupt_frac=0.8475 acc_corrupt=0.1910 loss_corrupt=4.8082 wrong_frac=0.7834 init_acc_corrupt=0.1358 acc_corrupt_t_0p0_0p2=0.0694 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=9.4903 out_g_norm=3.4038 acc_corrupt_t_0p2_0p4=0.2639 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.5268 corrupt_frac_t_0p4_0p6=1.0000 loss_all=4.8356 init_gold_top10=0.0982 init_gold_top100=0.2054
+step=875 epoch=438/500 epoch_step=1/2 micro_steps=875 elapsed=6.0s lr=2.000000e-03 loss=4.2792 loss_recon=4.2792 loss_meanflow=0.0000 mean_model_t=0.2682 mean_corrupt_t=0.2682 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3117 corrupt_frac=0.8500 acc_corrupt=0.2463 loss_corrupt=4.2792 wrong_frac=0.7365 init_acc_corrupt=0.1897 acc_corrupt_t_0p4_0p6=0.4762 corrupt_frac_t_0p4_0p6=1.0000 out_w_norm=9.6246 out_g_norm=3.7357 acc_corrupt_t_0p0_0p2=0.0808 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p2_0p4=0.2545 corrupt_frac_t_0p2_0p4=1.0000 loss_all=2.5656 init_gold_top10=0.4475 init_gold_top100=0.4487
+step=900 epoch=450/500 epoch_step=2/2 micro_steps=900 elapsed=3.2s lr=2.000000e-03 loss=5.0254 loss_recon=5.0254 loss_meanflow=0.0000 mean_model_t=0.1950 mean_corrupt_t=0.1950 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2067 corrupt_frac=0.8675 acc_corrupt=0.1626 loss_corrupt=5.0254 wrong_frac=0.8093 init_acc_corrupt=0.1086 acc_corrupt_t_0p0_0p2=0.0668 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=9.7542 out_g_norm=4.1217 acc_corrupt_t_0p2_0p4=0.2661 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p6_0p8=0.5938 corrupt_frac_t_0p6_0p8=1.0000 acc_corrupt_t_0p4_0p6=0.5379 corrupt_frac_t_0p4_0p6=1.0000 loss_all=6.6078 init_gold_top10=0.1440 init_gold_top100=0.2411
+step=925 epoch=463/500 epoch_step=1/2 micro_steps=925 elapsed=6.7s lr=2.000000e-03 loss=4.7291 loss_recon=4.7291 loss_meanflow=0.0000 mean_model_t=0.2037 mean_corrupt_t=0.2037 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2236 corrupt_frac=0.8550 acc_corrupt=0.1681 loss_corrupt=4.7291 wrong_frac=0.7979 init_acc_corrupt=0.1045 acc_corrupt_t_0p2_0p4=0.2907 corrupt_frac_t_0p2_0p4=1.0000 out_w_norm=9.8717 out_g_norm=4.3022 acc_corrupt_t_0p0_0p2=0.0891 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p4_0p6=0.4844 corrupt_frac_t_0p4_0p6=1.0000 loss_all=6.5996 init_gold_top10=0.0480 init_gold_top100=0.1797
+step=950 epoch=475/500 epoch_step=2/2 micro_steps=950 elapsed=3.2s lr=2.000000e-03 loss=4.8115 loss_recon=4.8115 loss_meanflow=0.0000 mean_model_t=0.1864 mean_corrupt_t=0.1864 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2011 corrupt_frac=0.8275 acc_corrupt=0.1441 loss_corrupt=4.8115 wrong_frac=0.8129 init_acc_corrupt=0.0838 acc_corrupt_t_0p0_0p2=0.0880 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=9.9749 out_g_norm=4.6037 acc_corrupt_t_0p2_0p4=0.2572 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.4464 corrupt_frac_t_0p4_0p6=1.0000 loss_all=4.9126 init_gold_top10=0.0368 init_gold_top100=0.1462
+step=975 epoch=488/500 epoch_step=1/2 micro_steps=975 elapsed=6.0s lr=2.000000e-03 loss=4.6119 loss_recon=4.6119 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2337 corrupt_frac=0.8600 acc_corrupt=0.1779 loss_corrupt=4.6119 wrong_frac=0.7927 init_acc_corrupt=0.1185 acc_corrupt_t_0p6_0p8=0.7279 corrupt_frac_t_0p6_0p8=1.0000 out_w_norm=10.0757 out_g_norm=4.2819 acc_corrupt_t_0p0_0p2=0.0761 corrupt_frac_t_0p0_0p2=1.0000 acc_corrupt_t_0p2_0p4=0.2466 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.5458 corrupt_frac_t_0p4_0p6=1.0000 loss_all=6.7924 init_gold_top10=0.0292 init_gold_top100=0.1271
+step=1000 epoch=500/500 epoch_step=2/2 micro_steps=1000 elapsed=3.2s lr=2.000000e-03 loss=4.3429 loss_recon=4.3429 loss_meanflow=0.0000 mean_model_t=0.2305 mean_corrupt_t=0.2305 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2578 corrupt_frac=0.8625 acc_corrupt=0.2071 loss_corrupt=4.3429 wrong_frac=0.7682 init_acc_corrupt=0.1369 acc_corrupt_t_0p0_0p2=0.0725 corrupt_frac_t_0p0_0p2=1.0000 out_w_norm=10.1581 out_g_norm=3.9652 acc_corrupt_t_0p2_0p4=0.2456 corrupt_frac_t_0p2_0p4=1.0000 acc_corrupt_t_0p4_0p6=0.5545 corrupt_frac_t_0p4_0p6=1.0000 loss_all=3.6109 init_gold_top10=0.2455 init_gold_top100=0.2902

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/hf_xet-1.5.0.dist-info/INSTALLER

@@ -0,0 +1 @@
+uv

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/hf_xet-1.5.0.dist-info/METADATA

@@ -0,0 +1,87 @@
+Metadata-Version: 2.4
+Name: hf-xet
+Version: 1.5.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: Free Threading
+Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Dist: pytest ; extra == 'tests'
+Provides-Extra: tests
+License-File: LICENSE
+Summary: Fast transfer of large files with the Hugging Face Hub.
+Maintainer-email: Rajat Arya <rajat@rajatarya.com>, Jared Sulzdorf <j.sulzdorf@gmail.com>, Di Xiao <di@huggingface.co>, Assaf Vayner <assaf@huggingface.co>, Hoyt Koepke <hoytak@gmail.com>
+License-Expression: Apache-2.0
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Documentation, https://huggingface.co/docs/hub/xet/index
+Project-URL: Homepage, https://github.com/huggingface/xet-core
+Project-URL: Issues, https://github.com/huggingface/xet-core/issues
+Project-URL: Repository, https://github.com/huggingface/xet-core.git
+
+<!---
+Copyright 2024 The HuggingFace Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<p align="center">
+    <a href="https://github.com/huggingface/xet-core/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/github/license/huggingface/xet-core.svg?color=blue"></a>
+    <a href="https://github.com/huggingface/xet-core/releases"><img alt="GitHub release" src="https://img.shields.io/github/release/huggingface/xet-core.svg"></a>
+    <a href="https://github.com/huggingface/xet-core/blob/main/CODE_OF_CONDUCT.md"><img alt="Contributor Covenant" src="https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg"></a>
+</p>
+
+<h3 align="center">
+  <p>🤗 hf-xet - xet client tech, used in <a target="_blank" href="https://github.com/huggingface/huggingface_hub/">huggingface_hub</a></p>
+</h3>
+
+## Welcome
+
+`hf-xet` enables `huggingface_hub` to utilize xet storage for uploading and downloading to HF Hub. Xet storage provides chunk-based deduplication, efficient storage/retrieval with local disk caching, and backwards compatibility with Git LFS. This library is not meant to be used directly, and is instead intended to be used from [huggingface_hub](https://pypi.org/project/huggingface-hub).
+
+## Key features
+
+♻ **chunk-based deduplication implementation**: avoid transferring and storing chunks that are shared across binary files (models, datasets, etc).
+
+🤗 **Python bindings**: bindings for [huggingface_hub](https://github.com/huggingface/huggingface_hub/) package.
+
+↔ **network communications**: concurrent communication to HF Hub Xet backend services (CAS).
+
+🔖 **local disk caching**: chunk-based cache that sits alongside the existing [huggingface_hub disk cache](https://huggingface.co/docs/huggingface_hub/guides/manage-cache).
+
+## Installation
+
+Install the `hf_xet` package with [pip](https://pypi.org/project/hf-xet/):
+
+```bash
+pip install hf_xet
+```
+
+## Quick Start
+
+`hf_xet` is not intended to be run independently as it is expected to be used from `huggingface_hub`, so to get started with `huggingface_hub` check out the documentation [here]("https://hf.co/docs/huggingface_hub").
+
+## Contributions (feature requests, bugs, etc.) are encouraged & appreciated 💙💚💛💜🧡❤️
+
+Please join us in making hf-xet better. We value everyone's contributions. Code is not the only way to help. Answering questions, helping each other, improving documentation, filing issues all help immensely. If you are interested in contributing (please do!), check out the [contribution guide](https://github.com/huggingface/xet-core/blob/main/CONTRIBUTING.md) for this repository.

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/typer/core.py

@@ -0,0 +1,820 @@
+import errno
+import inspect
+import os
+import sys
+from collections.abc import Callable, MutableMapping, Sequence
+from difflib import get_close_matches
+from enum import Enum
+from gettext import gettext as _
+from typing import (
+    Any,
+    TextIO,
+    Union,
+    cast,
+)
+
+import click
+import click.core
+import click.formatting
+import click.shell_completion
+import click.types
+import click.utils
+
+from ._typing import Literal
+from .utils import parse_boolean_env_var
+
+MarkupMode = Literal["markdown", "rich", None]
+MARKUP_MODE_KEY = "TYPER_RICH_MARKUP_MODE"
+
+HAS_RICH = parse_boolean_env_var(os.getenv("TYPER_USE_RICH"), default=True)
+
+if HAS_RICH:
+    DEFAULT_MARKUP_MODE: MarkupMode = "rich"
+else:
+    DEFAULT_MARKUP_MODE = None
+
+
+# Copy from click.parser._split_opt
+def _split_opt(opt: str) -> tuple[str, str]:
+    first = opt[:1]
+    if first.isalnum():
+        return "", opt
+    if opt[1:2] == first:
+        return opt[:2], opt[2:]
+    return first, opt[1:]
+
+
+def _typer_param_setup_autocompletion_compat(
+    self: click.Parameter,
+    *,
+    autocompletion: Callable[
+        [click.Context, list[str], str], list[tuple[str, str] | str]
+    ]
+    | None = None,
+) -> None:
+    if self._custom_shell_complete is not None:
+        import warnings
+
+        warnings.warn(
+            "In Typer, only the parameter 'autocompletion' is supported. "
+            "The support for 'shell_complete' is deprecated and will be removed in upcoming versions. ",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+    if autocompletion is not None:
+
+        def compat_autocompletion(
+            ctx: click.Context, param: click.core.Parameter, incomplete: str
+        ) -> list["click.shell_completion.CompletionItem"]:
+            from click.shell_completion import CompletionItem
+
+            out = []
+
+            for c in autocompletion(ctx, [], incomplete):
+                if isinstance(c, tuple):
+                    use_completion = CompletionItem(c[0], help=c[1])
+                else:
+                    assert isinstance(c, str)
+                    use_completion = CompletionItem(c)
+
+                if use_completion.value.startswith(incomplete):
+                    out.append(use_completion)
+
+            return out
+
+        self._custom_shell_complete = compat_autocompletion
+
+
+def _get_default_string(
+    obj: Union["TyperArgument", "TyperOption"],
+    *,
+    ctx: click.Context,
+    show_default_is_str: bool,
+    default_value: list[Any] | tuple[Any, ...] | str | Callable[..., Any] | Any,
+) -> str:
+    # Extracted from click.core.Option.get_help_record() to be reused by
+    # rich_utils avoiding RegEx hacks
+    if show_default_is_str:
+        default_string = f"({obj.show_default})"
+    elif isinstance(default_value, (list, tuple)):
+        default_string = ", ".join(
+            _get_default_string(
+                obj, ctx=ctx, show_default_is_str=show_default_is_str, default_value=d
+            )
+            for d in default_value
+        )
+    elif isinstance(default_value, Enum):
+        default_string = str(default_value.value)
+    elif inspect.isfunction(default_value):
+        default_string = _("(dynamic)")
+    elif isinstance(obj, TyperOption) and obj.is_bool_flag and obj.secondary_opts:
+        # For boolean flags that have distinct True/False opts,
+        # use the opt without prefix instead of the value.
+        # Typer override, original commented
+        # default_string = click.parser.split_opt(
+        #     (self.opts if self.default else self.secondary_opts)[0]
+        # )[1]
+        if obj.default:
+            if obj.opts:
+                default_string = _split_opt(obj.opts[0])[1]
+            else:
+                default_string = str(default_value)
+        else:
+            default_string = _split_opt(obj.secondary_opts[0])[1]
+        # Typer override end
+    elif (
+        isinstance(obj, TyperOption)
+        and obj.is_bool_flag
+        and not obj.secondary_opts
+        and not default_value
+    ):
+        default_string = ""
+    else:
+        default_string = str(default_value)
+    return default_string
+
+
+def _extract_default_help_str(
+    obj: Union["TyperArgument", "TyperOption"], *, ctx: click.Context
+) -> Any | Callable[[], Any] | None:
+    # Extracted from click.core.Option.get_help_record() to be reused by
+    # rich_utils avoiding RegEx hacks
+    # Temporarily enable resilient parsing to avoid type casting
+    # failing for the default. Might be possible to extend this to
+    # help formatting in general.
+    resilient = ctx.resilient_parsing
+    ctx.resilient_parsing = True
+
+    try:
+        default_value = obj.get_default(ctx, call=False)
+    finally:
+        ctx.resilient_parsing = resilient
+    return default_value
+
+
+def _main(
+    self: click.Command,
+    *,
+    args: Sequence[str] | None = None,
+    prog_name: str | None = None,
+    complete_var: str | None = None,
+    standalone_mode: bool = True,
+    windows_expand_args: bool = True,
+    rich_markup_mode: MarkupMode = DEFAULT_MARKUP_MODE,
+    **extra: Any,
+) -> Any:
+    # Typer override, duplicated from click.main() to handle custom rich exceptions
+    # Verify that the environment is configured correctly, or reject
+    # further execution to avoid a broken script.
+    if args is None:
+        args = sys.argv[1:]
+
+        # Covered in Click tests
+        if os.name == "nt" and windows_expand_args:  # pragma: no cover
+            args = click.utils._expand_args(args)
+    else:
+        args = list(args)
+
+    if prog_name is None:
+        prog_name = click.utils._detect_program_name()
+
+    # Process shell completion requests and exit early.
+    self._main_shell_completion(extra, prog_name, complete_var)
+
+    try:
+        try:
+            with self.make_context(prog_name, args, **extra) as ctx:
+                rv = self.invoke(ctx)
+                if not standalone_mode:
+                    return rv
+                # it's not safe to `ctx.exit(rv)` here!
+                # note that `rv` may actually contain data like "1" which
+                # has obvious effects
+                # more subtle case: `rv=[None, None]` can come out of
+                # chained commands which all returned `None` -- so it's not
+                # even always obvious that `rv` indicates success/failure
+                # by its truthiness/falsiness
+                ctx.exit()
+        except EOFError as e:
+            click.echo(file=sys.stderr)
+            raise click.Abort() from e
+        except KeyboardInterrupt as e:
+            raise click.exceptions.Exit(130) from e
+        except click.ClickException as e:
+            if not standalone_mode:
+                raise
+            # Typer override
+            if HAS_RICH and rich_markup_mode is not None:
+                from . import rich_utils
+
+                rich_utils.rich_format_error(e)
+            else:
+                e.show()
+            # Typer override end
+            sys.exit(e.exit_code)
+        except OSError as e:
+            if e.errno == errno.EPIPE:
+                sys.stdout = cast(TextIO, click.utils.PacifyFlushWrapper(sys.stdout))
+                sys.stderr = cast(TextIO, click.utils.PacifyFlushWrapper(sys.stderr))
+                sys.exit(1)
+            else:
+                raise
+    except click.exceptions.Exit as e:
+        if standalone_mode:
+            sys.exit(e.exit_code)
+        else:
+            # in non-standalone mode, return the exit code
+            # note that this is only reached if `self.invoke` above raises
+            # an Exit explicitly -- thus bypassing the check there which
+            # would return its result
+            # the results of non-standalone execution may therefore be
+            # somewhat ambiguous: if there are codepaths which lead to
+            # `ctx.exit(1)` and to `return 1`, the caller won't be able to
+            # tell the difference between the two
+            return e.exit_code
+    except click.Abort:
+        if not standalone_mode:
+            raise
+        # Typer override
+        if HAS_RICH and rich_markup_mode is not None:
+            from . import rich_utils
+
+            rich_utils.rich_abort_error()
+        else:
+            click.echo(_("Aborted!"), file=sys.stderr)
+        # Typer override end
+        sys.exit(1)
+
+
+class TyperArgument(click.core.Argument):
+    def __init__(
+        self,
+        *,
+        # Parameter
+        param_decls: list[str],
+        type: Any | None = None,
+        required: bool | None = None,
+        default: Any | None = None,
+        callback: Callable[..., Any] | None = None,
+        nargs: int | None = None,
+        metavar: str | None = None,
+        expose_value: bool = True,
+        is_eager: bool = False,
+        envvar: str | list[str] | None = None,
+        # Note that shell_complete is not fully supported and will be removed in future versions
+        # TODO: Remove shell_complete in a future version (after 0.16.0)
+        shell_complete: Callable[
+            [click.Context, click.Parameter, str],
+            list["click.shell_completion.CompletionItem"] | list[str],
+        ]
+        | None = None,
+        autocompletion: Callable[..., Any] | None = None,
+        # TyperArgument
+        show_default: bool | str = True,
+        show_choices: bool = True,
+        show_envvar: bool = True,
+        help: str | None = None,
+        hidden: bool = False,
+        # Rich settings
+        rich_help_panel: str | None = None,
+    ):
+        self.help = help
+        self.show_default = show_default
+        self.show_choices = show_choices
+        self.show_envvar = show_envvar
+        self.hidden = hidden
+        self.rich_help_panel = rich_help_panel
+
+        super().__init__(
+            param_decls=param_decls,
+            type=type,
+            required=required,
+            default=default,
+            callback=callback,
+            nargs=nargs,
+            metavar=metavar,
+            expose_value=expose_value,
+            is_eager=is_eager,
+            envvar=envvar,
+            shell_complete=shell_complete,
+        )
+        _typer_param_setup_autocompletion_compat(self, autocompletion=autocompletion)
+
+    def _get_default_string(
+        self,
+        *,
+        ctx: click.Context,
+        show_default_is_str: bool,
+        default_value: list[Any] | tuple[Any, ...] | str | Callable[..., Any] | Any,
+    ) -> str:
+        return _get_default_string(
+            self,
+            ctx=ctx,
+            show_default_is_str=show_default_is_str,
+            default_value=default_value,
+        )
+
+    def _extract_default_help_str(
+        self, *, ctx: click.Context
+    ) -> Any | Callable[[], Any] | None:
+        return _extract_default_help_str(self, ctx=ctx)
+
+    def get_help_record(self, ctx: click.Context) -> tuple[str, str] | None:
+        # Modified version of click.core.Option.get_help_record()
+        # to support Arguments
+        if self.hidden:
+            return None
+        name = self.make_metavar(ctx=ctx)
+        help = self.help or ""
+        extra = []
+        if self.show_envvar:
+            envvar = self.envvar
+            # allow_from_autoenv is currently not supported in Typer for CLI Arguments
+            if envvar is not None:
+                var_str = (
+                    ", ".join(str(d) for d in envvar)
+                    if isinstance(envvar, (list, tuple))
+                    else envvar
+                )
+                extra.append(f"env var: {var_str}")
+
+        # Typer override:
+        # Extracted to _extract_default_help_str() to allow re-using it in rich_utils
+        default_value = self._extract_default_help_str(ctx=ctx)
+        # Typer override end
+
+        show_default_is_str = isinstance(self.show_default, str)
+
+        if show_default_is_str or (
+            default_value is not None and (self.show_default or ctx.show_default)
+        ):
+            # Typer override:
+            # Extracted to _get_default_string() to allow re-using it in rich_utils
+            default_string = self._get_default_string(
+                ctx=ctx,
+                show_default_is_str=show_default_is_str,
+                default_value=default_value,
+            )
+            # Typer override end
+            if default_string:
+                extra.append(_("default: {default}").format(default=default_string))
+        if self.required:
+            extra.append(_("required"))
+        if extra:
+            extra_str = "; ".join(extra)
+            extra_str = f"[{extra_str}]"
+            rich_markup_mode = None
+            if hasattr(ctx, "obj") and isinstance(ctx.obj, dict):
+                rich_markup_mode = ctx.obj.get(MARKUP_MODE_KEY, None)
+            if HAS_RICH and rich_markup_mode == "rich":
+                # This is needed for when we want to export to HTML
+                from . import rich_utils
+
+                extra_str = rich_utils.escape_before_html_export(extra_str)
+
+            help = f"{help}  {extra_str}" if help else f"{extra_str}"
+        return name, help
+
+    def make_metavar(self, ctx: click.Context) -> str:
+        # Modified version of click.core.Argument.make_metavar()
+        # to include Argument name
+        if self.metavar is not None:
+            var = self.metavar
+            if not self.required and not var.startswith("["):
+                var = f"[{var}]"
+            return var
+        var = (self.name or "").upper()
+        if not self.required:
+            var = f"[{var}]"
+        type_var = self.type.get_metavar(self, ctx=ctx)
+        if type_var:
+            var += f":{type_var}"
+        if self.nargs != 1:
+            var += "..."
+        return var
+
+    def value_is_missing(self, value: Any) -> bool:
+        return _value_is_missing(self, value)
+
+
+class TyperOption(click.core.Option):
+    def __init__(
+        self,
+        *,
+        # Parameter
+        param_decls: list[str],
+        type: click.types.ParamType | Any | None = None,
+        required: bool | None = None,
+        default: Any | None = None,
+        callback: Callable[..., Any] | None = None,
+        nargs: int | None = None,
+        metavar: str | None = None,
+        expose_value: bool = True,
+        is_eager: bool = False,
+        envvar: str | list[str] | None = None,
+        # Note that shell_complete is not fully supported and will be removed in future versions
+        # TODO: Remove shell_complete in a future version (after 0.16.0)
+        shell_complete: Callable[
+            [click.Context, click.Parameter, str],
+            list["click.shell_completion.CompletionItem"] | list[str],
+        ]
+        | None = None,
+        autocompletion: Callable[..., Any] | None = None,
+        # Option
+        show_default: bool | str = False,
+        prompt: bool | str = False,
+        confirmation_prompt: bool | str = False,
+        prompt_required: bool = True,
+        hide_input: bool = False,
+        is_flag: bool | None = None,
+        multiple: bool = False,
+        count: bool = False,
+        allow_from_autoenv: bool = True,
+        help: str | None = None,
+        hidden: bool = False,
+        show_choices: bool = True,
+        show_envvar: bool = False,
+        # Rich settings
+        rich_help_panel: str | None = None,
+    ):
+        super().__init__(
+            param_decls=param_decls,
+            type=type,
+            required=required,
+            default=default,
+            callback=callback,
+            nargs=nargs,
+            metavar=metavar,
+            expose_value=expose_value,
+            is_eager=is_eager,
+            envvar=envvar,
+            show_default=show_default,
+            prompt=prompt,
+            confirmation_prompt=confirmation_prompt,
+            hide_input=hide_input,
+            is_flag=is_flag,
+            multiple=multiple,
+            count=count,
+            allow_from_autoenv=allow_from_autoenv,
+            help=help,
+            hidden=hidden,
+            show_choices=show_choices,
+            show_envvar=show_envvar,
+            prompt_required=prompt_required,
+            shell_complete=shell_complete,
+        )
+        _typer_param_setup_autocompletion_compat(self, autocompletion=autocompletion)
+        self.rich_help_panel = rich_help_panel
+
+    def _get_default_string(
+        self,
+        *,
+        ctx: click.Context,
+        show_default_is_str: bool,
+        default_value: list[Any] | tuple[Any, ...] | str | Callable[..., Any] | Any,
+    ) -> str:
+        return _get_default_string(
+            self,
+            ctx=ctx,
+            show_default_is_str=show_default_is_str,
+            default_value=default_value,
+        )
+
+    def _extract_default_help_str(
+        self, *, ctx: click.Context
+    ) -> Any | Callable[[], Any] | None:
+        return _extract_default_help_str(self, ctx=ctx)
+
+    def make_metavar(self, ctx: click.Context) -> str:
+        return super().make_metavar(ctx=ctx)
+
+    def get_help_record(self, ctx: click.Context) -> tuple[str, str] | None:
+        # Duplicate all of Click's logic only to modify a single line, to allow boolean
+        # flags with only names for False values as it's currently supported by Typer
+        # Ref: https://typer.tiangolo.com/tutorial/parameter-types/bool/#only-names-for-false
+        if self.hidden:
+            return None
+
+        any_prefix_is_slash = False
+
+        def _write_opts(opts: Sequence[str]) -> str:
+            nonlocal any_prefix_is_slash
+
+            rv, any_slashes = click.formatting.join_options(opts)
+
+            if any_slashes:
+                any_prefix_is_slash = True
+
+            if not self.is_flag and not self.count:
+                rv += f" {self.make_metavar(ctx=ctx)}"
+
+            return rv
+
+        rv = [_write_opts(self.opts)]
+
+        if self.secondary_opts:
+            rv.append(_write_opts(self.secondary_opts))
+
+        help = self.help or ""
+        extra = []
+
+        if self.show_envvar:
+            envvar = self.envvar
+
+            if envvar is None:
+                if (
+                    self.allow_from_autoenv
+                    and ctx.auto_envvar_prefix is not None
+                    and self.name is not None
+                ):
+                    envvar = f"{ctx.auto_envvar_prefix}_{self.name.upper()}"
+
+            if envvar is not None:
+                var_str = (
+                    envvar
+                    if isinstance(envvar, str)
+                    else ", ".join(str(d) for d in envvar)
+                )
+                extra.append(_("env var: {var}").format(var=var_str))
+
+        # Typer override:
+        # Extracted to _extract_default() to allow re-using it in rich_utils
+        default_value = self._extract_default_help_str(ctx=ctx)
+        # Typer override end
+
+        show_default_is_str = isinstance(self.show_default, str)
+
+        if show_default_is_str or (
+            default_value is not None and (self.show_default or ctx.show_default)
+        ):
+            # Typer override:
+            # Extracted to _get_default_string() to allow re-using it in rich_utils
+            default_string = self._get_default_string(
+                ctx=ctx,
+                show_default_is_str=show_default_is_str,
+                default_value=default_value,
+            )
+            # Typer override end
+            if default_string:
+                extra.append(_("default: {default}").format(default=default_string))
+
+        if isinstance(self.type, click.types._NumberRangeBase):
+            range_str = self.type._describe_range()
+
+            if range_str:
+                extra.append(range_str)
+
+        if self.required:
+            extra.append(_("required"))
+
+        if extra:
+            extra_str = "; ".join(extra)
+            extra_str = f"[{extra_str}]"
+            rich_markup_mode = None
+            if hasattr(ctx, "obj") and isinstance(ctx.obj, dict):
+                rich_markup_mode = ctx.obj.get(MARKUP_MODE_KEY, None)
+            if HAS_RICH and rich_markup_mode == "rich":
+                # This is needed for when we want to export to HTML
+                from . import rich_utils
+
+                extra_str = rich_utils.escape_before_html_export(extra_str)
+
+            help = f"{help}  {extra_str}" if help else f"{extra_str}"
+
+        return ("; " if any_prefix_is_slash else " / ").join(rv), help
+
+    def value_is_missing(self, value: Any) -> bool:
+        return _value_is_missing(self, value)
+
+
+def _value_is_missing(param: click.Parameter, value: Any) -> bool:
+    if value is None:
+        return True
+
+    # Click 8.3 and beyond
+    # if value is UNSET:
+    #     return True
+
+    if (param.nargs != 1 or param.multiple) and value == ():
+        return True  # pragma: no cover
+
+    return False
+
+
+def _typer_format_options(
+    self: click.core.Command, *, ctx: click.Context, formatter: click.HelpFormatter
+) -> None:
+    args = []
+    opts = []
+    for param in self.get_params(ctx):
+        rv = param.get_help_record(ctx)
+        if rv is not None:
+            if param.param_type_name == "argument":
+                args.append(rv)
+            elif param.param_type_name == "option":
+                opts.append(rv)
+
+    if args:
+        with formatter.section(_("Arguments")):
+            formatter.write_dl(args)
+    if opts:
+        with formatter.section(_("Options")):
+            formatter.write_dl(opts)
+
+
+def _typer_main_shell_completion(
+    self: click.core.Command,
+    *,
+    ctx_args: MutableMapping[str, Any],
+    prog_name: str,
+    complete_var: str | None = None,
+) -> None:
+    if complete_var is None:
+        complete_var = f"_{prog_name}_COMPLETE".replace("-", "_").upper()
+
+    instruction = os.environ.get(complete_var)
+
+    if not instruction:
+        return
+
+    from .completion import shell_complete
+
+    rv = shell_complete(self, ctx_args, prog_name, complete_var, instruction)
+    sys.exit(rv)
+
+
+class TyperCommand(click.core.Command):
+    def __init__(
+        self,
+        name: str | None,
+        *,
+        context_settings: dict[str, Any] | None = None,
+        callback: Callable[..., Any] | None = None,
+        params: list[click.Parameter] | None = None,
+        help: str | None = None,
+        epilog: str | None = None,
+        short_help: str | None = None,
+        options_metavar: str | None = "[OPTIONS]",
+        add_help_option: bool = True,
+        no_args_is_help: bool = False,
+        hidden: bool = False,
+        deprecated: bool = False,
+        # Rich settings
+        rich_markup_mode: MarkupMode = DEFAULT_MARKUP_MODE,
+        rich_help_panel: str | None = None,
+    ) -> None:
+        super().__init__(
+            name=name,
+            context_settings=context_settings,
+            callback=callback,
+            params=params,
+            help=help,
+            epilog=epilog,
+            short_help=short_help,
+            options_metavar=options_metavar,
+            add_help_option=add_help_option,
+            no_args_is_help=no_args_is_help,
+            hidden=hidden,
+            deprecated=deprecated,
+        )
+        self.rich_markup_mode: MarkupMode = rich_markup_mode
+        self.rich_help_panel = rich_help_panel
+
+    def format_options(
+        self, ctx: click.Context, formatter: click.HelpFormatter
+    ) -> None:
+        _typer_format_options(self, ctx=ctx, formatter=formatter)
+
+    def _main_shell_completion(
+        self,
+        ctx_args: MutableMapping[str, Any],
+        prog_name: str,
+        complete_var: str | None = None,
+    ) -> None:
+        _typer_main_shell_completion(
+            self, ctx_args=ctx_args, prog_name=prog_name, complete_var=complete_var
+        )
+
+    def main(
+        self,
+        args: Sequence[str] | None = None,
+        prog_name: str | None = None,
+        complete_var: str | None = None,
+        standalone_mode: bool = True,
+        windows_expand_args: bool = True,
+        **extra: Any,
+    ) -> Any:
+        return _main(
+            self,
+            args=args,
+            prog_name=prog_name,
+            complete_var=complete_var,
+            standalone_mode=standalone_mode,
+            windows_expand_args=windows_expand_args,
+            rich_markup_mode=self.rich_markup_mode,
+            **extra,
+        )
+
+    def format_help(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
+        if not HAS_RICH or self.rich_markup_mode is None:
+            if not hasattr(ctx, "obj") or ctx.obj is None:
+                ctx.ensure_object(dict)
+            if isinstance(ctx.obj, dict):
+                ctx.obj[MARKUP_MODE_KEY] = self.rich_markup_mode
+            return super().format_help(ctx, formatter)
+        from . import rich_utils
+
+        return rich_utils.rich_format_help(
+            obj=self,
+            ctx=ctx,
+            markup_mode=self.rich_markup_mode,
+        )
+
+
+class TyperGroup(click.core.Group):
+    def __init__(
+        self,
+        *,
+        name: str | None = None,
+        commands: dict[str, click.Command] | Sequence[click.Command] | None = None,
+        # Rich settings
+        rich_markup_mode: MarkupMode = DEFAULT_MARKUP_MODE,
+        rich_help_panel: str | None = None,
+        suggest_commands: bool = True,
+        **attrs: Any,
+    ) -> None:
+        super().__init__(name=name, commands=commands, **attrs)
+        self.rich_markup_mode: MarkupMode = rich_markup_mode
+        self.rich_help_panel = rich_help_panel
+        self.suggest_commands = suggest_commands
+
+    def format_options(
+        self, ctx: click.Context, formatter: click.HelpFormatter
+    ) -> None:
+        _typer_format_options(self, ctx=ctx, formatter=formatter)
+        self.format_commands(ctx, formatter)
+
+    def _main_shell_completion(
+        self,
+        ctx_args: MutableMapping[str, Any],
+        prog_name: str,
+        complete_var: str | None = None,
+    ) -> None:
+        _typer_main_shell_completion(
+            self, ctx_args=ctx_args, prog_name=prog_name, complete_var=complete_var
+        )
+
+    def resolve_command(
+        self, ctx: click.Context, args: list[str]
+    ) -> tuple[str | None, click.Command | None, list[str]]:
+        try:
+            return super().resolve_command(ctx, args)
+        except click.UsageError as e:
+            if self.suggest_commands:
+                available_commands = list(self.commands.keys())
+                if available_commands and args:
+                    typo = args[0]
+                    matches = get_close_matches(typo, available_commands)
+                    if matches:
+                        suggestions = ", ".join(f"{m!r}" for m in matches)
+                        message = e.message.rstrip(".")
+                        e.message = f"{message}. Did you mean {suggestions}?"
+            raise
+
+    def main(
+        self,
+        args: Sequence[str] | None = None,
+        prog_name: str | None = None,
+        complete_var: str | None = None,
+        standalone_mode: bool = True,
+        windows_expand_args: bool = True,
+        **extra: Any,
+    ) -> Any:
+        return _main(
+            self,
+            args=args,
+            prog_name=prog_name,
+            complete_var=complete_var,
+            standalone_mode=standalone_mode,
+            windows_expand_args=windows_expand_args,
+            rich_markup_mode=self.rich_markup_mode,
+            **extra,
+        )
+
+    def format_help(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
+        if not HAS_RICH or self.rich_markup_mode is None:
+            return super().format_help(ctx, formatter)
+        from . import rich_utils
+
+        return rich_utils.rich_format_help(
+            obj=self,
+            ctx=ctx,
+            markup_mode=self.rich_markup_mode,
+        )
+
+    def list_commands(self, ctx: click.Context) -> list[str]:
+        """Returns a list of subcommand names.
+        Note that in Click's Group class, these are sorted.
+        In Typer, we wish to maintain the original order of creation (cf Issue #933)"""
+        return [n for n, c in self.commands.items()]

LTA_openwebtext_dualt/scripts/dirichlet_support_decode_probe.py

@@ -0,0 +1,347 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import Sequence
+
+import torch
+import torch.nn.functional as F
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from eval import build_model_from_ckpt
+from flowtext_lab.decode import model_time_for_step, sample_noise_simplex, state_for_model
+from flowtext_lab.tokenization import BpeTextTokenizer
+
+
+WORD_RE = re.compile(r"[A-Za-z]+|\d+|[^\sA-Za-z\d]")
+
+
+def encode_prompt(tokenizer: BpeTextTokenizer, prompt: str, max_len: int) -> list[int]:
+    return list(tokenizer.tokenizer.encode(prompt, add_special_tokens=False).ids)[:max_len]
+
+
+def decode_text(tokenizer: BpeTextTokenizer, ids: Sequence[int]) -> str:
+    return tokenizer.decode(ids, stop_at_eos=False, skip_special_tokens=False)
+
+
+def text_metrics(text: str) -> dict[str, float]:
+    toks = WORD_RE.findall(text)
+    words = [t.lower() for t in toks if re.fullmatch(r"[A-Za-z]+", t)]
+    n_tok = max(len(toks), 1)
+    n_words = max(len(words), 1)
+    wc = Counter(words)
+    max_word_frac = wc.most_common(1)[0][1] / n_words if wc else 1.0
+    grams3 = list(zip(toks, toks[1:], toks[2:]))
+    rep3 = sum(v - 1 for v in Counter(grams3).values() if v > 1) / max(len(grams3), 1)
+    bigrams = list(zip(words, words[1:]))
+    distinct2 = len(set(bigrams)) / max(len(bigrams), 1) if bigrams else 0.0
+    punct_frac = sum(bool(re.fullmatch(r"[,.;:!?]+", t)) for t in toks) / n_tok
+    digit_frac = sum(t.isdigit() for t in toks) / n_tok
+    quality = (
+        min(len(text) / 700.0, 1.0)
+        + 0.35 * distinct2
+        - 2.6 * rep3
+        - 1.2 * max_word_frac
+        - 0.8 * punct_frac
+        - 1.0 * digit_frac
+        - 0.2 * text.count("<|endoftext|>")
+        - 0.5 * text.count("�")
+    )
+    return {
+        "quality": float(quality),
+        "chars": float(len(text)),
+        "words": float(len(words)),
+        "rep3": float(rep3),
+        "distinct2": float(distinct2),
+        "punct_frac": float(punct_frac),
+        "max_word_frac": float(max_word_frac),
+        "eot_count": float(text.count("<|endoftext|>")),
+    }
+
+
+def dirichlet_mean(endpoint: torch.Tensor, support_t: float, eps: float) -> torch.Tensor:
+    vocab = endpoint.size(-1)
+    mean = (1.0 - support_t) / float(vocab) + support_t * endpoint
+    mean = mean.clamp_min(eps)
+    return mean / mean.sum(dim=-1, keepdim=True).clamp_min(eps)
+
+
+def total_concentration(support_t: float, c_min: float, c_max: float) -> float:
+    log_min = math.log(max(c_min, 1e-8))
+    log_max = math.log(max(c_max, c_min))
+    return math.exp(log_min + support_t * (log_max - log_min))
+
+
+def dirichlet_resample(mean: torch.Tensor, support_t: float, c_min: float, c_max: float, eps: float) -> torch.Tensor:
+    conc = total_concentration(support_t, c_min, c_max)
+    alpha = (mean * conc).clamp_min(eps)
+    sample = torch._standard_gamma(alpha).clamp_min(eps)
+    return sample / sample.sum(dim=-1, keepdim=True).clamp_min(eps)
+
+
+def schedule_power(step: int, steps: int, power: float) -> float:
+    base = (step + 1) / max(steps, 1)
+    return float(max(0.0, min(1.0, base ** float(power))))
+
+
+def current_anchor(probs: torch.Tensor, mode: str, eps: float) -> torch.Tensor:
+    if mode == "state":
+        return probs
+    if mode == "onehot":
+        ids = probs.argmax(dim=-1)
+        return F.one_hot(ids, probs.size(-1)).to(dtype=probs.dtype, device=probs.device)
+    if mode == "sqrt_state":
+        x = probs.clamp_min(eps).sqrt()
+        return x / x.sum(dim=-1, keepdim=True).clamp_min(eps)
+    raise ValueError(f"unknown anchor mode: {mode}")
+
+
+@torch.no_grad()
+def build_initial(
+    tokenizer: BpeTextTokenizer,
+    prompts: list[str],
+    restarts: int,
+    max_len: int,
+    eps: float,
+    noise_init: str,
+    target_prob: float,
+    noise_sigma: float,
+    dirichlet_concentration: float,
+    lock_bos: bool,
+    lock_final_eos: bool,
+    device: torch.device,
+) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, list[str]]:
+    expanded = []
+    prompt_ids = []
+    for prompt in prompts:
+        ids = encode_prompt(tokenizer, prompt, max_len)
+        if lock_bos:
+            ids = [tokenizer.bos_id] + ids
+        ids = ids[:max_len]
+        for _ in range(restarts):
+            expanded.append(prompt)
+            prompt_ids.append(ids)
+    batch = len(prompt_ids)
+    probs = sample_noise_simplex(
+        (batch, max_len),
+        tokenizer.vocab_size,
+        device,
+        eps,
+        noise_mode=noise_init,
+        target_prob=target_prob,
+        noise_sigma=noise_sigma,
+        dirichlet_concentration=dirichlet_concentration,
+    )
+    lock = torch.zeros((batch, max_len), dtype=torch.bool, device=device)
+    lock_probs = torch.zeros((batch, max_len, tokenizer.vocab_size), dtype=torch.float32, device=device)
+    for row, ids in enumerate(prompt_ids):
+        if not ids:
+            continue
+        ids_t = torch.tensor(ids, dtype=torch.long, device=device)
+        onehot = F.one_hot(ids_t, tokenizer.vocab_size).float()
+        probs[row, : len(ids)] = onehot
+        lock_probs[row, : len(ids)] = onehot
+        lock[row, : len(ids)] = True
+    if lock_final_eos:
+        eos = torch.tensor([tokenizer.eos_id], dtype=torch.long, device=device)
+        eos_prob = F.one_hot(eos, tokenizer.vocab_size).float()[0]
+        probs[:, -1] = eos_prob
+        lock_probs[:, -1] = eos_prob
+        lock[:, -1] = True
+    attn = torch.ones((batch, max_len), dtype=torch.bool, device=device)
+    return probs, lock, lock_probs, attn, expanded
+
+
+@torch.no_grad()
+def decode_one_config(
+    model,
+    tokenizer,
+    init,
+    lock,
+    lock_probs,
+    attn,
+    args,
+    update: str,
+    final_from: str,
+    temp: float,
+    model_t_mode: str,
+    support_power: float,
+    semantic_power: float,
+    anchor_mode: str,
+):
+    probs = init.clone()
+    last_endpoint = probs
+    device = probs.device
+    for step in range(args.steps):
+        model_t = model_time_for_step(model_t_mode, step, args.steps, probs.size(0), device, dtype=torch.float32)
+        logits = model(state_for_model(model, probs, args.eps), model_t, attn).float() / temp
+        endpoint = F.softmax(logits, dim=-1)
+        last_endpoint = endpoint
+        support_t = schedule_power(step, args.steps, support_power)
+        semantic_t = schedule_power(step, args.steps, semantic_power)
+        if update.startswith("dual_line"):
+            anchor = current_anchor(probs, anchor_mode, args.eps)
+            forward_endpoint = (1.0 - semantic_t) * anchor + semantic_t * endpoint
+            forward_endpoint = forward_endpoint / forward_endpoint.sum(dim=-1, keepdim=True).clamp_min(args.eps)
+        else:
+            forward_endpoint = endpoint
+        mean = dirichlet_mean(forward_endpoint, support_t, args.eps)
+        if update == "mean":
+            new_probs = mean
+        elif update == "resample":
+            new_probs = dirichlet_resample(mean, support_t, args.concentration_min, args.concentration_max, args.eps)
+        elif update == "dual_line_mean":
+            new_probs = mean
+        elif update == "dual_line_resample":
+            new_probs = dirichlet_resample(mean, support_t, args.concentration_min, args.concentration_max, args.eps)
+        elif update == "ema_mean":
+            gamma = 1.0 / max(args.steps - step, 1)
+            new_probs = (1.0 - gamma) * probs + gamma * mean
+            new_probs = new_probs / new_probs.sum(dim=-1, keepdim=True).clamp_min(args.eps)
+        else:
+            raise ValueError(update)
+        probs = torch.where(lock.unsqueeze(-1), lock_probs, new_probs)
+    if final_from == "endpoint":
+        out = last_endpoint
+    elif final_from == "blend":
+        out = 0.5 * probs + 0.5 * last_endpoint
+    else:
+        out = probs
+    out = torch.where(lock.unsqueeze(-1), lock_probs, out)
+    return out / out.sum(dim=-1, keepdim=True).clamp_min(args.eps)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--checkpoint", required=True)
+    ap.add_argument("--tokenizer_path", required=True)
+    ap.add_argument("--output", required=True)
+    ap.add_argument("--max_len", type=int, default=256)
+    ap.add_argument("--steps", type=int, default=256)
+    ap.add_argument("--restarts", type=int, default=4)
+    ap.add_argument("--prompts", nargs="+", default=[""])
+    ap.add_argument("--noise_init", default="dirichlet")
+    ap.add_argument("--target_prob", type=float, default=0.99)
+    ap.add_argument("--noise_sigma", type=float, default=-1.0)
+    ap.add_argument("--dirichlet_init_concentration", type=float, default=1.0)
+    ap.add_argument("--concentration_min", type=float, default=1.0)
+    ap.add_argument("--concentration_max", type=float, default=1024.0)
+    ap.add_argument("--updates", nargs="+", default=["mean", "ema_mean", "resample"])
+    ap.add_argument("--finals", nargs="+", default=["state", "endpoint", "blend"])
+    ap.add_argument("--temps", nargs="+", type=float, default=[1.0, 1.2, 1.35])
+    ap.add_argument("--model_t_modes", nargs="+", default=["flow", "const05"])
+    ap.add_argument("--support_powers", nargs="+", type=float, default=[1.0])
+    ap.add_argument("--semantic_powers", nargs="+", type=float, default=[1.0])
+    ap.add_argument("--anchor_modes", nargs="+", default=["onehot"])
+    ap.add_argument("--lock_bos", action="store_true")
+    ap.add_argument("--lock_final_eos", action="store_true")
+    ap.add_argument("--eps", type=float, default=1e-8)
+    ap.add_argument("--seed", type=int, default=1234)
+    args = ap.parse_args()
+
+    torch.manual_seed(args.seed)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    tokenizer = BpeTextTokenizer.from_file(args.tokenizer_path)
+    ckpt = torch.load(args.checkpoint, map_location=device)
+    model = build_model_from_ckpt(ckpt, tokenizer.vocab_size, args.max_len, device)
+    model.eval()
+    init, lock, lock_probs, attn, expanded = build_initial(
+        tokenizer,
+        args.prompts,
+        args.restarts,
+        args.max_len,
+        args.eps,
+        args.noise_init,
+        args.target_prob,
+        args.noise_sigma,
+        args.dirichlet_init_concentration,
+        args.lock_bos,
+        args.lock_final_eos,
+        device,
+    )
+
+    configs = []
+    for update in args.updates:
+        for final_from in args.finals:
+            for temp in args.temps:
+                for model_t_mode in args.model_t_modes:
+                    for support_power in args.support_powers:
+                        for semantic_power in args.semantic_powers:
+                            for anchor_mode in args.anchor_modes:
+                                configs.append((update, final_from, temp, model_t_mode, support_power, semantic_power, anchor_mode))
+
+    out_path = Path(args.output)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    rows = []
+    with out_path.open("w") as f:
+        for update, final_from, temp, model_t_mode, support_power, semantic_power, anchor_mode in configs:
+            probs = decode_one_config(
+                model,
+                tokenizer,
+                init,
+                lock,
+                lock_probs,
+                attn,
+                args,
+                update,
+                final_from,
+                temp,
+                model_t_mode,
+                support_power,
+                semantic_power,
+                anchor_mode,
+            )
+            ids = probs.argmax(dim=-1).detach().cpu().tolist()
+            texts = [decode_text(tokenizer, row) for row in ids]
+            mets = [text_metrics(t) for t in texts]
+            mean_q = sum(m["quality"] for m in mets) / len(mets)
+            best_i = max(range(len(texts)), key=lambda i: mets[i]["quality"])
+            row = {
+                "update": update,
+                "final_from": final_from,
+                "temp": temp,
+                "model_t_mode": model_t_mode,
+                "support_power": support_power,
+                "semantic_power": semantic_power,
+                "anchor_mode": anchor_mode,
+                "mean_quality": mean_q,
+                "best_prompt": expanded[best_i],
+                "best_metrics": mets[best_i],
+                "best_text": texts[best_i],
+            }
+            rows.append(row)
+            print(
+                "\n====",
+                update,
+                final_from,
+                temp,
+                model_t_mode,
+                "support_p",
+                support_power,
+                "semantic_p",
+                semantic_power,
+                "anchor",
+                anchor_mode,
+                "mean_q",
+                round(mean_q, 4),
+                flush=True,
+            )
+            print(texts[best_i][:1600], flush=True)
+            f.write(json.dumps(row, ensure_ascii=False) + "\n")
+            f.flush()
+    best = max(rows, key=lambda r: r["mean_quality"])
+    print("\nBEST", json.dumps({k: best[k] for k in best if k != "best_text"}, ensure_ascii=False, indent=2), flush=True)
+    print(best["best_text"], flush=True)
+
+
+if __name__ == "__main__":
+    main()

LTA_openwebtext_dualt/scripts/infer_softkl_decode_probe.py.bak_lognsr_gumbel_20260519

@@ -0,0 +1,960 @@
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import sys
+from dataclasses import asdict, is_dataclass
+from pathlib import Path
+
+import torch
+import torch.nn.functional as F
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from flowtext_lab.decode import model_time_for_step, sample_noise_simplex, state_for_model
+from flowtext_lab.genppl import filter_generated_texts, summarize_token_diversity
+from flowtext_lab.model import EndpointPredictor
+from flowtext_lab.tokenization import BpeTextTokenizer
+
+
+def extend_pos_embed(sd: dict[str, torch.Tensor], max_len: int, mode: str) -> dict[str, torch.Tensor]:
+    sd = dict(sd)
+    key = "pos_embed"
+    if key not in sd:
+        return sd
+    pos = sd[key]
+    old_len = int(pos.size(1))
+    if old_len == max_len:
+        return sd
+    if old_len > max_len:
+        sd[key] = pos[:, :max_len].contiguous()
+        return sd
+    if mode == "repeat":
+        reps = math.ceil(max_len / old_len)
+        sd[key] = pos.repeat(1, reps, 1)[:, :max_len].contiguous()
+    elif mode == "interpolate":
+        x = pos.transpose(1, 2)
+        y = F.interpolate(x, size=max_len, mode="linear", align_corners=True)
+        sd[key] = y.transpose(1, 2).contiguous()
+    else:
+        raise ValueError(f"unknown pos_extend={mode}")
+    return sd
+
+
+def build_model(ckpt: dict, tokenizer: BpeTextTokenizer, max_len: int, device: torch.device, pos_extend: str) -> EndpointPredictor:
+    a = ckpt.get("args", {})
+    ckpt_state = ckpt["model"]
+    if "output_bias" in a:
+        output_bias = bool(a["output_bias"])
+    else:
+        output_bias = "output_layer.linear.bias" in ckpt_state or "out_proj.bias" in ckpt_state
+    model = EndpointPredictor(
+        vocab_size=tokenizer.vocab_size,
+        max_len=max_len,
+        d_model=int(a.get("d_model", 768)),
+        n_heads=int(a.get("n_heads", 12)),
+        n_layers=int(a.get("n_layers", 12)),
+        dim_ff=int(a.get("dim_ff", 3072)),
+        dropout=0.0,
+        model_type=str(a.get("model_type", "ddit")),
+        cond_dim=int(a.get("cond_dim", 128)),
+        input_format=str(a.get("state_format", a.get("input_format", "prob"))),
+        output_bias=output_bias,
+        norm_type=str(a.get("norm_type", "layernorm")),
+        elf_num_time_tokens=int(a.get("elf_num_time_tokens", 4)),
+        elf_num_model_mode_tokens=int(a.get("elf_num_model_mode_tokens", 0)),
+        qk_norm=bool(a.get("qk_norm", True)),
+        output_init_std=a.get("output_init_std", None),
+        ddit_mlp_type=str(a.get("ddit_mlp_type", "gelu")),
+    ).to(device)
+    state = extend_pos_embed(ckpt_state, max_len=max_len, mode=pos_extend)
+    model.load_state_dict(state, strict=True)
+    model.eval()
+    return model
+
+
+def total_concentration(support_t: float, c_min: float, c_max: float) -> float:
+    return math.exp(math.log(max(c_min, 1e-8)) + support_t * (math.log(max(c_max, c_min)) - math.log(max(c_min, 1e-8))))
+
+
+def dirichlet_path_mean(endpoint: torch.Tensor, support_t: float, eps: float) -> torch.Tensor:
+    vocab = endpoint.size(-1)
+    mean = (1.0 - support_t) / float(vocab) + support_t * endpoint
+    mean = mean.clamp_min(eps)
+    return mean / mean.sum(dim=-1, keepdim=True).clamp_min(eps)
+
+
+def dirichlet_resample(mean: torch.Tensor, support_t: float, c_min: float, c_max: float, eps: float) -> torch.Tensor:
+    conc = total_concentration(support_t, c_min, c_max)
+    alpha = (mean * conc).clamp_min(eps)
+    sample = torch._standard_gamma(alpha).clamp_min(eps)
+    return sample / sample.sum(dim=-1, keepdim=True).clamp_min(eps)
+
+
+def current_anchor(probs: torch.Tensor, mode: str, eps: float) -> torch.Tensor:
+    if mode == "onehot":
+        return F.one_hot(probs.argmax(dim=-1), probs.size(-1)).to(dtype=probs.dtype)
+    if mode == "sqrt_state":
+        anchor = probs.clamp_min(eps).sqrt()
+    else:
+        anchor = probs.clamp_min(eps)
+    return anchor / anchor.sum(dim=-1, keepdim=True).clamp_min(eps)
+
+
+def log_geodesic_mix(p: torch.Tensor, q: torch.Tensor, gamma: float, eps: float) -> torch.Tensor:
+    log_mix = (1.0 - gamma) * p.clamp_min(eps).log() + gamma * q.clamp_min(eps).log()
+    return torch.softmax(log_mix, dim=-1)
+
+
+def sqrt_geodesic_mix(p: torch.Tensor, q: torch.Tensor, gamma: float, eps: float) -> torch.Tensor:
+    root = (1.0 - gamma) * p.clamp_min(eps).sqrt() + gamma * q.clamp_min(eps).sqrt()
+    out = root.square().clamp_min(eps)
+    return out / out.sum(dim=-1, keepdim=True).clamp_min(eps)
+
+
+def fisher_rao_mix(p: torch.Tensor, q: torch.Tensor, gamma: float, eps: float) -> torch.Tensor:
+    a = p.clamp_min(eps).sqrt()
+    b = q.clamp_min(eps).sqrt()
+    dot = (a * b).sum(dim=-1, keepdim=True).clamp(-1.0 + 1e-6, 1.0 - 1e-6)
+    theta = torch.acos(dot)
+    sin_theta = torch.sin(theta).clamp_min(1e-6)
+    left = torch.sin((1.0 - gamma) * theta) / sin_theta
+    right = torch.sin(gamma * theta) / sin_theta
+    root = left * a + right * b
+    out = root.square().clamp_min(eps)
+    return out / out.sum(dim=-1, keepdim=True).clamp_min(eps)
+
+
+def simplex_mix(p: torch.Tensor, q: torch.Tensor, gamma: float, eps: float, geometry: str) -> torch.Tensor:
+    if geometry == "log":
+        return log_geodesic_mix(p, q, gamma, eps)
+    if geometry == "sqrt":
+        return sqrt_geodesic_mix(p, q, gamma, eps)
+    if geometry == "fisher":
+        return fisher_rao_mix(p, q, gamma, eps)
+    if geometry == "linear":
+        out = (1.0 - gamma) * p + gamma * q
+        out = out.clamp_min(eps)
+        return out / out.sum(dim=-1, keepdim=True).clamp_min(eps)
+    raise ValueError(geometry)
+
+
+def temperature(step: int, steps: int, early: float, late: float, temp_end: float, power: float) -> float:
+    progress = step / max(steps, 1)
+    if progress >= temp_end:
+        return late
+    rel = 1.0 - progress / max(temp_end, 1e-8)
+    return late + (early - late) * (rel ** power)
+
+
+def make_time_grid(
+    steps: int,
+    *,
+    schedule: str,
+    logit_mean: float,
+    logit_std: float,
+    power: float,
+    seed: int,
+    device: torch.device,
+) -> torch.Tensor:
+    if steps <= 0:
+        raise ValueError(f"steps must be positive, got {steps}")
+    if schedule == "uniform":
+        return torch.linspace(0.0, 1.0, steps + 1, device=device, dtype=torch.float32)
+    if schedule == "logit_normal":
+        if steps == 1:
+            return torch.tensor([0.0, 1.0], device=device, dtype=torch.float32)
+        generator = torch.Generator(device="cpu")
+        generator.manual_seed(int(seed))
+        z = torch.randn((steps - 1,), generator=generator, dtype=torch.float32)
+        middle = torch.sigmoid(z * float(logit_std) + float(logit_mean)).sort().values.to(device)
+        return torch.cat(
+            [
+                torch.zeros((1,), device=device, dtype=torch.float32),
+                middle,
+                torch.ones((1,), device=device, dtype=torch.float32),
+            ]
+        )
+    if schedule in {"power_low", "power_high"}:
+        if steps == 1:
+            return torch.tensor([0.0, 1.0], device=device, dtype=torch.float32)
+        generator = torch.Generator(device="cpu")
+        generator.manual_seed(int(seed))
+        u = torch.rand((steps - 1,), generator=generator, dtype=torch.float32)
+        exponent = max(float(power), 1e-8)
+        if schedule == "power_low":
+            middle = u.pow(exponent)
+        else:
+            middle = 1.0 - (1.0 - u).pow(exponent)
+        middle = middle.sort().values.to(device)
+        return torch.cat(
+            [
+                torch.zeros((1,), device=device, dtype=torch.float32),
+                middle,
+                torch.ones((1,), device=device, dtype=torch.float32),
+            ]
+        )
+    raise ValueError(f"unknown time schedule: {schedule}")
+
+
+def clamp_first_position(probs: torch.Tensor, first_ids: torch.Tensor | None) -> torch.Tensor:
+    if first_ids is None:
+        return probs
+    probs = probs.clone()
+    probs[:, 0, :].zero_()
+    probs[:, 0, :].scatter_(1, first_ids[:, None], 1.0)
+    return probs
+
+
+def final_decode_ids(
+    probs: torch.Tensor,
+    *,
+    mode: str,
+    temp: float,
+    top_k: int,
+    top_p: float,
+    eps: float,
+) -> torch.Tensor:
+    if mode == "argmax":
+        return probs.argmax(dim=-1)
+    if mode != "sample":
+        raise ValueError(mode)
+    logits = probs.clamp_min(eps).log() / max(float(temp), eps)
+    if top_k > 0 and top_k < logits.size(-1):
+        kth = logits.topk(top_k, dim=-1).values[..., -1, None]
+        logits = logits.masked_fill(logits < kth, -torch.inf)
+    if 0.0 < top_p < 1.0:
+        sorted_logits, sorted_idx = logits.sort(dim=-1, descending=True)
+        sorted_probs = F.softmax(sorted_logits, dim=-1)
+        remove = sorted_probs.cumsum(dim=-1) > float(top_p)
+        remove[..., 0] = False
+        sorted_logits = sorted_logits.masked_fill(remove, -torch.inf)
+        filtered = torch.full_like(logits, -torch.inf)
+        logits = filtered.scatter(-1, sorted_idx, sorted_logits)
+    sample_probs = F.softmax(logits, dim=-1)
+    flat = sample_probs.reshape(-1, sample_probs.size(-1))
+    return torch.multinomial(flat, num_samples=1).view(probs.shape[:-1])
+
+
+def onehot_from_ids(ids: torch.Tensor, vocab_size: int, dtype: torch.dtype) -> torch.Tensor:
+    return F.one_hot(ids.clamp_min(0), vocab_size).to(dtype=dtype)
+
+
+def apply_committed_state(
+    probs: torch.Tensor,
+    committed: torch.Tensor,
+    committed_ids: torch.Tensor,
+    eps: float,
+) -> torch.Tensor:
+    if not bool(committed.any()):
+        return probs
+    fixed = onehot_from_ids(committed_ids, probs.size(-1), probs.dtype)
+    out = torch.where(committed.unsqueeze(-1), fixed, probs)
+    out = out.clamp_min(eps)
+    return out / out.sum(dim=-1, keepdim=True).clamp_min(eps)
+
+
+def commit_target_ratio(
+    progress: float,
+    *,
+    start: float,
+    min_ratio: float,
+    max_ratio: float,
+    power: float,
+) -> float:
+    start = min(max(float(start), 0.0), 0.999999)
+    if progress < start:
+        return 0.0
+    rel = (min(max(float(progress), start), 1.0) - start) / max(1.0 - start, 1e-8)
+    ratio = float(min_ratio) + (float(max_ratio) - float(min_ratio)) * (rel ** max(float(power), 1e-8))
+    return min(max(ratio, 0.0), 1.0)
+
+
+def select_commit_positions(
+    endpoint: torch.Tensor,
+    committed: torch.Tensor,
+    *,
+    mode: str,
+    step: int,
+    steps: int,
+    progress: float,
+    threshold: float,
+    margin_threshold: float,
+    start: float,
+    min_ratio: float,
+    max_ratio: float,
+    power: float,
+    freq_max_frac: float,
+) -> tuple[torch.Tensor, torch.Tensor, dict[str, float]]:
+    top2 = endpoint.topk(k=2, dim=-1)
+    conf = top2.values[..., 0]
+    margin = top2.values[..., 0] - top2.values[..., 1]
+    pred_ids = top2.indices[..., 0]
+    candidates = ~committed
+    final_step = step == steps - 1
+    if mode == "off":
+        new_commit = torch.zeros_like(committed)
+        return new_commit, pred_ids, {"target_ratio": 0.0, "mean_conf_new": 0.0, "mean_margin_new": 0.0}
+    if final_step:
+        new_commit = candidates
+        return new_commit, pred_ids, {
+            "target_ratio": 1.0,
+            "mean_conf_new": float(conf[new_commit].mean().detach().cpu()) if bool(new_commit.any()) else 0.0,
+            "mean_margin_new": float(margin[new_commit].mean().detach().cpu()) if bool(new_commit.any()) else 0.0,
+        }
+    if progress < start:
+        new_commit = torch.zeros_like(committed)
+        return new_commit, pred_ids, {"target_ratio": 0.0, "mean_conf_new": 0.0, "mean_margin_new": 0.0}
+
+    if mode == "threshold":
+        new_commit = candidates & (conf >= float(threshold)) & (margin >= float(margin_threshold))
+    elif mode in {"ratio", "ratio_margin_freq"}:
+        target_ratio = commit_target_ratio(
+            progress,
+            start=start,
+            min_ratio=min_ratio,
+            max_ratio=max_ratio,
+            power=power,
+        )
+        bs, length = committed.shape
+        new_commit = torch.zeros_like(committed)
+        score = conf if mode == "ratio" else margin
+        for b in range(bs):
+            already = int(committed[b].sum().detach().cpu())
+            target_count = min(length, int(math.ceil(float(length) * target_ratio)))
+            need = max(0, target_count - already)
+            if need <= 0:
+                continue
+            cand_idx = torch.nonzero(candidates[b], as_tuple=False).flatten()
+            if cand_idx.numel() == 0:
+                continue
+            valid = cand_idx[(conf[b, cand_idx] >= float(threshold)) & (margin[b, cand_idx] >= float(margin_threshold))]
+            if valid.numel() == 0:
+                continue
+            order = torch.argsort(score[b, valid], descending=True)
+            valid = valid[order]
+            if mode == "ratio":
+                chosen = valid[:need]
+            else:
+                chosen_list: list[int] = []
+                max_per_token = max(1, int(math.floor(float(length) * float(freq_max_frac))))
+                counts: dict[int, int] = {}
+                if bool(committed[b].any()):
+                    ids = pred_ids[b, committed[b]].detach().cpu().tolist()
+                    for tok in ids:
+                        counts[int(tok)] = counts.get(int(tok), 0) + 1
+                for idx_t in valid.detach().cpu().tolist():
+                    tok = int(pred_ids[b, idx_t].detach().cpu())
+                    if counts.get(tok, 0) >= max_per_token:
+                        continue
+                    chosen_list.append(int(idx_t))
+                    counts[tok] = counts.get(tok, 0) + 1
+                    if len(chosen_list) >= need:
+                        break
+                if chosen_list:
+                    chosen = torch.tensor(chosen_list, dtype=torch.long, device=committed.device)
+                else:
+                    chosen = valid[:0]
+            if chosen.numel() > 0:
+                new_commit[b, chosen] = True
+    else:
+        raise ValueError(mode)
+
+    return new_commit, pred_ids, {
+        "target_ratio": (
+            0.0
+            if mode == "threshold"
+            else commit_target_ratio(progress, start=start, min_ratio=min_ratio, max_ratio=max_ratio, power=power)
+        ),
+        "mean_conf_new": float(conf[new_commit].mean().detach().cpu()) if bool(new_commit.any()) else 0.0,
+        "mean_margin_new": float(margin[new_commit].mean().detach().cpu()) if bool(new_commit.any()) else 0.0,
+    }
+
+
+def soften_endpoint_with_prior(
+    endpoint: torch.Tensor,
+    t: float,
+    *,
+    mode: str,
+    power: float,
+    min_conf: float,
+    max_conf: float,
+    eps: float,
+) -> tuple[torch.Tensor, float]:
+    if mode == "none":
+        return endpoint, 1.0
+    if mode != "uniform":
+        raise ValueError(mode)
+    alpha = float(min_conf) + (float(max_conf) - float(min_conf)) * (float(t) ** float(power))
+    alpha = max(0.0, min(1.0, alpha))
+    prior = 1.0 / float(endpoint.shape[-1])
+    softened = alpha * endpoint + (1.0 - alpha) * prior
+    softened = softened.clamp_min(eps)
+    softened = softened / softened.sum(dim=-1, keepdim=True).clamp_min(eps)
+    return softened, alpha
+
+
+def linear_soft_target_confidence(
+    t: float,
+    *,
+    power: float,
+    min_conf: float,
+    max_conf: float,
+) -> float:
+    alpha = float(min_conf) + (float(max_conf) - float(min_conf)) * (float(t) ** float(power))
+    return max(0.0, min(1.0, alpha))
+
+
+def decode_linear_soft_target_endpoint(
+    posterior: torch.Tensor,
+    t: float,
+    *,
+    mode: str,
+    power: float,
+    min_conf: float,
+    max_conf: float,
+    debias_start: float,
+    eps: float,
+) -> tuple[torch.Tensor, float]:
+    """Interpret logits trained with q_t=(1-alpha)Uniform+alpha*onehot.
+
+    posterior mode uses q_hat directly. Debias modes invert the linear target
+    only when alpha is large enough; early inversion amplifies noise.
+    """
+    alpha = linear_soft_target_confidence(t, power=power, min_conf=min_conf, max_conf=max_conf)
+    if mode == "posterior" or (mode == "late_debias" and float(t) < float(debias_start)):
+        return posterior, alpha
+    if mode not in {"debias", "late_debias"}:
+        raise ValueError(mode)
+    prior = 1.0 / float(posterior.shape[-1])
+    denom = max(alpha, eps)
+    endpoint = (posterior - (1.0 - alpha) * prior) / denom
+    endpoint = endpoint.clamp_min(eps)
+    endpoint = endpoint / endpoint.sum(dim=-1, keepdim=True).clamp_min(eps)
+    return endpoint, alpha
+
+
+@torch.inference_mode()
+def decode(
+    model: EndpointPredictor,
+    tokenizer: BpeTextTokenizer,
+    *,
+    max_len: int,
+    n_samples: int,
+    batch_size: int,
+    steps: int,
+    seed: int,
+    device: torch.device,
+    decode_rule: str,
+    support_power: float,
+    semantic_power: float,
+    early_temp: float,
+    late_temp: float,
+    temp_end: float,
+    temp_power: float,
+    hybrid_switch: float,
+    tail_temp: float,
+    c_min: float,
+    c_max: float,
+    anchor_mode: str,
+    model_t_mode: str,
+    time_schedule: str,
+    time_logit_mean: float,
+    time_logit_std: float,
+    time_power: float,
+    input_noise_scale: float,
+    input_noise_until: float,
+    input_noise_dirichlet_concentration: float,
+    endpoint_softening: str,
+    endpoint_soft_power: float,
+    endpoint_soft_min_conf: float,
+    endpoint_soft_max_conf: float,
+    soft_target_decode_mode: str,
+    soft_target_power: float,
+    soft_target_min_conf: float,
+    soft_target_max_conf: float,
+    soft_target_debias_start: float,
+    final_from: str,
+    final_decode: str,
+    final_sample_temp: float,
+    final_top_k: int,
+    final_top_p: float,
+    commit_mode: str,
+    commit_conf_threshold: float,
+    commit_margin_threshold: float,
+    commit_start: float,
+    commit_min_ratio: float,
+    commit_max_ratio: float,
+    commit_power: float,
+    commit_freq_max_frac: float,
+    eps: float,
+    fixed_first_token_id: int | None,
+    fixed_first_initial_argmax: bool,
+) -> tuple[list[list[int]], list[str], list[dict[str, object]]]:
+    torch.manual_seed(seed)
+    time_grid = make_time_grid(
+        steps,
+        schedule=time_schedule,
+        logit_mean=time_logit_mean,
+        logit_std=time_logit_std,
+        power=time_power,
+        seed=seed,
+        device=device,
+    )
+    all_ids: list[list[int]] = []
+    all_texts: list[str] = []
+    traces: list[dict[str, object]] = []
+    remaining = n_samples
+    while remaining > 0:
+        bs = min(batch_size, remaining)
+        probs = sample_noise_simplex(
+            (bs, max_len),
+            tokenizer.vocab_size,
+            device,
+            eps,
+            noise_mode="dirichlet",
+            target_prob=1.0,
+            noise_sigma=-1.0,
+            dirichlet_concentration=1.0,
+        )
+        fixed_first_ids: torch.Tensor | None = None
+        if fixed_first_initial_argmax:
+            fixed_first_ids = probs[:, 0, :].argmax(dim=-1)
+        elif fixed_first_token_id is not None:
+            fixed_first_ids = torch.full((bs,), int(fixed_first_token_id), dtype=torch.long, device=device)
+        probs = clamp_first_position(probs, fixed_first_ids)
+        committed = torch.zeros((bs, max_len), dtype=torch.bool, device=device)
+        committed_ids = torch.zeros((bs, max_len), dtype=torch.long, device=device)
+        if fixed_first_ids is not None:
+            committed[:, 0] = True
+            committed_ids[:, 0] = fixed_first_ids
+        attn = torch.ones((bs, max_len), dtype=torch.bool, device=device)
+        last_endpoint = probs
+        for step in range(steps):
+            progress = float(time_grid[step].item())
+            next_progress = float(time_grid[step + 1].item())
+            dt = max(next_progress - progress, 0.0)
+            if model_t_mode in {"pre", "flow"}:
+                t = torch.full((bs,), float(progress), dtype=torch.float32, device=device)
+            elif model_t_mode == "post":
+                t = torch.full((bs,), float(next_progress), dtype=torch.float32, device=device)
+            else:
+                t = model_time_for_step(model_t_mode, step, steps, bs, device, dtype=torch.float32)
+            temp = temperature(step, steps, early_temp, late_temp, temp_end, temp_power)
+            if tail_temp > 0 and progress >= hybrid_switch:
+                temp = tail_temp
+            model_probs = probs
+            if input_noise_scale > 0.0 and progress < input_noise_until:
+                fresh_noise = sample_noise_simplex(
+                    (bs, max_len),
+                    tokenizer.vocab_size,
+                    device,
+                    eps,
+                    noise_mode="dirichlet",
+                    target_prob=1.0,
+                    noise_sigma=-1.0,
+                    dirichlet_concentration=input_noise_dirichlet_concentration,
+                )
+                noisy = progress * probs + (1.0 - progress) * float(input_noise_scale) * fresh_noise
+                model_probs = noisy.clamp_min(eps)
+                model_probs = model_probs / model_probs.sum(dim=-1, keepdim=True).clamp_min(eps)
+            logits = model(state_for_model(model, model_probs, eps), t, attn).float()
+            raw_endpoint = F.softmax(logits / temp, dim=-1)
+            if soft_target_decode_mode == "off":
+                endpoint, endpoint_alpha = soften_endpoint_with_prior(
+                    raw_endpoint,
+                    next_progress,
+                    mode=endpoint_softening,
+                    power=endpoint_soft_power,
+                    min_conf=endpoint_soft_min_conf,
+                    max_conf=endpoint_soft_max_conf,
+                    eps=eps,
+                )
+            else:
+                endpoint, endpoint_alpha = decode_linear_soft_target_endpoint(
+                    raw_endpoint,
+                    next_progress,
+                    mode=soft_target_decode_mode,
+                    power=soft_target_power,
+                    min_conf=soft_target_min_conf,
+                    max_conf=soft_target_max_conf,
+                    debias_start=soft_target_debias_start,
+                    eps=eps,
+                )
+            new_commit, pred_ids, commit_stats = select_commit_positions(
+                endpoint,
+                committed,
+                mode=commit_mode,
+                step=step,
+                steps=steps,
+                progress=next_progress,
+                threshold=commit_conf_threshold,
+                margin_threshold=commit_margin_threshold,
+                start=commit_start,
+                min_ratio=commit_min_ratio,
+                max_ratio=commit_max_ratio,
+                power=commit_power,
+                freq_max_frac=commit_freq_max_frac,
+            )
+            if bool(new_commit.any()):
+                committed_ids = torch.where(new_commit, pred_ids, committed_ids)
+                committed = committed | new_commit
+            last_endpoint = endpoint
+            support_t = next_progress ** support_power
+            if decode_rule == "dirichlet_resample":
+                probs = dirichlet_resample(dirichlet_path_mean(endpoint, support_t, eps), support_t, c_min, c_max, eps)
+            elif decode_rule == "dual_line_resample":
+                semantic_t = next_progress ** semantic_power
+                anchor = current_anchor(probs, anchor_mode, eps)
+                forward_endpoint = (1.0 - semantic_t) * anchor + semantic_t * endpoint
+                forward_endpoint = forward_endpoint.clamp_min(eps)
+                forward_endpoint = forward_endpoint / forward_endpoint.sum(dim=-1, keepdim=True).clamp_min(eps)
+                probs = dirichlet_resample(dirichlet_path_mean(forward_endpoint, support_t, eps), support_t, c_min, c_max, eps)
+            elif decode_rule == "dual_replace_resample":
+                semantic_t = next_progress ** semantic_power
+                anchor = current_anchor(probs, anchor_mode, eps)
+                replace = torch.rand((bs, max_len, 1), device=device) < semantic_t
+                forward_endpoint = torch.where(replace, endpoint, anchor)
+                forward_endpoint = forward_endpoint.clamp_min(eps)
+                forward_endpoint = forward_endpoint / forward_endpoint.sum(dim=-1, keepdim=True).clamp_min(eps)
+                probs = dirichlet_resample(dirichlet_path_mean(forward_endpoint, support_t, eps), support_t, c_min, c_max, eps)
+            elif decode_rule in {"log_dual_resample", "sqrt_dual_resample", "fisher_dual_resample"}:
+                geometry = decode_rule.split("_", 1)[0]
+                semantic_t = next_progress ** semantic_power
+                anchor = current_anchor(probs, anchor_mode, eps)
+                forward_endpoint = simplex_mix(anchor, endpoint, semantic_t, eps, geometry)
+                probs = dirichlet_resample(dirichlet_path_mean(forward_endpoint, support_t, eps), support_t, c_min, c_max, eps)
+            elif decode_rule == "flowmap":
+                gamma = min(dt / max(1.0 - progress, eps), 1.0)
+                probs = probs + gamma * (endpoint - probs)
+                probs = probs.clamp_min(eps)
+                probs = probs / probs.sum(dim=-1, keepdim=True).clamp_min(eps)
+            elif decode_rule in {"log_geodesic", "sqrt_geodesic", "fisher_geodesic"}:
+                geometry = decode_rule.split("_", 1)[0]
+                gamma = min(dt / max(1.0 - progress, eps), 1.0)
+                probs = simplex_mix(probs, endpoint, gamma, eps, geometry)
+            elif decode_rule in {"hybrid_log_flowmap", "hybrid_log_dirres", "hybrid_log_logflow"}:
+                if progress < hybrid_switch:
+                    local = min(1.0, next_progress / max(hybrid_switch, 1e-8))
+                    semantic_t = local ** semantic_power
+                    anchor = current_anchor(probs, anchor_mode, eps)
+                    forward_endpoint = simplex_mix(anchor, endpoint, semantic_t, eps, "log")
+                    probs = dirichlet_resample(dirichlet_path_mean(forward_endpoint, support_t, eps), support_t, c_min, c_max, eps)
+                elif decode_rule == "hybrid_log_flowmap":
+                    gamma = min(dt / max(1.0 - progress, eps), 1.0)
+                    probs = simplex_mix(probs, endpoint, gamma, eps, "linear")
+                elif decode_rule == "hybrid_log_logflow":
+                    gamma = min(dt / max(1.0 - progress, eps), 1.0)
+                    probs = simplex_mix(probs, endpoint, gamma, eps, "log")
+                else:
+                    probs = dirichlet_resample(dirichlet_path_mean(endpoint, support_t, eps), support_t, c_min, c_max, eps)
+            else:
+                raise ValueError(decode_rule)
+            probs = clamp_first_position(probs, fixed_first_ids)
+            probs = apply_committed_state(probs, committed, committed_ids, eps)
+            if step in {0, 1, 3, 7, 15, 31, 63, steps - 1}:
+                ids0 = probs.argmax(dim=-1)[0].detach().cpu().tolist()
+                raw_maxprob = raw_endpoint[0].amax(dim=-1).mean().detach().item()
+                soft_maxprob = endpoint[0].amax(dim=-1).mean().detach().item()
+                commit_frac = committed[0].float().mean().detach().item()
+                new_commit_frac = new_commit[0].float().mean().detach().item()
+                traces.append({
+                    "step": step + 1,
+                    "progress": progress,
+                    "next_progress": next_progress,
+                    "dt": dt,
+                    "temperature": temp,
+                    "endpoint_alpha": endpoint_alpha,
+                    "raw_endpoint_mean_maxprob": raw_maxprob,
+                    "effective_endpoint_mean_maxprob": soft_maxprob,
+                    "commit_mode": commit_mode,
+                    "commit_frac": commit_frac,
+                    "new_commit_frac": new_commit_frac,
+                    **commit_stats,
+                    "sample0_text": tokenizer.decode(ids0, stop_at_eos=False, skip_special_tokens=False)[:480],
+                })
+        if final_from == "state":
+            final = probs
+        elif final_from == "endpoint":
+            final = last_endpoint
+        elif final_from == "blend":
+            final = 0.5 * probs + 0.5 * last_endpoint
+        else:
+            raise ValueError(final_from)
+        final = clamp_first_position(final, fixed_first_ids)
+        ids_tensor = final_decode_ids(
+            final,
+            mode=final_decode,
+            temp=final_sample_temp,
+            top_k=final_top_k,
+            top_p=final_top_p,
+            eps=eps,
+        )
+        ids = ids_tensor.detach().cpu().tolist()
+        all_ids.extend(ids)
+        all_texts.extend(tokenizer.decode(row, stop_at_eos=False, skip_special_tokens=False) for row in ids)
+        remaining -= bs
+        print(f"[decode] max_len={max_len} generated={n_samples-remaining}/{n_samples}", flush=True)
+    return all_ids, all_texts, traces
+
+
+def score_with_gpt2(texts: list[str], scorer_path: str, batch_size: int, max_length: int, device: torch.device) -> dict[str, float]:
+    scorer_tok = AutoTokenizer.from_pretrained(scorer_path, local_files_only=True)
+    if scorer_tok.pad_token is None:
+        scorer_tok.pad_token = scorer_tok.eos_token
+    scorer = AutoModelForCausalLM.from_pretrained(scorer_path, local_files_only=True).to(device).eval()
+    total_nll = 0.0
+    total_tokens = 0
+    for start in range(0, len(texts), batch_size):
+        batch = texts[start:start + batch_size]
+        enc = scorer_tok(batch, return_tensors="pt", padding=True, truncation=True, max_length=max_length).to(device)
+        input_ids = enc["input_ids"]
+        attn = enc["attention_mask"]
+        if input_ids.size(1) < 2:
+            continue
+        logits = scorer(input_ids=input_ids, attention_mask=attn).logits.transpose(-1, -2)
+        nll = F.cross_entropy(logits[..., :-1].float(), input_ids[..., 1:], reduction="none")
+        mask = attn[..., 1:].bool()
+        total_nll += float(nll[mask].sum().detach().cpu())
+        total_tokens += int(mask.sum().detach().cpu())
+    del scorer
+    if device.type == "cuda":
+        torch.cuda.empty_cache()
+    mean = total_nll / max(total_tokens, 1)
+    return {"gen_ppl": math.exp(min(20.0, mean)), "gen_nll": mean, "gen_tokens": total_tokens}
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--checkpoint", required=True)
+    ap.add_argument("--tokenizer_path", required=True)
+    ap.add_argument("--out_dir", required=True)
+    ap.add_argument("--max_lens", default="128,1024")
+    ap.add_argument("--n_samples", type=int, default=16)
+    ap.add_argument("--batch_size", type=int, default=2)
+    ap.add_argument("--steps", type=int, default=128)
+    ap.add_argument(
+        "--decode_rule",
+        choices=[
+            "dual_line_resample",
+            "dual_replace_resample",
+            "dirichlet_resample",
+            "flowmap",
+            "log_dual_resample",
+            "sqrt_dual_resample",
+            "fisher_dual_resample",
+            "log_geodesic",
+            "sqrt_geodesic",
+            "fisher_geodesic",
+            "hybrid_log_flowmap",
+            "hybrid_log_dirres",
+            "hybrid_log_logflow",
+        ],
+        default="dual_line_resample",
+    )
+    ap.add_argument("--pos_extend", choices=["repeat", "interpolate"], default="repeat")
+    ap.add_argument("--support_power", type=float, default=1.0)
+    ap.add_argument("--semantic_power", type=float, default=1.5)
+    ap.add_argument("--early_temp", type=float, default=2.8)
+    ap.add_argument("--late_temp", type=float, default=1.45)
+    ap.add_argument("--temp_end", type=float, default=0.55)
+    ap.add_argument("--temp_power", type=float, default=1.5)
+    ap.add_argument("--hybrid_switch", type=float, default=0.5)
+    ap.add_argument("--tail_temp", type=float, default=-1.0)
+    ap.add_argument("--c_min", type=float, default=1.0)
+    ap.add_argument("--c_max", type=float, default=1024.0)
+    ap.add_argument("--anchor_mode", choices=["state", "onehot", "sqrt_state"], default="state")
+    ap.add_argument(
+        "--model_t_mode",
+        choices=["pre", "post", "flow", "linear", "const0", "const05", "const1", "random"],
+        default="flow",
+    )
+    ap.add_argument("--time_schedule", choices=["uniform", "logit_normal", "power_low", "power_high"], default="uniform")
+    ap.add_argument("--time_logit_mean", type=float, default=-1.5)
+    ap.add_argument("--time_logit_std", type=float, default=0.8)
+    ap.add_argument("--time_power", type=float, default=2.0)
+    ap.add_argument("--input_noise_scale", type=float, default=0.0)
+    ap.add_argument("--input_noise_until", type=float, default=1.0)
+    ap.add_argument("--input_noise_dirichlet_concentration", type=float, default=1.0)
+    ap.add_argument("--endpoint_softening", choices=["none", "uniform"], default="none")
+    ap.add_argument("--endpoint_soft_power", type=float, default=2.0)
+    ap.add_argument("--endpoint_soft_min_conf", type=float, default=0.0)
+    ap.add_argument("--endpoint_soft_max_conf", type=float, default=1.0)
+    ap.add_argument("--soft_target_decode_mode", choices=["off", "posterior", "debias", "late_debias"], default="off")
+    ap.add_argument("--soft_target_power", type=float, default=1.0)
+    ap.add_argument("--soft_target_min_conf", type=float, default=0.0)
+    ap.add_argument("--soft_target_max_conf", type=float, default=1.0)
+    ap.add_argument("--soft_target_debias_start", type=float, default=0.7)
+    ap.add_argument("--final_from", choices=["state", "endpoint", "blend"], default="blend")
+    ap.add_argument("--final_decode", choices=["argmax", "sample"], default="argmax")
+    ap.add_argument("--final_sample_temp", type=float, default=1.0)
+    ap.add_argument("--final_top_k", type=int, default=0)
+    ap.add_argument("--final_top_p", type=float, default=1.0)
+    ap.add_argument("--commit_mode", choices=["off", "threshold", "ratio", "ratio_margin_freq"], default="off")
+    ap.add_argument("--commit_conf_threshold", type=float, default=0.0)
+    ap.add_argument("--commit_margin_threshold", type=float, default=0.0)
+    ap.add_argument("--commit_start", type=float, default=0.0)
+    ap.add_argument("--commit_min_ratio", type=float, default=0.0)
+    ap.add_argument("--commit_max_ratio", type=float, default=1.0)
+    ap.add_argument("--commit_power", type=float, default=2.0)
+    ap.add_argument("--commit_freq_max_frac", type=float, default=0.08)
+    ap.add_argument("--fixed_first_token_id", type=int, default=-1)
+    ap.add_argument("--fixed_first_token_text", default="")
+    ap.add_argument("--fixed_first_initial_argmax", action="store_true")
+    ap.add_argument("--scorer", default="/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard")
+    ap.add_argument("--score", action="store_true")
+    ap.add_argument("--use_ema", action="store_true", help="Use ema_model from checkpoint if present.")
+    ap.add_argument("--seed", type=int, default=20260514)
+    args = ap.parse_args()
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    tok = BpeTextTokenizer.from_file(args.tokenizer_path)
+    ckpt = torch.load(args.checkpoint, map_location="cpu", weights_only=False, mmap=True)
+    if args.use_ema and "ema_model" in ckpt:
+        ckpt = dict(ckpt)
+        ckpt["model"] = ckpt["ema_model"]
+    out_dir = Path(args.out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    fixed_first_token_id: int | None = None
+    if args.fixed_first_token_text:
+        encoded = tok.encode(args.fixed_first_token_text, add_eos=False, add_special_tokens=False)
+        if not encoded:
+            raise ValueError(f"fixed_first_token_text encoded to no tokens: {args.fixed_first_token_text!r}")
+        fixed_first_token_id = int(encoded[0])
+    elif args.fixed_first_token_id >= 0:
+        fixed_first_token_id = int(args.fixed_first_token_id)
+    summary = []
+    for max_len_s in args.max_lens.split(","):
+        max_len = int(max_len_s)
+        model = build_model(ckpt, tok, max_len, device, args.pos_extend)
+        ids, texts, traces = decode(
+            model,
+            tok,
+            max_len=max_len,
+            n_samples=args.n_samples,
+            batch_size=args.batch_size,
+            steps=args.steps,
+            seed=args.seed + max_len,
+            device=device,
+            decode_rule=args.decode_rule,
+            support_power=args.support_power,
+            semantic_power=args.semantic_power,
+            early_temp=args.early_temp,
+            late_temp=args.late_temp,
+            temp_end=args.temp_end,
+            temp_power=args.temp_power,
+            hybrid_switch=args.hybrid_switch,
+            tail_temp=args.tail_temp,
+            c_min=args.c_min,
+            c_max=args.c_max,
+            anchor_mode=args.anchor_mode,
+            model_t_mode=args.model_t_mode,
+            time_schedule=args.time_schedule,
+            time_logit_mean=args.time_logit_mean,
+            time_logit_std=args.time_logit_std,
+            time_power=args.time_power,
+            input_noise_scale=args.input_noise_scale,
+            input_noise_until=args.input_noise_until,
+            input_noise_dirichlet_concentration=args.input_noise_dirichlet_concentration,
+            endpoint_softening=args.endpoint_softening,
+            endpoint_soft_power=args.endpoint_soft_power,
+            endpoint_soft_min_conf=args.endpoint_soft_min_conf,
+            endpoint_soft_max_conf=args.endpoint_soft_max_conf,
+            soft_target_decode_mode=args.soft_target_decode_mode,
+            soft_target_power=args.soft_target_power,
+            soft_target_min_conf=args.soft_target_min_conf,
+            soft_target_max_conf=args.soft_target_max_conf,
+            soft_target_debias_start=args.soft_target_debias_start,
+            final_from=args.final_from,
+            final_decode=args.final_decode,
+            final_sample_temp=args.final_sample_temp,
+            final_top_k=args.final_top_k,
+            final_top_p=args.final_top_p,
+            commit_mode=args.commit_mode,
+            commit_conf_threshold=args.commit_conf_threshold,
+            commit_margin_threshold=args.commit_margin_threshold,
+            commit_start=args.commit_start,
+            commit_min_ratio=args.commit_min_ratio,
+            commit_max_ratio=args.commit_max_ratio,
+            commit_power=args.commit_power,
+            commit_freq_max_frac=args.commit_freq_max_frac,
+            eps=1e-8,
+            fixed_first_token_id=fixed_first_token_id,
+            fixed_first_initial_argmax=args.fixed_first_initial_argmax,
+        )
+        filt_result = filter_generated_texts(texts, min_chars=0, normalize_whitespace=True, drop_empty=False)
+        filt = filt_result[0] if isinstance(filt_result, tuple) else filt_result
+        diversity_result = summarize_token_diversity(ids)
+        diversity = asdict(diversity_result) if is_dataclass(diversity_result) else dict(diversity_result)
+        rec = {
+            "checkpoint": args.checkpoint,
+            "ckpt_step": int(ckpt.get("step", -1)),
+            "max_len": max_len,
+            "decode_rule": args.decode_rule,
+            "support_power": args.support_power,
+            "semantic_power": args.semantic_power,
+            "steps": args.steps,
+            "c_min": args.c_min,
+            "c_max": args.c_max,
+            "anchor_mode": args.anchor_mode,
+            "model_t_mode": args.model_t_mode,
+            "time_schedule": args.time_schedule,
+            "time_logit_mean": args.time_logit_mean,
+            "time_logit_std": args.time_logit_std,
+            "time_power": args.time_power,
+            "input_noise_scale": args.input_noise_scale,
+            "input_noise_until": args.input_noise_until,
+            "input_noise_dirichlet_concentration": args.input_noise_dirichlet_concentration,
+            "endpoint_softening": args.endpoint_softening,
+            "endpoint_soft_power": args.endpoint_soft_power,
+            "endpoint_soft_min_conf": args.endpoint_soft_min_conf,
+            "endpoint_soft_max_conf": args.endpoint_soft_max_conf,
+            "soft_target_decode_mode": args.soft_target_decode_mode,
+            "soft_target_power": args.soft_target_power,
+            "soft_target_min_conf": args.soft_target_min_conf,
+            "soft_target_max_conf": args.soft_target_max_conf,
+            "soft_target_debias_start": args.soft_target_debias_start,
+            "final_from": args.final_from,
+            "final_decode": args.final_decode,
+            "final_sample_temp": args.final_sample_temp,
+            "final_top_k": args.final_top_k,
+            "final_top_p": args.final_top_p,
+            "commit_mode": args.commit_mode,
+            "commit_conf_threshold": args.commit_conf_threshold,
+            "commit_margin_threshold": args.commit_margin_threshold,
+            "commit_start": args.commit_start,
+            "commit_min_ratio": args.commit_min_ratio,
+            "commit_max_ratio": args.commit_max_ratio,
+            "commit_power": args.commit_power,
+            "commit_freq_max_frac": args.commit_freq_max_frac,
+            "early_temp": args.early_temp,
+            "late_temp": args.late_temp,
+            "temp_end": args.temp_end,
+            "temp_power": args.temp_power,
+            "pos_extend": args.pos_extend,
+            "fixed_first_token_id": fixed_first_token_id,
+            "fixed_first_token_text": args.fixed_first_token_text,
+            "fixed_first_initial_argmax": bool(args.fixed_first_initial_argmax),
+            "use_ema": bool(args.use_ema and "ema_model" in ckpt),
+            "n_samples": len(texts),
+            **diversity,
+            "texts_preview": filt[:4],
+        }
+        (out_dir / f"context{max_len}_samples.txt").write_text("\n\n---\n\n".join(filt), encoding="utf-8")
+        (out_dir / f"context{max_len}_trace.json").write_text(json.dumps(traces, ensure_ascii=False, indent=2), encoding="utf-8")
+        del model
+        if device.type == "cuda":
+            torch.cuda.empty_cache()
+        if args.score:
+            rec.update(score_with_gpt2(filt, args.scorer, batch_size=2, max_length=min(max_len, 1024), device=device))
+        summary.append(rec)
+    (out_dir / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(json.dumps(summary, ensure_ascii=False, indent=2), flush=True)
+
+
+if __name__ == "__main__":
+    main()

LTA_openwebtext_dualt/scripts/launch_lta_lm1b_dualtline_cmax16_8gpu_duo_small_1m.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt
+export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}"
+export TOKENIZERS_PARALLELISM=false
+export PYTHONUNBUFFERED=1
+export NCCL_DEBUG="${NCCL_DEBUG:-WARN}"
+export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}"
+
+RUN_NAME="${RUN_NAME:-lta_lm1b_duo_aligned_dirichlet_true_dualtline_cmax16_flmpack_onehot_hardce_ddit_small_len128_gbs512_8gpu_1m_nw0}"
+DATA_PATH="${DATA_PATH:-data/lm1b_train_parquet}"
+TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json}"
+TEXT_COLUMN="${TEXT_COLUMN:-}"
+OPENWEBTEXT_SPLIT="${OPENWEBTEXT_SPLIT:-all}"
+SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}"
+LOG_FILE="${LOG_FILE:-logs/${RUN_NAME}.log}"
+
+NNODES="${NNODES:-1}"
+NPROC_PER_NODE="${NPROC_PER_NODE:-8}"
+NODE_RANK="${NODE_RANK:-0}"
+MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}"
+MASTER_PORT="${MASTER_PORT:-29621}"
+
+GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}"
+PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-64}"
+TOTAL_STEPS="${TOTAL_STEPS:-1000000}"
+WARMUP_STEPS="${WARMUP_STEPS:-2500}"
+MAX_LEN="${MAX_LEN:-128}"
+WRAP_MODE="${WRAP_MODE:-stream}"
+WRAP_RECORD_BUFFER_SIZE="${WRAP_RECORD_BUFFER_SIZE:-200}"
+NUM_WORKERS="${NUM_WORKERS:-0}"
+LOG_EVERY="${LOG_EVERY:-100}"
+SAVE_EVERY="${SAVE_EVERY:-20000}"
+LATEST_EVERY="${LATEST_EVERY:-1000}"
+EVAL_EVERY="${EVAL_EVERY:-0}"
+RESUME_PATH="${RESUME_PATH:-}"
+ALLOW_EXISTING_SAVE_DIR="${ALLOW_EXISTING_SAVE_DIR:-0}"
+ENABLE_TORCH_COMPILE="${ENABLE_TORCH_COMPILE:-0}"
+FORCE_DISABLE_TORCH_COMPILE="${FORCE_DISABLE_TORCH_COMPILE:-1}"
+if [[ "${FORCE_DISABLE_TORCH_COMPILE}" == "1" ]]; then
+  ENABLE_TORCH_COMPILE=0
+fi
+if [[ "${DATA_PATH}" == *"lm1b_train_parquet"* && "${NUM_WORKERS}" != "0" ]]; then
+  echo "LM1B has only 9 parquet shards; forcing NUM_WORKERS=0 to avoid empty DDP dataloader shards." >&2
+  NUM_WORKERS=0
+fi
+COMPILE_ARGS=()
+if [[ "${ENABLE_TORCH_COMPILE}" == "1" ]]; then
+  COMPILE_ARGS+=(--torch_compile --compile_mode reduce-overhead)
+fi
+RESUME_ARGS=()
+if [[ -n "${RESUME_PATH}" ]]; then
+  RESUME_ARGS+=(--resume_path "${RESUME_PATH}")
+fi
+TEXT_COLUMN_ARGS=()
+if [[ -n "${TEXT_COLUMN}" ]]; then
+  TEXT_COLUMN_ARGS+=(--text_column "${TEXT_COLUMN}")
+fi
+
+if [[ -f "${SAVE_DIR}/args.json" && -z "${RESUME_PATH}" && "${ALLOW_EXISTING_SAVE_DIR}" != "1" ]]; then
+  echo "Refusing to start because SAVE_DIR already contains args.json: ${SAVE_DIR}" >&2
+  echo "Use a new RUN_NAME/SAVE_DIR, set RESUME_PATH to resume, or set ALLOW_EXISTING_SAVE_DIR=1 intentionally." >&2
+  exit 2
+fi
+
+mkdir -p logs runs "${SAVE_DIR}"
+
+python -m torch.distributed.run \
+  --nnodes="${NNODES}" \
+  --nproc_per_node="${NPROC_PER_NODE}" \
+  --node_rank="${NODE_RANK}" \
+  --master_addr="${MASTER_ADDR}" \
+  --master_port="${MASTER_PORT}" \
+  train.py \
+  --data_path "${DATA_PATH}" \
+  "${TEXT_COLUMN_ARGS[@]}" \
+  --openwebtext_split "${OPENWEBTEXT_SPLIT}" \
+  --tokenizer_path "${TOKENIZER_PATH}" \
+  --save_dir "${SAVE_DIR}" \
+  --wrap \
+  --wrap_mode "${WRAP_MODE}" \
+  --wrap_record_buffer_size "${WRAP_RECORD_BUFFER_SIZE}" \
+  --max_len "${MAX_LEN}" \
+  --batch_size "${PER_GPU_BATCH_SIZE}" \
+  --num_workers "${NUM_WORKERS}" \
+  --global_batch_size "${GLOBAL_BATCH_SIZE}" \
+  --total_steps "${TOTAL_STEPS}" \
+  --log_every "${LOG_EVERY}" \
+  --eval_every "${EVAL_EVERY}" \
+  --save_every "${SAVE_EVERY}" \
+  --latest_every "${LATEST_EVERY}" \
+  --lr 3e-4 \
+  --weight_decay 0 \
+  --adam_beta1 0.9 \
+  --adam_beta2 0.999 \
+  --adam_eps 1e-8 \
+  --warmup_steps "${WARMUP_STEPS}" \
+  --lr_schedule constant_warmup \
+  --grad_clip 1.0 \
+  --seed 123 \
+  --d_model 768 \
+  --cond_dim 128 \
+  --n_layers 12 \
+  --n_heads 12 \
+  --dim_ff 3072 \
+  --dropout 0.1 \
+  --model_type ddit \
+  --state_format prob \
+  --bridge dirichlet \
+  --target_loss hard_ce \
+  --target_prob 1.0 \
+  --min_t 0.0 \
+  --max_t 1.0 \
+  --dual_t \
+  --corrupt_t_mode independent \
+  --corrupt_min_t 0.0 \
+  --corrupt_max_t 1.0 \
+  --min_mask_ratio 0.1 \
+  --max_mask_ratio 1.0 \
+  --wrong_token_replace_prob 1.0 \
+  --wrong_token_schedule linear_t \
+  --wrong_token_exp_k 1.0 \
+  --dirichlet_concentration_min 1.0 \
+  --dirichlet_concentration_max 16.0 \
+  --dirichlet_endpoint_mode dual_t_line \
+  --dirichlet_semantic_t_mode independent \
+  --dirichlet_semantic_t_value 0.0 \
+  --eps 1e-8 \
+  --infer_steps 128 \
+  --decode_damping 1.0 \
+  --max_gamma 1.0 \
+  --decode_solver flowmap \
+  --noise_init logistic_normal \
+  --bridge_noise_init logistic_normal \
+  --noise_sigma -1 \
+  "${RESUME_ARGS[@]}" \
+  "${COMPILE_ARGS[@]}" \
+  --bf16 2>&1 | tee -a "${LOG_FILE}"

LTA_openwebtext_dualt/scripts/launch_lta_owt_gamma2_8gpu.sh

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt
+export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}"
+export TOKENIZERS_PARALLELISM=false
+export PYTHONUNBUFFERED=1
+export NCCL_DEBUG="${NCCL_DEBUG:-WARN}"
+export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}"
+export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
+export OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}"
+
+RUN_NAME="${RUN_NAME:-lta_owt_gpt2cached_len1024_maskfloor_logitpow2gamma2_c1024_ddit768x12_muon_ema_gbs512_8gpu_1m_$(date +%Y%m%d_%H%M%S)}"
+SAVE_DIR="${SAVE_DIR:-runs/${RUN_NAME}}"
+LOG_DIR="${LOG_DIR:-logs/gamma2_8gpu}"
+LOG_FILE="${LOG_FILE:-${LOG_DIR}/${RUN_NAME}.log}"
+mkdir -p "${LOG_DIR}" "${SAVE_DIR}"
+
+OWT_CACHE="${OWT_CACHE:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train_minus_100k}"
+
+PERF_ARGS=()
+if [[ "${ALLOW_TF32:-1}" == "1" ]]; then
+  PERF_ARGS+=(--allow_tf32)
+else
+  PERF_ARGS+=(--no-allow_tf32)
+fi
+if [[ "${ACTIVATION_CHECKPOINTING:-1}" == "1" ]]; then
+  PERF_ARGS+=(
+    --activation_checkpointing
+    --activation_checkpoint_interval "${ACTIVATION_CHECKPOINT_INTERVAL:-2}"
+    --activation_checkpoint_scope "${ACTIVATION_CHECKPOINT_SCOPE:-block}"
+  )
+fi
+if [[ "${DDP_STATIC_GRAPH:-0}" == "1" ]]; then
+  PERF_ARGS+=(--ddp_static_graph)
+fi
+if [[ "${DDP_GRADIENT_AS_BUCKET_VIEW:-1}" == "1" ]]; then
+  PERF_ARGS+=(--ddp_gradient_as_bucket_view)
+else
+  PERF_ARGS+=(--no-ddp_gradient_as_bucket_view)
+fi
+if [[ "${BLOCKING_DATA_TRANSFER:-0}" == "1" ]]; then
+  PERF_ARGS+=(--blocking_data_transfer)
+fi
+
+python -m torch.distributed.run \
+  --nnodes="${NNODES:-1}" \
+  --nproc_per_node="${NPROC_PER_NODE:-8}" \
+  --node_rank="${NODE_RANK:-0}" \
+  --master_addr="${MASTER_ADDR:-127.0.0.1}" \
+  --master_port="${MASTER_PORT:-31995}" \
+  train.py \
+  --data_path /e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext \
+  --text_column text \
+  --openwebtext_split train_minus_100k \
+  --detokenizer auto \
+  --tokenizer_path /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-standard/tokenizer.json \
+  --save_dir "${SAVE_DIR}" \
+  --wrap \
+  --wrap_mode stream \
+  --wrap_record_buffer_size 200 \
+  --owt_cached_chunks \
+  --owt_chunk_cache_dir "${OWT_CACHE}" \
+  --owt_chunk_cache_write_batch 4096 \
+  --max_len "${MAX_LEN:-1024}" \
+  --batch_size "${PER_GPU_BATCH_SIZE:-32}" \
+  --num_workers "${NUM_WORKERS:-8}" \
+  --dataloader_prefetch_factor "${DATALOADER_PREFETCH_FACTOR:-4}" \
+  --global_batch_size "${GLOBAL_BATCH_SIZE:-512}" \
+  --total_steps "${TOTAL_STEPS:-1000000}" \
+  --warmup_steps "${WARMUP_STEPS:-2000}" \
+  --log_every "${LOG_EVERY:-50}" \
+  --eval_every "${EVAL_EVERY:-0}" \
+  --save_every "${SAVE_EVERY:-50000}" \
+  --latest_every "${LATEST_EVERY:-1000}" \
+  --lr "${LEARNING_RATE:-0.002}" \
+  --lr_schedule constant_warmup \
+  --min_lr 0 \
+  --weight_decay 0.0 \
+  --adam_beta1 0.9 \
+  --adam_beta2 0.95 \
+  --adam_eps 1e-8 \
+  --optimizer muon \
+  --muon_momentum 0.95 \
+  --muon_ns_steps 5 \
+  --muon_update_scale 1.0 \
+  --ema_decay 0.9999 \
+  --ema_start_step 0 \
+  --grad_clip 1.0 \
+  --adamw_param_groups nanogpt \
+  --seed 123 \
+  --d_model 768 \
+  --cond_dim 128 \
+  --n_layers 12 \
+  --n_heads 12 \
+  --dim_ff 3072 \
+  --dropout 0.0 \
+  --model_type ddit \
+  --state_format prob \
+  --bridge dirichlet \
+  --target_loss hard_ce \
+  --target_prob 1.0 \
+  --min_t 0.0 \
+  --max_t 1.0 \
+  --dual_t \
+  --corrupt_t_mode same \
+  --corrupt_min_t 0.0 \
+  --corrupt_max_t 1.0 \
+  --min_mask_ratio 0.1 \
+  --max_mask_ratio 1.0 \
+  --mask_ratio_floor_schedule one_minus_t \
+  --wrong_token_replace_prob 1.0 \
+  --wrong_token_schedule linear_t \
+  --wrong_token_exp_k 1.0 \
+  --dirichlet_concentration_min 1.0 \
+  --dirichlet_concentration_max 1024 \
+  --dirichlet_endpoint_mode categorical_dual_t \
+  --dirichlet_semantic_t_mode same \
+  --dirichlet_semantic_t_value 0.0 \
+  --dirichlet_semantic_t_curve logit_power \
+  --dirichlet_semantic_t_power 2.0 \
+  --endpoint_sequence_random_prob_alpha 0.0 \
+  --categorical_wrong_from_full_vocab \
+  --simplex_bridge_sampler dirichlet \
+  --eps 1e-8 \
+  --infer_steps 1024 \
+  --decode_damping 1.0 \
+  --max_gamma 1.0 \
+  --decode_solver flowmap \
+  --noise_init logistic_normal \
+  --bridge_noise_init logistic_normal \
+  --noise_sigma -1 \
+  "${PERF_ARGS[@]}" \
+  --bf16 2>&1 | tee -a "${LOG_FILE}"

LTA_openwebtext_dualt/scripts/launch_lta_owt_t5_adaln_adamw_wd0p1_rollin_p50_randk0_3_2node.sh

@@ -0,0 +1,186 @@
+#!/bin/bash
+set -e
+set -x
+set -o pipefail
+
+cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt
+
+# LTA's working torchrun on this cluster is /usr/local/bin/torchrun, which is
+# bound to /usr/bin/python.  Do not source the ConvNeXt my_env by default: that
+# env currently lacks the HuggingFace tokenizers package needed by T5 tokenized
+# data.  Set ACTIVATE_ENV=/path/to/bin/activate only when you know it has the
+# LTA dependencies installed.
+if [[ -n "${ACTIVATE_ENV:-}" ]]; then
+  source "${ACTIVATE_ENV}"
+fi
+
+# MLP multi-node launcher variables.  These can still be overridden manually.
+GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}"
+NNODES="${NNODES:-${MLP_WORKER_NUM:-2}}"
+NODE_RANK="${NODE_RANK:-${MLP_ROLE_INDEX:-0}}"
+MASTER_ADDR="${MASTER_ADDR:-${MLP_WORKER_0_HOST:-127.0.0.1}}"
+MASTER_PORT="${MASTER_PORT:-${MLP_WORKER_0_PORT:-32268}}"
+NPROC_PER_NODE="${NPROC_PER_NODE:-$(nvidia-smi --list-gpus | wc -l)}"
+TOTAL_GPUS=$(( NNODES * NPROC_PER_NODE ))
+PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-$(( GLOBAL_BATCH_SIZE / TOTAL_GPUS ))}"
+
+if [[ "${TOTAL_GPUS}" -le 0 || "${PER_GPU_BATCH_SIZE}" -le 0 ]]; then
+  echo "Invalid batch/GPU config: GLOBAL_BATCH_SIZE=${GLOBAL_BATCH_SIZE} NNODES=${NNODES} NPROC_PER_NODE=${NPROC_PER_NODE}" >&2
+  exit 2
+fi
+
+if [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then
+  export CUDA_VISIBLE_DEVICES="$(seq -s, 0 $((NPROC_PER_NODE - 1)))"
+else
+  export CUDA_VISIBLE_DEVICES
+fi
+export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}"
+export TOKENIZERS_PARALLELISM=false
+export PYTHONUNBUFFERED=1
+export NCCL_DEBUG="${NCCL_DEBUG:-INFO}"
+export TORCH_NCCL_AVOID_RECORD_STREAMS=1
+export TORCH_DISTRIBUTED_TIMEOUT="${TORCH_DISTRIBUTED_TIMEOUT:-3600}"
+
+python - <<'PY'
+import sys
+import torch
+import tokenizers
+
+print(f"[launch] python={sys.executable}")
+print(f"[launch] torch={torch.__version__} tokenizers={tokenizers.__version__}")
+PY
+
+export DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/embedded-language-flows/openwebtext-t5}"
+export TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json}"
+
+# RUN_NAME must match across nodes.  In MLP jobs, prefer the job id if present;
+# otherwise MLP_WORKER_0_PORT is usually stable across workers for one job.
+RUN_ID="${RUN_ID:-${MLP_TASK_ID:-${MLP_JOB_ID:-${MLP_RUN_ID:-${MLP_WORKER_0_PORT:-}}}}}"
+if [[ -z "${RUN_NAME:-}" ]]; then
+  if [[ -z "${RUN_ID}" ]]; then
+    RUN_ID="$(date +%Y%m%d_%H%M%S)"
+  fi
+  RUN_ID="$(printf "%s" "${RUN_ID}" | tr -c 'A-Za-z0-9_-' '_')"
+  export RUN_NAME="lta_owt_t5_adaln_adamw_wd0p1_rollin_p50_randk0_3_uniformt_temp1_synct_gbs${GLOBAL_BATCH_SIZE}_${NNODES}node${NPROC_PER_NODE}gpu_1m_${RUN_ID}"
+fi
+
+export LOG_DIR="${LOG_DIR:-logs/elfaligned_t5tokenized_2node}"
+export LOG_FILE="${LOG_FILE:-${LOG_DIR}/${RUN_NAME}.node${NODE_RANK}.log}"
+mkdir -p "${LOG_DIR}" "runs/${RUN_NAME}"
+
+echo "[launch] run_name=${RUN_NAME}"
+echo "[launch] node_rank=${NODE_RANK}/${NNODES} nproc_per_node=${NPROC_PER_NODE} master=${MASTER_ADDR}:${MASTER_PORT}"
+echo "[launch] total_gpus=${TOTAL_GPUS} global_batch_size=${GLOBAL_BATCH_SIZE} per_gpu_batch_size=${PER_GPU_BATCH_SIZE}"
+echo "[launch] data_path=${DATA_PATH}"
+echo "[launch] tokenizer=${TOKENIZER_PATH}"
+echo "[launch] log_file=${LOG_FILE}"
+
+torchrun \
+  --nnodes="${NNODES}" \
+  --nproc_per_node="${NPROC_PER_NODE}" \
+  --node_rank="${NODE_RANK}" \
+  --master_addr="${MASTER_ADDR}" \
+  --master_port="${MASTER_PORT}" \
+  train.py \
+  --data_path "${DATA_PATH}" \
+  --tokenized_hf \
+  --tokenized_pad_token pad \
+  --tokenizer_path "${TOKENIZER_PATH}" \
+  --save_dir "runs/${RUN_NAME}" \
+  --max_len 1024 \
+  --batch_size "${PER_GPU_BATCH_SIZE}" \
+  --global_batch_size "${GLOBAL_BATCH_SIZE}" \
+  --num_workers 8 \
+  --dataloader_prefetch_factor 4 \
+  --epochs 0 \
+  --total_steps 1000000 \
+  --warmup_steps 1000 \
+  --log_every 100 \
+  --eval_every 0 \
+  --save_every 10000 \
+  --latest_every 1000 \
+  --optimizer adamw \
+  --lr 6e-4 \
+  --lr_schedule cosine \
+  --min_lr 6e-5 \
+  --weight_decay 0.1 \
+  --output_weight_decay -1 \
+  --adamw_param_groups nanogpt \
+  --adam_beta1 0.9 \
+  --adam_beta2 0.999 \
+  --adam_eps 1e-8 \
+  --ema_decay 0.9999 \
+  --ema_start_step 0 \
+  --grad_clip 1.0 \
+  --seed 42 \
+  --d_model 768 \
+  --cond_dim 128 \
+  --n_layers 12 \
+  --n_heads 12 \
+  --dim_ff 3072 \
+  --dropout 0.0 \
+  --no-output_bias \
+  --norm_type rmsnorm \
+  --model_type ddit \
+  --ddit_mlp_type swiglu \
+  --state_format prob \
+  --bridge dirichlet \
+  --target_loss hard_ce \
+  --loss_t_weight_mode none \
+  --loss_t_min_weight 0.0 \
+  --rollout_train_prob 0.50 \
+  --rollout_train_time_mode sampled_path \
+  --rollout_train_steps 3 \
+  --rollout_train_steps_min 0 \
+  --rollout_train_infer_steps 1 \
+  --rollout_train_s_dist uniform \
+  --rollout_train_s_min_frac 0.0 \
+  --rollout_train_s_max_frac 0.25 \
+  --rollout_train_temp 1.0 \
+  --rollout_train_max_gamma 1.0 \
+  --rollout_train_corrupt_only \
+  --rollout_train_samplewise \
+  --rollout_train_selected_only \
+  --no-rollout_train_compute_always \
+  --rollout_train_sync_t \
+  --target_prob 1.0 \
+  --min_t 0.0 \
+  --max_t 1.0 \
+  --t_sampling_mode uniform \
+  --t_sampling_logit_mean -1.5 \
+  --t_sampling_logit_std 0.8 \
+  --t_sampling_eps 1e-4 \
+  --dual_t \
+  --corrupt_t_mode same \
+  --corrupt_min_t 0.0 \
+  --corrupt_max_t 1.0 \
+  --min_mask_ratio 1.0 \
+  --max_mask_ratio 1.0 \
+  --mask_mixture_original_prob 0.0 \
+  --mask_mixture_lowk_prob 0.0 \
+  --mask_mixture_lowcorrupt_prob 0.0 \
+  --mask_mixture_block_prob 0.0 \
+  --mask_mixture_all_prob 1.0 \
+  --wrong_token_replace_prob 1.0 \
+  --wrong_token_schedule linear_t \
+  --wrong_token_exp_k 1.0 \
+  --dirichlet_concentration_min 1.0 \
+  --dirichlet_concentration_max 1024 \
+  --dirichlet_endpoint_mode categorical_dual_t \
+  --dirichlet_semantic_t_mode same \
+  --dirichlet_semantic_t_value 0.0 \
+  --categorical_wrong_from_full_vocab \
+  --simplex_bridge_sampler dirichlet \
+  --eps 1e-8 \
+  --infer_steps 1024 \
+  --decode_damping 1.0 \
+  --max_gamma 1.0 \
+  --decode_solver flowmap \
+  --noise_init logistic_normal \
+  --bridge_noise_init logistic_normal \
+  --noise_sigma -1 \
+  --allow_tf32 \
+  --activation_checkpointing \
+  --activation_checkpoint_scope mlp \
+  --ddp_gradient_as_bucket_view \
+  2>&1 | tee -a "${LOG_FILE}"

LTA_openwebtext_dualt/scripts/qwen_transformers_openai_server.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import threading
+import time
+import uuid
+from typing import Any
+
+import torch
+import uvicorn
+from fastapi import FastAPI, Header, HTTPException
+from pydantic import BaseModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+
+class GenerateRequest(BaseModel):
+    model: str | None = None
+    messages: list[dict[str, Any]] | None = None
+    prompt: str | list[str] | None = None
+    max_tokens: int = 512
+    temperature: float = 0.7
+    top_p: float = 0.95
+    stop: str | list[str] | None = None
+    stream: bool = False
+    enable_thinking: bool | None = None
+    chat_template_kwargs: dict[str, Any] | None = None
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-path", required=True)
+    parser.add_argument("--served-model-name", default="qwen3.5-35b-a3b")
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument("--api-key", default=os.environ.get("API_KEY", ""))
+    parser.add_argument("--max-memory-gib", type=int, default=88)
+    parser.add_argument("--torch-dtype", default="bfloat16")
+    return parser.parse_args()
+
+
+args = parse_args()
+app = FastAPI()
+lock = threading.Lock()
+
+dtype = torch.bfloat16 if args.torch_dtype == "bfloat16" else torch.float16
+tokenizer = AutoTokenizer.from_pretrained(
+    args.model_path,
+    trust_remote_code=True,
+    local_files_only=True,
+)
+
+max_memory = {
+    0: f"{args.max_memory_gib}GiB",
+    1: f"{args.max_memory_gib}GiB",
+    "cpu": "128GiB",
+}
+model = AutoModelForCausalLM.from_pretrained(
+    args.model_path,
+    trust_remote_code=True,
+    local_files_only=True,
+    torch_dtype=dtype,
+    device_map="auto",
+    max_memory=max_memory,
+    low_cpu_mem_usage=True,
+)
+model.eval()
+
+
+def require_auth(authorization: str | None) -> None:
+    if not args.api_key:
+        return
+    expected = f"Bearer {args.api_key}"
+    if authorization != expected:
+        raise HTTPException(status_code=401, detail="Unauthorized")
+
+
+def make_prompt(req: GenerateRequest) -> str:
+    if req.messages is not None:
+        template_kwargs = dict(req.chat_template_kwargs or {})
+        if req.enable_thinking is not None:
+            template_kwargs["enable_thinking"] = req.enable_thinking
+        return tokenizer.apply_chat_template(
+            req.messages,
+            tokenize=False,
+            add_generation_prompt=True,
+            **template_kwargs,
+        )
+    if isinstance(req.prompt, list):
+        return req.prompt[0]
+    if isinstance(req.prompt, str):
+        return req.prompt
+    raise HTTPException(status_code=400, detail="messages or prompt is required")
+
+
+def strip_stop(text: str, stop: str | list[str] | None) -> str:
+    stops = [stop] if isinstance(stop, str) else (stop or [])
+    cut = len(text)
+    for s in stops:
+        if not s:
+            continue
+        idx = text.find(s)
+        if idx >= 0:
+            cut = min(cut, idx)
+    return text[:cut]
+
+
+def generate_text(req: GenerateRequest) -> tuple[str, int, int]:
+    prompt = make_prompt(req)
+    inputs = tokenizer(prompt, return_tensors="pt")
+    input_len = int(inputs.input_ids.shape[-1])
+    device = next(model.parameters()).device
+    inputs = {k: v.to(device) for k, v in inputs.items()}
+    do_sample = req.temperature is not None and req.temperature > 0
+    kwargs = {
+        "max_new_tokens": max(1, int(req.max_tokens)),
+        "do_sample": do_sample,
+        "pad_token_id": tokenizer.pad_token_id or tokenizer.eos_token_id,
+        "eos_token_id": tokenizer.eos_token_id,
+    }
+    if do_sample:
+        kwargs["temperature"] = float(req.temperature)
+        kwargs["top_p"] = float(req.top_p)
+    with lock, torch.inference_mode():
+        output = model.generate(**inputs, **kwargs)
+    new_tokens = output[0, input_len:]
+    text = tokenizer.decode(new_tokens, skip_special_tokens=True)
+    text = strip_stop(text, req.stop)
+    return text, input_len, int(new_tokens.numel())
+
+
+@app.get("/healthz")
+def healthz() -> dict[str, Any]:
+    return {"ok": True, "model": args.served_model_name}
+
+
+@app.get("/v1/models")
+def models(authorization: str | None = Header(default=None)) -> dict[str, Any]:
+    require_auth(authorization)
+    return {
+        "object": "list",
+        "data": [
+            {
+                "id": args.served_model_name,
+                "object": "model",
+                "created": 0,
+                "owned_by": "local",
+            }
+        ],
+    }
+
+
+@app.post("/v1/chat/completions")
+def chat(req: GenerateRequest, authorization: str | None = Header(default=None)) -> dict[str, Any]:
+    require_auth(authorization)
+    if req.stream:
+        raise HTTPException(status_code=400, detail="stream=true is not implemented")
+    text, prompt_tokens, completion_tokens = generate_text(req)
+    return {
+        "id": f"chatcmpl-{uuid.uuid4().hex}",
+        "object": "chat.completion",
+        "created": int(time.time()),
+        "model": req.model or args.served_model_name,
+        "choices": [
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": text},
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "total_tokens": prompt_tokens + completion_tokens,
+        },
+    }
+
+
+@app.post("/v1/completions")
+def completions(req: GenerateRequest, authorization: str | None = Header(default=None)) -> dict[str, Any]:
+    require_auth(authorization)
+    if req.stream:
+        raise HTTPException(status_code=400, detail="stream=true is not implemented")
+    text, prompt_tokens, completion_tokens = generate_text(req)
+    return {
+        "id": f"cmpl-{uuid.uuid4().hex}",
+        "object": "text_completion",
+        "created": int(time.time()),
+        "model": req.model or args.served_model_name,
+        "choices": [{"text": text, "index": 0, "finish_reason": "stop"}],
+        "usage": {
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "total_tokens": prompt_tokens + completion_tokens,
+        },
+    }
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host=args.host, port=args.port, log_level="info")

LTA_openwebtext_dualt/scripts/run_lta_owt_bert_absrope_time4_dirichlet_len1024_Cv_to_2v_8gpu_1m_mask1_sameT_save10k.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt
+
+# OWT raw text + BERT tokenizer, FLM wrapped stream:
+#   [CLS] + 1022 payload tokens + [SEP]
+#
+# Backbone:
+#   ddit_elf = RMSNorm/SwiGLU/QK-norm + RoPE + 4 prefix time tokens.
+#   We also add learned absolute position embeddings before RoPE.
+#
+# Bridge:
+#   Dirichlet C=V->2V, mask_ratio=1.0, model t and corruption t are shared.
+
+export DATA_PATH="${DATA_PATH:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext}"
+export TEXT_COLUMN="${TEXT_COLUMN:-text}"
+export OPENWEBTEXT_SPLIT="${OPENWEBTEXT_SPLIT:-train_minus_100k}"
+export TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json}"
+export TOKENIZED_HF=0
+export WRAP_MODE="${WRAP_MODE:-stream}"
+
+export VOCAB_SIZE="${VOCAB_SIZE:-30522}"
+export CMIN="${CMIN:-30522}"
+export CMAX="${CMAX:-61044}"
+
+export MODEL_TYPE=ddit_elf
+export ELF_NUM_TIME_TOKENS="${ELF_NUM_TIME_TOKENS:-4}"
+export ELF_NUM_MODEL_MODE_TOKENS="${ELF_NUM_MODEL_MODE_TOKENS:-0}"
+export QK_NORM="${QK_NORM:-1}"
+export ABS_POS_EMBED=1
+export CORRUPT_T_MODE=same
+export MIN_MASK_RATIO=1.0
+export MAX_MASK_RATIO=1.0
+export CATEGORICAL_WRONG_PROB_FLOOR="${CATEGORICAL_WRONG_PROB_FLOOR:-0.0}"
+
+_ngpus_avail=$(nvidia-smi --query-gpu=index --format=csv,noheader 2>/dev/null | wc -l || echo 1)
+if [[ "${_ngpus_avail}" -le 0 ]]; then _ngpus_avail=1; fi
+_default_cvd=$(seq -s, 0 $((_ngpus_avail - 1)))
+export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-${_default_cvd}}"
+IFS=',' read -ra _cvd_arr <<< "${CUDA_VISIBLE_DEVICES}"
+export NPROC_PER_NODE="${NPROC_PER_NODE:-${#_cvd_arr[@]}}"
+unset _ngpus_avail _default_cvd _cvd_arr
+export NNODES="${NNODES:-${MLP_WORKER_NUM:-1}}"
+export NODE_RANK="${NODE_RANK:-${MLP_ROLE_INDEX:-0}}"
+export MASTER_ADDR="${MASTER_ADDR:-${MLP_WORKER_0_HOST:-127.0.0.1}}"
+export MASTER_PORT="${MASTER_PORT:-${MLP_WORKER_0_PORT:-29500}}"
+export GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}"
+export PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-32}"
+export TOTAL_STEPS="${TOTAL_STEPS:-1000000}"
+export WARMUP_STEPS="${WARMUP_STEPS:-2500}"
+export SAVE_EVERY="${SAVE_EVERY:-10000}"
+export LATEST_EVERY="${LATEST_EVERY:-1000}"
+export LOG_EVERY="${LOG_EVERY:-100}"
+
+export DATE_TAG="${DATE_TAG:-$(date +%Y%m%d)}"
+export RUN_NAME="${RUN_NAME:-lta_owt_bert_absrope_time4_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_1m_save10k_${DATE_TAG}}"
+
+export WATCH_ENABLED="${WATCH_ENABLED:-1}"
+export WATCH_STEP_INTERVAL="${WATCH_STEP_INTERVAL:-10000}"
+export WATCH_N_SAMPLES="${WATCH_N_SAMPLES:-128}"
+export WATCH_CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES:-7}"
+export WATCH_ENDPOINT_TEMP="${WATCH_ENDPOINT_TEMP:-1.45}"
+export WATCH_ENDPOINT_TOP_P="${WATCH_ENDPOINT_TOP_P:-0.95}"
+export WATCH_GUMBEL_TAU_START="${WATCH_GUMBEL_TAU_START:-1.0}"
+export WATCH_GUMBEL_TAU_END="${WATCH_GUMBEL_TAU_END:-0.2}"
+export WATCH_OUT_BASE="${WATCH_OUT_BASE:-docs/lta_samples/metrics_${DATE_TAG}/owt_bert_absrope_time4_Cv_to_2v_mask1_sameT_sde_gumbel_topp${WATCH_ENDPOINT_TOP_P//./p}_tau${WATCH_GUMBEL_TAU_START//./p}_to_${WATCH_GUMBEL_TAU_END//./p}_blend_c${CMIN}_${CMAX}_n${WATCH_N_SAMPLES}/${RUN_NAME}}"
+export WATCH_LOG_DIR="${WATCH_LOG_DIR:-logs/owt_bert_absrope_time4_Cv_to_2v_mask1_sameT_gumbel_sde_watch}"
+
+bash scripts/run_lta_owt_dirichlet_len1024_Cv_to_2v_8gpu_save1k_with_gumbel_watch.sh

LTA_openwebtext_dualt/scripts/run_train8_rollin_focused_pilots_4gpu.sh

@@ -0,0 +1,272 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt
+
+export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}"
+export TOKENIZERS_PARALLELISM=false
+export PYTHONUNBUFFERED=1
+
+BASE_CACHE="${BASE_CACHE:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks}"
+TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-standard/tokenizer.json}"
+MAX_LEN="${MAX_LEN:-256}"
+N_SAMPLES="${N_SAMPLES:-64}"
+INFER_STEPS="${INFER_STEPS:-128}"
+STEP_CHUNK="${STEP_CHUNK:-500}"
+MAX_TOTAL_STEPS="${MAX_TOTAL_STEPS:-12000}"
+STOP_EXACT_COUNT="${STOP_EXACT_COUNT:-64}"
+STOP_EXACT_REF_COUNT="${STOP_EXACT_REF_COUNT:-8}"
+PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-128}"
+GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}"
+GROUP_STAMP="${GROUP_STAMP:-$(date +%Y%m%d_%H%M%S)}"
+OUT_ROOT="${OUT_ROOT:-docs/lta_samples/metrics_20260517/rollin_focused_len${MAX_LEN}_bs512_ode128_${GROUP_STAMP}}"
+DRIVER_LOG="${DRIVER_LOG:-logs/rollin_focused_4gpu/${GROUP_STAMP}.log}"
+CURVE_CSV="${CURVE_CSV:-${OUT_ROOT}/hit_ratio_curve.csv}"
+mkdir -p "$(dirname "${DRIVER_LOG}")" "${OUT_ROOT}"
+
+cache="${BASE_CACHE}/gpt2_len${MAX_LEN}_train8_compact_overfit"
+vocab_size="$(
+python - "$cache" <<'PY'
+import json
+import sys
+from pathlib import Path
+meta = json.loads((Path(sys.argv[1]) / "meta.json").read_text())
+print(int(meta.get("compact_vocab_size", meta.get("vocab_size"))))
+PY
+)"
+
+if [[ ! -f "${CURVE_CSV}" ]]; then
+  echo "config,ckpt_step,train_views_seen,train_tokens_seen,token_acc_mean,exact_count,exact_ref_count,exact_ref_hits" > "${CURVE_CSV}"
+fi
+
+latest_step() {
+  local run_name="$1"
+  python - "$run_name" <<'PY'
+import re
+import sys
+from pathlib import Path
+run = Path("runs") / sys.argv[1]
+steps = []
+for path in run.glob("step_*.pt"):
+    m = re.search(r"step_(\d+)\.pt$", path.name)
+    if m:
+        steps.append(int(m.group(1)))
+print(max(steps) if steps else 0)
+PY
+}
+
+free_port() {
+  python - <<'PY'
+import socket
+with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+    s.bind(("127.0.0.1", 0))
+    print(s.getsockname()[1])
+PY
+}
+
+eval_latest() {
+  local config="$1"
+  local run_name="$2"
+  local target_step="$3"
+  local out_dir="${OUT_ROOT}/${config}/step_${target_step}"
+  mkdir -p "${out_dir}"
+  CUDA_VISIBLE_DEVICES="${EVAL_CUDA_VISIBLE_DEVICES:-0}" python scripts/eval_train8_decode_acc.py \
+    --runs_glob "runs/${run_name}" \
+    --data_dir "${cache}" \
+    --tokenizer_path "${TOKENIZER_PATH}" \
+    --out_dir "${out_dir}" \
+    --max_len "${MAX_LEN}" \
+    --n_samples "${N_SAMPLES}" \
+    --batch_size "${N_SAMPLES}" \
+    --latest_only \
+    --endpoint_softenings none \
+    --steps "${INFER_STEPS}" \
+    --decode_rule flowmap \
+    --time_schedule logit_normal \
+    --time_logit_mean -1.5 \
+    --time_logit_std 0.8 \
+    --model_t_mode post \
+    --c_min 1 \
+    --c_max 512 \
+    --late_temp 1.0 \
+    --final_from state \
+    --final_decode argmax
+  python - "$config" "$out_dir" "$N_SAMPLES" "$GLOBAL_BATCH_SIZE" "$MAX_LEN" "$CURVE_CSV" "$STOP_EXACT_COUNT" "$STOP_EXACT_REF_COUNT" <<'PY'
+import json
+import sys
+from pathlib import Path
+config = sys.argv[1]
+out = Path(sys.argv[2])
+n = int(sys.argv[3])
+global_batch = int(sys.argv[4])
+max_len = int(sys.argv[5])
+curve = Path(sys.argv[6])
+stop_exact_count = int(sys.argv[7])
+stop_exact_ref_count = int(sys.argv[8])
+row = json.loads((out / "decode_token_acc.jsonl").read_text().splitlines()[-1])
+views = int(row["ckpt_step"]) * global_batch
+tokens = views * max_len
+print(
+    "RESULT "
+    f"config={config} ckpt_step={row['ckpt_step']} views={views} "
+    f"token_acc={row['token_acc_mean']:.4f} exact={row['exact_count']}/{n} "
+    f"exact_refs={row['exact_ref_count']} hits={row['exact_ref_hits']}",
+    flush=True,
+)
+with curve.open("a", encoding="utf-8") as f:
+    f.write(
+        f"{config},{row['ckpt_step']},{views},{tokens},{row['token_acc_mean']},"
+        f"{row['exact_count']},{row['exact_ref_count']},\"{row['exact_ref_hits']}\"\n"
+    )
+raise SystemExit(
+    0
+    if int(row["exact_count"]) >= stop_exact_count
+    and int(row["exact_ref_count"]) >= stop_exact_ref_count
+    else 1
+)
+PY
+}
+
+reset_defaults() {
+  export MIN_MASK_RATIO=1.0
+  export MAX_MASK_RATIO=1.0
+  export MASK_MIXTURE_LOWK_PROB=0.0
+  export MASK_MIXTURE_ALL_PROB=1.0
+  export LOWK_CLEAN_TOKENS=0
+  export CLEAN_STATE_MODE=onehot
+  export TARGET_LOSS=hard_ce
+  export DIRICHLET_CONCENTRATION_MIN=1.0
+  export DIRICHLET_CONCENTRATION_MAX=1024
+  export SIMPLEX_BRIDGE_SAMPLER=dirichlet
+  export CATEGORICAL_WRONG_UNIGRAM_PROB=0.0
+  export CATEGORICAL_WRONG_UNIGRAM_SHARED_PROB=0.0
+  export CATEGORICAL_WRONG_PROB_FLOOR=0.0
+  export ROLLOUT_TRAIN_PROB=0.0
+  export ROLLOUT_TRAIN_STEPS=1
+  export ROLLOUT_TRAIN_INFER_STEPS=64
+  export ROLLOUT_TRAIN_TEMP=1.45
+  export ROLLOUT_TRAIN_MAX_GAMMA=1.0
+  export ROLLOUT_TRAIN_CORRUPT_ONLY=1
+  export ROLLOUT_TRAIN_SAMPLEWISE=1
+  export ROLLOUT_TRAIN_COMPUTE_ALWAYS=0
+}
+
+configure() {
+  local config="$1"
+  reset_defaults
+  case "${config}" in
+    rollin_p50_s4_i32)
+      export ROLLOUT_TRAIN_PROB=0.50
+      export ROLLOUT_TRAIN_STEPS=4
+      export ROLLOUT_TRAIN_INFER_STEPS=32
+      ;;
+    rollin_p75_s4_i32)
+      export ROLLOUT_TRAIN_PROB=0.75
+      export ROLLOUT_TRAIN_STEPS=4
+      export ROLLOUT_TRAIN_INFER_STEPS=32
+      ;;
+    rollin_p100_s4_i32)
+      export ROLLOUT_TRAIN_PROB=1.00
+      export ROLLOUT_TRAIN_STEPS=4
+      export ROLLOUT_TRAIN_INFER_STEPS=32
+      ;;
+    rollin_p50_s8_i64)
+      export ROLLOUT_TRAIN_PROB=0.50
+      export ROLLOUT_TRAIN_STEPS=8
+      export ROLLOUT_TRAIN_INFER_STEPS=64
+      ;;
+    rollin_p75_s8_i64)
+      export ROLLOUT_TRAIN_PROB=0.75
+      export ROLLOUT_TRAIN_STEPS=8
+      export ROLLOUT_TRAIN_INFER_STEPS=64
+      ;;
+    rollin_p50_s4_i32_temp1p0)
+      export ROLLOUT_TRAIN_PROB=0.50
+      export ROLLOUT_TRAIN_STEPS=4
+      export ROLLOUT_TRAIN_INFER_STEPS=32
+      export ROLLOUT_TRAIN_TEMP=1.0
+      ;;
+    *)
+      echo "unknown config: ${config}" >&2
+      return 2
+      ;;
+  esac
+}
+
+configs=(
+  rollin_p50_s4_i32
+  rollin_p75_s4_i32
+  rollin_p100_s4_i32
+  rollin_p50_s8_i64
+  rollin_p75_s8_i64
+  rollin_p50_s4_i32_temp1p0
+)
+
+echo "[rollin-focused] start stamp=${GROUP_STAMP} len=${MAX_LEN} vocab=${vocab_size} out=${OUT_ROOT}" | tee -a "${DRIVER_LOG}"
+
+round_idx=0
+while :; do
+  round_idx=$((round_idx + 1))
+  active=0
+  echo "[rollin-focused] round=${round_idx} $(date)" | tee -a "${DRIVER_LOG}"
+  for config in "${configs[@]}"; do
+    run_name="train8_rollin_focused_len${MAX_LEN}_${config}_${GROUP_STAMP}"
+    done_flag="${OUT_ROOT}/${config}/DONE"
+    if [[ -f "${done_flag}" ]]; then
+      echo "[rollin-focused] skip done config=${config}" | tee -a "${DRIVER_LOG}"
+      continue
+    fi
+    step_now="$(latest_step "${run_name}")"
+    if [[ "${step_now}" -ge "${MAX_TOTAL_STEPS}" ]]; then
+      echo "[rollin-focused] capped config=${config} step=${step_now}" | tee -a "${DRIVER_LOG}"
+      continue
+    fi
+    active=1
+    target_step=$((step_now + STEP_CHUNK))
+    if [[ "${target_step}" -gt "${MAX_TOTAL_STEPS}" ]]; then
+      target_step="${MAX_TOTAL_STEPS}"
+    fi
+    resume_path=""
+    if [[ -f "runs/${run_name}/latest.pt" ]]; then
+      resume_path="runs/${run_name}/latest.pt"
+    fi
+    configure "${config}"
+    echo "[rollin-focused] train config=${config} from=${step_now} to=${target_step} rollout=${ROLLOUT_TRAIN_PROB}/s${ROLLOUT_TRAIN_STEPS}/i${ROLLOUT_TRAIN_INFER_STEPS}/temp${ROLLOUT_TRAIN_TEMP}" | tee -a "${DRIVER_LOG}"
+    CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}" \
+    NPROC_PER_NODE="${NPROC_PER_NODE:-4}" \
+    MASTER_PORT="$(free_port)" \
+    OWT_CHUNK_CACHE_DIR="${cache}" \
+    OWT_EXACT_REPEAT_PER_CHUNK="${OWT_EXACT_REPEAT_PER_CHUNK:-64}" \
+    MAX_LEN="${MAX_LEN}" \
+    VOCAB_SIZE_OVERRIDE="${vocab_size}" \
+    D_MODEL="${D_MODEL:-192}" \
+    COND_DIM="${COND_DIM:-64}" \
+    N_LAYERS="${N_LAYERS:-3}" \
+    N_HEADS="${N_HEADS:-3}" \
+    DIM_FF="${DIM_FF:-768}" \
+    TOTAL_STEPS="${target_step}" \
+    PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE}" \
+    GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE}" \
+    NUM_WORKERS="${NUM_WORKERS:-0}" \
+    LOG_EVERY="${LOG_EVERY:-100}" \
+    SAVE_EVERY="${STEP_CHUNK}" \
+    LATEST_EVERY="${STEP_CHUNK}" \
+    WARMUP_STEPS="${WARMUP_STEPS:-10}" \
+    LEARNING_RATE="${LEARNING_RATE:-0.002}" \
+    WEIGHT_DECAY="${WEIGHT_DECAY:-0.1}" \
+    MUON_IMPL="${MUON_IMPL:-legacy}" \
+    OUTPUT_WEIGHT_DECAY="${OUTPUT_WEIGHT_DECAY:--1}" \
+    RUN_NAME="${run_name}" \
+    RESUME_PATH="${resume_path}" \
+    bash scripts/launch_lta_owt_gpt2_softendpoint_mn_pilot_4gpu.sh
+    echo "[rollin-focused] eval config=${config} step=${target_step}" | tee -a "${DRIVER_LOG}"
+    if eval_latest "${config}" "${run_name}" "${target_step}" | tee -a "${DRIVER_LOG}"; then
+      touch "${done_flag}"
+      echo "[rollin-focused] done config=${config}" | tee -a "${DRIVER_LOG}"
+    fi
+  done
+  if [[ "${active}" -eq 0 ]]; then
+    echo "[rollin-focused] all capped/done $(date)" | tee -a "${DRIVER_LOG}"
+    break
+  fi
+done

LTA_openwebtext_dualt/scripts/run_train8_selected_long20k_4gpu.sh

@@ -0,0 +1,228 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt
+
+export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}"
+export TOKENIZERS_PARALLELISM=false
+export PYTHONUNBUFFERED=1
+
+BASE_CACHE="${BASE_CACHE:-/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks}"
+TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-standard/tokenizer.json}"
+MAX_LEN="${MAX_LEN:-256}"
+N_SAMPLES="${N_SAMPLES:-64}"
+INFER_STEPS="${INFER_STEPS:-128}"
+STEP_CHUNK="${STEP_CHUNK:-1000}"
+MAX_TOTAL_STEPS="${MAX_TOTAL_STEPS:-20000}"
+PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE:-128}"
+GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-512}"
+GROUP_STAMP="${GROUP_STAMP:-$(date +%Y%m%d_%H%M%S)}"
+OUT_ROOT="${OUT_ROOT:-docs/lta_samples/metrics_20260517/selected_long20k_len${MAX_LEN}_bs512_ode128_${GROUP_STAMP}}"
+DRIVER_LOG="${DRIVER_LOG:-logs/selected_long20k_4gpu/${GROUP_STAMP}.log}"
+CURVE_CSV="${CURVE_CSV:-${OUT_ROOT}/hit_ratio_curve.csv}"
+mkdir -p "$(dirname "${DRIVER_LOG}")" "${OUT_ROOT}"
+
+cache="${BASE_CACHE}/gpt2_len${MAX_LEN}_train8_compact_overfit"
+vocab_size="$(
+python - "$cache" <<'PY'
+import json
+import sys
+from pathlib import Path
+meta = json.loads((Path(sys.argv[1]) / "meta.json").read_text())
+print(int(meta.get("compact_vocab_size", meta.get("vocab_size"))))
+PY
+)"
+
+if [[ ! -f "${CURVE_CSV}" ]]; then
+  echo "config,run_name,ckpt_step,train_views_seen,train_tokens_seen,token_acc_mean,exact_count,exact_ref_count,exact_ref_hits" > "${CURVE_CSV}"
+fi
+
+latest_step() {
+  local run_name="$1"
+  python - "$run_name" <<'PY'
+import re
+import sys
+from pathlib import Path
+run = Path("runs") / sys.argv[1]
+steps = []
+for path in run.glob("step_*.pt"):
+    m = re.search(r"step_(\d+)\.pt$", path.name)
+    if m:
+        steps.append(int(m.group(1)))
+print(max(steps) if steps else 0)
+PY
+}
+
+free_port() {
+  python - <<'PY'
+import socket
+with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+    s.bind(("127.0.0.1", 0))
+    print(s.getsockname()[1])
+PY
+}
+
+eval_latest() {
+  local config="$1"
+  local run_name="$2"
+  local target_step="$3"
+  local out_dir="${OUT_ROOT}/${config}/step_${target_step}"
+  mkdir -p "${out_dir}"
+  CUDA_VISIBLE_DEVICES="${EVAL_CUDA_VISIBLE_DEVICES:-0}" python scripts/eval_train8_decode_acc.py \
+    --runs_glob "runs/${run_name}" \
+    --data_dir "${cache}" \
+    --tokenizer_path "${TOKENIZER_PATH}" \
+    --out_dir "${out_dir}" \
+    --max_len "${MAX_LEN}" \
+    --n_samples "${N_SAMPLES}" \
+    --batch_size "${N_SAMPLES}" \
+    --latest_only \
+    --endpoint_softenings none \
+    --steps "${INFER_STEPS}" \
+    --decode_rule flowmap \
+    --time_schedule logit_normal \
+    --time_logit_mean -1.5 \
+    --time_logit_std 0.8 \
+    --model_t_mode post \
+    --c_min 1 \
+    --c_max 512 \
+    --late_temp 1.0 \
+    --final_from state \
+    --final_decode argmax
+  python - "$config" "$run_name" "$out_dir" "$N_SAMPLES" "$GLOBAL_BATCH_SIZE" "$MAX_LEN" "$CURVE_CSV" <<'PY'
+import json
+import sys
+from pathlib import Path
+config = sys.argv[1]
+run_name = sys.argv[2]
+out = Path(sys.argv[3])
+n = int(sys.argv[4])
+global_batch = int(sys.argv[5])
+max_len = int(sys.argv[6])
+curve = Path(sys.argv[7])
+row = json.loads((out / "decode_token_acc.jsonl").read_text().splitlines()[-1])
+views = int(row["ckpt_step"]) * global_batch
+tokens = views * max_len
+print(
+    "RESULT "
+    f"config={config} run={run_name} ckpt_step={row['ckpt_step']} views={views} "
+    f"token_acc={row['token_acc_mean']:.4f} exact={row['exact_count']}/{n} "
+    f"exact_refs={row['exact_ref_count']} hits={row['exact_ref_hits']}",
+    flush=True,
+)
+with curve.open("a", encoding="utf-8") as f:
+    f.write(
+        f"{config},{run_name},{row['ckpt_step']},{views},{tokens},{row['token_acc_mean']},"
+        f"{row['exact_count']},{row['exact_ref_count']},\"{row['exact_ref_hits']}\"\n"
+    )
+PY
+}
+
+reset_common() {
+  export MIN_MASK_RATIO=1.0
+  export MAX_MASK_RATIO=1.0
+  export MASK_MIXTURE_LOWK_PROB=0.0
+  export MASK_MIXTURE_ALL_PROB=1.0
+  export LOWK_CLEAN_TOKENS=0
+  export CLEAN_STATE_MODE=onehot
+  export TARGET_LOSS=hard_ce
+  export DIRICHLET_CONCENTRATION_MIN=1.0
+  export DIRICHLET_CONCENTRATION_MAX=1024
+  export SIMPLEX_BRIDGE_SAMPLER=dirichlet
+  export CATEGORICAL_WRONG_PROB_FLOOR=0.0
+  export CATEGORICAL_WRONG_UNIGRAM_PROB=0.0
+  export CATEGORICAL_WRONG_UNIGRAM_SHARED_PROB=0.0
+  export ROLLOUT_TRAIN_PROB=0.0
+  export ROLLOUT_TRAIN_STEPS=1
+  export ROLLOUT_TRAIN_INFER_STEPS=64
+  export ROLLOUT_TRAIN_TEMP=1.45
+  export ROLLOUT_TRAIN_MAX_GAMMA=1.0
+  export ROLLOUT_TRAIN_CORRUPT_ONLY=1
+  export ROLLOUT_TRAIN_SAMPLEWISE=1
+  export ROLLOUT_TRAIN_COMPUTE_ALWAYS=0
+  export ROLLOUT_TRAIN_SYNC_T=0
+}
+
+configure() {
+  local config="$1"
+  reset_common
+  case "${config}" in
+    baseline_allcorrupt)
+      RUN_NAME_SELECTED="train8_noisegeo_len256_allcorrupt_fullvocab_dirC1_1024_20260517_163805"
+      ;;
+    rollin_p50_s4_old)
+      RUN_NAME_SELECTED="train8_rollin_len256_rollin_p50_s4_i32_20260517_171654"
+      export ROLLOUT_TRAIN_PROB=0.50
+      export ROLLOUT_TRAIN_STEPS=4
+      export ROLLOUT_TRAIN_INFER_STEPS=32
+      export ROLLOUT_TRAIN_TEMP=1.45
+      export ROLLOUT_TRAIN_SYNC_T=0
+      ;;
+    *)
+      echo "unknown config: ${config}" >&2
+      return 2
+      ;;
+  esac
+}
+
+configs=(
+  baseline_allcorrupt
+  rollin_p50_s4_old
+)
+
+echo "[selected-long20k] start stamp=${GROUP_STAMP} len=${MAX_LEN} vocab=${vocab_size} out=${OUT_ROOT}" | tee -a "${DRIVER_LOG}"
+while :; do
+  active=0
+  for config in "${configs[@]}"; do
+    configure "${config}"
+    run_name="${RUN_NAME_SELECTED}"
+    step_now="$(latest_step "${run_name}")"
+    if [[ "${step_now}" -ge "${MAX_TOTAL_STEPS}" ]]; then
+      echo "[selected-long20k] capped config=${config} run=${run_name} step=${step_now}" | tee -a "${DRIVER_LOG}"
+      continue
+    fi
+    active=1
+    target_step=$((step_now + STEP_CHUNK))
+    if [[ "${target_step}" -gt "${MAX_TOTAL_STEPS}" ]]; then
+      target_step="${MAX_TOTAL_STEPS}"
+    fi
+    resume_path=""
+    if [[ -f "runs/${run_name}/latest.pt" ]]; then
+      resume_path="runs/${run_name}/latest.pt"
+    fi
+    echo "[selected-long20k] train config=${config} run=${run_name} from=${step_now} to=${target_step} rollout=${ROLLOUT_TRAIN_PROB}/s${ROLLOUT_TRAIN_STEPS}/i${ROLLOUT_TRAIN_INFER_STEPS}" | tee -a "${DRIVER_LOG}"
+    CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}" \
+    NPROC_PER_NODE="${NPROC_PER_NODE:-4}" \
+    MASTER_PORT="$(free_port)" \
+    OWT_CHUNK_CACHE_DIR="${cache}" \
+    OWT_EXACT_REPEAT_PER_CHUNK="${OWT_EXACT_REPEAT_PER_CHUNK:-64}" \
+    MAX_LEN="${MAX_LEN}" \
+    VOCAB_SIZE_OVERRIDE="${vocab_size}" \
+    D_MODEL="${D_MODEL:-192}" \
+    COND_DIM="${COND_DIM:-64}" \
+    N_LAYERS="${N_LAYERS:-3}" \
+    N_HEADS="${N_HEADS:-3}" \
+    DIM_FF="${DIM_FF:-768}" \
+    TOTAL_STEPS="${target_step}" \
+    PER_GPU_BATCH_SIZE="${PER_GPU_BATCH_SIZE}" \
+    GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE}" \
+    NUM_WORKERS="${NUM_WORKERS:-0}" \
+    LOG_EVERY="${LOG_EVERY:-100}" \
+    SAVE_EVERY="${STEP_CHUNK}" \
+    LATEST_EVERY="${STEP_CHUNK}" \
+    WARMUP_STEPS="${WARMUP_STEPS:-10}" \
+    LEARNING_RATE="${LEARNING_RATE:-0.002}" \
+    WEIGHT_DECAY="${WEIGHT_DECAY:-0.1}" \
+    MUON_IMPL="${MUON_IMPL:-legacy}" \
+    OUTPUT_WEIGHT_DECAY="${OUTPUT_WEIGHT_DECAY:--1}" \
+    RUN_NAME="${run_name}" \
+    RESUME_PATH="${resume_path}" \
+    bash scripts/launch_lta_owt_gpt2_softendpoint_mn_pilot_4gpu.sh
+    echo "[selected-long20k] eval config=${config} step=${target_step}" | tee -a "${DRIVER_LOG}"
+    eval_latest "${config}" "${run_name}" "${target_step}" | tee -a "${DRIVER_LOG}"
+  done
+  if [[ "${active}" -eq 0 ]]; then
+    echo "[selected-long20k] all capped $(date)" | tee -a "${DRIVER_LOG}"
+    break
+  fi
+done

LTA_openwebtext_dualt/scripts/score_lta_decode_strategy.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import torch
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from eval import build_model_from_ckpt
+from flowtext_lab.decode import sample_noise_simplex
+from flowtext_lab.genppl import filter_generated_texts, score_gen_ppl, summarize_token_diversity
+from flowtext_lab.tokenization import BpeTextTokenizer
+from scripts.flowtext_decode_lab import DecodeConfig, decode_batch
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser()
+    p.add_argument("--checkpoint", required=True)
+    p.add_argument("--tokenizer_path", required=True)
+    p.add_argument("--max_len", type=int, default=128)
+    p.add_argument("--samples", type=int, default=128)
+    p.add_argument("--batch_size", type=int, default=4)
+    p.add_argument("--steps", type=int, default=128)
+    p.add_argument("--seed", type=int, default=20260502)
+    p.add_argument("--noise_init", choices=["uniform", "logistic_normal", "dirichlet"], default="dirichlet")
+    p.add_argument("--dirichlet_init_concentration", type=float, default=1.0)
+    p.add_argument("--target_prob", type=float, default=1.0)
+    p.add_argument("--noise_sigma", type=float, default=-1.0)
+    p.add_argument("--eps", type=float, default=1e-8)
+    p.add_argument("--rule", choices=["flowmap", "replace", "geometric", "centered_residual"], default="flowmap")
+    p.add_argument("--model_t_mode", choices=["linear", "flow", "const0", "const05", "const1", "random"], default="flow")
+    p.add_argument("--damping", type=float, default=1.0)
+    p.add_argument("--max_gamma", type=float, default=1.0)
+    p.add_argument("--eta", type=float, default=0.5)
+    p.add_argument("--endpoint_temp", type=float, default=1.0)
+    p.add_argument("--final_from", choices=["state", "endpoint", "blend"], default="state")
+    p.add_argument("--state_floor", type=float, default=1e-8)
+    p.add_argument("--eos_logit_bias", type=float, default=0.0)
+    p.add_argument("--output", required=True)
+    p.add_argument("--gen_ppl_model", required=True)
+    p.add_argument("--gen_ppl_batch_size", type=int, default=4)
+    p.add_argument("--gen_ppl_max_length", type=int, default=1024)
+    return p.parse_args()
+
+
+@torch.no_grad()
+def main() -> None:
+    args = parse_args()
+    torch.manual_seed(args.seed)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    tokenizer = BpeTextTokenizer.from_file(args.tokenizer_path)
+    ckpt = torch.load(args.checkpoint, map_location="cpu")
+    model = build_model_from_ckpt(ckpt, tokenizer.vocab_size, args.max_len, device)
+    model.eval()
+
+    cfg = DecodeConfig(
+        label=f"{args.rule}_t{args.endpoint_temp:g}_{args.final_from}",
+        rule=args.rule,
+        steps=args.steps,
+        model_t_mode=args.model_t_mode,
+        eta=args.eta,
+        damping=args.damping,
+        max_gamma=args.max_gamma,
+        endpoint_temp=args.endpoint_temp,
+        state_floor=args.state_floor,
+        final_from=args.final_from,
+        eos_logit_bias=args.eos_logit_bias,
+    )
+
+    output = Path(args.output)
+    if not output.is_absolute():
+        output = Path(args.checkpoint).resolve().parent / output
+    output.parent.mkdir(parents=True, exist_ok=True)
+
+    generated_texts: list[str] = []
+    generated_ids: list[list[int]] = []
+    remaining = args.samples
+    with output.open("w", encoding="utf-8") as f:
+        sample_index = 0
+        while remaining > 0:
+            cur_bs = min(args.batch_size, remaining)
+            init = sample_noise_simplex(
+                (cur_bs, args.max_len),
+                tokenizer.vocab_size,
+                device,
+                args.eps,
+                noise_mode=args.noise_init,
+                target_prob=args.target_prob,
+                noise_sigma=args.noise_sigma,
+                dirichlet_concentration=args.dirichlet_init_concentration,
+            )
+            attn = torch.ones((cur_bs, args.max_len), dtype=torch.bool, device=device)
+            lock = torch.zeros((cur_bs, args.max_len), dtype=torch.bool, device=device)
+            lock_probs = torch.zeros((cur_bs, args.max_len, tokenizer.vocab_size), dtype=torch.float32, device=device)
+            decoded = decode_batch(model, init, attn, lock, lock_probs, cfg, args.eps, tokenizer.eos_id)
+            ids_batch = decoded.argmax(dim=-1).detach().cpu().tolist()
+            for ids in ids_batch:
+                text = tokenizer.decode(ids, stop_at_eos=False, skip_special_tokens=False)
+                generated_ids.append(ids)
+                generated_texts.append(text)
+                f.write(json.dumps({"index": sample_index, "text": text, "ids": ids}, ensure_ascii=False) + "\n")
+                sample_index += 1
+            remaining -= cur_bs
+
+    kept_texts, keep_mask = filter_generated_texts(generated_texts, drop_empty=True)
+    diversity = summarize_token_diversity(generated_ids)
+    gen_result = score_gen_ppl(
+        kept_texts,
+        model_name_or_path=args.gen_ppl_model,
+        batch_size=args.gen_ppl_batch_size,
+        max_length=args.gen_ppl_max_length,
+        device=device,
+        drop_remainder=False,
+    )
+    print(json.dumps(
+        {
+            "checkpoint": args.checkpoint,
+            "output": str(output),
+            "samples": args.samples,
+            "kept_samples": len(kept_texts),
+            "decode": vars(args),
+            "gen_ppl": gen_result.ppl,
+            "gen_nll_per_token": gen_result.nll_per_token,
+            "gen_tokens": gen_result.tokens,
+            "gen_scored_samples": gen_result.kept_samples - gen_result.skipped_samples,
+            "sample_entropy": diversity.sample_entropy,
+            "distinct_1": diversity.distinct_1,
+            "distinct_2": diversity.distinct_2,
+            "top_token_mass": diversity.top_token_mass,
+            "preview": kept_texts[:5],
+        },
+        ensure_ascii=False,
+        indent=2,
+    ))
+
+
+if __name__ == "__main__":
+    main()

LTA_openwebtext_dualt/scripts/watch_infer_lm1b_classic_c1024_every1k_t1p45.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt
+
+export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}"
+export TOKENIZERS_PARALLELISM=false
+export PYTHONUNBUFFERED=1
+
+RUN_DIR="${RUN_DIR:?RUN_DIR is required, e.g. runs/<run_name>}"
+TOKENIZER_PATH="${TOKENIZER_PATH:-/e2e-data/evad-tech-vla/wanghan58/workspace/imagenet_handoff_20260327/nlp_dts_light/assets/distilbert-base-uncased/tokenizer.json}"
+SCORER="${SCORER:-/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard}"
+
+CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-3}"
+N_SAMPLES="${N_SAMPLES:-256}"
+STEPS="${STEPS:-128}"
+CMAX="${CMAX:-1024}"
+TEMP="${TEMP:-1.45}"
+MAX_LEN="${MAX_LEN:-128}"
+DECODE_BATCH="${DECODE_BATCH:-16}"
+SCORE_BATCH="${SCORE_BATCH:-8}"
+SCORE_MAX_LENGTH="${SCORE_MAX_LENGTH:-256}"
+STEP_INTERVAL="${STEP_INTERVAL:-1000}"
+SLEEP_SECONDS="${SLEEP_SECONDS:-30}"
+DATE_TAG="${DATE_TAG:-$(date +%Y%m%d)}"
+
+RUN_STEM="$(basename "${RUN_DIR}")"
+TEMP_TAG="${TEMP//./p}"
+OUT_BASE="${OUT_BASE:-docs/lta_samples/metrics_${DATE_TAG}/lm1b_classic_dirichlet_len${MAX_LEN}_every1k_normal_steps_state_t${TEMP_TAG}_c${CMAX}_n${N_SAMPLES}/${RUN_STEM}}"
+LOG_DIR="${LOG_DIR:-logs/lm1b_classic_dirichlet_every1k_infer_watch}"
+PROCESSED_FILE="${PROCESSED_FILE:-${LOG_DIR}/processed_${RUN_STEM}_steps${STEPS}_c${CMAX}_t${TEMP_TAG}_n${N_SAMPLES}.txt}"
+
+mkdir -p "${OUT_BASE}" "${LOG_DIR}"
+touch "${PROCESSED_FILE}"
+
+echo "[watch-classic-1k] run_dir=${RUN_DIR}"
+echo "[watch-classic-1k] out_base=${OUT_BASE}"
+echo "[watch-classic-1k] processed_file=${PROCESSED_FILE}"
+echo "[watch-classic-1k] interval=${STEP_INTERVAL} decode=normal_steps_sweep steps=${STEPS} cmax=${CMAX} temp=${TEMP} final_from=state n=${N_SAMPLES}"
+
+while true; do
+  shopt -s nullglob
+  ckpts=("${RUN_DIR}"/step_*.pt)
+  shopt -u nullglob
+
+  if (( ${#ckpts[@]} == 0 )); then
+    echo "[watch-classic-1k] $(date +%F_%T) no step_*.pt yet"
+    sleep "${SLEEP_SECONDS}"
+    continue
+  fi
+
+  printf '%s\n' "${ckpts[@]}" | sort | while read -r ckpt; do
+    base="$(basename "${ckpt}")"
+    step="${base#step_}"
+    step="${step%.pt}"
+    step_num="$((10#${step}))"
+    if (( step_num % STEP_INTERVAL != 0 )); then
+      continue
+    fi
+    if grep -Fxq "${ckpt}" "${PROCESSED_FILE}"; then
+      continue
+    fi
+
+    out_dir="${OUT_BASE}/step_${step}"
+    log_file="${LOG_DIR}/infer_${RUN_STEM}_step_${step}_t${TEMP_TAG}.log"
+    mkdir -p "${out_dir}"
+
+    echo "[watch-classic-1k] $(date +%F_%T) infer ${ckpt} -> ${out_dir}" | tee -a "${log_file}"
+    CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" python scripts/eval_owt_normal_steps_sweep_20260515.py \
+      --checkpoint "${ckpt}" \
+      --tokenizer_path "${TOKENIZER_PATH}" \
+      --scorer "${SCORER}" \
+      --out_dir "${out_dir}" \
+      --steps_list "${STEPS}" \
+      --cmax_list "${CMAX}" \
+      --endpoint_temps "${TEMP}" \
+      --n_samples "${N_SAMPLES}" \
+      --max_len "${MAX_LEN}" \
+      --decode_batch "${DECODE_BATCH}" \
+      --score_batch "${SCORE_BATCH}" \
+      --score_max_length "${SCORE_MAX_LENGTH}" \
+      --detokenizer lm1b \
+      --seed 20260511 \
+      --save_samples 16 \
+      2>&1 | tee -a "${log_file}"
+
+    echo "${ckpt}" >> "${PROCESSED_FILE}"
+    echo "[watch-classic-1k] $(date +%F_%T) done step_${step}" | tee -a "${log_file}"
+  done
+
+  sleep "${SLEEP_SECONDS}"
+done