diff --git a/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/baseline_allcorrupt/step_6000/decode_token_acc_summary.json b/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/baseline_allcorrupt/step_6000/decode_token_acc_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7cf43d13e9c210508ce66f576d5b91fba5bd6add --- /dev/null +++ b/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/baseline_allcorrupt/step_6000/decode_token_acc_summary.json @@ -0,0 +1,159 @@ +{ + "num_rows": 1, + "best_by_run": { + "train8_noisegeo_len256_allcorrupt_fullvocab_dirC1_1024_20260517_163805::none": { + "run": "train8_noisegeo_len256_allcorrupt_fullvocab_dirC1_1024_20260517_163805", + "checkpoint": "runs/train8_noisegeo_len256_allcorrupt_fullvocab_dirC1_1024_20260517_163805/step_0006000.pt", + "ckpt_step": 6000, + "endpoint_softening": "none", + "decode_rule": "flowmap", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.1416015625, + "token_acc_min": 0.0625, + "token_acc_max": 0.34375, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 2, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 0, + 5, + 2, + 2, + 4, + 2, + 5, + 1, + 1, + 1, + 1, + 1, + 5, + 1, + 4, + 2, + 0, + 2, + 0, + 1, + 2, + 5, + 2, + 1, + 2, + 1, + 2, + 5, + 0, + 1, + 2, + 1, + 2, + 2, + 1, + 5, + 5, + 0, + 2, + 2, + 2, + 5, + 1, + 1, + 1, + 2, + 5, + 1, + 2, + 1, + 2, + 2, + 0, + 1, + 2, + 1 + ], + "best_token_acc": [ + 0.109375, + 0.16796875, + 0.08203125, + 0.0625, + 0.13671875, + 0.1640625, + 0.07421875, + 0.13671875, + 0.19921875, + 0.13671875, + 0.0859375, + 0.078125, + 0.24609375, + 0.12109375, + 0.140625, + 0.078125, + 0.07421875, + 0.21484375, + 0.16796875, + 0.234375, + 0.109375, + 0.3125, + 0.20703125, + 0.09375, + 0.24609375, + 0.078125, + 0.171875, + 0.1796875, + 0.09765625, + 0.109375, + 0.09765625, + 0.296875, + 0.0703125, + 0.08984375, + 0.0703125, + 0.19921875, + 0.17578125, + 0.2421875, + 0.0859375, + 0.2578125, + 0.10546875, + 0.21484375, + 0.1640625, + 0.17578125, + 0.125, + 0.078125, + 0.078125, + 0.1015625, + 0.14453125, + 0.0859375, + 0.34375, + 0.06640625, + 0.1015625, + 0.09375, + 0.109375, + 0.2421875, + 0.08984375, + 0.109375, + 0.14453125, + 0.08984375, + 0.17578125, + 0.08984375, + 0.11328125, + 0.1171875 + ] + } + }, + "first_exact_by_run": {} +} \ No newline at end of file diff --git a/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/baseline_allcorrupt/step_7000/decode_token_acc.jsonl b/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/baseline_allcorrupt/step_7000/decode_token_acc.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d36b04e14b37fd6b6c4050db0fd8f7d9713d91b6 --- /dev/null +++ b/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/baseline_allcorrupt/step_7000/decode_token_acc.jsonl @@ -0,0 +1 @@ +{"run": "train8_noisegeo_len256_allcorrupt_fullvocab_dirC1_1024_20260517_163805", "checkpoint": "runs/train8_noisegeo_len256_allcorrupt_fullvocab_dirC1_1024_20260517_163805/step_0007000.pt", "ckpt_step": 7000, "endpoint_softening": "none", "decode_rule": "flowmap", "steps": 128, "time_schedule": "logit_normal", "model_t_mode": "post", "final_from": "state", "n_gen": 64, "n_refs": 8, "token_acc_mean": 0.27874755859375, "token_acc_min": 0.07421875, "token_acc_max": 0.58984375, "exact_acc": 0.0, "exact_count": 0, "exact_ref_coverage": 0.0, "exact_ref_count": 0, "exact_ref_hits": [], "best_ref_idx": [3, 0, 3, 4, 1, 2, 4, 7, 2, 7, 1, 0, 0, 4, 2, 4, 2, 4, 7, 2, 4, 4, 4, 5, 4, 5, 2, 4, 4, 4, 2, 1, 2, 1, 4, 2, 2, 1, 4, 2, 1, 3, 1, 1, 4, 4, 1, 2, 4, 2, 1, 0, 2, 5, 0, 1, 1, 4, 4, 2, 1, 4, 4, 5], "best_token_acc": [0.13671875, 0.07421875, 0.1015625, 0.25390625, 0.4609375, 0.28125, 0.1484375, 0.203125, 0.109375, 0.15234375, 0.10546875, 0.40625, 0.25390625, 0.5390625, 0.3203125, 0.32421875, 0.44921875, 0.23828125, 0.11328125, 0.26171875, 0.49609375, 0.390625, 0.56640625, 0.13671875, 0.16015625, 0.203125, 0.078125, 0.12109375, 0.53125, 0.58984375, 0.40234375, 0.0859375, 0.43359375, 0.203125, 0.4921875, 0.4375, 0.4375, 0.1015625, 0.15234375, 0.46875, 0.078125, 0.1015625, 0.0859375, 0.37109375, 0.56640625, 0.5546875, 0.3203125, 0.08984375, 0.47265625, 0.109375, 0.0859375, 0.33203125, 0.11328125, 0.17578125, 0.37890625, 0.30078125, 0.2109375, 0.20703125, 0.1796875, 0.375, 0.1015625, 0.5390625, 0.46875, 0.19921875]} diff --git a/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/baseline_allcorrupt/step_7000/decode_token_acc.tsv b/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/baseline_allcorrupt/step_7000/decode_token_acc.tsv new file mode 100644 index 0000000000000000000000000000000000000000..0b2d2d1682d29f9b32d313c955155cfc6ac0cbd0 --- /dev/null +++ b/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/baseline_allcorrupt/step_7000/decode_token_acc.tsv @@ -0,0 +1,2 @@ +run ckpt_step endpoint_softening token_acc_mean token_acc_min token_acc_max exact_acc exact_count exact_ref_coverage exact_ref_count +train8_noisegeo_len256_allcorrupt_fullvocab_dirC1_1024_20260517_163805 7000 none 0.27874755859375 0.07421875 0.58984375 0.0 0 0.0 0 diff --git a/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/baseline_allcorrupt/step_7000/decode_token_acc_summary.json b/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/baseline_allcorrupt/step_7000/decode_token_acc_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..713b019600b20bae2f3b2f4692f68ee5314171a9 --- /dev/null +++ b/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/baseline_allcorrupt/step_7000/decode_token_acc_summary.json @@ -0,0 +1,159 @@ +{ + "num_rows": 1, + "best_by_run": { + "train8_noisegeo_len256_allcorrupt_fullvocab_dirC1_1024_20260517_163805::none": { + "run": "train8_noisegeo_len256_allcorrupt_fullvocab_dirC1_1024_20260517_163805", + "checkpoint": "runs/train8_noisegeo_len256_allcorrupt_fullvocab_dirC1_1024_20260517_163805/step_0007000.pt", + "ckpt_step": 7000, + "endpoint_softening": "none", + "decode_rule": "flowmap", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.27874755859375, + "token_acc_min": 0.07421875, + "token_acc_max": 0.58984375, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 3, + 0, + 3, + 4, + 1, + 2, + 4, + 7, + 2, + 7, + 1, + 0, + 0, + 4, + 2, + 4, + 2, + 4, + 7, + 2, + 4, + 4, + 4, + 5, + 4, + 5, + 2, + 4, + 4, + 4, + 2, + 1, + 2, + 1, + 4, + 2, + 2, + 1, + 4, + 2, + 1, + 3, + 1, + 1, + 4, + 4, + 1, + 2, + 4, + 2, + 1, + 0, + 2, + 5, + 0, + 1, + 1, + 4, + 4, + 2, + 1, + 4, + 4, + 5 + ], + "best_token_acc": [ + 0.13671875, + 0.07421875, + 0.1015625, + 0.25390625, + 0.4609375, + 0.28125, + 0.1484375, + 0.203125, + 0.109375, + 0.15234375, + 0.10546875, + 0.40625, + 0.25390625, + 0.5390625, + 0.3203125, + 0.32421875, + 0.44921875, + 0.23828125, + 0.11328125, + 0.26171875, + 0.49609375, + 0.390625, + 0.56640625, + 0.13671875, + 0.16015625, + 0.203125, + 0.078125, + 0.12109375, + 0.53125, + 0.58984375, + 0.40234375, + 0.0859375, + 0.43359375, + 0.203125, + 0.4921875, + 0.4375, + 0.4375, + 0.1015625, + 0.15234375, + 0.46875, + 0.078125, + 0.1015625, + 0.0859375, + 0.37109375, + 0.56640625, + 0.5546875, + 0.3203125, + 0.08984375, + 0.47265625, + 0.109375, + 0.0859375, + 0.33203125, + 0.11328125, + 0.17578125, + 0.37890625, + 0.30078125, + 0.2109375, + 0.20703125, + 0.1796875, + 0.375, + 0.1015625, + 0.5390625, + 0.46875, + 0.19921875 + ] + } + }, + "first_exact_by_run": {} +} \ No newline at end of file diff --git a/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/rollin_p50_s4_old/step_3000/decode_token_acc.jsonl b/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/rollin_p50_s4_old/step_3000/decode_token_acc.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..af2ea79771c68f0dd4f6c425a2c2d106289bcb10 --- /dev/null +++ b/LTA_openwebtext_dualt/docs/lta_samples/metrics_20260517/selected_long20k_len256_bs512_ode128_20260517_1830long20k/rollin_p50_s4_old/step_3000/decode_token_acc.jsonl @@ -0,0 +1 @@ +{"run": "train8_rollin_len256_rollin_p50_s4_i32_20260517_171654", "checkpoint": "runs/train8_rollin_len256_rollin_p50_s4_i32_20260517_171654/step_0003000.pt", "ckpt_step": 3000, "endpoint_softening": "none", "decode_rule": "flowmap", "steps": 128, "time_schedule": "logit_normal", "model_t_mode": "post", "final_from": "state", "n_gen": 64, "n_refs": 8, "token_acc_mean": 0.9420166015625, "token_acc_min": 0.12109375, "token_acc_max": 1.0, "exact_acc": 0.140625, "exact_count": 9, "exact_ref_coverage": 0.125, "exact_ref_count": 1, "exact_ref_hits": [6], "best_ref_idx": [6, 6, 6, 6, 6, 6, 4, 6, 1, 6, 7, 6, 6, 6, 7, 6, 6, 6, 6, 4, 6, 6, 6, 6, 1, 6, 6, 4, 6, 4, 6, 6, 6, 6, 6, 6, 6, 1, 6, 6, 6, 6, 4, 6, 6, 6, 6, 2, 6, 6, 4, 6, 6, 6, 4, 6, 4, 6, 6, 6, 1, 6, 6, 4], "best_token_acc": [1.0, 1.0, 0.98046875, 0.99609375, 0.9453125, 0.921875, 0.4765625, 0.98828125, 0.953125, 0.953125, 0.98828125, 0.99609375, 0.91015625, 0.94140625, 0.9921875, 0.99609375, 0.98828125, 0.99609375, 0.9921875, 0.859375, 0.99609375, 0.9921875, 0.984375, 0.98828125, 0.9765625, 0.99609375, 0.9921875, 0.98046875, 0.99609375, 0.9453125, 1.0, 0.98046875, 0.984375, 0.99609375, 0.984375, 0.99609375, 0.8125, 0.73828125, 0.9921875, 0.99609375, 0.93359375, 1.0, 0.96875, 1.0, 0.98828125, 0.9921875, 1.0, 0.96875, 1.0, 0.953125, 0.42578125, 0.9921875, 0.9765625, 0.984375, 0.96875, 0.984375, 0.12109375, 1.0, 0.98828125, 1.0, 0.88671875, 0.99609375, 0.98046875, 0.9765625]} diff --git a/LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/ctx1024_core_tradeoff_dual_20260517_230929.log b/LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/ctx1024_core_tradeoff_dual_20260517_230929.log new file mode 100644 index 0000000000000000000000000000000000000000..5216938f81727ff7441656670386f9b5447b644d --- /dev/null +++ b/LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/ctx1024_core_tradeoff_dual_20260517_230929.log @@ -0,0 +1,2435 @@ +[ctx1024-sampleds] start stamp=ctx1024_core_tradeoff_dual_20260517_230929 len=1024 vocab=2664 out=docs/lta_samples/metrics_20260517/ctx1024_sampleds_sweep_bs512_ode128_ctx1024_core_tradeoff_dual_20260517_230929 +[ctx1024-sampleds] config=p50_unif0_0p25_outwdm1 run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 p=0.50 mode=sampled_s steps=1 s_dist=uniform s_frac=0.0->0.25 beta=2.0,6.0 outwd=-1 sync_t=1 +[ctx1024-sampleds] train config=p50_unif0_0p25_outwdm1 from=0 to=1000 +[ctx1024-sampleds] eval config=p50_unif0_0p25_outwdm1 step=1000 +[eval-decode-acc] train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 step=1000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0001000.pt", + "ckpt_step": 1000, + "endpoint_softening": "none", + "decode_rule": "flowmap", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.045379638671875, + "token_acc_min": 0.029296875, + "token_acc_max": 0.1044921875, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 0, + 5, + 2, + 5, + 5, + 0, + 5, + 2, + 5, + 7, + 3, + 5, + 5, + 5, + 5, + 5, + 5, + 7, + 5, + 5, + 2, + 5, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 7, + 7, + 0, + 2, + 5, + 5, + 5, + 5, + 2, + 5, + 2, + 5, + 5, + 0, + 5, + 5, + 5, + 2, + 2, + 2, + 5, + 5, + 2 + ], + "best_token_acc": [ + 0.0380859375, + 0.0439453125, + 0.0419921875, + 0.0537109375, + 0.046875, + 0.0400390625, + 0.0498046875, + 0.0498046875, + 0.041015625, + 0.0439453125, + 0.046875, + 0.0341796875, + 0.037109375, + 0.0458984375, + 0.0478515625, + 0.0302734375, + 0.0380859375, + 0.037109375, + 0.044921875, + 0.03515625, + 0.0673828125, + 0.041015625, + 0.0458984375, + 0.068359375, + 0.0458984375, + 0.0439453125, + 0.0361328125, + 0.0390625, + 0.04296875, + 0.0419921875, + 0.1044921875, + 0.052734375, + 0.048828125, + 0.0390625, + 0.0458984375, + 0.072265625, + 0.052734375, + 0.0400390625, + 0.0751953125, + 0.0419921875, + 0.0380859375, + 0.041015625, + 0.041015625, + 0.0419921875, + 0.0361328125, + 0.033203125, + 0.07421875, + 0.0390625, + 0.033203125, + 0.029296875, + 0.0478515625, + 0.0341796875, + 0.03125, + 0.04296875, + 0.041015625, + 0.056640625, + 0.078125, + 0.0419921875, + 0.03515625, + 0.0400390625, + 0.0341796875, + 0.0498046875, + 0.0361328125, + 0.03515625 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p50_unif0_0p25_outwdm1 decode=flowmap run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 ckpt_step=1000 views=512000 token_acc=0.0454 exact=0/64 exact_refs=0 hits=[] +[eval-decode-acc] train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 step=1000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0001000.pt", + "ckpt_step": 1000, + "endpoint_softening": "none", + "decode_rule": "dual_line_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.0696868896484375, + "token_acc_min": 0.021484375, + "token_acc_max": 0.203125, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 2, + 2, + 2, + 5, + 5, + 2, + 2, + 5, + 5, + 2, + 2, + 7, + 4, + 5, + 2, + 5, + 2, + 5, + 2, + 2, + 5, + 2, + 2, + 5, + 5, + 2, + 2, + 3, + 5, + 2, + 2, + 5, + 0, + 2, + 2, + 2, + 2, + 5, + 4, + 0, + 4, + 2, + 2, + 2, + 2, + 3, + 5, + 0, + 5, + 5, + 5, + 5, + 2, + 5, + 2, + 0, + 5, + 5, + 2, + 5, + 2, + 2, + 5, + 3 + ], + "best_token_acc": [ + 0.1494140625, + 0.0439453125, + 0.1435546875, + 0.0322265625, + 0.072265625, + 0.138671875, + 0.046875, + 0.0283203125, + 0.1748046875, + 0.046875, + 0.048828125, + 0.04296875, + 0.0322265625, + 0.1650390625, + 0.0927734375, + 0.029296875, + 0.029296875, + 0.0498046875, + 0.0283203125, + 0.072265625, + 0.0390625, + 0.1142578125, + 0.068359375, + 0.0439453125, + 0.03515625, + 0.056640625, + 0.12109375, + 0.0498046875, + 0.021484375, + 0.0478515625, + 0.0361328125, + 0.05078125, + 0.0283203125, + 0.0810546875, + 0.1201171875, + 0.04296875, + 0.0888671875, + 0.1640625, + 0.0302734375, + 0.0263671875, + 0.0302734375, + 0.0517578125, + 0.0283203125, + 0.0390625, + 0.0498046875, + 0.0322265625, + 0.0634765625, + 0.0322265625, + 0.203125, + 0.103515625, + 0.1015625, + 0.03125, + 0.0322265625, + 0.048828125, + 0.0302734375, + 0.1142578125, + 0.1953125, + 0.0751953125, + 0.1484375, + 0.10546875, + 0.033203125, + 0.03125, + 0.0986328125, + 0.0458984375 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p50_unif0_0p25_outwdm1 decode=dual_line_resample run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 ckpt_step=1000 views=512000 token_acc=0.0697 exact=0/64 exact_refs=0 hits=[] +[ctx1024-sampleds] train config=p50_unif0_0p25_outwdm1 from=1000 to=2000 +[ctx1024-sampleds] eval config=p50_unif0_0p25_outwdm1 step=2000 +[eval-decode-acc] train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 step=2000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0002000.pt", + "ckpt_step": 2000, + "endpoint_softening": "none", + "decode_rule": "flowmap", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.87042236328125, + "token_acc_min": 0.12109375, + "token_acc_max": 0.96875, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 3, + 3, + 2, + 2, + 2, + 3, + 4, + 2, + 7, + 2, + 2, + 3, + 2, + 3, + 4, + 3, + 4, + 3, + 3, + 5, + 2, + 2, + 3, + 2, + 0, + 4, + 0, + 2, + 3, + 3, + 4, + 2, + 2, + 2, + 3, + 7, + 3, + 0, + 3, + 3, + 2, + 3, + 3, + 3, + 3, + 7, + 2, + 3, + 2, + 2, + 3, + 3, + 5, + 3, + 0, + 0, + 7, + 3, + 3, + 3, + 3, + 2, + 3, + 2 + ], + "best_token_acc": [ + 0.9091796875, + 0.91015625, + 0.6552734375, + 0.96875, + 0.94921875, + 0.908203125, + 0.966796875, + 0.958984375, + 0.9306640625, + 0.560546875, + 0.9638671875, + 0.912109375, + 0.9599609375, + 0.9140625, + 0.21484375, + 0.912109375, + 0.966796875, + 0.9169921875, + 0.9052734375, + 0.927734375, + 0.9599609375, + 0.96484375, + 0.9140625, + 0.958984375, + 0.740234375, + 0.96484375, + 0.72265625, + 0.365234375, + 0.8994140625, + 0.9052734375, + 0.7998046875, + 0.94921875, + 0.958984375, + 0.9609375, + 0.9052734375, + 0.9052734375, + 0.912109375, + 0.72265625, + 0.916015625, + 0.912109375, + 0.958984375, + 0.9140625, + 0.9072265625, + 0.908203125, + 0.9052734375, + 0.94921875, + 0.96484375, + 0.90625, + 0.962890625, + 0.9677734375, + 0.9150390625, + 0.8974609375, + 0.12109375, + 0.904296875, + 0.724609375, + 0.7197265625, + 0.955078125, + 0.90625, + 0.9033203125, + 0.9140625, + 0.919921875, + 0.9599609375, + 0.9072265625, + 0.96484375 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p50_unif0_0p25_outwdm1 decode=flowmap run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 ckpt_step=2000 views=1024000 token_acc=0.8704 exact=0/64 exact_refs=0 hits=[] +[eval-decode-acc] train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 step=2000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0002000.pt", + "ckpt_step": 2000, + "endpoint_softening": "none", + "decode_rule": "dual_line_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.910552978515625, + "token_acc_min": 0.2138671875, + "token_acc_max": 0.96484375, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 5, + 3, + 5, + 2, + 2, + 2, + 5, + 5, + 2, + 3, + 6, + 5, + 5, + 7, + 7, + 2, + 5, + 7, + 3, + 2, + 5, + 3, + 2, + 7, + 2, + 7, + 2, + 2, + 3, + 3, + 1, + 5, + 3, + 2, + 5, + 3, + 2, + 3, + 2, + 3, + 0, + 3, + 2, + 5, + 2, + 2, + 3, + 2, + 3, + 3, + 7, + 5, + 3, + 3, + 3, + 2, + 3, + 5, + 5, + 2, + 3, + 0, + 2, + 2 + ], + "best_token_acc": [ + 0.916015625, + 0.9052734375, + 0.9248046875, + 0.96484375, + 0.9599609375, + 0.958984375, + 0.91796875, + 0.921875, + 0.958984375, + 0.896484375, + 0.2138671875, + 0.921875, + 0.9189453125, + 0.9443359375, + 0.947265625, + 0.9599609375, + 0.9130859375, + 0.9443359375, + 0.900390625, + 0.9619140625, + 0.9189453125, + 0.9033203125, + 0.9609375, + 0.943359375, + 0.9609375, + 0.9423828125, + 0.9619140625, + 0.9609375, + 0.900390625, + 0.8935546875, + 0.888671875, + 0.916015625, + 0.9013671875, + 0.962890625, + 0.9287109375, + 0.896484375, + 0.9599609375, + 0.896484375, + 0.9580078125, + 0.90234375, + 0.6923828125, + 0.8994140625, + 0.962890625, + 0.919921875, + 0.9619140625, + 0.962890625, + 0.8955078125, + 0.958984375, + 0.8955078125, + 0.8994140625, + 0.947265625, + 0.919921875, + 0.892578125, + 0.8955078125, + 0.9013671875, + 0.9580078125, + 0.900390625, + 0.9189453125, + 0.91796875, + 0.9599609375, + 0.8974609375, + 0.6845703125, + 0.9638671875, + 0.9599609375 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p50_unif0_0p25_outwdm1 decode=dual_line_resample run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 ckpt_step=2000 views=1024000 token_acc=0.9106 exact=0/64 exact_refs=0 hits=[] +[ctx1024-sampleds] train config=p50_unif0_0p25_outwdm1 from=2000 to=3000 +[ctx1024-sampleds] eval config=p50_unif0_0p25_outwdm1 step=3000 +[eval-decode-acc] train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 step=3000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0003000.pt", + "ckpt_step": 3000, + "endpoint_softening": "none", + "decode_rule": "flowmap", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9010009765625, + "token_acc_min": 0.326171875, + "token_acc_max": 0.998046875, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 7, + 3, + 7, + 3, + 1, + 1, + 3, + 1, + 7, + 0, + 5, + 3, + 3, + 3, + 3, + 3, + 1, + 7, + 3, + 3, + 3, + 3, + 1, + 1, + 7, + 3, + 5, + 0, + 5, + 7, + 2, + 3, + 7, + 7, + 7, + 3, + 3, + 1, + 3, + 2, + 7, + 3, + 3, + 4, + 7, + 6, + 5, + 1, + 4, + 7, + 4, + 3, + 7, + 6, + 3, + 3, + 4, + 3, + 3, + 3, + 7, + 1, + 3, + 7 + ], + "best_token_acc": [ + 0.8759765625, + 0.7099609375, + 0.978515625, + 0.86328125, + 0.9892578125, + 0.9453125, + 0.908203125, + 0.9814453125, + 0.9169921875, + 0.994140625, + 0.9716796875, + 0.3310546875, + 0.9951171875, + 0.9892578125, + 0.5888671875, + 0.9912109375, + 0.8974609375, + 0.9853515625, + 0.6357421875, + 0.98046875, + 0.986328125, + 0.9921875, + 0.9853515625, + 0.998046875, + 0.9951171875, + 0.6083984375, + 0.998046875, + 0.99609375, + 0.9921875, + 0.9677734375, + 0.99609375, + 0.9921875, + 0.9970703125, + 0.9970703125, + 0.9755859375, + 0.9951171875, + 0.7470703125, + 0.892578125, + 0.9951171875, + 0.5029296875, + 0.904296875, + 0.92578125, + 0.99609375, + 0.9814453125, + 0.99609375, + 0.8505859375, + 0.998046875, + 0.9912109375, + 0.9619140625, + 0.998046875, + 0.970703125, + 0.9931640625, + 0.9873046875, + 0.947265625, + 0.9921875, + 0.8095703125, + 0.8603515625, + 0.4853515625, + 0.326171875, + 0.99609375, + 0.998046875, + 0.9912109375, + 0.7861328125, + 0.77734375 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p50_unif0_0p25_outwdm1 decode=flowmap run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 ckpt_step=3000 views=1536000 token_acc=0.9010 exact=0/64 exact_refs=0 hits=[] +[eval-decode-acc] train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 step=3000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0003000.pt", + "ckpt_step": 3000, + "endpoint_softening": "none", + "decode_rule": "dual_line_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9642333984375, + "token_acc_min": 0.4638671875, + "token_acc_max": 0.9892578125, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 5, + 3, + 6, + 5, + 3, + 7, + 4, + 7, + 7, + 3, + 3, + 7, + 7, + 7, + 2, + 5, + 7, + 7, + 5, + 3, + 7, + 5, + 3, + 7, + 3, + 1, + 2, + 5, + 3, + 3, + 3, + 7, + 5, + 5, + 3, + 2, + 7, + 3, + 5, + 2, + 7, + 3, + 7, + 7, + 7, + 7, + 2, + 7, + 2, + 7, + 7, + 3, + 5, + 5, + 3, + 2, + 7, + 7, + 2, + 2, + 7, + 3, + 2, + 7 + ], + "best_token_acc": [ + 0.9619140625, + 0.97265625, + 0.4638671875, + 0.9677734375, + 0.98046875, + 0.9765625, + 0.900390625, + 0.966796875, + 0.9658203125, + 0.9814453125, + 0.9833984375, + 0.9765625, + 0.9697265625, + 0.9775390625, + 0.9892578125, + 0.9716796875, + 0.974609375, + 0.9658203125, + 0.974609375, + 0.98046875, + 0.96484375, + 0.9609375, + 0.9736328125, + 0.97265625, + 0.97265625, + 0.947265625, + 0.982421875, + 0.96875, + 0.9716796875, + 0.9755859375, + 0.9833984375, + 0.9677734375, + 0.9609375, + 0.9677734375, + 0.98046875, + 0.9775390625, + 0.9697265625, + 0.9814453125, + 0.951171875, + 0.9833984375, + 0.9716796875, + 0.9755859375, + 0.9736328125, + 0.97265625, + 0.9658203125, + 0.9638671875, + 0.982421875, + 0.97265625, + 0.9833984375, + 0.9736328125, + 0.97265625, + 0.9794921875, + 0.9638671875, + 0.9658203125, + 0.9755859375, + 0.9873046875, + 0.9716796875, + 0.9736328125, + 0.982421875, + 0.9775390625, + 0.97265625, + 0.978515625, + 0.9853515625, + 0.9736328125 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p50_unif0_0p25_outwdm1 decode=dual_line_resample run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 ckpt_step=3000 views=1536000 token_acc=0.9642 exact=0/64 exact_refs=0 hits=[] +[ctx1024-sampleds] train config=p50_unif0_0p25_outwdm1 from=3000 to=4000 +[ctx1024-sampleds] eval config=p50_unif0_0p25_outwdm1 step=4000 +[eval-decode-acc] train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 step=4000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0004000.pt", + "ckpt_step": 4000, + "endpoint_softening": "none", + "decode_rule": "flowmap", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.8270416259765625, + "token_acc_min": 0.052734375, + "token_acc_max": 1.0, + "exact_acc": 0.03125, + "exact_count": 2, + "exact_ref_coverage": 0.125, + "exact_ref_count": 1, + "exact_ref_hits": [ + 7 + ], + "best_ref_idx": [ + 7, + 0, + 0, + 7, + 7, + 3, + 7, + 3, + 3, + 7, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 7, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 2, + 7, + 3, + 3, + 3, + 2, + 7, + 3, + 7, + 3, + 3, + 3, + 3, + 7, + 7, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 7, + 3, + 3, + 3, + 6, + 7, + 7, + 3 + ], + "best_token_acc": [ + 0.9462890625, + 0.052734375, + 0.9970703125, + 0.9736328125, + 0.814453125, + 0.716796875, + 0.998046875, + 0.962890625, + 0.8505859375, + 0.9296875, + 0.5390625, + 0.97265625, + 0.890625, + 0.677734375, + 0.9365234375, + 0.5703125, + 0.5673828125, + 0.939453125, + 0.998046875, + 0.890625, + 0.4892578125, + 0.966796875, + 0.9521484375, + 0.9169921875, + 0.970703125, + 0.921875, + 0.646484375, + 1.0, + 0.7109375, + 0.9697265625, + 0.9658203125, + 0.6962890625, + 0.998046875, + 0.556640625, + 0.5048828125, + 0.9599609375, + 0.779296875, + 0.7666015625, + 0.7333984375, + 0.771484375, + 0.94921875, + 0.90625, + 0.939453125, + 0.8203125, + 0.974609375, + 0.873046875, + 0.599609375, + 0.8740234375, + 0.99609375, + 0.9443359375, + 0.87890625, + 0.80859375, + 0.69921875, + 0.7021484375, + 0.98828125, + 0.896484375, + 1.0, + 0.9541015625, + 0.953125, + 0.5869140625, + 0.9755859375, + 0.9677734375, + 0.5693359375, + 0.5712890625 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0004000.pt", + "ckpt_step": 4000, + "endpoint_softening": "none", + "decode_rule": "flowmap", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.8270416259765625, + "token_acc_min": 0.052734375, + "token_acc_max": 1.0, + "exact_acc": 0.03125, + "exact_count": 2, + "exact_ref_coverage": 0.125, + "exact_ref_count": 1, + "exact_ref_hits": [ + 7 + ], + "best_ref_idx": [ + 7, + 0, + 0, + 7, + 7, + 3, + 7, + 3, + 3, + 7, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 7, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 2, + 7, + 3, + 3, + 3, + 2, + 7, + 3, + 7, + 3, + 3, + 3, + 3, + 7, + 7, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 7, + 3, + 3, + 3, + 6, + 7, + 7, + 3 + ], + "best_token_acc": [ + 0.9462890625, + 0.052734375, + 0.9970703125, + 0.9736328125, + 0.814453125, + 0.716796875, + 0.998046875, + 0.962890625, + 0.8505859375, + 0.9296875, + 0.5390625, + 0.97265625, + 0.890625, + 0.677734375, + 0.9365234375, + 0.5703125, + 0.5673828125, + 0.939453125, + 0.998046875, + 0.890625, + 0.4892578125, + 0.966796875, + 0.9521484375, + 0.9169921875, + 0.970703125, + 0.921875, + 0.646484375, + 1.0, + 0.7109375, + 0.9697265625, + 0.9658203125, + 0.6962890625, + 0.998046875, + 0.556640625, + 0.5048828125, + 0.9599609375, + 0.779296875, + 0.7666015625, + 0.7333984375, + 0.771484375, + 0.94921875, + 0.90625, + 0.939453125, + 0.8203125, + 0.974609375, + 0.873046875, + 0.599609375, + 0.8740234375, + 0.99609375, + 0.9443359375, + 0.87890625, + 0.80859375, + 0.69921875, + 0.7021484375, + 0.98828125, + 0.896484375, + 1.0, + 0.9541015625, + 0.953125, + 0.5869140625, + 0.9755859375, + 0.9677734375, + 0.5693359375, + 0.5712890625 + ] + } + } +} +RESULT config=p50_unif0_0p25_outwdm1 decode=flowmap run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 ckpt_step=4000 views=2048000 token_acc=0.8270 exact=2/64 exact_refs=1 hits=[7] +[eval-decode-acc] train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 step=4000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0004000.pt", + "ckpt_step": 4000, + "endpoint_softening": "none", + "decode_rule": "dual_line_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9935760498046875, + "token_acc_min": 0.9326171875, + "token_acc_max": 1.0, + "exact_acc": 0.078125, + "exact_count": 5, + "exact_ref_coverage": 0.25, + "exact_ref_count": 2, + "exact_ref_hits": [ + 0, + 7 + ], + "best_ref_idx": [ + 0, + 2, + 0, + 1, + 0, + 0, + 2, + 7, + 3, + 3, + 2, + 1, + 3, + 6, + 3, + 2, + 3, + 1, + 6, + 0, + 2, + 3, + 3, + 3, + 2, + 0, + 0, + 1, + 0, + 3, + 3, + 7, + 3, + 2, + 2, + 0, + 1, + 3, + 2, + 4, + 3, + 3, + 3, + 6, + 0, + 0, + 6, + 2, + 3, + 3, + 3, + 0, + 0, + 1, + 2, + 3, + 5, + 2, + 3, + 0, + 0, + 0, + 3, + 3 + ], + "best_token_acc": [ + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9931640625, + 0.998046875, + 0.9990234375, + 0.9990234375, + 1.0, + 0.99609375, + 0.9970703125, + 0.998046875, + 0.994140625, + 0.99609375, + 0.9365234375, + 0.9970703125, + 0.9990234375, + 0.99609375, + 0.994140625, + 0.94140625, + 0.9990234375, + 0.9970703125, + 0.99609375, + 0.99609375, + 0.9970703125, + 0.998046875, + 1.0, + 1.0, + 0.994140625, + 0.9990234375, + 0.99609375, + 0.99609375, + 1.0, + 0.9970703125, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.994140625, + 0.9970703125, + 0.998046875, + 0.9912109375, + 0.99609375, + 0.9970703125, + 0.9970703125, + 0.94140625, + 0.9990234375, + 0.9990234375, + 0.9326171875, + 0.998046875, + 0.9970703125, + 0.994140625, + 0.99609375, + 0.9990234375, + 0.9990234375, + 0.99609375, + 0.9990234375, + 0.9951171875, + 0.994140625, + 0.998046875, + 0.998046875, + 0.9990234375, + 1.0, + 0.998046875, + 0.99609375, + 0.994140625 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0004000.pt", + "ckpt_step": 4000, + "endpoint_softening": "none", + "decode_rule": "dual_line_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9935760498046875, + "token_acc_min": 0.9326171875, + "token_acc_max": 1.0, + "exact_acc": 0.078125, + "exact_count": 5, + "exact_ref_coverage": 0.25, + "exact_ref_count": 2, + "exact_ref_hits": [ + 0, + 7 + ], + "best_ref_idx": [ + 0, + 2, + 0, + 1, + 0, + 0, + 2, + 7, + 3, + 3, + 2, + 1, + 3, + 6, + 3, + 2, + 3, + 1, + 6, + 0, + 2, + 3, + 3, + 3, + 2, + 0, + 0, + 1, + 0, + 3, + 3, + 7, + 3, + 2, + 2, + 0, + 1, + 3, + 2, + 4, + 3, + 3, + 3, + 6, + 0, + 0, + 6, + 2, + 3, + 3, + 3, + 0, + 0, + 1, + 2, + 3, + 5, + 2, + 3, + 0, + 0, + 0, + 3, + 3 + ], + "best_token_acc": [ + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9931640625, + 0.998046875, + 0.9990234375, + 0.9990234375, + 1.0, + 0.99609375, + 0.9970703125, + 0.998046875, + 0.994140625, + 0.99609375, + 0.9365234375, + 0.9970703125, + 0.9990234375, + 0.99609375, + 0.994140625, + 0.94140625, + 0.9990234375, + 0.9970703125, + 0.99609375, + 0.99609375, + 0.9970703125, + 0.998046875, + 1.0, + 1.0, + 0.994140625, + 0.9990234375, + 0.99609375, + 0.99609375, + 1.0, + 0.9970703125, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.994140625, + 0.9970703125, + 0.998046875, + 0.9912109375, + 0.99609375, + 0.9970703125, + 0.9970703125, + 0.94140625, + 0.9990234375, + 0.9990234375, + 0.9326171875, + 0.998046875, + 0.9970703125, + 0.994140625, + 0.99609375, + 0.9990234375, + 0.9990234375, + 0.99609375, + 0.9990234375, + 0.9951171875, + 0.994140625, + 0.998046875, + 0.998046875, + 0.9990234375, + 1.0, + 0.998046875, + 0.99609375, + 0.994140625 + ] + } + } +} +RESULT config=p50_unif0_0p25_outwdm1 decode=dual_line_resample run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 ckpt_step=4000 views=2048000 token_acc=0.9936 exact=5/64 exact_refs=2 hits=[0, 7] +[ctx1024-sampleds] train config=p50_unif0_0p25_outwdm1 from=4000 to=5000 +[ctx1024-sampleds] eval config=p50_unif0_0p25_outwdm1 step=5000 +[eval-decode-acc] train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 step=5000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0005000.pt", + "ckpt_step": 5000, + "endpoint_softening": "none", + "decode_rule": "flowmap", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.6998748779296875, + "token_acc_min": 0.3828125, + "token_acc_max": 0.9990234375, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 4, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 4, + 4, + 4, + 3, + 3, + 3, + 4, + 3, + 3, + 3, + 3, + 3, + 4, + 1, + 4, + 3, + 3, + 3, + 3, + 3, + 1, + 3, + 3, + 1, + 4, + 3, + 3, + 3, + 4, + 4, + 1, + 3, + 4, + 3, + 3, + 4, + 3, + 4, + 4, + 3, + 3, + 3, + 1, + 4, + 3, + 3, + 4, + 6, + 3, + 3, + 6, + 3, + 3, + 4, + 3 + ], + "best_token_acc": [ + 0.98046875, + 0.6328125, + 0.486328125, + 0.3828125, + 0.6083984375, + 0.5205078125, + 0.54296875, + 0.798828125, + 0.544921875, + 0.9951171875, + 0.9970703125, + 0.896484375, + 0.5927734375, + 0.5908203125, + 0.541015625, + 0.9951171875, + 0.6259765625, + 0.642578125, + 0.5029296875, + 0.568359375, + 0.5556640625, + 0.8525390625, + 0.97265625, + 0.994140625, + 0.4609375, + 0.50390625, + 0.822265625, + 0.796875, + 0.4970703125, + 0.92578125, + 0.51953125, + 0.7138671875, + 0.955078125, + 0.759765625, + 0.59765625, + 0.7197265625, + 0.865234375, + 0.431640625, + 0.9990234375, + 0.4658203125, + 0.552734375, + 0.841796875, + 0.6142578125, + 0.603515625, + 0.9892578125, + 0.4833984375, + 0.8916015625, + 0.8662109375, + 0.607421875, + 0.5234375, + 0.6083984375, + 0.513671875, + 0.9970703125, + 0.5595703125, + 0.654296875, + 0.703125, + 0.8779296875, + 0.93359375, + 0.6201171875, + 0.8134765625, + 0.46484375, + 0.7587890625, + 0.974609375, + 0.4833984375 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p50_unif0_0p25_outwdm1 decode=flowmap run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 ckpt_step=5000 views=2560000 token_acc=0.6999 exact=0/64 exact_refs=0 hits=[] +[eval-decode-acc] train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 step=5000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0005000.pt", + "ckpt_step": 5000, + "endpoint_softening": "none", + "decode_rule": "dual_line_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9088897705078125, + "token_acc_min": 0.6416015625, + "token_acc_max": 0.9892578125, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 3, + 6, + 6, + 1, + 7, + 6, + 6, + 3, + 6, + 3, + 3, + 7, + 6, + 6, + 6, + 3, + 3, + 6, + 6, + 6, + 3, + 1, + 1, + 6, + 3, + 1, + 1, + 6, + 7, + 3, + 6, + 6, + 6, + 3, + 1, + 3, + 3, + 3, + 6, + 2, + 6, + 3, + 3, + 6, + 1, + 3, + 1, + 6, + 6, + 3, + 6, + 0, + 3, + 6, + 6, + 6, + 3, + 6, + 6, + 6, + 1, + 6, + 1, + 1 + ], + "best_token_acc": [ + 0.984375, + 0.8955078125, + 0.9150390625, + 0.7880859375, + 0.9755859375, + 0.9052734375, + 0.8994140625, + 0.9814453125, + 0.908203125, + 0.9765625, + 0.9814453125, + 0.966796875, + 0.87890625, + 0.9130859375, + 0.923828125, + 0.9853515625, + 0.984375, + 0.900390625, + 0.9091796875, + 0.9072265625, + 0.98046875, + 0.8125, + 0.7685546875, + 0.912109375, + 0.9814453125, + 0.771484375, + 0.767578125, + 0.892578125, + 0.966796875, + 0.9833984375, + 0.9052734375, + 0.8984375, + 0.9072265625, + 0.986328125, + 0.8037109375, + 0.9833984375, + 0.9765625, + 0.9892578125, + 0.9033203125, + 0.6416015625, + 0.91796875, + 0.9892578125, + 0.9833984375, + 0.9453125, + 0.775390625, + 0.986328125, + 0.80078125, + 0.9091796875, + 0.9111328125, + 0.984375, + 0.884765625, + 0.923828125, + 0.9892578125, + 0.9111328125, + 0.9208984375, + 0.931640625, + 0.9892578125, + 0.9091796875, + 0.9228515625, + 0.87890625, + 0.80859375, + 0.8984375, + 0.802734375, + 0.7822265625 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p50_unif0_0p25_outwdm1 decode=dual_line_resample run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 ckpt_step=5000 views=2560000 token_acc=0.9089 exact=0/64 exact_refs=0 hits=[] +[ctx1024-sampleds] train config=p50_unif0_0p25_outwdm1 from=5000 to=6000 +[ctx1024-sampleds] eval config=p50_unif0_0p25_outwdm1 step=6000 +[eval-decode-acc] train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 step=6000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0006000.pt", + "ckpt_step": 6000, + "endpoint_softening": "none", + "decode_rule": "flowmap", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.3032989501953125, + "token_acc_min": 0.001953125, + "token_acc_max": 0.9462890625, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 6, + 6, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 6, + 2, + 2, + 1, + 4, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 4, + 2, + 6, + 3, + 4, + 4, + 4, + 6, + 2, + 4, + 3, + 4, + 4, + 4, + 4, + 4, + 4, + 2, + 2, + 1, + 6, + 4, + 4, + 4, + 6, + 4, + 4, + 4, + 3, + 4, + 4, + 4, + 2, + 4, + 4, + 4, + 6, + 6, + 4, + 4, + 4, + 2 + ], + "best_token_acc": [ + 0.517578125, + 0.326171875, + 0.001953125, + 0.37109375, + 0.373046875, + 0.4052734375, + 0.3037109375, + 0.369140625, + 0.40234375, + 0.0458984375, + 0.3310546875, + 0.0302734375, + 0.017578125, + 0.91796875, + 0.2734375, + 0.41796875, + 0.30859375, + 0.388671875, + 0.3037109375, + 0.376953125, + 0.375, + 0.029296875, + 0.298828125, + 0.02734375, + 0.3544921875, + 0.8994140625, + 0.1884765625, + 0.4306640625, + 0.3720703125, + 0.28515625, + 0.0478515625, + 0.359375, + 0.099609375, + 0.3427734375, + 0.3759765625, + 0.330078125, + 0.3583984375, + 0.35546875, + 0.3056640625, + 0.04296875, + 0.0537109375, + 0.9462890625, + 0.00390625, + 0.0986328125, + 0.359375, + 0.3974609375, + 0.30078125, + 0.357421875, + 0.326171875, + 0.2978515625, + 0.06640625, + 0.37109375, + 0.3583984375, + 0.353515625, + 0.0380859375, + 0.3076171875, + 0.3388671875, + 0.35546875, + 0.4736328125, + 0.31640625, + 0.33984375, + 0.2080078125, + 0.3388671875, + 0.0419921875 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p50_unif0_0p25_outwdm1 decode=flowmap run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 ckpt_step=6000 views=3072000 token_acc=0.3033 exact=0/64 exact_refs=0 hits=[] +[eval-decode-acc] train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 step=6000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0006000.pt", + "ckpt_step": 6000, + "endpoint_softening": "none", + "decode_rule": "dual_line_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.922149658203125, + "token_acc_min": 0.716796875, + "token_acc_max": 1.0, + "exact_acc": 0.015625, + "exact_count": 1, + "exact_ref_coverage": 0.125, + "exact_ref_count": 1, + "exact_ref_hits": [ + 6 + ], + "best_ref_idx": [ + 7, + 0, + 7, + 0, + 0, + 4, + 4, + 0, + 2, + 1, + 2, + 3, + 1, + 0, + 6, + 7, + 4, + 5, + 6, + 0, + 6, + 3, + 5, + 6, + 1, + 4, + 7, + 2, + 3, + 2, + 7, + 1, + 0, + 7, + 0, + 5, + 0, + 0, + 0, + 0, + 0, + 4, + 1, + 4, + 0, + 0, + 0, + 3, + 0, + 0, + 7, + 3, + 0, + 6, + 6, + 6, + 0, + 0, + 7, + 0, + 1, + 1, + 0, + 5 + ], + "best_token_acc": [ + 0.84765625, + 0.994140625, + 0.796875, + 0.99609375, + 0.9921875, + 0.970703125, + 0.9443359375, + 0.994140625, + 0.78515625, + 0.755859375, + 0.751953125, + 0.9677734375, + 0.7392578125, + 0.994140625, + 0.9970703125, + 0.8046875, + 0.955078125, + 0.9814453125, + 0.9990234375, + 0.9951171875, + 1.0, + 0.970703125, + 0.982421875, + 0.9990234375, + 0.732421875, + 0.958984375, + 0.7578125, + 0.794921875, + 0.9697265625, + 0.7646484375, + 0.7294921875, + 0.8017578125, + 0.990234375, + 0.759765625, + 0.9951171875, + 0.9794921875, + 0.994140625, + 0.99609375, + 0.9921875, + 0.990234375, + 0.9912109375, + 0.9580078125, + 0.716796875, + 0.94140625, + 0.994140625, + 0.9921875, + 0.9931640625, + 0.9755859375, + 0.9970703125, + 0.9970703125, + 0.81640625, + 0.9775390625, + 0.998046875, + 0.998046875, + 0.998046875, + 0.998046875, + 0.9873046875, + 0.994140625, + 0.77734375, + 0.9931640625, + 0.779296875, + 0.740234375, + 0.9921875, + 0.9892578125 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929::none": { + "run": "train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929", + "checkpoint": "runs/train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929/step_0006000.pt", + "ckpt_step": 6000, + "endpoint_softening": "none", + "decode_rule": "dual_line_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.922149658203125, + "token_acc_min": 0.716796875, + "token_acc_max": 1.0, + "exact_acc": 0.015625, + "exact_count": 1, + "exact_ref_coverage": 0.125, + "exact_ref_count": 1, + "exact_ref_hits": [ + 6 + ], + "best_ref_idx": [ + 7, + 0, + 7, + 0, + 0, + 4, + 4, + 0, + 2, + 1, + 2, + 3, + 1, + 0, + 6, + 7, + 4, + 5, + 6, + 0, + 6, + 3, + 5, + 6, + 1, + 4, + 7, + 2, + 3, + 2, + 7, + 1, + 0, + 7, + 0, + 5, + 0, + 0, + 0, + 0, + 0, + 4, + 1, + 4, + 0, + 0, + 0, + 3, + 0, + 0, + 7, + 3, + 0, + 6, + 6, + 6, + 0, + 0, + 7, + 0, + 1, + 1, + 0, + 5 + ], + "best_token_acc": [ + 0.84765625, + 0.994140625, + 0.796875, + 0.99609375, + 0.9921875, + 0.970703125, + 0.9443359375, + 0.994140625, + 0.78515625, + 0.755859375, + 0.751953125, + 0.9677734375, + 0.7392578125, + 0.994140625, + 0.9970703125, + 0.8046875, + 0.955078125, + 0.9814453125, + 0.9990234375, + 0.9951171875, + 1.0, + 0.970703125, + 0.982421875, + 0.9990234375, + 0.732421875, + 0.958984375, + 0.7578125, + 0.794921875, + 0.9697265625, + 0.7646484375, + 0.7294921875, + 0.8017578125, + 0.990234375, + 0.759765625, + 0.9951171875, + 0.9794921875, + 0.994140625, + 0.99609375, + 0.9921875, + 0.990234375, + 0.9912109375, + 0.9580078125, + 0.716796875, + 0.94140625, + 0.994140625, + 0.9921875, + 0.9931640625, + 0.9755859375, + 0.9970703125, + 0.9970703125, + 0.81640625, + 0.9775390625, + 0.998046875, + 0.998046875, + 0.998046875, + 0.998046875, + 0.9873046875, + 0.994140625, + 0.77734375, + 0.9931640625, + 0.779296875, + 0.740234375, + 0.9921875, + 0.9892578125 + ] + } + } +} +RESULT config=p50_unif0_0p25_outwdm1 decode=dual_line_resample run=train8_ctx1024_core_p50_unif0_0p25_outwdm1_ctx1024_core_tradeoff_dual_20260517_230929 ckpt_step=6000 views=3072000 token_acc=0.9221 exact=1/64 exact_refs=1 hits=[6] +[ctx1024-sampleds] train config=p50_unif0_0p25_outwdm1 from=6000 to=7000 diff --git a/LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/t5tok_ctx1024_k03_20260518_022728.log b/LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/t5tok_ctx1024_k03_20260518_022728.log new file mode 100644 index 0000000000000000000000000000000000000000..1336c497b2cf6202a3f177c5007b6863223454a0 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/t5tok_ctx1024_k03_20260518_022728.log @@ -0,0 +1,1808 @@ +[ctx1024-sampleds] start stamp=t5tok_ctx1024_k03_20260518_022728 len=1024 vocab=2423 out=docs/lta_samples/metrics_20260518/t5tok_ctx1024_k03_t5tok_ctx1024_k03_20260518_022728 +[ctx1024-sampleds] config=p50_rand0_3_unif0_0p25_outwdm1 run=train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 p=0.50 mode=sampled_path steps=3 steps_min=0 s_dist=uniform s_frac=0.0->0.25 beta=2.0,6.0 outwd=-1 sync_t=1 +[ctx1024-sampleds] train config=p50_rand0_3_unif0_0p25_outwdm1 from=0 to=1000 +[ctx1024-sampleds] eval config=p50_rand0_3_unif0_0p25_outwdm1 step=1000 +[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 step=1000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/step_0001000.pt", + "ckpt_step": 1000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.064483642578125, + "token_acc_min": 0.013671875, + "token_acc_max": 0.7587890625, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 0, + 5, + 5, + 5, + 5, + 5, + 3, + 0, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 7, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 7, + 5, + 5, + 3, + 5, + 5, + 3, + 5, + 0, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 5, + 5, + 4, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 0.0166015625, + 0.015625, + 0.046875, + 0.021484375, + 0.0576171875, + 0.115234375, + 0.0185546875, + 0.013671875, + 0.046875, + 0.0400390625, + 0.041015625, + 0.0419921875, + 0.015625, + 0.029296875, + 0.0576171875, + 0.0224609375, + 0.0390625, + 0.478515625, + 0.7587890625, + 0.0166015625, + 0.0185546875, + 0.0234375, + 0.03125, + 0.64453125, + 0.03515625, + 0.0205078125, + 0.03125, + 0.0283203125, + 0.0361328125, + 0.0244140625, + 0.0283203125, + 0.02734375, + 0.02734375, + 0.02734375, + 0.0361328125, + 0.2294921875, + 0.0380859375, + 0.0302734375, + 0.03515625, + 0.0302734375, + 0.0205078125, + 0.021484375, + 0.0302734375, + 0.0224609375, + 0.013671875, + 0.041015625, + 0.0234375, + 0.025390625, + 0.0185546875, + 0.0244140625, + 0.0361328125, + 0.0205078125, + 0.099609375, + 0.060546875, + 0.0517578125, + 0.0234375, + 0.037109375, + 0.029296875, + 0.052734375, + 0.0185546875, + 0.056640625, + 0.0302734375, + 0.03515625, + 0.037109375 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p50_rand0_3_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 ckpt_step=1000 views=512000 token_acc=0.0645 exact=0/64 exact_refs=0 hits=[] +[ctx1024-sampleds] train config=p50_rand0_3_unif0_0p25_outwdm1 from=1000 to=2000 +[ctx1024-sampleds] eval config=p50_rand0_3_unif0_0p25_outwdm1 step=2000 +[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 step=2000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/step_0002000.pt", + "ckpt_step": 2000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.922027587890625, + "token_acc_min": 0.0126953125, + "token_acc_max": 0.9990234375, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 7, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 3, + 5, + 5, + 5, + 5, + 5, + 3, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 5, + 5, + 2, + 7, + 2, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 5, + 7, + 5, + 5, + 7, + 5, + 5, + 5, + 5, + 2, + 2, + 5, + 5, + 2, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 0.9990234375, + 0.9970703125, + 0.998046875, + 0.9970703125, + 0.9970703125, + 0.998046875, + 0.9970703125, + 0.998046875, + 0.998046875, + 0.9970703125, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.998046875, + 0.998046875, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.9970703125, + 0.9990234375, + 0.998046875, + 0.9970703125, + 0.998046875, + 0.998046875, + 0.029296875, + 0.9970703125, + 0.029296875, + 0.9990234375, + 0.998046875, + 0.9970703125, + 0.9970703125, + 0.9970703125, + 0.9970703125, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.0322265625, + 0.998046875, + 0.998046875, + 0.998046875, + 0.9970703125, + 0.998046875, + 0.9970703125, + 0.029296875, + 0.998046875, + 0.998046875, + 0.998046875, + 0.998046875, + 0.0126953125, + 0.998046875, + 0.9970703125, + 0.9970703125, + 0.9990234375, + 0.9990234375, + 0.9970703125, + 0.998046875, + 0.9990234375, + 0.9970703125, + 0.998046875, + 0.9970703125, + 0.998046875 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p50_rand0_3_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 ckpt_step=2000 views=1024000 token_acc=0.9220 exact=0/64 exact_refs=0 hits=[] +[ctx1024-sampleds] train config=p50_rand0_3_unif0_0p25_outwdm1 from=2000 to=3000 +[ctx1024-sampleds] eval config=p50_rand0_3_unif0_0p25_outwdm1 step=3000 +[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 step=3000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/step_0003000.pt", + "ckpt_step": 3000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9999847412109375, + "token_acc_min": 0.9990234375, + "token_acc_max": 1.0, + "exact_acc": 0.984375, + "exact_count": 63, + "exact_ref_coverage": 0.375, + "exact_ref_count": 3, + "exact_ref_hits": [ + 1, + 3, + 5 + ], + "best_ref_idx": [ + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 3, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 3, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/step_0003000.pt", + "ckpt_step": 3000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9999847412109375, + "token_acc_min": 0.9990234375, + "token_acc_max": 1.0, + "exact_acc": 0.984375, + "exact_count": 63, + "exact_ref_coverage": 0.375, + "exact_ref_count": 3, + "exact_ref_hits": [ + 1, + 3, + 5 + ], + "best_ref_idx": [ + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 3, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 3, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ] + } + } +} +RESULT config=p50_rand0_3_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 ckpt_step=3000 views=1536000 token_acc=1.0000 exact=63/64 exact_refs=3 hits=[1, 3, 5] +[ctx1024-sampleds] early-hit config=p50_rand0_3_unif0_0p25_outwdm1 +[ctx1024-sampleds] config=p35_rand0_3_unif0_0p25_outwdm1 run=train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 p=0.35 mode=sampled_path steps=3 steps_min=0 s_dist=uniform s_frac=0.0->0.25 beta=2.0,6.0 outwd=-1 sync_t=1 +[ctx1024-sampleds] train config=p35_rand0_3_unif0_0p25_outwdm1 from=0 to=1000 +[ctx1024-sampleds] eval config=p35_rand0_3_unif0_0p25_outwdm1 step=1000 +[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 step=1000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/step_0001000.pt", + "ckpt_step": 1000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.0539703369140625, + "token_acc_min": 0.0205078125, + "token_acc_max": 0.6650390625, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 5, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 1, + 5, + 1, + 5, + 5, + 5, + 5, + 3, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 1, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 4, + 5, + 5 + ], + "best_token_acc": [ + 0.0322265625, + 0.0439453125, + 0.033203125, + 0.029296875, + 0.0234375, + 0.03515625, + 0.033203125, + 0.029296875, + 0.0205078125, + 0.0244140625, + 0.029296875, + 0.0400390625, + 0.037109375, + 0.044921875, + 0.0361328125, + 0.0224609375, + 0.0224609375, + 0.0205078125, + 0.462890625, + 0.0244140625, + 0.0751953125, + 0.05078125, + 0.03125, + 0.0654296875, + 0.0732421875, + 0.0302734375, + 0.033203125, + 0.0205078125, + 0.02734375, + 0.02734375, + 0.0234375, + 0.029296875, + 0.6650390625, + 0.03515625, + 0.0400390625, + 0.0234375, + 0.056640625, + 0.0380859375, + 0.107421875, + 0.0400390625, + 0.029296875, + 0.0380859375, + 0.0263671875, + 0.0439453125, + 0.033203125, + 0.0400390625, + 0.0400390625, + 0.044921875, + 0.0302734375, + 0.0390625, + 0.0244140625, + 0.025390625, + 0.0244140625, + 0.07421875, + 0.0263671875, + 0.029296875, + 0.025390625, + 0.044921875, + 0.1220703125, + 0.025390625, + 0.044921875, + 0.021484375, + 0.0361328125, + 0.0263671875 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p35_rand0_3_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 ckpt_step=1000 views=512000 token_acc=0.0540 exact=0/64 exact_refs=0 hits=[] +[ctx1024-sampleds] train config=p35_rand0_3_unif0_0p25_outwdm1 from=1000 to=2000 +[ctx1024-sampleds] eval config=p35_rand0_3_unif0_0p25_outwdm1 step=2000 +[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 step=2000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/step_0002000.pt", + "ckpt_step": 2000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.8461456298828125, + "token_acc_min": 0.0302734375, + "token_acc_max": 0.998046875, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 0.99609375, + 0.9970703125, + 0.9970703125, + 0.9970703125, + 0.9970703125, + 0.9970703125, + 0.9970703125, + 0.9970703125, + 0.9951171875, + 0.998046875, + 0.99609375, + 0.998046875, + 0.0302734375, + 0.9970703125, + 0.99609375, + 0.9970703125, + 0.998046875, + 0.9970703125, + 0.0302734375, + 0.998046875, + 0.03125, + 0.998046875, + 0.0302734375, + 0.998046875, + 0.0302734375, + 0.9970703125, + 0.998046875, + 0.0302734375, + 0.0302734375, + 0.998046875, + 0.9970703125, + 0.99609375, + 0.9970703125, + 0.9970703125, + 0.998046875, + 0.9970703125, + 0.998046875, + 0.99609375, + 0.9970703125, + 0.0302734375, + 0.9970703125, + 0.998046875, + 0.9970703125, + 0.998046875, + 0.9970703125, + 0.9970703125, + 0.9970703125, + 0.9970703125, + 0.9951171875, + 0.998046875, + 0.9970703125, + 0.99609375, + 0.998046875, + 0.033203125, + 0.998046875, + 0.99609375, + 0.9970703125, + 0.99609375, + 0.9970703125, + 0.03125, + 0.99609375, + 0.998046875, + 0.9970703125, + 0.998046875 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p35_rand0_3_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 ckpt_step=2000 views=1024000 token_acc=0.8461 exact=0/64 exact_refs=0 hits=[] +[ctx1024-sampleds] train config=p35_rand0_3_unif0_0p25_outwdm1 from=2000 to=3000 +[ctx1024-sampleds] eval config=p35_rand0_3_unif0_0p25_outwdm1 step=3000 +[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 step=3000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/step_0003000.pt", + "ckpt_step": 3000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.99884033203125, + "token_acc_min": 0.998046875, + "token_acc_max": 0.9990234375, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p35_rand0_3_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 ckpt_step=3000 views=1536000 token_acc=0.9988 exact=0/64 exact_refs=0 hits=[] +[ctx1024-sampleds] train config=p35_rand0_3_unif0_0p25_outwdm1 from=3000 to=4000 +[ctx1024-sampleds] eval config=p35_rand0_3_unif0_0p25_outwdm1 step=4000 +[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 step=4000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/step_0004000.pt", + "ckpt_step": 4000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.999755859375, + "token_acc_min": 0.9990234375, + "token_acc_max": 1.0, + "exact_acc": 0.75, + "exact_count": 48, + "exact_ref_coverage": 0.375, + "exact_ref_count": 3, + "exact_ref_hits": [ + 2, + 4, + 5 + ], + "best_ref_idx": [ + 5, + 5, + 5, + 4, + 5, + 4, + 4, + 4, + 5, + 4, + 5, + 2, + 5, + 5, + 5, + 5, + 4, + 5, + 5, + 4, + 5, + 5, + 5, + 5, + 4, + 5, + 5, + 5, + 5, + 4, + 5, + 4, + 4, + 4, + 5, + 4, + 4, + 4, + 4, + 5, + 5, + 4, + 4, + 4, + 5, + 5, + 4, + 5, + 5, + 4, + 5, + 5, + 4, + 5, + 4, + 5, + 5, + 5, + 4, + 4, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/step_0004000.pt", + "ckpt_step": 4000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.999755859375, + "token_acc_min": 0.9990234375, + "token_acc_max": 1.0, + "exact_acc": 0.75, + "exact_count": 48, + "exact_ref_coverage": 0.375, + "exact_ref_count": 3, + "exact_ref_hits": [ + 2, + 4, + 5 + ], + "best_ref_idx": [ + 5, + 5, + 5, + 4, + 5, + 4, + 4, + 4, + 5, + 4, + 5, + 2, + 5, + 5, + 5, + 5, + 4, + 5, + 5, + 4, + 5, + 5, + 5, + 5, + 4, + 5, + 5, + 5, + 5, + 4, + 5, + 4, + 4, + 4, + 5, + 4, + 4, + 4, + 4, + 5, + 5, + 4, + 4, + 4, + 5, + 5, + 4, + 5, + 5, + 4, + 5, + 5, + 4, + 5, + 4, + 5, + 5, + 5, + 4, + 4, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375 + ] + } + } +} +RESULT config=p35_rand0_3_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 ckpt_step=4000 views=2048000 token_acc=0.9998 exact=48/64 exact_refs=3 hits=[2, 4, 5] +[ctx1024-sampleds] train config=p35_rand0_3_unif0_0p25_outwdm1 from=4000 to=5000 +[ctx1024-sampleds] eval config=p35_rand0_3_unif0_0p25_outwdm1 step=5000 +[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 step=5000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/step_0005000.pt", + "ckpt_step": 5000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9998321533203125, + "token_acc_min": 0.998046875, + "token_acc_max": 1.0, + "exact_acc": 0.84375, + "exact_count": 54, + "exact_ref_coverage": 0.625, + "exact_ref_count": 5, + "exact_ref_hits": [ + 1, + 2, + 3, + 5, + 7 + ], + "best_ref_idx": [ + 5, + 5, + 5, + 5, + 5, + 3, + 2, + 5, + 5, + 5, + 3, + 2, + 3, + 5, + 5, + 3, + 5, + 5, + 1, + 5, + 5, + 5, + 3, + 3, + 3, + 3, + 5, + 2, + 5, + 5, + 5, + 1, + 5, + 3, + 1, + 5, + 5, + 5, + 5, + 5, + 7, + 3, + 5, + 5, + 5, + 3, + 5, + 5, + 3, + 7, + 1, + 5, + 7, + 5, + 5, + 3, + 5, + 5, + 3, + 5, + 5, + 3, + 5, + 5 + ], + "best_token_acc": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.998046875, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728/step_0005000.pt", + "ckpt_step": 5000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9998321533203125, + "token_acc_min": 0.998046875, + "token_acc_max": 1.0, + "exact_acc": 0.84375, + "exact_count": 54, + "exact_ref_coverage": 0.625, + "exact_ref_count": 5, + "exact_ref_hits": [ + 1, + 2, + 3, + 5, + 7 + ], + "best_ref_idx": [ + 5, + 5, + 5, + 5, + 5, + 3, + 2, + 5, + 5, + 5, + 3, + 2, + 3, + 5, + 5, + 3, + 5, + 5, + 1, + 5, + 5, + 5, + 3, + 3, + 3, + 3, + 5, + 2, + 5, + 5, + 5, + 1, + 5, + 3, + 1, + 5, + 5, + 5, + 5, + 5, + 7, + 3, + 5, + 5, + 5, + 3, + 5, + 5, + 3, + 7, + 1, + 5, + 7, + 5, + 5, + 3, + 5, + 5, + 3, + 5, + 5, + 3, + 5, + 5 + ], + "best_token_acc": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.998046875, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ] + } + } +} +RESULT config=p35_rand0_3_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_3_unif0_0p25_outwdm1_t5tok_ctx1024_k03_20260518_022728 ckpt_step=5000 views=2560000 token_acc=0.9998 exact=54/64 exact_refs=5 hits=[1, 2, 3, 5, 7] +[ctx1024-sampleds] capped config=p35_rand0_3_unif0_0p25_outwdm1 step=5000 +[ctx1024-sampleds] done diff --git a/LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/t5tok_ctx1024_randk_20260518_014800.nohup b/LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/t5tok_ctx1024_randk_20260518_014800.nohup new file mode 100644 index 0000000000000000000000000000000000000000..862f06433941c18f62bf805e12abc917b45cfe27 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/t5tok_ctx1024_randk_20260518_014800.nohup @@ -0,0 +1,4830 @@ +[ctx1024-sampleds] start stamp=t5tok_ctx1024_randk_20260518_014800 len=1024 vocab=2423 out=docs/lta_samples/metrics_20260518/t5tok_ctx1024_randk_t5tok_ctx1024_randk_20260518_014800 +[ctx1024-sampleds] config=p50_rand0_4_unif0_0p25_outwdm1 run=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 p=0.50 mode=sampled_path steps=4 steps_min=0 s_dist=uniform s_frac=0.0->0.25 beta=2.0,6.0 outwd=-1 sync_t=1 +[ctx1024-sampleds] train config=p50_rand0_4_unif0_0p25_outwdm1 from=0 to=1000 +[launch] gpt2 cached OWT soft-endpoint m/n pilot +[launch] run_name=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] save_dir=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] n=1024 m=0 clean_state_mode=onehot +[launch] mask_mixture lowk=0.0 all=1.0 +[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=2423 +[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1 +[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0 +[launch] mask_ratio=1.0->1.0 +[launch] mask_ratio_floor_schedule=none +[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet +[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids= +[launch] rollout_train prob=0.50 mode=sampled_path steps=4 steps_min=0 infer_steps=1 s_dist=uniform s_frac=0.0->0.25 temp=1.0 corrupt_only=1 samplewise=1 selected_only=1 sync_t=1 +[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit exact_repeat_per_chunk=64 +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=26.2s lr=2.000000e-03 loss=7.3399 loss_recon=7.3399 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5002 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3325 corrupt_frac=1.0000 acc_corrupt=0.3325 loss_corrupt=7.3399 wrong_frac=0.4986 init_acc_corrupt=0.4674 acc_corrupt_t_0p0_0p2=0.0457 corrupt_frac_t_0p0_0p2=0.1952 acc_corrupt_t_0p2_0p4=0.1646 corrupt_frac_t_0p2_0p4=0.2063 acc_corrupt_t_0p4_0p6=0.3267 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=0.4815 corrupt_frac_t_0p6_0p8=0.1976 acc_corrupt_t_0p8_1p0=0.6387 corrupt_frac_t_0p8_1p0=0.2036 out_w_norm=1.0906 out_g_norm=1.0044 loss_all=6.6981 init_gold_top10=0.5044 init_gold_top100=0.6198 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.4979 init_acc_rollout_kept=0.4387 logit_acc_rollout_applied=0.3337 logit_acc_rollout_kept=0.2999 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=25.3s lr=2.000000e-03 loss=5.8172 loss_recon=5.8172 loss_meanflow=0.0000 mean_model_t=0.4985 mean_corrupt_t=0.4985 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5030 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3271 corrupt_frac=1.0000 acc_corrupt=0.3271 loss_corrupt=5.8172 wrong_frac=0.5014 init_acc_corrupt=0.4646 acc_corrupt_t_0p0_0p2=0.0526 corrupt_frac_t_0p0_0p2=0.2037 acc_corrupt_t_0p2_0p4=0.1621 corrupt_frac_t_0p2_0p4=0.1982 acc_corrupt_t_0p4_0p6=0.3284 corrupt_frac_t_0p4_0p6=0.1956 acc_corrupt_t_0p6_0p8=0.4722 corrupt_frac_t_0p6_0p8=0.2056 acc_corrupt_t_0p8_1p0=0.6244 corrupt_frac_t_0p8_1p0=0.1969 out_w_norm=3.4900 out_g_norm=1.3272 loss_all=5.0546 init_gold_top10=0.5050 init_gold_top100=0.6441 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.5127 init_acc_rollout_kept=0.4286 logit_acc_rollout_applied=0.3773 logit_acc_rollout_kept=0.3212 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=25.5s lr=2.000000e-03 loss=4.7556 loss_recon=4.7556 loss_meanflow=0.0000 mean_model_t=0.4953 mean_corrupt_t=0.4953 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5102 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3619 corrupt_frac=1.0000 acc_corrupt=0.3619 loss_corrupt=4.7556 wrong_frac=0.5048 init_acc_corrupt=0.4615 acc_corrupt_t_0p0_0p2=0.0555 corrupt_frac_t_0p0_0p2=0.2036 acc_corrupt_t_0p2_0p4=0.1875 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.3634 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=0.5244 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=0.6984 corrupt_frac_t_0p8_1p0=0.1934 out_w_norm=5.5768 out_g_norm=0.5510 loss_all=4.3231 init_gold_top10=0.5280 init_gold_top100=0.6625 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.5230 init_acc_rollout_kept=0.4573 logit_acc_rollout_applied=0.4251 logit_acc_rollout_kept=0.3738 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=25.4s lr=2.000000e-03 loss=4.1317 loss_recon=4.1317 loss_meanflow=0.0000 mean_model_t=0.4980 mean_corrupt_t=0.4980 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5031 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4209 corrupt_frac=1.0000 acc_corrupt=0.4209 loss_corrupt=4.1317 wrong_frac=0.5019 init_acc_corrupt=0.4652 acc_corrupt_t_0p0_0p2=0.0581 corrupt_frac_t_0p0_0p2=0.2061 acc_corrupt_t_0p2_0p4=0.2100 corrupt_frac_t_0p2_0p4=0.1960 acc_corrupt_t_0p4_0p6=0.4188 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=0.6132 corrupt_frac_t_0p6_0p8=0.2020 acc_corrupt_t_0p8_1p0=0.8121 corrupt_frac_t_0p8_1p0=0.1985 out_w_norm=7.1061 out_g_norm=0.2755 loss_all=3.8732 init_gold_top10=0.5097 init_gold_top100=0.6643 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.4337 init_acc_rollout_kept=0.4870 logit_acc_rollout_applied=0.4383 logit_acc_rollout_kept=0.4931 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=25.4s lr=2.000000e-03 loss=3.5371 loss_recon=3.5371 loss_meanflow=0.0000 mean_model_t=0.4998 mean_corrupt_t=0.4998 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5031 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4846 corrupt_frac=1.0000 acc_corrupt=0.4846 loss_corrupt=3.5371 wrong_frac=0.5002 init_acc_corrupt=0.4678 acc_corrupt_t_0p0_0p2=0.0593 corrupt_frac_t_0p0_0p2=0.1998 acc_corrupt_t_0p2_0p4=0.2383 corrupt_frac_t_0p2_0p4=0.1971 acc_corrupt_t_0p4_0p6=0.5041 corrupt_frac_t_0p4_0p6=0.2008 acc_corrupt_t_0p6_0p8=0.7115 corrupt_frac_t_0p6_0p8=0.2002 acc_corrupt_t_0p8_1p0=0.9013 corrupt_frac_t_0p8_1p0=0.2020 out_w_norm=8.4353 out_g_norm=0.2381 loss_all=3.1644 init_gold_top10=0.5262 init_gold_top100=0.6726 rollout_applied_pos_frac=0.4531 init_acc_rollout_applied=0.4713 init_acc_rollout_kept=0.4937 logit_acc_rollout_applied=0.4920 logit_acc_rollout_kept=0.5176 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=25.3s lr=2.000000e-03 loss=3.0897 loss_recon=3.0897 loss_meanflow=0.0000 mean_model_t=0.5009 mean_corrupt_t=0.5009 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4992 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4958 corrupt_frac=1.0000 acc_corrupt=0.4958 loss_corrupt=3.0897 wrong_frac=0.4992 init_acc_corrupt=0.4690 acc_corrupt_t_0p0_0p2=0.0612 corrupt_frac_t_0p0_0p2=0.1963 acc_corrupt_t_0p2_0p4=0.2685 corrupt_frac_t_0p2_0p4=0.2016 acc_corrupt_t_0p4_0p6=0.5235 corrupt_frac_t_0p4_0p6=0.2056 acc_corrupt_t_0p6_0p8=0.7170 corrupt_frac_t_0p6_0p8=0.1941 acc_corrupt_t_0p8_1p0=0.9033 corrupt_frac_t_0p8_1p0=0.2024 out_w_norm=9.6981 out_g_norm=0.2439 loss_all=2.8362 init_gold_top10=0.5378 init_gold_top100=0.6937 rollout_applied_pos_frac=0.5469 init_acc_rollout_applied=0.5065 init_acc_rollout_kept=0.4697 logit_acc_rollout_applied=0.5309 logit_acc_rollout_kept=0.5021 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=25.3s lr=2.000000e-03 loss=2.7597 loss_recon=2.7597 loss_meanflow=0.0000 mean_model_t=0.5003 mean_corrupt_t=0.5003 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4918 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5027 corrupt_frac=1.0000 acc_corrupt=0.5027 loss_corrupt=2.7597 wrong_frac=0.4998 init_acc_corrupt=0.4685 acc_corrupt_t_0p0_0p2=0.0631 corrupt_frac_t_0p0_0p2=0.2006 acc_corrupt_t_0p2_0p4=0.2830 corrupt_frac_t_0p2_0p4=0.1945 acc_corrupt_t_0p4_0p6=0.5334 corrupt_frac_t_0p4_0p6=0.2064 acc_corrupt_t_0p6_0p8=0.7221 corrupt_frac_t_0p6_0p8=0.1962 acc_corrupt_t_0p8_1p0=0.9057 corrupt_frac_t_0p8_1p0=0.2023 out_w_norm=10.6934 out_g_norm=0.2836 loss_all=2.4361 init_gold_top10=0.5494 init_gold_top100=0.7213 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.4773 init_acc_rollout_kept=0.5063 logit_acc_rollout_applied=0.5171 logit_acc_rollout_kept=0.5495 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=25.3s lr=2.000000e-03 loss=2.2898 loss_recon=2.2898 loss_meanflow=0.0000 mean_model_t=0.4990 mean_corrupt_t=0.4990 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4960 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5367 corrupt_frac=1.0000 acc_corrupt=0.5367 loss_corrupt=2.2898 wrong_frac=0.5010 init_acc_corrupt=0.4684 acc_corrupt_t_0p0_0p2=0.0625 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.3151 corrupt_frac_t_0p2_0p4=0.2004 acc_corrupt_t_0p4_0p6=0.5996 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=0.7792 corrupt_frac_t_0p6_0p8=0.2018 acc_corrupt_t_0p8_1p0=0.9301 corrupt_frac_t_0p8_1p0=0.1982 out_w_norm=11.2230 out_g_norm=0.3424 loss_all=1.8891 init_gold_top10=0.5743 init_gold_top100=0.7283 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.4767 init_acc_rollout_kept=0.4894 logit_acc_rollout_applied=0.5812 logit_acc_rollout_kept=0.6061 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=25.5s lr=2.000000e-03 loss=1.8022 loss_recon=1.8022 loss_meanflow=0.0000 mean_model_t=0.5027 mean_corrupt_t=0.5027 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6154 corrupt_frac=1.0000 acc_corrupt=0.6154 loss_corrupt=1.8022 wrong_frac=0.4975 init_acc_corrupt=0.4773 acc_corrupt_t_0p0_0p2=0.0643 corrupt_frac_t_0p0_0p2=0.1991 acc_corrupt_t_0p2_0p4=0.3909 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.7435 corrupt_frac_t_0p4_0p6=0.1958 acc_corrupt_t_0p6_0p8=0.8955 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=0.9732 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=11.6942 out_g_norm=0.4741 loss_all=1.5768 init_gold_top10=0.5957 init_gold_top100=0.7365 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.5000 init_acc_rollout_kept=0.4748 logit_acc_rollout_applied=0.6691 logit_acc_rollout_kept=0.6601 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=25.3s lr=2.000000e-03 loss=1.4777 loss_recon=1.4777 loss_meanflow=0.0000 mean_model_t=0.5005 mean_corrupt_t=0.5005 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4931 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6819 corrupt_frac=1.0000 acc_corrupt=0.6819 loss_corrupt=1.4777 wrong_frac=0.4995 init_acc_corrupt=0.4862 acc_corrupt_t_0p0_0p2=0.0711 corrupt_frac_t_0p0_0p2=0.2029 acc_corrupt_t_0p2_0p4=0.5129 corrupt_frac_t_0p2_0p4=0.1965 acc_corrupt_t_0p4_0p6=0.8696 corrupt_frac_t_0p4_0p6=0.1988 acc_corrupt_t_0p6_0p8=0.9661 corrupt_frac_t_0p6_0p8=0.2028 acc_corrupt_t_0p8_1p0=0.9944 corrupt_frac_t_0p8_1p0=0.1990 out_w_norm=12.0477 out_g_norm=0.5497 loss_all=0.9938 init_gold_top10=0.6508 init_gold_top100=0.7640 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.5264 init_acc_rollout_kept=0.5098 logit_acc_rollout_applied=0.7689 logit_acc_rollout_kept=0.7548 +[ctx1024-sampleds] eval config=p50_rand0_4_unif0_0p25_outwdm1 step=1000 +[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=1000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0001000.pt", + "ckpt_step": 1000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.0328826904296875, + "token_acc_min": 0.015625, + "token_acc_max": 0.0576171875, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 5, + 2, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 0.037109375, + 0.017578125, + 0.0185546875, + 0.041015625, + 0.0283203125, + 0.029296875, + 0.0234375, + 0.0302734375, + 0.025390625, + 0.0322265625, + 0.0341796875, + 0.0400390625, + 0.0263671875, + 0.025390625, + 0.0419921875, + 0.0263671875, + 0.03515625, + 0.0302734375, + 0.025390625, + 0.025390625, + 0.021484375, + 0.0302734375, + 0.0341796875, + 0.041015625, + 0.037109375, + 0.0400390625, + 0.0419921875, + 0.037109375, + 0.0302734375, + 0.02734375, + 0.0458984375, + 0.0390625, + 0.03125, + 0.037109375, + 0.015625, + 0.03515625, + 0.015625, + 0.0341796875, + 0.048828125, + 0.041015625, + 0.037109375, + 0.0283203125, + 0.0380859375, + 0.0576171875, + 0.0361328125, + 0.0361328125, + 0.037109375, + 0.0185546875, + 0.037109375, + 0.0224609375, + 0.033203125, + 0.03125, + 0.029296875, + 0.029296875, + 0.033203125, + 0.0400390625, + 0.041015625, + 0.0322265625, + 0.0361328125, + 0.029296875, + 0.0361328125, + 0.04296875, + 0.0439453125, + 0.01953125 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p50_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=1000 views=512000 token_acc=0.0329 exact=0/64 exact_refs=0 hits=[] +[ctx1024-sampleds] train config=p50_rand0_4_unif0_0p25_outwdm1 from=1000 to=2000 +[launch] gpt2 cached OWT soft-endpoint m/n pilot +[launch] run_name=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] save_dir=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] n=1024 m=0 clean_state_mode=onehot +[launch] mask_mixture lowk=0.0 all=1.0 +[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=2423 +[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1 +[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0 +[launch] mask_ratio=1.0->1.0 +[launch] mask_ratio_floor_schedule=none +[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet +[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids= +[launch] rollout_train prob=0.50 mode=sampled_path steps=4 steps_min=0 infer_steps=1 s_dist=uniform s_frac=0.0->0.25 temp=1.0 corrupt_only=1 samplewise=1 selected_only=1 sync_t=1 +[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit exact_repeat_per_chunk=64 +[launch] resume_path=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt" +} +step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=1100 elapsed=26.2s lr=2.000000e-03 loss=1.2344 loss_recon=1.2344 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5002 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7302 corrupt_frac=1.0000 acc_corrupt=0.7302 loss_corrupt=1.2344 wrong_frac=0.4986 init_acc_corrupt=0.4980 acc_corrupt_t_0p0_0p2=0.0838 corrupt_frac_t_0p0_0p2=0.1952 acc_corrupt_t_0p2_0p4=0.6282 corrupt_frac_t_0p2_0p4=0.2063 acc_corrupt_t_0p4_0p6=0.9396 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=0.9896 corrupt_frac_t_0p6_0p8=0.1976 acc_corrupt_t_0p8_1p0=0.9987 corrupt_frac_t_0p8_1p0=0.2036 out_w_norm=12.2959 out_g_norm=0.5515 loss_all=0.9902 init_gold_top10=0.6298 init_gold_top100=0.7388 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.5386 init_acc_rollout_kept=0.4387 logit_acc_rollout_applied=0.7955 logit_acc_rollout_kept=0.7412 +step=1200 epoch=1200/2000 epoch_step=1/1 micro_steps=1200 elapsed=25.3s lr=2.000000e-03 loss=1.0733 loss_recon=1.0733 loss_meanflow=0.0000 mean_model_t=0.4985 mean_corrupt_t=0.4985 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5030 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7618 corrupt_frac=1.0000 acc_corrupt=0.7618 loss_corrupt=1.0733 wrong_frac=0.5014 init_acc_corrupt=0.5039 acc_corrupt_t_0p0_0p2=0.1136 corrupt_frac_t_0p0_0p2=0.2037 acc_corrupt_t_0p2_0p4=0.7417 corrupt_frac_t_0p2_0p4=0.1982 acc_corrupt_t_0p4_0p6=0.9715 corrupt_frac_t_0p4_0p6=0.1956 acc_corrupt_t_0p6_0p8=0.9962 corrupt_frac_t_0p6_0p8=0.2056 acc_corrupt_t_0p8_1p0=0.9995 corrupt_frac_t_0p8_1p0=0.1969 out_w_norm=12.4860 out_g_norm=0.5688 loss_all=0.9003 init_gold_top10=0.6437 init_gold_top100=0.7551 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.6043 init_acc_rollout_kept=0.4286 logit_acc_rollout_applied=0.8600 logit_acc_rollout_kept=0.7247 +step=1300 epoch=1300/2000 epoch_step=1/1 micro_steps=1300 elapsed=25.5s lr=2.000000e-03 loss=0.9386 loss_recon=0.9386 loss_meanflow=0.0000 mean_model_t=0.4953 mean_corrupt_t=0.4953 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5102 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7883 corrupt_frac=1.0000 acc_corrupt=0.7883 loss_corrupt=0.9386 wrong_frac=0.5048 init_acc_corrupt=0.5082 acc_corrupt_t_0p0_0p2=0.1491 corrupt_frac_t_0p0_0p2=0.2036 acc_corrupt_t_0p2_0p4=0.8254 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.9864 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=0.9985 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=0.9998 corrupt_frac_t_0p8_1p0=0.1934 out_w_norm=12.6527 out_g_norm=0.5836 loss_all=0.9096 init_gold_top10=0.6388 init_gold_top100=0.7579 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.5984 init_acc_rollout_kept=0.4573 logit_acc_rollout_applied=0.8193 logit_acc_rollout_kept=0.7818 +step=1400 epoch=1400/2000 epoch_step=1/1 micro_steps=1400 elapsed=25.3s lr=2.000000e-03 loss=0.8326 loss_recon=0.8326 loss_meanflow=0.0000 mean_model_t=0.4980 mean_corrupt_t=0.4980 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5031 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8089 corrupt_frac=1.0000 acc_corrupt=0.8089 loss_corrupt=0.8326 wrong_frac=0.5019 init_acc_corrupt=0.5159 acc_corrupt_t_0p0_0p2=0.1867 corrupt_frac_t_0p0_0p2=0.2061 acc_corrupt_t_0p2_0p4=0.8874 corrupt_frac_t_0p2_0p4=0.1960 acc_corrupt_t_0p4_0p6=0.9936 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=0.9992 corrupt_frac_t_0p6_0p8=0.2020 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.1985 out_w_norm=12.7900 out_g_norm=0.6058 loss_all=0.9790 init_gold_top10=0.6331 init_gold_top100=0.7760 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.5342 init_acc_rollout_kept=0.4870 logit_acc_rollout_applied=0.7610 logit_acc_rollout_kept=0.8135 +step=1500 epoch=1500/2000 epoch_step=1/1 micro_steps=1500 elapsed=25.4s lr=2.000000e-03 loss=0.7346 loss_recon=0.7346 loss_meanflow=0.0000 mean_model_t=0.4998 mean_corrupt_t=0.4998 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5031 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8291 corrupt_frac=1.0000 acc_corrupt=0.8291 loss_corrupt=0.7346 wrong_frac=0.5002 init_acc_corrupt=0.5247 acc_corrupt_t_0p0_0p2=0.2215 corrupt_frac_t_0p0_0p2=0.1998 acc_corrupt_t_0p2_0p4=0.9265 corrupt_frac_t_0p2_0p4=0.1971 acc_corrupt_t_0p4_0p6=0.9967 corrupt_frac_t_0p4_0p6=0.2008 acc_corrupt_t_0p6_0p8=0.9995 corrupt_frac_t_0p6_0p8=0.2002 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2020 out_w_norm=12.8940 out_g_norm=0.5915 loss_all=0.7378 init_gold_top10=0.6498 init_gold_top100=0.7620 rollout_applied_pos_frac=0.4531 init_acc_rollout_applied=0.5844 init_acc_rollout_kept=0.4937 logit_acc_rollout_applied=0.8150 logit_acc_rollout_kept=0.8507 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=25.2s lr=2.000000e-03 loss=0.6538 loss_recon=0.6538 loss_meanflow=0.0000 mean_model_t=0.5009 mean_corrupt_t=0.5009 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4992 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8438 corrupt_frac=1.0000 acc_corrupt=0.8438 loss_corrupt=0.6538 wrong_frac=0.4992 init_acc_corrupt=0.5289 acc_corrupt_t_0p0_0p2=0.2541 corrupt_frac_t_0p0_0p2=0.1963 acc_corrupt_t_0p2_0p4=0.9533 corrupt_frac_t_0p2_0p4=0.2016 acc_corrupt_t_0p4_0p6=0.9982 corrupt_frac_t_0p4_0p6=0.2056 acc_corrupt_t_0p6_0p8=0.9997 corrupt_frac_t_0p6_0p8=0.1941 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2024 out_w_norm=12.9664 out_g_norm=0.5899 loss_all=0.6217 init_gold_top10=0.6633 init_gold_top100=0.7844 rollout_applied_pos_frac=0.5469 init_acc_rollout_applied=0.6083 init_acc_rollout_kept=0.4697 logit_acc_rollout_applied=0.8460 logit_acc_rollout_kept=0.8638 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=25.3s lr=2.000000e-03 loss=0.6057 loss_recon=0.6057 loss_meanflow=0.0000 mean_model_t=0.5003 mean_corrupt_t=0.5003 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4918 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8511 corrupt_frac=1.0000 acc_corrupt=0.8511 loss_corrupt=0.6057 wrong_frac=0.4998 init_acc_corrupt=0.5276 acc_corrupt_t_0p0_0p2=0.2917 corrupt_frac_t_0p0_0p2=0.2006 acc_corrupt_t_0p2_0p4=0.9667 corrupt_frac_t_0p2_0p4=0.1945 acc_corrupt_t_0p4_0p6=0.9989 corrupt_frac_t_0p4_0p6=0.2064 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.1962 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2023 out_w_norm=13.0069 out_g_norm=0.5721 loss_all=0.5100 init_gold_top10=0.6651 init_gold_top100=0.7683 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.5736 init_acc_rollout_kept=0.5063 logit_acc_rollout_applied=0.8628 logit_acc_rollout_kept=0.8789 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=25.3s lr=2.000000e-03 loss=0.5628 loss_recon=0.5628 loss_meanflow=0.0000 mean_model_t=0.4990 mean_corrupt_t=0.4990 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4960 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8568 corrupt_frac=1.0000 acc_corrupt=0.8568 loss_corrupt=0.5628 wrong_frac=0.5010 init_acc_corrupt=0.5297 acc_corrupt_t_0p0_0p2=0.3077 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9789 corrupt_frac_t_0p2_0p4=0.2004 acc_corrupt_t_0p4_0p6=0.9993 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.2018 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.1982 out_w_norm=13.0385 out_g_norm=0.5779 loss_all=0.4891 init_gold_top10=0.6652 init_gold_top100=0.7587 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.5679 init_acc_rollout_kept=0.4894 logit_acc_rollout_applied=0.8559 logit_acc_rollout_kept=0.9019 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=25.5s lr=2.000000e-03 loss=0.5038 loss_recon=0.5038 loss_meanflow=0.0000 mean_model_t=0.5027 mean_corrupt_t=0.5027 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8672 corrupt_frac=1.0000 acc_corrupt=0.8672 loss_corrupt=0.5038 wrong_frac=0.4975 init_acc_corrupt=0.5357 acc_corrupt_t_0p0_0p2=0.3506 corrupt_frac_t_0p0_0p2=0.1991 acc_corrupt_t_0p2_0p4=0.9836 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.9994 corrupt_frac_t_0p4_0p6=0.1958 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=13.0550 out_g_norm=0.5923 loss_all=0.5111 init_gold_top10=0.6586 init_gold_top100=0.7661 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.5827 init_acc_rollout_kept=0.4748 logit_acc_rollout_applied=0.8607 logit_acc_rollout_kept=0.8673 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=25.3s lr=2.000000e-03 loss=0.4619 loss_recon=0.4619 loss_meanflow=0.0000 mean_model_t=0.5005 mean_corrupt_t=0.5005 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4931 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8734 corrupt_frac=1.0000 acc_corrupt=0.8734 loss_corrupt=0.4619 wrong_frac=0.4995 init_acc_corrupt=0.5314 acc_corrupt_t_0p0_0p2=0.3878 corrupt_frac_t_0p0_0p2=0.2029 acc_corrupt_t_0p2_0p4=0.9887 corrupt_frac_t_0p2_0p4=0.1965 acc_corrupt_t_0p4_0p6=0.9996 corrupt_frac_t_0p4_0p6=0.1988 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2028 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1990 out_w_norm=13.0738 out_g_norm=0.5136 loss_all=0.2064 init_gold_top10=0.7059 init_gold_top100=0.7647 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.6530 init_acc_rollout_kept=0.5098 logit_acc_rollout_applied=0.9759 logit_acc_rollout_kept=0.9126 +[ctx1024-sampleds] eval config=p50_rand0_4_unif0_0p25_outwdm1 step=2000 +[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=2000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0002000.pt", + "ckpt_step": 2000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9068603515625, + "token_acc_min": 0.0107421875, + "token_acc_max": 1.0, + "exact_acc": 0.171875, + "exact_count": 11, + "exact_ref_coverage": 0.25, + "exact_ref_count": 2, + "exact_ref_hits": [ + 2, + 5 + ], + "best_ref_idx": [ + 5, + 5, + 1, + 1, + 1, + 5, + 1, + 1, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 2, + 3, + 5, + 5, + 5, + 3, + 5, + 3, + 1, + 2, + 5, + 1, + 1, + 5, + 5, + 2, + 1, + 5, + 5, + 5, + 2, + 2, + 1, + 2, + 5, + 2, + 5, + 5, + 5, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.029296875, + 0.029296875, + 0.01171875, + 0.998046875, + 1.0, + 0.9970703125, + 1.0, + 0.9951171875, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.9970703125, + 0.9970703125, + 0.99609375, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.0107421875, + 0.9990234375, + 0.9970703125, + 0.9970703125, + 0.9970703125, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 0.9970703125, + 1.0, + 0.998046875, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.0126953125, + 0.029296875, + 0.9990234375, + 0.998046875, + 1.0, + 0.9990234375, + 0.9990234375, + 0.998046875, + 1.0, + 0.998046875, + 1.0, + 0.998046875, + 0.9990234375 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0002000.pt", + "ckpt_step": 2000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9068603515625, + "token_acc_min": 0.0107421875, + "token_acc_max": 1.0, + "exact_acc": 0.171875, + "exact_count": 11, + "exact_ref_coverage": 0.25, + "exact_ref_count": 2, + "exact_ref_hits": [ + 2, + 5 + ], + "best_ref_idx": [ + 5, + 5, + 1, + 1, + 1, + 5, + 1, + 1, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 2, + 3, + 5, + 5, + 5, + 3, + 5, + 3, + 1, + 2, + 5, + 1, + 1, + 5, + 5, + 2, + 1, + 5, + 5, + 5, + 2, + 2, + 1, + 2, + 5, + 2, + 5, + 5, + 5, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.029296875, + 0.029296875, + 0.01171875, + 0.998046875, + 1.0, + 0.9970703125, + 1.0, + 0.9951171875, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.9970703125, + 0.9970703125, + 0.99609375, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.0107421875, + 0.9990234375, + 0.9970703125, + 0.9970703125, + 0.9970703125, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 0.9970703125, + 1.0, + 0.998046875, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.0126953125, + 0.029296875, + 0.9990234375, + 0.998046875, + 1.0, + 0.9990234375, + 0.9990234375, + 0.998046875, + 1.0, + 0.998046875, + 1.0, + 0.998046875, + 0.9990234375 + ] + } + } +} +RESULT config=p50_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=2000 views=1024000 token_acc=0.9069 exact=11/64 exact_refs=2 hits=[2, 5] +[ctx1024-sampleds] train config=p50_rand0_4_unif0_0p25_outwdm1 from=2000 to=3000 +[launch] gpt2 cached OWT soft-endpoint m/n pilot +[launch] run_name=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] save_dir=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] n=1024 m=0 clean_state_mode=onehot +[launch] mask_mixture lowk=0.0 all=1.0 +[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=2423 +[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1 +[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0 +[launch] mask_ratio=1.0->1.0 +[launch] mask_ratio_floor_schedule=none +[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet +[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids= +[launch] rollout_train prob=0.50 mode=sampled_path steps=4 steps_min=0 infer_steps=1 s_dist=uniform s_frac=0.0->0.25 temp=1.0 corrupt_only=1 samplewise=1 selected_only=1 sync_t=1 +[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit exact_repeat_per_chunk=64 +[launch] resume_path=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt" +} +step=2100 epoch=2100/3000 epoch_step=1/1 micro_steps=2100 elapsed=26.2s lr=2.000000e-03 loss=0.4190 loss_recon=0.4190 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5002 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8812 corrupt_frac=1.0000 acc_corrupt=0.8812 loss_corrupt=0.4190 wrong_frac=0.4986 init_acc_corrupt=0.5341 acc_corrupt_t_0p0_0p2=0.4008 corrupt_frac_t_0p0_0p2=0.1952 acc_corrupt_t_0p2_0p4=0.9915 corrupt_frac_t_0p2_0p4=0.2063 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1976 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2036 out_w_norm=13.0937 out_g_norm=0.5156 loss_all=0.3296 init_gold_top10=0.6719 init_gold_top100=0.7541 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.6163 init_acc_rollout_kept=0.4387 logit_acc_rollout_applied=0.9030 logit_acc_rollout_kept=0.9081 +step=2200 epoch=2200/3000 epoch_step=1/1 micro_steps=2200 elapsed=25.3s lr=2.000000e-03 loss=0.4020 loss_recon=0.4020 loss_meanflow=0.0000 mean_model_t=0.4985 mean_corrupt_t=0.4985 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5030 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8834 corrupt_frac=1.0000 acc_corrupt=0.8834 loss_corrupt=0.4020 wrong_frac=0.5014 init_acc_corrupt=0.5317 acc_corrupt_t_0p0_0p2=0.4348 corrupt_frac_t_0p0_0p2=0.2037 acc_corrupt_t_0p2_0p4=0.9932 corrupt_frac_t_0p2_0p4=0.1982 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.1956 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2056 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1969 out_w_norm=13.0841 out_g_norm=0.5157 loss_all=0.3613 init_gold_top10=0.6713 init_gold_top100=0.7618 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.6304 init_acc_rollout_kept=0.4286 logit_acc_rollout_applied=0.9367 logit_acc_rollout_kept=0.8423 +step=2300 epoch=2300/3000 epoch_step=1/1 micro_steps=2300 elapsed=25.5s lr=2.000000e-03 loss=0.3683 loss_recon=0.3683 loss_meanflow=0.0000 mean_model_t=0.4953 mean_corrupt_t=0.4953 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5102 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8895 corrupt_frac=1.0000 acc_corrupt=0.8895 loss_corrupt=0.3683 wrong_frac=0.5048 init_acc_corrupt=0.5306 acc_corrupt_t_0p0_0p2=0.4633 corrupt_frac_t_0p0_0p2=0.2036 acc_corrupt_t_0p2_0p4=0.9942 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1934 out_w_norm=13.0576 out_g_norm=0.5447 loss_all=0.3975 init_gold_top10=0.6742 init_gold_top100=0.7632 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.6516 init_acc_rollout_kept=0.4573 logit_acc_rollout_applied=0.8970 logit_acc_rollout_kept=0.8564 +step=2400 epoch=2400/3000 epoch_step=1/1 micro_steps=2400 elapsed=25.3s lr=2.000000e-03 loss=0.3434 loss_recon=0.3434 loss_meanflow=0.0000 mean_model_t=0.4980 mean_corrupt_t=0.4980 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5031 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8949 corrupt_frac=1.0000 acc_corrupt=0.8949 loss_corrupt=0.3434 wrong_frac=0.5019 init_acc_corrupt=0.5332 acc_corrupt_t_0p0_0p2=0.4940 corrupt_frac_t_0p0_0p2=0.2061 acc_corrupt_t_0p2_0p4=0.9960 corrupt_frac_t_0p2_0p4=0.1960 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2020 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1985 out_w_norm=13.0340 out_g_norm=0.5392 loss_all=0.4300 init_gold_top10=0.6882 init_gold_top100=0.7822 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.5630 init_acc_rollout_kept=0.4870 logit_acc_rollout_applied=0.8407 logit_acc_rollout_kept=0.8916 +step=2500 epoch=2500/3000 epoch_step=1/1 micro_steps=2500 elapsed=25.4s lr=2.000000e-03 loss=0.3163 loss_recon=0.3163 loss_meanflow=0.0000 mean_model_t=0.4998 mean_corrupt_t=0.4998 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5031 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8987 corrupt_frac=1.0000 acc_corrupt=0.8987 loss_corrupt=0.3163 wrong_frac=0.5002 init_acc_corrupt=0.5391 acc_corrupt_t_0p0_0p2=0.4964 corrupt_frac_t_0p0_0p2=0.1998 acc_corrupt_t_0p2_0p4=0.9967 corrupt_frac_t_0p2_0p4=0.1971 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.2008 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2002 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2020 out_w_norm=12.9838 out_g_norm=0.4663 loss_all=0.4043 init_gold_top10=0.6901 init_gold_top100=0.7652 rollout_applied_pos_frac=0.4531 init_acc_rollout_applied=0.6117 init_acc_rollout_kept=0.4937 logit_acc_rollout_applied=0.8817 logit_acc_rollout_kept=0.9003 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=25.3s lr=2.000000e-03 loss=0.2946 loss_recon=0.2946 loss_meanflow=0.0000 mean_model_t=0.5009 mean_corrupt_t=0.5009 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4992 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9046 corrupt_frac=1.0000 acc_corrupt=0.9046 loss_corrupt=0.2946 wrong_frac=0.4992 init_acc_corrupt=0.5390 acc_corrupt_t_0p0_0p2=0.5172 corrupt_frac_t_0p0_0p2=0.1963 acc_corrupt_t_0p2_0p4=0.9971 corrupt_frac_t_0p2_0p4=0.2016 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.2056 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1941 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2024 out_w_norm=12.9131 out_g_norm=0.4678 loss_all=0.2708 init_gold_top10=0.7101 init_gold_top100=0.7854 rollout_applied_pos_frac=0.5469 init_acc_rollout_applied=0.6213 init_acc_rollout_kept=0.4697 logit_acc_rollout_applied=0.8978 logit_acc_rollout_kept=0.9240 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=25.3s lr=2.000000e-03 loss=0.2811 loss_recon=0.2811 loss_meanflow=0.0000 mean_model_t=0.5003 mean_corrupt_t=0.5003 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4918 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9079 corrupt_frac=1.0000 acc_corrupt=0.9079 loss_corrupt=0.2811 wrong_frac=0.4998 init_acc_corrupt=0.5370 acc_corrupt_t_0p0_0p2=0.5432 corrupt_frac_t_0p0_0p2=0.2006 acc_corrupt_t_0p2_0p4=0.9978 corrupt_frac_t_0p2_0p4=0.1945 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.2064 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1962 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2023 out_w_norm=12.8539 out_g_norm=0.4151 loss_all=0.2373 init_gold_top10=0.6927 init_gold_top100=0.7685 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.5863 init_acc_rollout_kept=0.5063 logit_acc_rollout_applied=0.8911 logit_acc_rollout_kept=0.9464 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=25.3s lr=2.000000e-03 loss=0.2840 loss_recon=0.2840 loss_meanflow=0.0000 mean_model_t=0.4990 mean_corrupt_t=0.4990 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4960 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9050 corrupt_frac=1.0000 acc_corrupt=0.9050 loss_corrupt=0.2840 wrong_frac=0.5010 init_acc_corrupt=0.5367 acc_corrupt_t_0p0_0p2=0.5279 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9985 corrupt_frac_t_0p2_0p4=0.2004 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2018 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1982 out_w_norm=12.7871 out_g_norm=0.3989 loss_all=0.2110 init_gold_top10=0.6883 init_gold_top100=0.7588 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.5831 init_acc_rollout_kept=0.4894 logit_acc_rollout_applied=0.9010 logit_acc_rollout_kept=0.9581 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=25.5s lr=2.000000e-03 loss=0.2628 loss_recon=0.2628 loss_meanflow=0.0000 mean_model_t=0.5027 mean_corrupt_t=0.5027 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9106 corrupt_frac=1.0000 acc_corrupt=0.9106 loss_corrupt=0.2628 wrong_frac=0.4975 init_acc_corrupt=0.5428 acc_corrupt_t_0p0_0p2=0.5528 corrupt_frac_t_0p0_0p2=0.1991 acc_corrupt_t_0p2_0p4=0.9986 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1958 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=12.7316 out_g_norm=0.3606 loss_all=0.3097 init_gold_top10=0.6875 init_gold_top100=0.7661 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.5933 init_acc_rollout_kept=0.4748 logit_acc_rollout_applied=0.8714 logit_acc_rollout_kept=0.9071 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=25.3s lr=2.000000e-03 loss=0.2591 loss_recon=0.2591 loss_meanflow=0.0000 mean_model_t=0.5005 mean_corrupt_t=0.5005 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4931 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9125 corrupt_frac=1.0000 acc_corrupt=0.9125 loss_corrupt=0.2591 wrong_frac=0.4995 init_acc_corrupt=0.5377 acc_corrupt_t_0p0_0p2=0.5697 corrupt_frac_t_0p0_0p2=0.2029 acc_corrupt_t_0p2_0p4=0.9991 corrupt_frac_t_0p2_0p4=0.1965 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1988 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2028 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1990 out_w_norm=12.6787 out_g_norm=0.3601 loss_all=0.1232 init_gold_top10=0.7090 init_gold_top100=0.7647 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.6620 init_acc_rollout_kept=0.5098 logit_acc_rollout_applied=0.9970 logit_acc_rollout_kept=0.9235 +[ctx1024-sampleds] eval config=p50_rand0_4_unif0_0p25_outwdm1 step=3000 +[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=3000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0003000.pt", + "ckpt_step": 3000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.999725341796875, + "token_acc_min": 0.998046875, + "token_acc_max": 1.0, + "exact_acc": 0.734375, + "exact_count": 47, + "exact_ref_coverage": 0.25, + "exact_ref_count": 2, + "exact_ref_hits": [ + 2, + 5 + ], + "best_ref_idx": [ + 2, + 1, + 1, + 2, + 5, + 5, + 5, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 5, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 2, + 5, + 2, + 2, + 5, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 5, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 5, + 2, + 5, + 5, + 5, + 2, + 2, + 1, + 2, + 2, + 2, + 1, + 5, + 7, + 5, + 5, + 2, + 2 + ], + "best_token_acc": [ + 1.0, + 0.9990234375, + 0.998046875, + 1.0, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 0.9990234375, + 1.0, + 0.9990234375, + 1.0, + 1.0 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0003000.pt", + "ckpt_step": 3000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.999725341796875, + "token_acc_min": 0.998046875, + "token_acc_max": 1.0, + "exact_acc": 0.734375, + "exact_count": 47, + "exact_ref_coverage": 0.25, + "exact_ref_count": 2, + "exact_ref_hits": [ + 2, + 5 + ], + "best_ref_idx": [ + 2, + 1, + 1, + 2, + 5, + 5, + 5, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 5, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 2, + 5, + 2, + 2, + 5, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 5, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 5, + 2, + 5, + 5, + 5, + 2, + 2, + 1, + 2, + 2, + 2, + 1, + 5, + 7, + 5, + 5, + 2, + 2 + ], + "best_token_acc": [ + 1.0, + 0.9990234375, + 0.998046875, + 1.0, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 0.9990234375, + 1.0, + 0.9990234375, + 1.0, + 1.0 + ] + } + } +} +RESULT config=p50_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=3000 views=1536000 token_acc=0.9997 exact=47/64 exact_refs=2 hits=[2, 5] +[ctx1024-sampleds] train config=p50_rand0_4_unif0_0p25_outwdm1 from=3000 to=4000 +[launch] gpt2 cached OWT soft-endpoint m/n pilot +[launch] run_name=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] save_dir=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] n=1024 m=0 clean_state_mode=onehot +[launch] mask_mixture lowk=0.0 all=1.0 +[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=2423 +[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1 +[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0 +[launch] mask_ratio=1.0->1.0 +[launch] mask_ratio_floor_schedule=none +[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet +[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids= +[launch] rollout_train prob=0.50 mode=sampled_path steps=4 steps_min=0 infer_steps=1 s_dist=uniform s_frac=0.0->0.25 temp=1.0 corrupt_only=1 samplewise=1 selected_only=1 sync_t=1 +[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit exact_repeat_per_chunk=64 +[launch] resume_path=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt" +} +step=3100 epoch=3100/4000 epoch_step=1/1 micro_steps=3100 elapsed=26.3s lr=2.000000e-03 loss=0.2585 loss_recon=0.2585 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5002 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9118 corrupt_frac=1.0000 acc_corrupt=0.9118 loss_corrupt=0.2585 wrong_frac=0.4986 init_acc_corrupt=0.5384 acc_corrupt_t_0p0_0p2=0.5490 corrupt_frac_t_0p0_0p2=0.1952 acc_corrupt_t_0p2_0p4=0.9993 corrupt_frac_t_0p2_0p4=0.2063 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1976 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2036 out_w_norm=12.6446 out_g_norm=0.3371 loss_all=0.1788 init_gold_top10=0.6953 init_gold_top100=0.7541 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.6225 init_acc_rollout_kept=0.4387 logit_acc_rollout_applied=0.9237 logit_acc_rollout_kept=0.9519 +step=3200 epoch=3200/4000 epoch_step=1/1 micro_steps=3200 elapsed=25.6s lr=2.000000e-03 loss=0.2490 loss_recon=0.2490 loss_meanflow=0.0000 mean_model_t=0.4985 mean_corrupt_t=0.4985 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5030 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9133 corrupt_frac=1.0000 acc_corrupt=0.9133 loss_corrupt=0.2490 wrong_frac=0.5014 init_acc_corrupt=0.5363 acc_corrupt_t_0p0_0p2=0.5749 corrupt_frac_t_0p0_0p2=0.2037 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.1982 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1956 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2056 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1969 out_w_norm=12.6089 out_g_norm=0.2866 loss_all=0.2228 init_gold_top10=0.6804 init_gold_top100=0.7618 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.6477 init_acc_rollout_kept=0.4286 logit_acc_rollout_applied=0.9706 logit_acc_rollout_kept=0.8768 +step=3300 epoch=3300/4000 epoch_step=1/1 micro_steps=3300 elapsed=25.7s lr=2.000000e-03 loss=0.2485 loss_recon=0.2485 loss_meanflow=0.0000 mean_model_t=0.4953 mean_corrupt_t=0.4953 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5102 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9144 corrupt_frac=1.0000 acc_corrupt=0.9144 loss_corrupt=0.2485 wrong_frac=0.5048 init_acc_corrupt=0.5349 acc_corrupt_t_0p0_0p2=0.5806 corrupt_frac_t_0p0_0p2=0.2036 acc_corrupt_t_0p2_0p4=0.9991 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1934 out_w_norm=12.5783 out_g_norm=0.3324 loss_all=0.2909 init_gold_top10=0.6843 init_gold_top100=0.7632 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.6723 init_acc_rollout_kept=0.4573 logit_acc_rollout_applied=0.9145 logit_acc_rollout_kept=0.8771 +step=3400 epoch=3400/4000 epoch_step=1/1 micro_steps=3400 elapsed=25.5s lr=2.000000e-03 loss=0.2423 loss_recon=0.2423 loss_meanflow=0.0000 mean_model_t=0.4980 mean_corrupt_t=0.4980 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5031 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9170 corrupt_frac=1.0000 acc_corrupt=0.9170 loss_corrupt=0.2423 wrong_frac=0.5019 init_acc_corrupt=0.5367 acc_corrupt_t_0p0_0p2=0.5984 corrupt_frac_t_0p0_0p2=0.2061 acc_corrupt_t_0p2_0p4=0.9990 corrupt_frac_t_0p2_0p4=0.1960 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2020 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1985 out_w_norm=12.5616 out_g_norm=0.2825 loss_all=0.3823 init_gold_top10=0.7051 init_gold_top100=0.7822 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.5722 init_acc_rollout_kept=0.4870 logit_acc_rollout_applied=0.8564 logit_acc_rollout_kept=0.9133 +step=3500 epoch=3500/4000 epoch_step=1/1 micro_steps=3500 elapsed=25.5s lr=2.000000e-03 loss=0.2440 loss_recon=0.2440 loss_meanflow=0.0000 mean_model_t=0.4998 mean_corrupt_t=0.4998 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5031 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9168 corrupt_frac=1.0000 acc_corrupt=0.9168 loss_corrupt=0.2440 wrong_frac=0.5002 init_acc_corrupt=0.5414 acc_corrupt_t_0p0_0p2=0.5851 corrupt_frac_t_0p0_0p2=0.1998 acc_corrupt_t_0p2_0p4=0.9987 corrupt_frac_t_0p2_0p4=0.1971 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2008 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.2002 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2020 out_w_norm=12.5491 out_g_norm=0.3061 loss_all=0.3157 init_gold_top10=0.7029 init_gold_top100=0.7652 rollout_applied_pos_frac=0.4531 init_acc_rollout_applied=0.6169 init_acc_rollout_kept=0.4937 logit_acc_rollout_applied=0.8954 logit_acc_rollout_kept=0.9080 +step=3600 epoch=3600/4000 epoch_step=1/1 micro_steps=3600 elapsed=25.3s lr=2.000000e-03 loss=0.2269 loss_recon=0.2269 loss_meanflow=0.0000 mean_model_t=0.5009 mean_corrupt_t=0.5009 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4992 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9208 corrupt_frac=1.0000 acc_corrupt=0.9208 loss_corrupt=0.2269 wrong_frac=0.4992 init_acc_corrupt=0.5419 acc_corrupt_t_0p0_0p2=0.5973 corrupt_frac_t_0p0_0p2=0.1963 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.2016 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2056 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1941 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2024 out_w_norm=12.5320 out_g_norm=0.2724 loss_all=0.2221 init_gold_top10=0.7132 init_gold_top100=0.7854 rollout_applied_pos_frac=0.5469 init_acc_rollout_applied=0.6239 init_acc_rollout_kept=0.4697 logit_acc_rollout_applied=0.9083 logit_acc_rollout_kept=0.9323 +step=3700 epoch=3700/4000 epoch_step=1/1 micro_steps=3700 elapsed=25.3s lr=2.000000e-03 loss=0.2286 loss_recon=0.2286 loss_meanflow=0.0000 mean_model_t=0.5003 mean_corrupt_t=0.5003 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4918 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9222 corrupt_frac=1.0000 acc_corrupt=0.9222 loss_corrupt=0.2286 wrong_frac=0.4998 init_acc_corrupt=0.5391 acc_corrupt_t_0p0_0p2=0.6131 corrupt_frac_t_0p0_0p2=0.2006 acc_corrupt_t_0p2_0p4=0.9992 corrupt_frac_t_0p2_0p4=0.1945 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.2064 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1962 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2023 out_w_norm=12.5313 out_g_norm=0.2704 loss_all=0.1737 init_gold_top10=0.7026 init_gold_top100=0.7685 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.6008 init_acc_rollout_kept=0.5063 logit_acc_rollout_applied=0.9206 logit_acc_rollout_kept=0.9552 +step=3800 epoch=3800/4000 epoch_step=1/1 micro_steps=3800 elapsed=25.3s lr=2.000000e-03 loss=0.2299 loss_recon=0.2299 loss_meanflow=0.0000 mean_model_t=0.4990 mean_corrupt_t=0.4990 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4960 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9201 corrupt_frac=1.0000 acc_corrupt=0.9201 loss_corrupt=0.2299 wrong_frac=0.5010 init_acc_corrupt=0.5390 acc_corrupt_t_0p0_0p2=0.6021 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.2004 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2018 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1982 out_w_norm=12.5312 out_g_norm=0.2116 loss_all=0.1417 init_gold_top10=0.6969 init_gold_top100=0.7588 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.5894 init_acc_rollout_kept=0.4894 logit_acc_rollout_applied=0.9411 logit_acc_rollout_kept=0.9625 +step=3900 epoch=3900/4000 epoch_step=1/1 micro_steps=3900 elapsed=25.5s lr=2.000000e-03 loss=0.2197 loss_recon=0.2197 loss_meanflow=0.0000 mean_model_t=0.5027 mean_corrupt_t=0.5027 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9244 corrupt_frac=1.0000 acc_corrupt=0.9244 loss_corrupt=0.2197 wrong_frac=0.4975 init_acc_corrupt=0.5450 acc_corrupt_t_0p0_0p2=0.6217 corrupt_frac_t_0p0_0p2=0.1991 acc_corrupt_t_0p2_0p4=0.9991 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.1958 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=12.5317 out_g_norm=0.2390 loss_all=0.3058 init_gold_top10=0.6858 init_gold_top100=0.7661 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.5908 init_acc_rollout_kept=0.4748 logit_acc_rollout_applied=0.8692 logit_acc_rollout_kept=0.9259 +step=4000 epoch=4000/4000 epoch_step=1/1 micro_steps=4000 elapsed=25.4s lr=2.000000e-03 loss=0.2222 loss_recon=0.2222 loss_meanflow=0.0000 mean_model_t=0.5005 mean_corrupt_t=0.5005 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4931 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9236 corrupt_frac=1.0000 acc_corrupt=0.9236 loss_corrupt=0.2222 wrong_frac=0.4995 init_acc_corrupt=0.5391 acc_corrupt_t_0p0_0p2=0.6240 corrupt_frac_t_0p0_0p2=0.2029 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.1965 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1988 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2028 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1990 out_w_norm=12.5445 out_g_norm=0.2244 loss_all=0.0821 init_gold_top10=0.7090 init_gold_top100=0.7647 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.6604 init_acc_rollout_kept=0.5098 logit_acc_rollout_applied=0.9968 logit_acc_rollout_kept=0.9536 +[ctx1024-sampleds] eval config=p50_rand0_4_unif0_0p25_outwdm1 step=4000 +[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=4000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0004000.pt", + "ckpt_step": 4000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.99932861328125, + "token_acc_min": 0.9970703125, + "token_acc_max": 1.0, + "exact_acc": 0.578125, + "exact_count": 37, + "exact_ref_coverage": 0.75, + "exact_ref_count": 6, + "exact_ref_hits": [ + 1, + 2, + 3, + 4, + 5, + 7 + ], + "best_ref_idx": [ + 5, + 5, + 5, + 1, + 4, + 1, + 1, + 5, + 7, + 5, + 5, + 5, + 1, + 4, + 5, + 5, + 7, + 3, + 2, + 5, + 7, + 2, + 1, + 4, + 1, + 1, + 5, + 5, + 4, + 5, + 5, + 2, + 2, + 5, + 1, + 1, + 3, + 5, + 4, + 5, + 5, + 5, + 1, + 4, + 1, + 5, + 3, + 5, + 1, + 4, + 5, + 5, + 5, + 3, + 1, + 5, + 5, + 5, + 3, + 4, + 1, + 7, + 1, + 1 + ], + "best_token_acc": [ + 0.9990234375, + 0.998046875, + 0.998046875, + 1.0, + 1.0, + 1.0, + 1.0, + 0.998046875, + 1.0, + 0.9990234375, + 0.998046875, + 0.998046875, + 1.0, + 1.0, + 0.998046875, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 0.998046875, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.998046875, + 1.0, + 0.998046875, + 0.998046875, + 1.0, + 1.0, + 0.998046875, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 0.998046875, + 0.9990234375, + 0.998046875, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.9970703125, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 0.998046875, + 0.998046875, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0004000.pt", + "ckpt_step": 4000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.99932861328125, + "token_acc_min": 0.9970703125, + "token_acc_max": 1.0, + "exact_acc": 0.578125, + "exact_count": 37, + "exact_ref_coverage": 0.75, + "exact_ref_count": 6, + "exact_ref_hits": [ + 1, + 2, + 3, + 4, + 5, + 7 + ], + "best_ref_idx": [ + 5, + 5, + 5, + 1, + 4, + 1, + 1, + 5, + 7, + 5, + 5, + 5, + 1, + 4, + 5, + 5, + 7, + 3, + 2, + 5, + 7, + 2, + 1, + 4, + 1, + 1, + 5, + 5, + 4, + 5, + 5, + 2, + 2, + 5, + 1, + 1, + 3, + 5, + 4, + 5, + 5, + 5, + 1, + 4, + 1, + 5, + 3, + 5, + 1, + 4, + 5, + 5, + 5, + 3, + 1, + 5, + 5, + 5, + 3, + 4, + 1, + 7, + 1, + 1 + ], + "best_token_acc": [ + 0.9990234375, + 0.998046875, + 0.998046875, + 1.0, + 1.0, + 1.0, + 1.0, + 0.998046875, + 1.0, + 0.9990234375, + 0.998046875, + 0.998046875, + 1.0, + 1.0, + 0.998046875, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 0.998046875, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.998046875, + 1.0, + 0.998046875, + 0.998046875, + 1.0, + 1.0, + 0.998046875, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 0.998046875, + 0.9990234375, + 0.998046875, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.9970703125, + 0.9990234375, + 1.0, + 1.0, + 0.9990234375, + 0.998046875, + 0.998046875, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0 + ] + } + } +} +RESULT config=p50_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=4000 views=2048000 token_acc=0.9993 exact=37/64 exact_refs=6 hits=[1, 2, 3, 4, 5, 7] +[ctx1024-sampleds] train config=p50_rand0_4_unif0_0p25_outwdm1 from=4000 to=5000 +[launch] gpt2 cached OWT soft-endpoint m/n pilot +[launch] run_name=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] save_dir=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] n=1024 m=0 clean_state_mode=onehot +[launch] mask_mixture lowk=0.0 all=1.0 +[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=2423 +[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1 +[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0 +[launch] mask_ratio=1.0->1.0 +[launch] mask_ratio_floor_schedule=none +[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet +[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids= +[launch] rollout_train prob=0.50 mode=sampled_path steps=4 steps_min=0 infer_steps=1 s_dist=uniform s_frac=0.0->0.25 temp=1.0 corrupt_only=1 samplewise=1 selected_only=1 sync_t=1 +[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit exact_repeat_per_chunk=64 +[launch] resume_path=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt start_step=4001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.5, + "rollout_train_steps": 4, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt" +} +step=4100 epoch=4100/5000 epoch_step=1/1 micro_steps=4100 elapsed=26.2s lr=2.000000e-03 loss=0.2186 loss_recon=0.2186 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5002 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9238 corrupt_frac=1.0000 acc_corrupt=0.9238 loss_corrupt=0.2186 wrong_frac=0.4986 init_acc_corrupt=0.5406 acc_corrupt_t_0p0_0p2=0.6100 corrupt_frac_t_0p0_0p2=0.1952 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.2063 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1976 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2036 out_w_norm=12.5634 out_g_norm=0.2004 loss_all=0.1765 init_gold_top10=0.6897 init_gold_top100=0.7541 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.6232 init_acc_rollout_kept=0.4387 logit_acc_rollout_applied=0.9170 logit_acc_rollout_kept=0.9681 +step=4200 epoch=4200/5000 epoch_step=1/1 micro_steps=4200 elapsed=25.3s lr=2.000000e-03 loss=0.2225 loss_recon=0.2225 loss_meanflow=0.0000 mean_model_t=0.4985 mean_corrupt_t=0.4985 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5030 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9240 corrupt_frac=1.0000 acc_corrupt=0.9240 loss_corrupt=0.2225 wrong_frac=0.5014 init_acc_corrupt=0.5372 acc_corrupt_t_0p0_0p2=0.6275 corrupt_frac_t_0p0_0p2=0.2037 acc_corrupt_t_0p2_0p4=0.9993 corrupt_frac_t_0p2_0p4=0.1982 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.1956 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2056 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1969 out_w_norm=12.5838 out_g_norm=0.1905 loss_all=0.1620 init_gold_top10=0.6844 init_gold_top100=0.7618 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.6516 init_acc_rollout_kept=0.4286 logit_acc_rollout_applied=0.9779 logit_acc_rollout_kept=0.8964 +step=4300 epoch=4300/5000 epoch_step=1/1 micro_steps=4300 elapsed=25.5s lr=2.000000e-03 loss=0.2157 loss_recon=0.2157 loss_meanflow=0.0000 mean_model_t=0.4953 mean_corrupt_t=0.4953 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5102 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9254 corrupt_frac=1.0000 acc_corrupt=0.9254 loss_corrupt=0.2157 wrong_frac=0.5048 init_acc_corrupt=0.5364 acc_corrupt_t_0p0_0p2=0.6343 corrupt_frac_t_0p0_0p2=0.2036 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1934 out_w_norm=12.6004 out_g_norm=0.1667 loss_all=0.2117 init_gold_top10=0.6878 init_gold_top100=0.7632 rollout_applied_pos_frac=0.4922 init_acc_rollout_applied=0.6725 init_acc_rollout_kept=0.4573 logit_acc_rollout_applied=0.9250 logit_acc_rollout_kept=0.9231 +step=4400 epoch=4400/5000 epoch_step=1/1 micro_steps=4400 elapsed=25.4s lr=2.000000e-03 loss=0.2121 loss_recon=0.2121 loss_meanflow=0.0000 mean_model_t=0.4980 mean_corrupt_t=0.4980 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5031 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9258 corrupt_frac=1.0000 acc_corrupt=0.9258 loss_corrupt=0.2121 wrong_frac=0.5019 init_acc_corrupt=0.5376 acc_corrupt_t_0p0_0p2=0.6403 corrupt_frac_t_0p0_0p2=0.2061 acc_corrupt_t_0p2_0p4=0.9998 corrupt_frac_t_0p2_0p4=0.1960 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2020 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1985 out_w_norm=12.6099 out_g_norm=0.1469 loss_all=0.2892 init_gold_top10=0.7065 init_gold_top100=0.7822 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.5722 init_acc_rollout_kept=0.4870 logit_acc_rollout_applied=0.8531 logit_acc_rollout_kept=0.9403 +step=4500 epoch=4500/5000 epoch_step=1/1 micro_steps=4500 elapsed=25.4s lr=2.000000e-03 loss=0.2133 loss_recon=0.2133 loss_meanflow=0.0000 mean_model_t=0.4998 mean_corrupt_t=0.4998 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5031 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9265 corrupt_frac=1.0000 acc_corrupt=0.9265 loss_corrupt=0.2133 wrong_frac=0.5002 init_acc_corrupt=0.5429 acc_corrupt_t_0p0_0p2=0.6329 corrupt_frac_t_0p0_0p2=0.1998 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.1971 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2008 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2002 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2020 out_w_norm=12.6294 out_g_norm=0.1496 loss_all=0.2702 init_gold_top10=0.7049 init_gold_top100=0.7652 rollout_applied_pos_frac=0.4531 init_acc_rollout_applied=0.6152 init_acc_rollout_kept=0.4937 logit_acc_rollout_applied=0.8901 logit_acc_rollout_kept=0.9134 +step=4600 epoch=4600/5000 epoch_step=1/1 micro_steps=4600 elapsed=25.3s lr=2.000000e-03 loss=0.2022 loss_recon=0.2022 loss_meanflow=0.0000 mean_model_t=0.5009 mean_corrupt_t=0.5009 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4992 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9301 corrupt_frac=1.0000 acc_corrupt=0.9301 loss_corrupt=0.2022 wrong_frac=0.4992 init_acc_corrupt=0.5427 acc_corrupt_t_0p0_0p2=0.6440 corrupt_frac_t_0p0_0p2=0.1963 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.2016 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2056 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1941 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2024 out_w_norm=12.6419 out_g_norm=0.1513 loss_all=0.1736 init_gold_top10=0.7198 init_gold_top100=0.7854 rollout_applied_pos_frac=0.5469 init_acc_rollout_applied=0.6244 init_acc_rollout_kept=0.4697 logit_acc_rollout_applied=0.9318 logit_acc_rollout_kept=0.9491 +step=4700 epoch=4700/5000 epoch_step=1/1 micro_steps=4700 elapsed=25.3s lr=2.000000e-03 loss=0.1975 loss_recon=0.1975 loss_meanflow=0.0000 mean_model_t=0.5003 mean_corrupt_t=0.5003 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4918 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9328 corrupt_frac=1.0000 acc_corrupt=0.9328 loss_corrupt=0.1975 wrong_frac=0.4998 init_acc_corrupt=0.5406 acc_corrupt_t_0p0_0p2=0.6657 corrupt_frac_t_0p0_0p2=0.2006 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.1945 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2064 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1962 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2023 out_w_norm=12.6608 out_g_norm=0.1390 loss_all=0.1708 init_gold_top10=0.7059 init_gold_top100=0.7685 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.6019 init_acc_rollout_kept=0.5063 logit_acc_rollout_applied=0.9325 logit_acc_rollout_kept=0.9396 +step=4800 epoch=4800/5000 epoch_step=1/1 micro_steps=4800 elapsed=25.3s lr=2.000000e-03 loss=0.2091 loss_recon=0.2091 loss_meanflow=0.0000 mean_model_t=0.4990 mean_corrupt_t=0.4990 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4960 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9280 corrupt_frac=1.0000 acc_corrupt=0.9280 loss_corrupt=0.2091 wrong_frac=0.5010 init_acc_corrupt=0.5398 acc_corrupt_t_0p0_0p2=0.6413 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.2004 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2018 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1982 out_w_norm=12.6854 out_g_norm=0.1459 loss_all=0.1538 init_gold_top10=0.6980 init_gold_top100=0.7588 rollout_applied_pos_frac=0.5391 init_acc_rollout_applied=0.5934 init_acc_rollout_kept=0.4894 logit_acc_rollout_applied=0.9379 logit_acc_rollout_kept=0.9556 +step=4900 epoch=4900/5000 epoch_step=1/1 micro_steps=4900 elapsed=25.5s lr=2.000000e-03 loss=0.1977 loss_recon=0.1977 loss_meanflow=0.0000 mean_model_t=0.5027 mean_corrupt_t=0.5027 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9315 corrupt_frac=1.0000 acc_corrupt=0.9315 loss_corrupt=0.1977 wrong_frac=0.4975 init_acc_corrupt=0.5459 acc_corrupt_t_0p0_0p2=0.6575 corrupt_frac_t_0p0_0p2=0.1991 acc_corrupt_t_0p2_0p4=0.9993 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.1958 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=12.7066 out_g_norm=0.1458 loss_all=0.2784 init_gold_top10=0.6868 init_gold_top100=0.7661 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.5915 init_acc_rollout_kept=0.4748 logit_acc_rollout_applied=0.8609 logit_acc_rollout_kept=0.9276 +step=5000 epoch=5000/5000 epoch_step=1/1 micro_steps=5000 elapsed=25.3s lr=2.000000e-03 loss=0.2020 loss_recon=0.2020 loss_meanflow=0.0000 mean_model_t=0.5005 mean_corrupt_t=0.5005 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4931 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9310 corrupt_frac=1.0000 acc_corrupt=0.9310 loss_corrupt=0.2020 wrong_frac=0.4995 init_acc_corrupt=0.5398 acc_corrupt_t_0p0_0p2=0.6608 corrupt_frac_t_0p0_0p2=0.2029 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.1965 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.1988 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2028 acc_corrupt_t_0p8_1p0=0.9998 corrupt_frac_t_0p8_1p0=0.1990 out_w_norm=12.7400 out_g_norm=0.1335 loss_all=0.0640 init_gold_top10=0.7090 init_gold_top100=0.7647 rollout_applied_pos_frac=0.4609 init_acc_rollout_applied=0.6647 init_acc_rollout_kept=0.5098 logit_acc_rollout_applied=0.9999 logit_acc_rollout_kept=0.9611 +[ctx1024-sampleds] eval config=p50_rand0_4_unif0_0p25_outwdm1 step=5000 +[eval-decode-acc] train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=5000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0005000.pt", + "ckpt_step": 5000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9999237060546875, + "token_acc_min": 0.9990234375, + "token_acc_max": 1.0, + "exact_acc": 0.921875, + "exact_count": 59, + "exact_ref_coverage": 0.75, + "exact_ref_count": 6, + "exact_ref_hits": [ + 1, + 2, + 3, + 4, + 5, + 7 + ], + "best_ref_idx": [ + 7, + 5, + 1, + 5, + 2, + 3, + 2, + 3, + 2, + 2, + 3, + 3, + 2, + 3, + 3, + 2, + 1, + 2, + 1, + 2, + 2, + 3, + 2, + 1, + 2, + 3, + 3, + 2, + 3, + 2, + 2, + 2, + 5, + 1, + 3, + 3, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 2, + 3, + 2, + 2, + 2, + 2, + 5, + 2, + 2, + 2, + 3, + 2, + 2, + 2 + ], + "best_token_acc": [ + 1.0, + 0.9990234375, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0005000.pt", + "ckpt_step": 5000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9999237060546875, + "token_acc_min": 0.9990234375, + "token_acc_max": 1.0, + "exact_acc": 0.921875, + "exact_count": 59, + "exact_ref_coverage": 0.75, + "exact_ref_count": 6, + "exact_ref_hits": [ + 1, + 2, + 3, + 4, + 5, + 7 + ], + "best_ref_idx": [ + 7, + 5, + 1, + 5, + 2, + 3, + 2, + 3, + 2, + 2, + 3, + 3, + 2, + 3, + 3, + 2, + 1, + 2, + 1, + 2, + 2, + 3, + 2, + 1, + 2, + 3, + 3, + 2, + 3, + 2, + 2, + 2, + 5, + 1, + 3, + 3, + 2, + 2, + 2, + 2, + 2, + 2, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 2, + 3, + 2, + 2, + 2, + 2, + 5, + 2, + 2, + 2, + 3, + 2, + 2, + 2 + ], + "best_token_acc": [ + 1.0, + 0.9990234375, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ] + } + } +} +RESULT config=p50_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p50_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=5000 views=2560000 token_acc=0.9999 exact=59/64 exact_refs=6 hits=[1, 2, 3, 4, 5, 7] +[ctx1024-sampleds] capped config=p50_rand0_4_unif0_0p25_outwdm1 step=5000 +[ctx1024-sampleds] config=p35_rand0_4_unif0_0p25_outwdm1 run=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 p=0.35 mode=sampled_path steps=4 steps_min=0 s_dist=uniform s_frac=0.0->0.25 beta=2.0,6.0 outwd=-1 sync_t=1 +[ctx1024-sampleds] train config=p35_rand0_4_unif0_0p25_outwdm1 from=0 to=1000 +[launch] gpt2 cached OWT soft-endpoint m/n pilot +[launch] run_name=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] save_dir=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] n=1024 m=0 clean_state_mode=onehot +[launch] mask_mixture lowk=0.0 all=1.0 +[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=2423 +[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1 +[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0 +[launch] mask_ratio=1.0->1.0 +[launch] mask_ratio_floor_schedule=none +[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet +[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids= +[launch] rollout_train prob=0.35 mode=sampled_path steps=4 steps_min=0 infer_steps=1 s_dist=uniform s_frac=0.0->0.25 temp=1.0 corrupt_only=1 samplewise=1 selected_only=1 sync_t=1 +[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit exact_repeat_per_chunk=64 +NCCL version 2.25.1+cuda12.8 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 1000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 4, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +step=100 epoch=100/1000 epoch_step=1/1 micro_steps=100 elapsed=22.3s lr=2.000000e-03 loss=7.3398 loss_recon=7.3398 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3505 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3321 corrupt_frac=1.0000 acc_corrupt=0.3321 loss_corrupt=7.3398 wrong_frac=0.4986 init_acc_corrupt=0.4674 acc_corrupt_t_0p0_0p2=0.0458 corrupt_frac_t_0p0_0p2=0.1952 acc_corrupt_t_0p2_0p4=0.1644 corrupt_frac_t_0p2_0p4=0.2063 acc_corrupt_t_0p4_0p6=0.3261 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=0.4811 corrupt_frac_t_0p6_0p8=0.1976 acc_corrupt_t_0p8_1p0=0.6381 corrupt_frac_t_0p8_1p0=0.2036 out_w_norm=1.0907 out_g_norm=1.0054 loss_all=6.6983 init_gold_top10=0.5036 init_gold_top100=0.6154 rollout_applied_pos_frac=0.3672 init_acc_rollout_applied=0.5092 init_acc_rollout_kept=0.4438 logit_acc_rollout_applied=0.3436 logit_acc_rollout_kept=0.3040 +step=200 epoch=200/1000 epoch_step=1/1 micro_steps=200 elapsed=21.4s lr=2.000000e-03 loss=5.8158 loss_recon=5.8158 loss_meanflow=0.0000 mean_model_t=0.4985 mean_corrupt_t=0.4985 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3512 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3270 corrupt_frac=1.0000 acc_corrupt=0.3270 loss_corrupt=5.8158 wrong_frac=0.5014 init_acc_corrupt=0.4644 acc_corrupt_t_0p0_0p2=0.0525 corrupt_frac_t_0p0_0p2=0.2037 acc_corrupt_t_0p2_0p4=0.1627 corrupt_frac_t_0p2_0p4=0.1982 acc_corrupt_t_0p4_0p6=0.3281 corrupt_frac_t_0p4_0p6=0.1956 acc_corrupt_t_0p6_0p8=0.4717 corrupt_frac_t_0p6_0p8=0.2056 acc_corrupt_t_0p8_1p0=0.6239 corrupt_frac_t_0p8_1p0=0.1969 out_w_norm=3.4902 out_g_norm=1.3276 loss_all=5.0556 init_gold_top10=0.5033 init_gold_top100=0.6302 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5130 init_acc_rollout_kept=0.4511 logit_acc_rollout_applied=0.3732 logit_acc_rollout_kept=0.3346 +step=300 epoch=300/1000 epoch_step=1/1 micro_steps=300 elapsed=21.5s lr=2.000000e-03 loss=4.7529 loss_recon=4.7529 loss_meanflow=0.0000 mean_model_t=0.4953 mean_corrupt_t=0.4953 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3561 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3619 corrupt_frac=1.0000 acc_corrupt=0.3619 loss_corrupt=4.7529 wrong_frac=0.5048 init_acc_corrupt=0.4611 acc_corrupt_t_0p0_0p2=0.0554 corrupt_frac_t_0p0_0p2=0.2036 acc_corrupt_t_0p2_0p4=0.1879 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.3636 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=0.5241 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=0.6983 corrupt_frac_t_0p8_1p0=0.1934 out_w_norm=5.5748 out_g_norm=0.5497 loss_all=4.3187 init_gold_top10=0.5207 init_gold_top100=0.6468 rollout_applied_pos_frac=0.3203 init_acc_rollout_applied=0.5246 init_acc_rollout_kept=0.4730 logit_acc_rollout_applied=0.4309 logit_acc_rollout_kept=0.3892 +step=400 epoch=400/1000 epoch_step=1/1 micro_steps=400 elapsed=21.3s lr=2.000000e-03 loss=4.1311 loss_recon=4.1311 loss_meanflow=0.0000 mean_model_t=0.4980 mean_corrupt_t=0.4980 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3477 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4208 corrupt_frac=1.0000 acc_corrupt=0.4208 loss_corrupt=4.1311 wrong_frac=0.5019 init_acc_corrupt=0.4646 acc_corrupt_t_0p0_0p2=0.0580 corrupt_frac_t_0p0_0p2=0.2061 acc_corrupt_t_0p2_0p4=0.2099 corrupt_frac_t_0p2_0p4=0.1960 acc_corrupt_t_0p4_0p6=0.4188 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=0.6131 corrupt_frac_t_0p6_0p8=0.2020 acc_corrupt_t_0p8_1p0=0.8120 corrupt_frac_t_0p8_1p0=0.1985 out_w_norm=7.1079 out_g_norm=0.2788 loss_all=3.8756 init_gold_top10=0.4993 init_gold_top100=0.6349 rollout_applied_pos_frac=0.3203 init_acc_rollout_applied=0.4744 init_acc_rollout_kept=0.4485 logit_acc_rollout_applied=0.4807 logit_acc_rollout_kept=0.4541 +step=500 epoch=500/1000 epoch_step=1/1 micro_steps=500 elapsed=21.6s lr=2.000000e-03 loss=3.5338 loss_recon=3.5338 loss_meanflow=0.0000 mean_model_t=0.4998 mean_corrupt_t=0.4998 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3556 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4850 corrupt_frac=1.0000 acc_corrupt=0.4850 loss_corrupt=3.5338 wrong_frac=0.5002 init_acc_corrupt=0.4672 acc_corrupt_t_0p0_0p2=0.0595 corrupt_frac_t_0p0_0p2=0.1998 acc_corrupt_t_0p2_0p4=0.2389 corrupt_frac_t_0p2_0p4=0.1971 acc_corrupt_t_0p4_0p6=0.5047 corrupt_frac_t_0p4_0p6=0.2008 acc_corrupt_t_0p6_0p8=0.7118 corrupt_frac_t_0p6_0p8=0.2002 acc_corrupt_t_0p8_1p0=0.9014 corrupt_frac_t_0p8_1p0=0.2020 out_w_norm=8.4429 out_g_norm=0.2396 loss_all=3.1683 init_gold_top10=0.5190 init_gold_top100=0.6517 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.4787 init_acc_rollout_kept=0.4850 logit_acc_rollout_applied=0.4965 logit_acc_rollout_kept=0.5102 +step=600 epoch=600/1000 epoch_step=1/1 micro_steps=600 elapsed=21.2s lr=2.000000e-03 loss=3.0899 loss_recon=3.0899 loss_meanflow=0.0000 mean_model_t=0.5009 mean_corrupt_t=0.5009 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3408 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4960 corrupt_frac=1.0000 acc_corrupt=0.4960 loss_corrupt=3.0899 wrong_frac=0.4992 init_acc_corrupt=0.4683 acc_corrupt_t_0p0_0p2=0.0615 corrupt_frac_t_0p0_0p2=0.1963 acc_corrupt_t_0p2_0p4=0.2689 corrupt_frac_t_0p2_0p4=0.2016 acc_corrupt_t_0p4_0p6=0.5237 corrupt_frac_t_0p4_0p6=0.2056 acc_corrupt_t_0p6_0p8=0.7171 corrupt_frac_t_0p6_0p8=0.1941 acc_corrupt_t_0p8_1p0=0.9034 corrupt_frac_t_0p8_1p0=0.2024 out_w_norm=9.7107 out_g_norm=0.2350 loss_all=2.8169 init_gold_top10=0.5280 init_gold_top100=0.6629 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.5225 init_acc_rollout_kept=0.4687 logit_acc_rollout_applied=0.5480 logit_acc_rollout_kept=0.5012 +step=700 epoch=700/1000 epoch_step=1/1 micro_steps=700 elapsed=21.5s lr=2.000000e-03 loss=2.7701 loss_recon=2.7701 loss_meanflow=0.0000 mean_model_t=0.5003 mean_corrupt_t=0.5003 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3456 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5024 corrupt_frac=1.0000 acc_corrupt=0.5024 loss_corrupt=2.7701 wrong_frac=0.4998 init_acc_corrupt=0.4679 acc_corrupt_t_0p0_0p2=0.0632 corrupt_frac_t_0p0_0p2=0.2006 acc_corrupt_t_0p2_0p4=0.2825 corrupt_frac_t_0p2_0p4=0.1945 acc_corrupt_t_0p4_0p6=0.5328 corrupt_frac_t_0p4_0p6=0.2064 acc_corrupt_t_0p6_0p8=0.7215 corrupt_frac_t_0p6_0p8=0.1962 acc_corrupt_t_0p8_1p0=0.9055 corrupt_frac_t_0p8_1p0=0.2023 out_w_norm=10.7214 out_g_norm=0.2984 loss_all=2.4240 init_gold_top10=0.5449 init_gold_top100=0.6994 rollout_applied_pos_frac=0.3359 init_acc_rollout_applied=0.4338 init_acc_rollout_kept=0.5231 logit_acc_rollout_applied=0.4746 logit_acc_rollout_kept=0.5618 +step=800 epoch=800/1000 epoch_step=1/1 micro_steps=800 elapsed=21.5s lr=2.000000e-03 loss=2.3140 loss_recon=2.3140 loss_meanflow=0.0000 mean_model_t=0.4990 mean_corrupt_t=0.4990 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3480 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5340 corrupt_frac=1.0000 acc_corrupt=0.5340 loss_corrupt=2.3140 wrong_frac=0.5010 init_acc_corrupt=0.4673 acc_corrupt_t_0p0_0p2=0.0627 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.3125 corrupt_frac_t_0p2_0p4=0.2004 acc_corrupt_t_0p4_0p6=0.5945 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=0.7750 corrupt_frac_t_0p6_0p8=0.2018 acc_corrupt_t_0p8_1p0=0.9285 corrupt_frac_t_0p8_1p0=0.1982 out_w_norm=11.2606 out_g_norm=0.3675 loss_all=1.9381 init_gold_top10=0.5523 init_gold_top100=0.6877 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.4343 init_acc_rollout_kept=0.5056 logit_acc_rollout_applied=0.5323 logit_acc_rollout_kept=0.6172 +step=900 epoch=900/1000 epoch_step=1/1 micro_steps=900 elapsed=21.5s lr=2.000000e-03 loss=1.8331 loss_recon=1.8331 loss_meanflow=0.0000 mean_model_t=0.5027 mean_corrupt_t=0.5027 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3570 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6092 corrupt_frac=1.0000 acc_corrupt=0.6092 loss_corrupt=1.8331 wrong_frac=0.4975 init_acc_corrupt=0.4744 acc_corrupt_t_0p0_0p2=0.0647 corrupt_frac_t_0p0_0p2=0.1991 acc_corrupt_t_0p2_0p4=0.3814 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.7313 corrupt_frac_t_0p4_0p6=0.1958 acc_corrupt_t_0p6_0p8=0.8878 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=0.9707 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=11.7391 out_g_norm=0.4679 loss_all=1.6215 init_gold_top10=0.5723 init_gold_top100=0.7072 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5013 init_acc_rollout_kept=0.4759 logit_acc_rollout_applied=0.6607 logit_acc_rollout_kept=0.6511 +step=1000 epoch=1000/1000 epoch_step=1/1 micro_steps=1000 elapsed=21.4s lr=2.000000e-03 loss=1.5052 loss_recon=1.5052 loss_meanflow=0.0000 mean_model_t=0.5005 mean_corrupt_t=0.5005 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3489 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6756 corrupt_frac=1.0000 acc_corrupt=0.6756 loss_corrupt=1.5052 wrong_frac=0.4995 init_acc_corrupt=0.4803 acc_corrupt_t_0p0_0p2=0.0712 corrupt_frac_t_0p0_0p2=0.2029 acc_corrupt_t_0p2_0p4=0.4982 corrupt_frac_t_0p2_0p4=0.1965 acc_corrupt_t_0p4_0p6=0.8585 corrupt_frac_t_0p4_0p6=0.1988 acc_corrupt_t_0p6_0p8=0.9612 corrupt_frac_t_0p6_0p8=0.2028 acc_corrupt_t_0p8_1p0=0.9934 corrupt_frac_t_0p8_1p0=0.1990 out_w_norm=12.1101 out_g_norm=0.5194 loss_all=1.0420 init_gold_top10=0.6055 init_gold_top100=0.7165 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.5002 init_acc_rollout_kept=0.5045 logit_acc_rollout_applied=0.7523 logit_acc_rollout_kept=0.7499 +[ctx1024-sampleds] eval config=p35_rand0_4_unif0_0p25_outwdm1 step=1000 +[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=1000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0001000.pt", + "ckpt_step": 1000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.1269989013671875, + "token_acc_min": 0.013671875, + "token_acc_max": 0.7060546875, + "exact_acc": 0.0, + "exact_count": 0, + "exact_ref_coverage": 0.0, + "exact_ref_count": 0, + "exact_ref_hits": [], + "best_ref_idx": [ + 1, + 5, + 0, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 4, + 5, + 5, + 1, + 5, + 5, + 7, + 5, + 5, + 5, + 5, + 5, + 3, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 3, + 5, + 3, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 0.017578125, + 0.0302734375, + 0.0166015625, + 0.0234375, + 0.0283203125, + 0.0234375, + 0.6455078125, + 0.0263671875, + 0.0224609375, + 0.03515625, + 0.037109375, + 0.03125, + 0.51953125, + 0.0361328125, + 0.04296875, + 0.0283203125, + 0.0458984375, + 0.1630859375, + 0.099609375, + 0.0224609375, + 0.0419921875, + 0.03125, + 0.6328125, + 0.7060546875, + 0.0263671875, + 0.142578125, + 0.0361328125, + 0.041015625, + 0.013671875, + 0.1201171875, + 0.0302734375, + 0.021484375, + 0.142578125, + 0.0478515625, + 0.0166015625, + 0.041015625, + 0.041015625, + 0.0361328125, + 0.21484375, + 0.03125, + 0.013671875, + 0.158203125, + 0.6884765625, + 0.03515625, + 0.0322265625, + 0.0322265625, + 0.04296875, + 0.0234375, + 0.6162109375, + 0.5224609375, + 0.0166015625, + 0.0419921875, + 0.0146484375, + 0.0361328125, + 0.0185546875, + 0.04296875, + 0.0283203125, + 0.0537109375, + 0.51953125, + 0.0703125, + 0.2080078125, + 0.53515625, + 0.048828125, + 0.017578125 + ] + } + }, + "first_exact_by_run": {} +} +RESULT config=p35_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=1000 views=512000 token_acc=0.1270 exact=0/64 exact_refs=0 hits=[] +[ctx1024-sampleds] train config=p35_rand0_4_unif0_0p25_outwdm1 from=1000 to=2000 +[launch] gpt2 cached OWT soft-endpoint m/n pilot +[launch] run_name=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] save_dir=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] n=1024 m=0 clean_state_mode=onehot +[launch] mask_mixture lowk=0.0 all=1.0 +[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=2423 +[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1 +[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0 +[launch] mask_ratio=1.0->1.0 +[launch] mask_ratio_floor_schedule=none +[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet +[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids= +[launch] rollout_train prob=0.35 mode=sampled_path steps=4 steps_min=0 infer_steps=1 s_dist=uniform s_frac=0.0->0.25 temp=1.0 corrupt_only=1 samplewise=1 selected_only=1 sync_t=1 +[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit exact_repeat_per_chunk=64 +[launch] resume_path=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt start_step=1001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 2000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 4, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt" +} +step=1100 epoch=1100/2000 epoch_step=1/1 micro_steps=1100 elapsed=22.4s lr=2.000000e-03 loss=1.2767 loss_recon=1.2767 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3505 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7230 corrupt_frac=1.0000 acc_corrupt=0.7230 loss_corrupt=1.2767 wrong_frac=0.4986 init_acc_corrupt=0.4886 acc_corrupt_t_0p0_0p2=0.0808 corrupt_frac_t_0p0_0p2=0.1952 acc_corrupt_t_0p2_0p4=0.6060 corrupt_frac_t_0p2_0p4=0.2063 acc_corrupt_t_0p4_0p6=0.9317 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=0.9879 corrupt_frac_t_0p6_0p8=0.1976 acc_corrupt_t_0p8_1p0=0.9984 corrupt_frac_t_0p8_1p0=0.2036 out_w_norm=12.3788 out_g_norm=0.6042 loss_all=1.0626 init_gold_top10=0.5895 init_gold_top100=0.6964 rollout_applied_pos_frac=0.3672 init_acc_rollout_applied=0.5416 init_acc_rollout_kept=0.4438 logit_acc_rollout_applied=0.7898 logit_acc_rollout_kept=0.7326 +step=1200 epoch=1200/2000 epoch_step=1/1 micro_steps=1200 elapsed=21.5s lr=2.000000e-03 loss=1.1235 loss_recon=1.1235 loss_meanflow=0.0000 mean_model_t=0.4985 mean_corrupt_t=0.4985 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3512 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7536 corrupt_frac=1.0000 acc_corrupt=0.7536 loss_corrupt=1.1235 wrong_frac=0.5014 init_acc_corrupt=0.4914 acc_corrupt_t_0p0_0p2=0.1035 corrupt_frac_t_0p0_0p2=0.2037 acc_corrupt_t_0p2_0p4=0.7157 corrupt_frac_t_0p2_0p4=0.1982 acc_corrupt_t_0p4_0p6=0.9669 corrupt_frac_t_0p4_0p6=0.1956 acc_corrupt_t_0p6_0p8=0.9956 corrupt_frac_t_0p6_0p8=0.2056 acc_corrupt_t_0p8_1p0=0.9995 corrupt_frac_t_0p8_1p0=0.1969 out_w_norm=12.5808 out_g_norm=0.6078 loss_all=0.9406 init_gold_top10=0.5981 init_gold_top100=0.7016 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.6352 init_acc_rollout_kept=0.4511 logit_acc_rollout_applied=0.8745 logit_acc_rollout_kept=0.7457 +step=1300 epoch=1300/2000 epoch_step=1/1 micro_steps=1300 elapsed=21.6s lr=2.000000e-03 loss=0.9872 loss_recon=0.9872 loss_meanflow=0.0000 mean_model_t=0.4953 mean_corrupt_t=0.4953 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3561 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7801 corrupt_frac=1.0000 acc_corrupt=0.7801 loss_corrupt=0.9872 wrong_frac=0.5048 init_acc_corrupt=0.4923 acc_corrupt_t_0p0_0p2=0.1360 corrupt_frac_t_0p0_0p2=0.2036 acc_corrupt_t_0p2_0p4=0.8013 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.9834 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=0.9982 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=0.9998 corrupt_frac_t_0p8_1p0=0.1934 out_w_norm=12.7435 out_g_norm=0.6096 loss_all=0.9355 init_gold_top10=0.5925 init_gold_top100=0.7038 rollout_applied_pos_frac=0.3203 init_acc_rollout_applied=0.6028 init_acc_rollout_kept=0.4730 logit_acc_rollout_applied=0.8285 logit_acc_rollout_kept=0.7765 +step=1400 epoch=1400/2000 epoch_step=1/1 micro_steps=1400 elapsed=21.4s lr=2.000000e-03 loss=0.8762 loss_recon=0.8762 loss_meanflow=0.0000 mean_model_t=0.4980 mean_corrupt_t=0.4980 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3477 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8016 corrupt_frac=1.0000 acc_corrupt=0.8016 loss_corrupt=0.8762 wrong_frac=0.5019 init_acc_corrupt=0.4999 acc_corrupt_t_0p0_0p2=0.1713 corrupt_frac_t_0p0_0p2=0.2061 acc_corrupt_t_0p2_0p4=0.8680 corrupt_frac_t_0p2_0p4=0.1960 acc_corrupt_t_0p4_0p6=0.9922 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=0.9990 corrupt_frac_t_0p6_0p8=0.2020 acc_corrupt_t_0p8_1p0=0.9998 corrupt_frac_t_0p8_1p0=0.1985 out_w_norm=12.8779 out_g_norm=0.6266 loss_all=1.0431 init_gold_top10=0.5569 init_gold_top100=0.6933 rollout_applied_pos_frac=0.3203 init_acc_rollout_applied=0.5252 init_acc_rollout_kept=0.4485 logit_acc_rollout_applied=0.7440 logit_acc_rollout_kept=0.7830 +step=1500 epoch=1500/2000 epoch_step=1/1 micro_steps=1500 elapsed=21.7s lr=2.000000e-03 loss=0.7811 loss_recon=0.7811 loss_meanflow=0.0000 mean_model_t=0.4998 mean_corrupt_t=0.4998 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3556 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8210 corrupt_frac=1.0000 acc_corrupt=0.8210 loss_corrupt=0.7811 wrong_frac=0.5002 init_acc_corrupt=0.5047 acc_corrupt_t_0p0_0p2=0.1993 corrupt_frac_t_0p0_0p2=0.1998 acc_corrupt_t_0p2_0p4=0.9085 corrupt_frac_t_0p2_0p4=0.1971 acc_corrupt_t_0p4_0p6=0.9959 corrupt_frac_t_0p4_0p6=0.2008 acc_corrupt_t_0p6_0p8=0.9994 corrupt_frac_t_0p6_0p8=0.2002 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2020 out_w_norm=12.9795 out_g_norm=0.6605 loss_all=0.7407 init_gold_top10=0.6019 init_gold_top100=0.7049 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.5921 init_acc_rollout_kept=0.4850 logit_acc_rollout_applied=0.8213 logit_acc_rollout_kept=0.8415 +step=1600 epoch=1600/2000 epoch_step=1/1 micro_steps=1600 elapsed=21.3s lr=2.000000e-03 loss=0.6912 loss_recon=0.6912 loss_meanflow=0.0000 mean_model_t=0.5009 mean_corrupt_t=0.5009 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3408 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8386 corrupt_frac=1.0000 acc_corrupt=0.8386 loss_corrupt=0.6912 wrong_frac=0.4992 init_acc_corrupt=0.5091 acc_corrupt_t_0p0_0p2=0.2368 corrupt_frac_t_0p0_0p2=0.1963 acc_corrupt_t_0p2_0p4=0.9448 corrupt_frac_t_0p2_0p4=0.2016 acc_corrupt_t_0p4_0p6=0.9979 corrupt_frac_t_0p4_0p6=0.2056 acc_corrupt_t_0p6_0p8=0.9996 corrupt_frac_t_0p6_0p8=0.1941 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2024 out_w_norm=13.0625 out_g_norm=0.6266 loss_all=0.6583 init_gold_top10=0.6102 init_gold_top100=0.7189 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.5999 init_acc_rollout_kept=0.4687 logit_acc_rollout_applied=0.8509 logit_acc_rollout_kept=0.8435 +step=1700 epoch=1700/2000 epoch_step=1/1 micro_steps=1700 elapsed=21.6s lr=2.000000e-03 loss=0.6448 loss_recon=0.6448 loss_meanflow=0.0000 mean_model_t=0.5003 mean_corrupt_t=0.5003 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3456 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8462 corrupt_frac=1.0000 acc_corrupt=0.8462 loss_corrupt=0.6448 wrong_frac=0.4998 init_acc_corrupt=0.5116 acc_corrupt_t_0p0_0p2=0.2707 corrupt_frac_t_0p0_0p2=0.2006 acc_corrupt_t_0p2_0p4=0.9633 corrupt_frac_t_0p2_0p4=0.1945 acc_corrupt_t_0p4_0p6=0.9987 corrupt_frac_t_0p4_0p6=0.2064 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.1962 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2023 out_w_norm=13.1160 out_g_norm=0.5624 loss_all=0.5461 init_gold_top10=0.6344 init_gold_top100=0.7312 rollout_applied_pos_frac=0.3359 init_acc_rollout_applied=0.5986 init_acc_rollout_kept=0.5231 logit_acc_rollout_applied=0.8390 logit_acc_rollout_kept=0.8845 +step=1800 epoch=1800/2000 epoch_step=1/1 micro_steps=1800 elapsed=21.6s lr=2.000000e-03 loss=0.6028 loss_recon=0.6028 loss_meanflow=0.0000 mean_model_t=0.4990 mean_corrupt_t=0.4990 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3480 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8527 corrupt_frac=1.0000 acc_corrupt=0.8527 loss_corrupt=0.6028 wrong_frac=0.5010 init_acc_corrupt=0.5109 acc_corrupt_t_0p0_0p2=0.2903 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9758 corrupt_frac_t_0p2_0p4=0.2004 acc_corrupt_t_0p4_0p6=0.9992 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=0.9998 corrupt_frac_t_0p6_0p8=0.2018 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.1982 out_w_norm=13.1521 out_g_norm=0.5877 loss_all=0.5062 init_gold_top10=0.6120 init_gold_top100=0.7186 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5046 init_acc_rollout_kept=0.5056 logit_acc_rollout_applied=0.8045 logit_acc_rollout_kept=0.9070 +step=1900 epoch=1900/2000 epoch_step=1/1 micro_steps=1900 elapsed=21.7s lr=2.000000e-03 loss=0.5329 loss_recon=0.5329 loss_meanflow=0.0000 mean_model_t=0.5027 mean_corrupt_t=0.5027 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3570 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8641 corrupt_frac=1.0000 acc_corrupt=0.8641 loss_corrupt=0.5329 wrong_frac=0.4975 init_acc_corrupt=0.5146 acc_corrupt_t_0p0_0p2=0.3359 corrupt_frac_t_0p0_0p2=0.1991 acc_corrupt_t_0p2_0p4=0.9823 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.9994 corrupt_frac_t_0p4_0p6=0.1958 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=13.1712 out_g_norm=0.5743 loss_all=0.5300 init_gold_top10=0.6270 init_gold_top100=0.7250 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.6050 init_acc_rollout_kept=0.4759 logit_acc_rollout_applied=0.8912 logit_acc_rollout_kept=0.8507 +step=2000 epoch=2000/2000 epoch_step=1/1 micro_steps=2000 elapsed=21.6s lr=2.000000e-03 loss=0.4949 loss_recon=0.4949 loss_meanflow=0.0000 mean_model_t=0.5005 mean_corrupt_t=0.5005 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3489 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8686 corrupt_frac=1.0000 acc_corrupt=0.8686 loss_corrupt=0.4949 wrong_frac=0.4995 init_acc_corrupt=0.5134 acc_corrupt_t_0p0_0p2=0.3647 corrupt_frac_t_0p0_0p2=0.2029 acc_corrupt_t_0p2_0p4=0.9877 corrupt_frac_t_0p2_0p4=0.1965 acc_corrupt_t_0p4_0p6=0.9996 corrupt_frac_t_0p4_0p6=0.1988 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2028 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1990 out_w_norm=13.1851 out_g_norm=0.5259 loss_all=0.2158 init_gold_top10=0.6500 init_gold_top100=0.7168 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.6294 init_acc_rollout_kept=0.5045 logit_acc_rollout_applied=0.9802 logit_acc_rollout_kept=0.9204 +[ctx1024-sampleds] eval config=p35_rand0_4_unif0_0p25_outwdm1 step=2000 +[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=2000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0002000.pt", + "ckpt_step": 2000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.650299072265625, + "token_acc_min": 0.0126953125, + "token_acc_max": 1.0, + "exact_acc": 0.453125, + "exact_count": 29, + "exact_ref_coverage": 0.125, + "exact_ref_count": 1, + "exact_ref_hits": [ + 5 + ], + "best_ref_idx": [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 7, + 5, + 5, + 5, + 5, + 7, + 5, + 5, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 5, + 5, + 3, + 2, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 3, + 5, + 7, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 7, + 5, + 5, + 5, + 2 + ], + "best_token_acc": [ + 0.029296875, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.029296875, + 1.0, + 1.0, + 0.998046875, + 0.029296875, + 1.0, + 1.0, + 0.029296875, + 0.998046875, + 0.0302734375, + 0.029296875, + 0.029296875, + 0.9990234375, + 0.029296875, + 0.9990234375, + 1.0, + 0.029296875, + 0.0302734375, + 0.029296875, + 1.0, + 0.029296875, + 0.029296875, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 0.029296875, + 1.0, + 0.0166015625, + 0.9990234375, + 1.0, + 0.9990234375, + 0.029296875, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0166015625, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 0.029296875, + 1.0, + 0.029296875, + 1.0, + 1.0, + 0.0126953125, + 1.0, + 0.029296875, + 0.029296875, + 0.9990234375, + 1.0, + 0.029296875, + 1.0, + 0.998046875 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0002000.pt", + "ckpt_step": 2000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.650299072265625, + "token_acc_min": 0.0126953125, + "token_acc_max": 1.0, + "exact_acc": 0.453125, + "exact_count": 29, + "exact_ref_coverage": 0.125, + "exact_ref_count": 1, + "exact_ref_hits": [ + 5 + ], + "best_ref_idx": [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 7, + 5, + 5, + 5, + 5, + 7, + 5, + 5, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 2, + 5, + 5, + 5, + 5, + 5, + 3, + 2, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 3, + 5, + 7, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 7, + 5, + 5, + 5, + 2 + ], + "best_token_acc": [ + 0.029296875, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 0.029296875, + 1.0, + 1.0, + 0.998046875, + 0.029296875, + 1.0, + 1.0, + 0.029296875, + 0.998046875, + 0.0302734375, + 0.029296875, + 0.029296875, + 0.9990234375, + 0.029296875, + 0.9990234375, + 1.0, + 0.029296875, + 0.0302734375, + 0.029296875, + 1.0, + 0.029296875, + 0.029296875, + 0.9990234375, + 0.9990234375, + 1.0, + 1.0, + 0.029296875, + 1.0, + 0.0166015625, + 0.9990234375, + 1.0, + 0.9990234375, + 0.029296875, + 1.0, + 1.0, + 1.0, + 1.0, + 0.0166015625, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 0.029296875, + 1.0, + 0.029296875, + 1.0, + 1.0, + 0.0126953125, + 1.0, + 0.029296875, + 0.029296875, + 0.9990234375, + 1.0, + 0.029296875, + 1.0, + 0.998046875 + ] + } + } +} +RESULT config=p35_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=2000 views=1024000 token_acc=0.6503 exact=29/64 exact_refs=1 hits=[5] +[ctx1024-sampleds] train config=p35_rand0_4_unif0_0p25_outwdm1 from=2000 to=3000 +[launch] gpt2 cached OWT soft-endpoint m/n pilot +[launch] run_name=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] save_dir=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] n=1024 m=0 clean_state_mode=onehot +[launch] mask_mixture lowk=0.0 all=1.0 +[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=2423 +[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1 +[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0 +[launch] mask_ratio=1.0->1.0 +[launch] mask_ratio_floor_schedule=none +[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet +[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids= +[launch] rollout_train prob=0.35 mode=sampled_path steps=4 steps_min=0 infer_steps=1 s_dist=uniform s_frac=0.0->0.25 temp=1.0 corrupt_only=1 samplewise=1 selected_only=1 sync_t=1 +[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit exact_repeat_per_chunk=64 +[launch] resume_path=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt start_step=2001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 3000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 4, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt" +} +step=2100 epoch=2100/3000 epoch_step=1/1 micro_steps=2100 elapsed=22.3s lr=2.000000e-03 loss=0.4462 loss_recon=0.4462 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3505 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8772 corrupt_frac=1.0000 acc_corrupt=0.8772 loss_corrupt=0.4462 wrong_frac=0.4986 init_acc_corrupt=0.5139 acc_corrupt_t_0p0_0p2=0.3826 corrupt_frac_t_0p0_0p2=0.1952 acc_corrupt_t_0p2_0p4=0.9897 corrupt_frac_t_0p2_0p4=0.2063 acc_corrupt_t_0p4_0p6=0.9996 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1976 acc_corrupt_t_0p8_1p0=0.9999 corrupt_frac_t_0p8_1p0=0.2036 out_w_norm=13.1855 out_g_norm=0.5866 loss_all=0.3201 init_gold_top10=0.6185 init_gold_top100=0.7053 rollout_applied_pos_frac=0.3672 init_acc_rollout_applied=0.5589 init_acc_rollout_kept=0.4438 logit_acc_rollout_applied=0.9093 logit_acc_rollout_kept=0.9095 +step=2200 epoch=2200/3000 epoch_step=1/1 micro_steps=2200 elapsed=21.4s lr=2.000000e-03 loss=0.4130 loss_recon=0.4130 loss_meanflow=0.0000 mean_model_t=0.4985 mean_corrupt_t=0.4985 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3512 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8820 corrupt_frac=1.0000 acc_corrupt=0.8820 loss_corrupt=0.4130 wrong_frac=0.5014 init_acc_corrupt=0.5131 acc_corrupt_t_0p0_0p2=0.4279 corrupt_frac_t_0p0_0p2=0.2037 acc_corrupt_t_0p2_0p4=0.9927 corrupt_frac_t_0p2_0p4=0.1982 acc_corrupt_t_0p4_0p6=0.9997 corrupt_frac_t_0p4_0p6=0.1956 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2056 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1969 out_w_norm=13.1924 out_g_norm=0.5322 loss_all=0.3586 init_gold_top10=0.6108 init_gold_top100=0.7017 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.6741 init_acc_rollout_kept=0.4511 logit_acc_rollout_applied=0.9561 logit_acc_rollout_kept=0.8641 +step=2300 epoch=2300/3000 epoch_step=1/1 micro_steps=2300 elapsed=21.4s lr=2.000000e-03 loss=0.3803 loss_recon=0.3803 loss_meanflow=0.0000 mean_model_t=0.4953 mean_corrupt_t=0.4953 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3561 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8884 corrupt_frac=1.0000 acc_corrupt=0.8884 loss_corrupt=0.3803 wrong_frac=0.5048 init_acc_corrupt=0.5092 acc_corrupt_t_0p0_0p2=0.4584 corrupt_frac_t_0p0_0p2=0.2036 acc_corrupt_t_0p2_0p4=0.9940 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1934 out_w_norm=13.1910 out_g_norm=0.5184 loss_all=0.4000 init_gold_top10=0.6118 init_gold_top100=0.7075 rollout_applied_pos_frac=0.3203 init_acc_rollout_applied=0.6476 init_acc_rollout_kept=0.4730 logit_acc_rollout_applied=0.9479 logit_acc_rollout_kept=0.8544 +step=2400 epoch=2400/3000 epoch_step=1/1 micro_steps=2400 elapsed=21.3s lr=2.000000e-03 loss=0.3508 loss_recon=0.3508 loss_meanflow=0.0000 mean_model_t=0.4980 mean_corrupt_t=0.4980 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3477 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8934 corrupt_frac=1.0000 acc_corrupt=0.8934 loss_corrupt=0.3508 wrong_frac=0.5019 init_acc_corrupt=0.5133 acc_corrupt_t_0p0_0p2=0.4868 corrupt_frac_t_0p0_0p2=0.2061 acc_corrupt_t_0p2_0p4=0.9958 corrupt_frac_t_0p2_0p4=0.1960 acc_corrupt_t_0p4_0p6=0.9998 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=0.9999 corrupt_frac_t_0p6_0p8=0.2020 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1985 out_w_norm=13.1887 out_g_norm=0.5057 loss_all=0.4408 init_gold_top10=0.5895 init_gold_top100=0.7003 rollout_applied_pos_frac=0.3203 init_acc_rollout_applied=0.5455 init_acc_rollout_kept=0.4485 logit_acc_rollout_applied=0.8244 logit_acc_rollout_kept=0.8776 +step=2500 epoch=2500/3000 epoch_step=1/1 micro_steps=2500 elapsed=21.6s lr=2.000000e-03 loss=0.3328 loss_recon=0.3328 loss_meanflow=0.0000 mean_model_t=0.4998 mean_corrupt_t=0.4998 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3556 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.8969 corrupt_frac=1.0000 acc_corrupt=0.8969 loss_corrupt=0.3328 wrong_frac=0.5002 init_acc_corrupt=0.5156 acc_corrupt_t_0p0_0p2=0.4878 corrupt_frac_t_0p0_0p2=0.1998 acc_corrupt_t_0p2_0p4=0.9964 corrupt_frac_t_0p2_0p4=0.1971 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2008 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2002 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2020 out_w_norm=13.1803 out_g_norm=0.4942 loss_all=0.3619 init_gold_top10=0.6216 init_gold_top100=0.7085 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.6158 init_acc_rollout_kept=0.4850 logit_acc_rollout_applied=0.8643 logit_acc_rollout_kept=0.8886 +step=2600 epoch=2600/3000 epoch_step=1/1 micro_steps=2600 elapsed=21.2s lr=2.000000e-03 loss=0.3044 loss_recon=0.3044 loss_meanflow=0.0000 mean_model_t=0.5009 mean_corrupt_t=0.5009 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3408 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9041 corrupt_frac=1.0000 acc_corrupt=0.9041 loss_corrupt=0.3044 wrong_frac=0.4992 init_acc_corrupt=0.5175 acc_corrupt_t_0p0_0p2=0.5144 corrupt_frac_t_0p0_0p2=0.1963 acc_corrupt_t_0p2_0p4=0.9973 corrupt_frac_t_0p2_0p4=0.2016 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2056 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1941 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2024 out_w_norm=13.1441 out_g_norm=0.4960 loss_all=0.3029 init_gold_top10=0.6351 init_gold_top100=0.7199 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.6223 init_acc_rollout_kept=0.4687 logit_acc_rollout_applied=0.8984 logit_acc_rollout_kept=0.9060 +step=2700 epoch=2700/3000 epoch_step=1/1 micro_steps=2700 elapsed=21.5s lr=2.000000e-03 loss=0.2965 loss_recon=0.2965 loss_meanflow=0.0000 mean_model_t=0.5003 mean_corrupt_t=0.5003 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3456 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9053 corrupt_frac=1.0000 acc_corrupt=0.9053 loss_corrupt=0.2965 wrong_frac=0.4998 init_acc_corrupt=0.5185 acc_corrupt_t_0p0_0p2=0.5301 corrupt_frac_t_0p0_0p2=0.2006 acc_corrupt_t_0p2_0p4=0.9979 corrupt_frac_t_0p2_0p4=0.1945 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2064 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1962 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2023 out_w_norm=13.0921 out_g_norm=0.4470 loss_all=0.2555 init_gold_top10=0.6564 init_gold_top100=0.7317 rollout_applied_pos_frac=0.3359 init_acc_rollout_applied=0.6218 init_acc_rollout_kept=0.5231 logit_acc_rollout_applied=0.8883 logit_acc_rollout_kept=0.9308 +step=2800 epoch=2800/3000 epoch_step=1/1 micro_steps=2800 elapsed=21.5s lr=2.000000e-03 loss=0.2918 loss_recon=0.2918 loss_meanflow=0.0000 mean_model_t=0.4990 mean_corrupt_t=0.4990 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3480 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9048 corrupt_frac=1.0000 acc_corrupt=0.9048 loss_corrupt=0.2918 wrong_frac=0.5010 init_acc_corrupt=0.5162 acc_corrupt_t_0p0_0p2=0.5270 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9982 corrupt_frac_t_0p2_0p4=0.2004 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2018 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1982 out_w_norm=13.0408 out_g_norm=0.4282 loss_all=0.2638 init_gold_top10=0.6356 init_gold_top100=0.7187 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5152 init_acc_rollout_kept=0.5056 logit_acc_rollout_applied=0.8472 logit_acc_rollout_kept=0.9478 +step=2900 epoch=2900/3000 epoch_step=1/1 micro_steps=2900 elapsed=21.5s lr=2.000000e-03 loss=0.2759 loss_recon=0.2759 loss_meanflow=0.0000 mean_model_t=0.5027 mean_corrupt_t=0.5027 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3570 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9100 corrupt_frac=1.0000 acc_corrupt=0.9100 loss_corrupt=0.2759 wrong_frac=0.4975 init_acc_corrupt=0.5199 acc_corrupt_t_0p0_0p2=0.5494 corrupt_frac_t_0p0_0p2=0.1991 acc_corrupt_t_0p2_0p4=0.9986 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1958 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=12.9782 out_g_norm=0.4290 loss_all=0.3209 init_gold_top10=0.6401 init_gold_top100=0.7250 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.6212 init_acc_rollout_kept=0.4759 logit_acc_rollout_applied=0.9051 logit_acc_rollout_kept=0.8887 +step=3000 epoch=3000/3000 epoch_step=1/1 micro_steps=3000 elapsed=21.5s lr=2.000000e-03 loss=0.2653 loss_recon=0.2653 loss_meanflow=0.0000 mean_model_t=0.5005 mean_corrupt_t=0.5005 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3489 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9119 corrupt_frac=1.0000 acc_corrupt=0.9119 loss_corrupt=0.2653 wrong_frac=0.4995 init_acc_corrupt=0.5176 acc_corrupt_t_0p0_0p2=0.5668 corrupt_frac_t_0p0_0p2=0.2029 acc_corrupt_t_0p2_0p4=0.9990 corrupt_frac_t_0p2_0p4=0.1965 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1988 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2028 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1990 out_w_norm=12.9209 out_g_norm=0.3901 loss_all=0.1247 init_gold_top10=0.6494 init_gold_top100=0.7168 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.6353 init_acc_rollout_kept=0.5045 logit_acc_rollout_applied=0.9789 logit_acc_rollout_kept=0.9466 +[ctx1024-sampleds] eval config=p35_rand0_4_unif0_0p25_outwdm1 step=3000 +[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=3000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0003000.pt", + "ckpt_step": 3000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.99871826171875, + "token_acc_min": 0.9970703125, + "token_acc_max": 1.0, + "exact_acc": 0.265625, + "exact_count": 17, + "exact_ref_coverage": 0.5, + "exact_ref_count": 4, + "exact_ref_hits": [ + 1, + 2, + 3, + 5 + ], + "best_ref_idx": [ + 5, + 1, + 5, + 5, + 5, + 1, + 5, + 1, + 1, + 1, + 1, + 3, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 1, + 1, + 7, + 1, + 5, + 1, + 1, + 1, + 2, + 5, + 5, + 5, + 5, + 5, + 1, + 5, + 5, + 1, + 5, + 1, + 5, + 1, + 1, + 5, + 5, + 1, + 5, + 3, + 5, + 5, + 5, + 3, + 5, + 1, + 2, + 5, + 5, + 1, + 1, + 3, + 5, + 1, + 1, + 5 + ], + "best_token_acc": [ + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.998046875, + 1.0, + 0.9970703125, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 1.0, + 1.0, + 1.0, + 0.998046875, + 0.998046875, + 0.9970703125, + 0.998046875, + 0.998046875, + 0.9970703125, + 0.998046875, + 0.998046875, + 1.0, + 0.9970703125, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 0.998046875, + 0.9990234375, + 1.0, + 0.9970703125, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.9990234375, + 1.0, + 0.9970703125, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 0.998046875, + 0.9970703125, + 1.0, + 0.998046875, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.998046875, + 1.0 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0003000.pt", + "ckpt_step": 3000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.99871826171875, + "token_acc_min": 0.9970703125, + "token_acc_max": 1.0, + "exact_acc": 0.265625, + "exact_count": 17, + "exact_ref_coverage": 0.5, + "exact_ref_count": 4, + "exact_ref_hits": [ + 1, + 2, + 3, + 5 + ], + "best_ref_idx": [ + 5, + 1, + 5, + 5, + 5, + 1, + 5, + 1, + 1, + 1, + 1, + 3, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 1, + 1, + 7, + 1, + 5, + 1, + 1, + 1, + 2, + 5, + 5, + 5, + 5, + 5, + 1, + 5, + 5, + 1, + 5, + 1, + 5, + 1, + 1, + 5, + 5, + 1, + 5, + 3, + 5, + 5, + 5, + 3, + 5, + 1, + 2, + 5, + 5, + 1, + 1, + 3, + 5, + 1, + 1, + 5 + ], + "best_token_acc": [ + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.998046875, + 1.0, + 0.9970703125, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 1.0, + 1.0, + 1.0, + 0.998046875, + 0.998046875, + 0.9970703125, + 0.998046875, + 0.998046875, + 0.9970703125, + 0.998046875, + 0.998046875, + 1.0, + 0.9970703125, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 0.998046875, + 0.9990234375, + 1.0, + 0.9970703125, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.998046875, + 0.9990234375, + 1.0, + 0.9970703125, + 0.9990234375, + 1.0, + 1.0, + 1.0, + 0.9990234375, + 1.0, + 1.0, + 0.998046875, + 0.9970703125, + 1.0, + 0.998046875, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.998046875, + 0.998046875, + 1.0 + ] + } + } +} +RESULT config=p35_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=3000 views=1536000 token_acc=0.9987 exact=17/64 exact_refs=4 hits=[1, 2, 3, 5] +[ctx1024-sampleds] train config=p35_rand0_4_unif0_0p25_outwdm1 from=3000 to=4000 +[launch] gpt2 cached OWT soft-endpoint m/n pilot +[launch] run_name=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] save_dir=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] n=1024 m=0 clean_state_mode=onehot +[launch] mask_mixture lowk=0.0 all=1.0 +[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=2423 +[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1 +[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0 +[launch] mask_ratio=1.0->1.0 +[launch] mask_ratio_floor_schedule=none +[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet +[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids= +[launch] rollout_train prob=0.35 mode=sampled_path steps=4 steps_min=0 infer_steps=1 s_dist=uniform s_frac=0.0->0.25 temp=1.0 corrupt_only=1 samplewise=1 selected_only=1 sync_t=1 +[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit exact_repeat_per_chunk=64 +[launch] resume_path=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt start_step=3001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 4000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 4, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt" +} +step=3100 epoch=3100/4000 epoch_step=1/1 micro_steps=3100 elapsed=22.3s lr=2.000000e-03 loss=0.2609 loss_recon=0.2609 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3505 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9123 corrupt_frac=1.0000 acc_corrupt=0.9123 loss_corrupt=0.2609 wrong_frac=0.4986 init_acc_corrupt=0.5173 acc_corrupt_t_0p0_0p2=0.5519 corrupt_frac_t_0p0_0p2=0.1952 acc_corrupt_t_0p2_0p4=0.9991 corrupt_frac_t_0p2_0p4=0.2063 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1976 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2036 out_w_norm=12.8783 out_g_norm=0.4025 loss_all=0.1703 init_gold_top10=0.6252 init_gold_top100=0.7053 rollout_applied_pos_frac=0.3672 init_acc_rollout_applied=0.5608 init_acc_rollout_kept=0.4438 logit_acc_rollout_applied=0.9224 logit_acc_rollout_kept=0.9513 +step=3200 epoch=3200/4000 epoch_step=1/1 micro_steps=3200 elapsed=21.4s lr=2.000000e-03 loss=0.2604 loss_recon=0.2604 loss_meanflow=0.0000 mean_model_t=0.4985 mean_corrupt_t=0.4985 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3512 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9118 corrupt_frac=1.0000 acc_corrupt=0.9118 loss_corrupt=0.2604 wrong_frac=0.5014 init_acc_corrupt=0.5157 acc_corrupt_t_0p0_0p2=0.5677 corrupt_frac_t_0p0_0p2=0.2037 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.1982 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.1956 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2056 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1969 out_w_norm=12.8221 out_g_norm=0.3907 loss_all=0.2038 init_gold_top10=0.6119 init_gold_top100=0.7017 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.6757 init_acc_rollout_kept=0.4511 logit_acc_rollout_applied=0.9773 logit_acc_rollout_kept=0.9039 +step=3300 epoch=3300/4000 epoch_step=1/1 micro_steps=3300 elapsed=21.5s lr=2.000000e-03 loss=0.2517 loss_recon=0.2517 loss_meanflow=0.0000 mean_model_t=0.4953 mean_corrupt_t=0.4953 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3561 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9148 corrupt_frac=1.0000 acc_corrupt=0.9148 loss_corrupt=0.2517 wrong_frac=0.5048 init_acc_corrupt=0.5118 acc_corrupt_t_0p0_0p2=0.5823 corrupt_frac_t_0p0_0p2=0.2036 acc_corrupt_t_0p2_0p4=0.9993 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2005 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1934 out_w_norm=12.7718 out_g_norm=0.3434 loss_all=0.2658 init_gold_top10=0.6161 init_gold_top100=0.7075 rollout_applied_pos_frac=0.3203 init_acc_rollout_applied=0.6542 init_acc_rollout_kept=0.4730 logit_acc_rollout_applied=0.9581 logit_acc_rollout_kept=0.8839 +step=3400 epoch=3400/4000 epoch_step=1/1 micro_steps=3400 elapsed=21.3s lr=2.000000e-03 loss=0.2479 loss_recon=0.2479 loss_meanflow=0.0000 mean_model_t=0.4980 mean_corrupt_t=0.4980 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3477 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9155 corrupt_frac=1.0000 acc_corrupt=0.9155 loss_corrupt=0.2479 wrong_frac=0.5019 init_acc_corrupt=0.5150 acc_corrupt_t_0p0_0p2=0.5904 corrupt_frac_t_0p0_0p2=0.2061 acc_corrupt_t_0p2_0p4=0.9995 corrupt_frac_t_0p2_0p4=0.1960 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2020 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1985 out_w_norm=12.7430 out_g_norm=0.3736 loss_all=0.3522 init_gold_top10=0.5957 init_gold_top100=0.7003 rollout_applied_pos_frac=0.3203 init_acc_rollout_applied=0.5486 init_acc_rollout_kept=0.4485 logit_acc_rollout_applied=0.8328 logit_acc_rollout_kept=0.9001 +step=3500 epoch=3500/4000 epoch_step=1/1 micro_steps=3500 elapsed=21.6s lr=2.000000e-03 loss=0.2511 loss_recon=0.2511 loss_meanflow=0.0000 mean_model_t=0.4998 mean_corrupt_t=0.4998 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3556 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9152 corrupt_frac=1.0000 acc_corrupt=0.9152 loss_corrupt=0.2511 wrong_frac=0.5002 init_acc_corrupt=0.5169 acc_corrupt_t_0p0_0p2=0.5763 corrupt_frac_t_0p0_0p2=0.1998 acc_corrupt_t_0p2_0p4=0.9994 corrupt_frac_t_0p2_0p4=0.1971 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.2008 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2002 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2020 out_w_norm=12.7032 out_g_norm=0.3187 loss_all=0.3310 init_gold_top10=0.6211 init_gold_top100=0.7085 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.6220 init_acc_rollout_kept=0.4850 logit_acc_rollout_applied=0.8659 logit_acc_rollout_kept=0.9094 +step=3600 epoch=3600/4000 epoch_step=1/1 micro_steps=3600 elapsed=21.2s lr=2.000000e-03 loss=0.2338 loss_recon=0.2338 loss_meanflow=0.0000 mean_model_t=0.5009 mean_corrupt_t=0.5009 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3408 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9201 corrupt_frac=1.0000 acc_corrupt=0.9201 loss_corrupt=0.2338 wrong_frac=0.4992 init_acc_corrupt=0.5187 acc_corrupt_t_0p0_0p2=0.5936 corrupt_frac_t_0p0_0p2=0.1963 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.2016 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.2056 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1941 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2024 out_w_norm=12.6731 out_g_norm=0.3229 loss_all=0.2204 init_gold_top10=0.6387 init_gold_top100=0.7199 rollout_applied_pos_frac=0.3828 init_acc_rollout_applied=0.6243 init_acc_rollout_kept=0.4687 logit_acc_rollout_applied=0.8953 logit_acc_rollout_kept=0.9395 +step=3700 epoch=3700/4000 epoch_step=1/1 micro_steps=3700 elapsed=21.5s lr=2.000000e-03 loss=0.2356 loss_recon=0.2356 loss_meanflow=0.0000 mean_model_t=0.5003 mean_corrupt_t=0.5003 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3456 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9198 corrupt_frac=1.0000 acc_corrupt=0.9198 loss_corrupt=0.2356 wrong_frac=0.4998 init_acc_corrupt=0.5195 acc_corrupt_t_0p0_0p2=0.6006 corrupt_frac_t_0p0_0p2=0.2006 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.1945 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.2064 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1962 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2023 out_w_norm=12.6559 out_g_norm=0.3148 loss_all=0.2045 init_gold_top10=0.6618 init_gold_top100=0.7317 rollout_applied_pos_frac=0.3359 init_acc_rollout_applied=0.6307 init_acc_rollout_kept=0.5231 logit_acc_rollout_applied=0.9134 logit_acc_rollout_kept=0.9381 +step=3800 epoch=3800/4000 epoch_step=1/1 micro_steps=3800 elapsed=21.6s lr=2.000000e-03 loss=0.2395 loss_recon=0.2395 loss_meanflow=0.0000 mean_model_t=0.4990 mean_corrupt_t=0.4990 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3480 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9184 corrupt_frac=1.0000 acc_corrupt=0.9184 loss_corrupt=0.2395 wrong_frac=0.5010 init_acc_corrupt=0.5169 acc_corrupt_t_0p0_0p2=0.5933 corrupt_frac_t_0p0_0p2=0.2005 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.2004 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.1991 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2018 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1982 out_w_norm=12.6400 out_g_norm=0.3101 loss_all=0.2123 init_gold_top10=0.6421 init_gold_top100=0.7187 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.5164 init_acc_rollout_kept=0.5056 logit_acc_rollout_applied=0.8528 logit_acc_rollout_kept=0.9597 +step=3900 epoch=3900/4000 epoch_step=1/1 micro_steps=3900 elapsed=21.6s lr=2.000000e-03 loss=0.2277 loss_recon=0.2277 loss_meanflow=0.0000 mean_model_t=0.5027 mean_corrupt_t=0.5027 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3570 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9211 corrupt_frac=1.0000 acc_corrupt=0.9211 loss_corrupt=0.2277 wrong_frac=0.4975 init_acc_corrupt=0.5205 acc_corrupt_t_0p0_0p2=0.6044 corrupt_frac_t_0p0_0p2=0.1991 acc_corrupt_t_0p2_0p4=0.9996 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.1958 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=12.6312 out_g_norm=0.2864 loss_all=0.2865 init_gold_top10=0.6438 init_gold_top100=0.7250 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.6188 init_acc_rollout_kept=0.4759 logit_acc_rollout_applied=0.9056 logit_acc_rollout_kept=0.8930 +step=4000 epoch=4000/4000 epoch_step=1/1 micro_steps=4000 elapsed=21.5s lr=2.000000e-03 loss=0.2278 loss_recon=0.2278 loss_meanflow=0.0000 mean_model_t=0.5005 mean_corrupt_t=0.5005 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3489 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9225 corrupt_frac=1.0000 acc_corrupt=0.9225 loss_corrupt=0.2278 wrong_frac=0.4995 init_acc_corrupt=0.5186 acc_corrupt_t_0p0_0p2=0.6184 corrupt_frac_t_0p0_0p2=0.2029 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.1965 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.1988 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2028 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1990 out_w_norm=12.6244 out_g_norm=0.2732 loss_all=0.1031 init_gold_top10=0.6493 init_gold_top100=0.7168 rollout_applied_pos_frac=0.3125 init_acc_rollout_applied=0.6364 init_acc_rollout_kept=0.5045 logit_acc_rollout_applied=0.9772 logit_acc_rollout_kept=0.9571 +[ctx1024-sampleds] eval config=p35_rand0_4_unif0_0p25_outwdm1 step=4000 +[eval-decode-acc] train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 step=4000 soft=none +[decode] max_len=1024 generated=64/64 +{ + "num_rows": 1, + "best_by_run": { + "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0004000.pt", + "ckpt_step": 4000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9990234375, + "token_acc_min": 0.9970703125, + "token_acc_max": 1.0, + "exact_acc": 0.109375, + "exact_count": 7, + "exact_ref_coverage": 0.375, + "exact_ref_count": 3, + "exact_ref_hits": [ + 3, + 4, + 7 + ], + "best_ref_idx": [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 4, + 5, + 5, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 1, + 3, + 1, + 5, + 5, + 7, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 5, + 7, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 1, + 4, + 5, + 5, + 5, + 2, + 5, + 5, + 7, + 5, + 5, + 7, + 5, + 5, + 1, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 1.0, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 1.0, + 0.998046875, + 0.9990234375, + 0.9990234375, + 1.0, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9970703125, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 1.0, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9970703125, + 1.0, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 1.0, + 0.9990234375, + 0.9990234375, + 1.0, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375 + ] + } + }, + "first_exact_by_run": { + "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800::none": { + "run": "train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "checkpoint": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/step_0004000.pt", + "ckpt_step": 4000, + "endpoint_softening": "none", + "decode_rule": "dirichlet_resample", + "steps": 128, + "time_schedule": "logit_normal", + "model_t_mode": "post", + "final_from": "state", + "n_gen": 64, + "n_refs": 8, + "token_acc_mean": 0.9990234375, + "token_acc_min": 0.9970703125, + "token_acc_max": 1.0, + "exact_acc": 0.109375, + "exact_count": 7, + "exact_ref_coverage": 0.375, + "exact_ref_count": 3, + "exact_ref_hits": [ + 3, + 4, + 7 + ], + "best_ref_idx": [ + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 4, + 5, + 5, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 1, + 3, + 1, + 5, + 5, + 7, + 5, + 5, + 5, + 1, + 5, + 5, + 5, + 5, + 5, + 7, + 5, + 5, + 5, + 5, + 5, + 5, + 5, + 1, + 4, + 5, + 5, + 5, + 2, + 5, + 5, + 7, + 5, + 5, + 7, + 5, + 5, + 1, + 5, + 5, + 5, + 5 + ], + "best_token_acc": [ + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 1.0, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.998046875, + 1.0, + 0.998046875, + 0.9990234375, + 0.9990234375, + 1.0, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9970703125, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 1.0, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9970703125, + 1.0, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 1.0, + 0.9990234375, + 0.9990234375, + 1.0, + 0.9990234375, + 0.9990234375, + 0.998046875, + 0.9990234375, + 0.9990234375, + 0.9990234375, + 0.9990234375 + ] + } + } +} +RESULT config=p35_rand0_4_unif0_0p25_outwdm1 decode=dirichlet_resample run=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 ckpt_step=4000 views=2048000 token_acc=0.9990 exact=7/64 exact_refs=3 hits=[3, 4, 7] +[ctx1024-sampleds] train config=p35_rand0_4_unif0_0p25_outwdm1 from=4000 to=5000 +[launch] gpt2 cached OWT soft-endpoint m/n pilot +[launch] run_name=train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] save_dir=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800 +[launch] n=1024 m=0 clean_state_mode=onehot +[launch] mask_mixture lowk=0.0 all=1.0 +[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=2423 +[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1 +[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0 +[launch] mask_ratio=1.0->1.0 +[launch] mask_ratio_floor_schedule=none +[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet +[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids= +[launch] rollout_train prob=0.35 mode=sampled_path steps=4 steps_min=0 infer_steps=1 s_dist=uniform s_frac=0.0->0.25 temp=1.0 corrupt_only=1 samplewise=1 selected_only=1 sync_t=1 +[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit exact_repeat_per_chunk=64 +[launch] resume_path=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt +NCCL version 2.25.1+cuda12.8 +resumed_from=runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt start_step=4001 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "owt_cached_chunks:8", + "vocab_size": 2423, + "tokenizer_vocab_size": 32100, + "save_dir": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800", + "batch_size": 128, + "grad_accum": 1, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "muon", + "epochs": 0.0, + "steps_per_epoch": 1, + "total_steps": 5000, + "warmup_steps": 10, + "warmup_epochs": -1.0, + "min_lr": 0.0, + "weight_decay": 0.1, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "legacy_dim_ge_2", + "muon_param_count": 2523776, + "muon_adam_param_count": 8192, + "muon_param_names": [ + "vocab_embed.embedding", + "sigma_map.net.0.weight", + "sigma_map.net.2.weight", + "blocks.0.attn_qkv.weight", + "blocks.0.attn_out.weight", + "blocks.0.mlp.0.weight", + "blocks.0.mlp.2.weight", + "blocks.0.adaLN_modulation.weight", + "blocks.1.attn_qkv.weight", + "blocks.1.attn_out.weight", + "blocks.1.mlp.0.weight", + "blocks.1.mlp.2.weight", + "blocks.1.adaLN_modulation.weight", + "blocks.2.attn_qkv.weight", + "blocks.2.attn_out.weight", + "blocks.2.mlp.0.weight", + "blocks.2.mlp.2.weight", + "blocks.2.adaLN_modulation.weight", + "output_layer.linear.weight", + "output_layer.adaLN_modulation.weight" + ], + "muon_adam_param_names": [ + "sigma_map.net.0.bias", + "sigma_map.net.2.bias", + "blocks.0.norm1.weight", + "blocks.0.norm2.weight", + "blocks.0.mlp.0.bias", + "blocks.0.mlp.2.bias", + "blocks.0.adaLN_modulation.bias", + "blocks.1.norm1.weight", + "blocks.1.norm2.weight", + "blocks.1.mlp.0.bias", + "blocks.1.mlp.2.bias", + "blocks.1.adaLN_modulation.bias", + "blocks.2.norm1.weight", + "blocks.2.norm2.weight", + "blocks.2.mlp.0.bias", + "blocks.2.mlp.2.bias", + "blocks.2.adaLN_modulation.bias", + "output_layer.norm_final.weight", + "output_layer.adaLN_modulation.bias" + ], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.1, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.1, + "ema_decay": 0.9999, + "ema_start_step": 0, + "model_type": "ddit", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 4, + "elf_num_model_mode_tokens": 0, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 1.0, + "mask_mixture_lowk_clean_tokens": "0", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.1, + "logistic_normal_sigma_max": 1.0, + "logistic_normal_tau_min": 1.0, + "logistic_normal_tau_max": 1.0, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.35, + "rollout_train_steps": 4, + "rollout_train_steps_min": 0, + "rollout_train_infer_steps": 1, + "rollout_train_time_mode": "sampled_path", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.25, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": true, + "rollout_train_compute_always": false, + "rollout_train_sync_t": true, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": true, + "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/t5_len1024_train8_compact_overfit", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 64, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "runs/train8_ctx1024_t5tok_p35_rand0_4_unif0_0p25_outwdm1_t5tok_ctx1024_randk_20260518_014800/latest.pt" +} +step=4100 epoch=4100/5000 epoch_step=1/1 micro_steps=4100 elapsed=22.5s lr=2.000000e-03 loss=0.2292 loss_recon=0.2292 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3505 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9208 corrupt_frac=1.0000 acc_corrupt=0.9208 loss_corrupt=0.2292 wrong_frac=0.4986 init_acc_corrupt=0.5174 acc_corrupt_t_0p0_0p2=0.5949 corrupt_frac_t_0p0_0p2=0.1952 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.2063 acc_corrupt_t_0p4_0p6=1.0000 corrupt_frac_t_0p4_0p6=0.1973 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.1976 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.2036 out_w_norm=12.6267 out_g_norm=0.2705 loss_all=0.1427 init_gold_top10=0.6265 init_gold_top100=0.7053 rollout_applied_pos_frac=0.3672 init_acc_rollout_applied=0.5611 init_acc_rollout_kept=0.4438 logit_acc_rollout_applied=0.9235 logit_acc_rollout_kept=0.9667 +step=4200 epoch=4200/5000 epoch_step=1/1 micro_steps=4200 elapsed=21.6s lr=2.000000e-03 loss=0.2282 loss_recon=0.2282 loss_meanflow=0.0000 mean_model_t=0.4985 mean_corrupt_t=0.4985 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.3512 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.9205 corrupt_frac=1.0000 acc_corrupt=0.9205 loss_corrupt=0.2282 wrong_frac=0.5014 init_acc_corrupt=0.5162 acc_corrupt_t_0p0_0p2=0.6100 corrupt_frac_t_0p0_0p2=0.2037 acc_corrupt_t_0p2_0p4=0.9997 corrupt_frac_t_0p2_0p4=0.1982 acc_corrupt_t_0p4_0p6=0.9999 corrupt_frac_t_0p4_0p6=0.1956 acc_corrupt_t_0p6_0p8=1.0000 corrupt_frac_t_0p6_0p8=0.2056 acc_corrupt_t_0p8_1p0=1.0000 corrupt_frac_t_0p8_1p0=0.1969 out_w_norm=12.6309 out_g_norm=0.2424 loss_all=0.1838 init_gold_top10=0.6120 init_gold_top100=0.7017 rollout_applied_pos_frac=0.3438 init_acc_rollout_applied=0.6763 init_acc_rollout_kept=0.4511 logit_acc_rollout_applied=0.9769 logit_acc_rollout_kept=0.9125 +Terminated diff --git a/LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/t5tok_ctx1024_randk_20260518_014800.pid b/LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/t5tok_ctx1024_randk_20260518_014800.pid new file mode 100644 index 0000000000000000000000000000000000000000..d07cc323f1c7b7b1d87a23b5c28c580194548adb --- /dev/null +++ b/LTA_openwebtext_dualt/logs/ctx1024_sampleds_sweep_4gpu/t5tok_ctx1024_randk_20260518_014800.pid @@ -0,0 +1 @@ +481284 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d23c5a5a27878561b79551da9513cf600dce5005 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/__init__.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from ._core._contextmanagers import AsyncContextManagerMixin as AsyncContextManagerMixin +from ._core._contextmanagers import ContextManagerMixin as ContextManagerMixin +from ._core._eventloop import current_time as current_time +from ._core._eventloop import get_all_backends as get_all_backends +from ._core._eventloop import get_available_backends as get_available_backends +from ._core._eventloop import get_cancelled_exc_class as get_cancelled_exc_class +from ._core._eventloop import run as run +from ._core._eventloop import sleep as sleep +from ._core._eventloop import sleep_forever as sleep_forever +from ._core._eventloop import sleep_until as sleep_until +from ._core._exceptions import BrokenResourceError as BrokenResourceError +from ._core._exceptions import BrokenWorkerInterpreter as BrokenWorkerInterpreter +from ._core._exceptions import BrokenWorkerProcess as BrokenWorkerProcess +from ._core._exceptions import BusyResourceError as BusyResourceError +from ._core._exceptions import ClosedResourceError as ClosedResourceError +from ._core._exceptions import ConnectionFailed as ConnectionFailed +from ._core._exceptions import DelimiterNotFound as DelimiterNotFound +from ._core._exceptions import EndOfStream as EndOfStream +from ._core._exceptions import IncompleteRead as IncompleteRead +from ._core._exceptions import NoEventLoopError as NoEventLoopError +from ._core._exceptions import RunFinishedError as RunFinishedError +from ._core._exceptions import TypedAttributeLookupError as TypedAttributeLookupError +from ._core._exceptions import WouldBlock as WouldBlock +from ._core._fileio import AsyncFile as AsyncFile +from ._core._fileio import Path as Path +from ._core._fileio import open_file as open_file +from ._core._fileio import wrap_file as wrap_file +from ._core._resources import aclose_forcefully as aclose_forcefully +from ._core._signals import open_signal_receiver as open_signal_receiver +from ._core._sockets import TCPConnectable as TCPConnectable +from ._core._sockets import UNIXConnectable as UNIXConnectable +from ._core._sockets import as_connectable as as_connectable +from ._core._sockets import connect_tcp as connect_tcp +from ._core._sockets import connect_unix as connect_unix +from ._core._sockets import create_connected_udp_socket as create_connected_udp_socket +from ._core._sockets import ( + create_connected_unix_datagram_socket as create_connected_unix_datagram_socket, +) +from ._core._sockets import create_tcp_listener as create_tcp_listener +from ._core._sockets import create_udp_socket as create_udp_socket +from ._core._sockets import create_unix_datagram_socket as create_unix_datagram_socket +from ._core._sockets import create_unix_listener as create_unix_listener +from ._core._sockets import getaddrinfo as getaddrinfo +from ._core._sockets import getnameinfo as getnameinfo +from ._core._sockets import notify_closing as notify_closing +from ._core._sockets import wait_readable as wait_readable +from ._core._sockets import wait_socket_readable as wait_socket_readable +from ._core._sockets import wait_socket_writable as wait_socket_writable +from ._core._sockets import wait_writable as wait_writable +from ._core._streams import create_memory_object_stream as create_memory_object_stream +from ._core._subprocesses import open_process as open_process +from ._core._subprocesses import run_process as run_process +from ._core._synchronization import CapacityLimiter as CapacityLimiter +from ._core._synchronization import ( + CapacityLimiterStatistics as CapacityLimiterStatistics, +) +from ._core._synchronization import Condition as Condition +from ._core._synchronization import ConditionStatistics as ConditionStatistics +from ._core._synchronization import Event as Event +from ._core._synchronization import EventStatistics as EventStatistics +from ._core._synchronization import Lock as Lock +from ._core._synchronization import LockStatistics as LockStatistics +from ._core._synchronization import ResourceGuard as ResourceGuard +from ._core._synchronization import Semaphore as Semaphore +from ._core._synchronization import SemaphoreStatistics as SemaphoreStatistics +from ._core._tasks import TASK_STATUS_IGNORED as TASK_STATUS_IGNORED +from ._core._tasks import CancelScope as CancelScope +from ._core._tasks import create_task_group as create_task_group +from ._core._tasks import current_effective_deadline as current_effective_deadline +from ._core._tasks import fail_after as fail_after +from ._core._tasks import move_on_after as move_on_after +from ._core._tempfile import NamedTemporaryFile as NamedTemporaryFile +from ._core._tempfile import SpooledTemporaryFile as SpooledTemporaryFile +from ._core._tempfile import TemporaryDirectory as TemporaryDirectory +from ._core._tempfile import TemporaryFile as TemporaryFile +from ._core._tempfile import gettempdir as gettempdir +from ._core._tempfile import gettempdirb as gettempdirb +from ._core._tempfile import mkdtemp as mkdtemp +from ._core._tempfile import mkstemp as mkstemp +from ._core._testing import TaskInfo as TaskInfo +from ._core._testing import get_current_task as get_current_task +from ._core._testing import get_running_tasks as get_running_tasks +from ._core._testing import wait_all_tasks_blocked as wait_all_tasks_blocked +from ._core._typedattr import TypedAttributeProvider as TypedAttributeProvider +from ._core._typedattr import TypedAttributeSet as TypedAttributeSet +from ._core._typedattr import typed_attribute as typed_attribute + +# Re-export imports so they look like they live directly in this package +for __value in list(locals().values()): + if getattr(__value, "__module__", "").startswith("anyio."): + __value.__module__ = __name__ + + +del __value + + +def __getattr__(attr: str) -> type[BrokenWorkerInterpreter]: + """Support deprecated aliases.""" + if attr == "BrokenWorkerIntepreter": + import warnings + + warnings.warn( + "The 'BrokenWorkerIntepreter' alias is deprecated, use 'BrokenWorkerInterpreter' instead.", + DeprecationWarning, + stacklevel=2, + ) + return BrokenWorkerInterpreter + + raise AttributeError(f"module {__name__!r} has no attribute {attr!r}") diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/functools.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/functools.py new file mode 100644 index 0000000000000000000000000000000000000000..f1d6c7c55c17b041f0b010dca5a4b8f1bcb5da7a --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/functools.py @@ -0,0 +1,409 @@ +from __future__ import annotations + +__all__ = ( + "AsyncCacheInfo", + "AsyncCacheParameters", + "AsyncLRUCacheWrapper", + "cache", + "lru_cache", + "reduce", +) + +import functools +import sys +from collections import OrderedDict +from collections.abc import ( + AsyncIterable, + Awaitable, + Callable, + Coroutine, + Hashable, + Iterable, +) +from functools import update_wrapper +from inspect import iscoroutinefunction +from typing import ( + Any, + Generic, + NamedTuple, + TypedDict, + TypeVar, + cast, + final, + overload, +) +from weakref import WeakKeyDictionary + +from ._core._eventloop import current_time +from ._core._synchronization import Lock +from .lowlevel import RunVar, checkpoint + +if sys.version_info >= (3, 11): + from typing import ParamSpec +else: + from typing_extensions import ParamSpec + +T = TypeVar("T") +S = TypeVar("S") +P = ParamSpec("P") +lru_cache_items: RunVar[ + WeakKeyDictionary[ + AsyncLRUCacheWrapper[Any, Any], + OrderedDict[ + Hashable, + tuple[_InitialMissingType, Lock, float | None] + | tuple[Any, None, float | None], + ], + ] +] = RunVar("lru_cache_items") + + +class _InitialMissingType: + pass + + +initial_missing: _InitialMissingType = _InitialMissingType() + + +class AsyncCacheInfo(NamedTuple): + hits: int + misses: int + maxsize: int | None + currsize: int + ttl: int | None + + +class AsyncCacheParameters(TypedDict): + maxsize: int | None + typed: bool + always_checkpoint: bool + ttl: int | None + + +class _LRUMethodWrapper(Generic[T]): + def __init__(self, wrapper: AsyncLRUCacheWrapper[..., T], instance: object): + self.__wrapper = wrapper + self.__instance = instance + + def cache_info(self) -> AsyncCacheInfo: + return self.__wrapper.cache_info() + + def cache_parameters(self) -> AsyncCacheParameters: + return self.__wrapper.cache_parameters() + + def cache_clear(self) -> None: + self.__wrapper.cache_clear() + + async def __call__(self, *args: Any, **kwargs: Any) -> T: + if self.__instance is None: + return await self.__wrapper(*args, **kwargs) + + return await self.__wrapper(self.__instance, *args, **kwargs) + + +@final +class AsyncLRUCacheWrapper(Generic[P, T]): + def __init__( + self, + func: Callable[P, Awaitable[T]], + maxsize: int | None, + typed: bool, + always_checkpoint: bool, + ttl: int | None, + ): + self.__wrapped__ = func + self._hits: int = 0 + self._misses: int = 0 + self._maxsize = max(maxsize, 0) if maxsize is not None else None + self._currsize: int = 0 + self._typed = typed + self._always_checkpoint = always_checkpoint + self._ttl = ttl + update_wrapper(self, func) + + def cache_info(self) -> AsyncCacheInfo: + return AsyncCacheInfo( + self._hits, self._misses, self._maxsize, self._currsize, self._ttl + ) + + def cache_parameters(self) -> AsyncCacheParameters: + return { + "maxsize": self._maxsize, + "typed": self._typed, + "always_checkpoint": self._always_checkpoint, + "ttl": self._ttl, + } + + def cache_clear(self) -> None: + if cache := lru_cache_items.get(None): + cache.pop(self, None) + self._hits = self._misses = self._currsize = 0 + + async def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T: + # Easy case first: if maxsize == 0, no caching is done + if self._maxsize == 0: + value = await self.__wrapped__(*args, **kwargs) + self._misses += 1 + return value + + # The key is constructed as a flat tuple to avoid memory overhead + key: tuple[Any, ...] = args + if kwargs: + # initial_missing is used as a separator + key += (initial_missing,) + sum(kwargs.items(), ()) + + if self._typed: + key += tuple(type(arg) for arg in args) + if kwargs: + key += (initial_missing,) + tuple(type(val) for val in kwargs.values()) + + try: + cache = lru_cache_items.get() + except LookupError: + cache = WeakKeyDictionary() + lru_cache_items.set(cache) + + try: + cache_entry = cache[self] + except KeyError: + cache_entry = cache[self] = OrderedDict() + + cached_value: T | _InitialMissingType + try: + cached_value, lock, expires_at = cache_entry[key] + except KeyError: + # We're the first task to call this function + cached_value, lock, expires_at = ( + initial_missing, + Lock(fast_acquire=not self._always_checkpoint), + None, + ) + cache_entry[key] = cached_value, lock, expires_at + + if lock is None: + if expires_at is not None and current_time() >= expires_at: + self._currsize -= 1 + cached_value, lock, expires_at = ( + initial_missing, + Lock(fast_acquire=not self._always_checkpoint), + None, + ) + cache_entry[key] = cached_value, lock, expires_at + else: + # The value was already cached + self._hits += 1 + cache_entry.move_to_end(key) + if self._always_checkpoint: + await checkpoint() + + return cast(T, cached_value) + + async with lock: + # Check if another task filled the cache while we acquired the lock + if (cached_value := cache_entry[key][0]) is initial_missing: + self._misses += 1 + if self._maxsize is not None and self._currsize >= self._maxsize: + cache_entry.popitem(last=False) + else: + self._currsize += 1 + + value = await self.__wrapped__(*args, **kwargs) + expires_at = ( + current_time() + self._ttl if self._ttl is not None else None + ) + cache_entry[key] = value, None, expires_at + else: + # Another task filled the cache while we were waiting for the lock + self._hits += 1 + cache_entry.move_to_end(key) + value = cast(T, cached_value) + + return value + + def __get__( + self, instance: object, owner: type | None = None + ) -> _LRUMethodWrapper[T]: + wrapper = _LRUMethodWrapper(self, instance) + update_wrapper(wrapper, self.__wrapped__) + return wrapper + + +class _LRUCacheWrapper(Generic[T]): + def __init__( + self, maxsize: int | None, typed: bool, always_checkpoint: bool, ttl: int | None + ): + self._maxsize = maxsize + self._typed = typed + self._always_checkpoint = always_checkpoint + self._ttl = ttl + + @overload + def __call__( # type: ignore[overload-overlap] + self, func: Callable[P, Coroutine[Any, Any, T]], / + ) -> AsyncLRUCacheWrapper[P, T]: ... + + @overload + def __call__( + self, func: Callable[..., T], / + ) -> functools._lru_cache_wrapper[T]: ... + + def __call__( + self, f: Callable[P, Coroutine[Any, Any, T]] | Callable[..., T], / + ) -> AsyncLRUCacheWrapper[P, T] | functools._lru_cache_wrapper[T]: + if iscoroutinefunction(f): + return AsyncLRUCacheWrapper( + f, self._maxsize, self._typed, self._always_checkpoint, self._ttl + ) + + return functools.lru_cache(maxsize=self._maxsize, typed=self._typed)(f) # type: ignore[arg-type] + + +@overload +def cache( # type: ignore[overload-overlap] + func: Callable[P, Coroutine[Any, Any, T]], / +) -> AsyncLRUCacheWrapper[P, T]: ... + + +@overload +def cache(func: Callable[..., T], /) -> functools._lru_cache_wrapper[T]: ... + + +def cache( + func: Callable[..., T] | Callable[P, Coroutine[Any, Any, T]], / +) -> AsyncLRUCacheWrapper[P, T] | functools._lru_cache_wrapper[T]: + """ + A convenient shortcut for :func:`lru_cache` with ``maxsize=None``. + + This is the asynchronous equivalent to :func:`functools.cache`. + + """ + return lru_cache(maxsize=None)(func) + + +@overload +def lru_cache( + *, + maxsize: int | None = ..., + typed: bool = ..., + always_checkpoint: bool = ..., + ttl: int | None = ..., +) -> _LRUCacheWrapper[Any]: ... + + +@overload +def lru_cache( # type: ignore[overload-overlap] + func: Callable[P, Coroutine[Any, Any, T]], / +) -> AsyncLRUCacheWrapper[P, T]: ... + + +@overload +def lru_cache(func: Callable[..., T], /) -> functools._lru_cache_wrapper[T]: ... + + +def lru_cache( + func: Callable[P, Coroutine[Any, Any, T]] | Callable[..., T] | None = None, + /, + *, + maxsize: int | None = 128, + typed: bool = False, + always_checkpoint: bool = False, + ttl: int | None = None, +) -> ( + AsyncLRUCacheWrapper[P, T] | functools._lru_cache_wrapper[T] | _LRUCacheWrapper[Any] +): + """ + An asynchronous version of :func:`functools.lru_cache`. + + If a synchronous function is passed, the standard library + :func:`functools.lru_cache` is applied instead. + + :param always_checkpoint: if ``True``, every call to the cached function will be + guaranteed to yield control to the event loop at least once + :param ttl: time in seconds after which to invalidate cache entries + + .. note:: Caches and locks are managed on a per-event loop basis. + + """ + if func is None: + return _LRUCacheWrapper[Any](maxsize, typed, always_checkpoint, ttl) + + if not callable(func): + raise TypeError("the first argument must be callable") + + return _LRUCacheWrapper[T](maxsize, typed, always_checkpoint, ttl)(func) + + +@overload +async def reduce( + function: Callable[[T, S], Awaitable[T]], + iterable: Iterable[S] | AsyncIterable[S], + /, + initial: T, +) -> T: ... + + +@overload +async def reduce( + function: Callable[[T, T], Awaitable[T]], + iterable: Iterable[T] | AsyncIterable[T], + /, +) -> T: ... + + +async def reduce( # type: ignore[misc] + function: Callable[[T, T], Awaitable[T]] | Callable[[T, S], Awaitable[T]], + iterable: Iterable[T] | Iterable[S] | AsyncIterable[T] | AsyncIterable[S], + /, + initial: T | _InitialMissingType = initial_missing, +) -> T: + """ + Asynchronous version of :func:`functools.reduce`. + + :param function: a coroutine function that takes two arguments: the accumulated + value and the next element from the iterable + :param iterable: an iterable or async iterable + :param initial: the initial value (if missing, the first element of the iterable is + used as the initial value) + + """ + element: Any + function_called = False + if isinstance(iterable, AsyncIterable): + async_it = iterable.__aiter__() + if initial is initial_missing: + try: + value = cast(T, await async_it.__anext__()) + except StopAsyncIteration: + raise TypeError( + "reduce() of empty sequence with no initial value" + ) from None + else: + value = cast(T, initial) + + async for element in async_it: + value = await function(value, element) + function_called = True + elif isinstance(iterable, Iterable): + it = iter(iterable) + if initial is initial_missing: + try: + value = cast(T, next(it)) + except StopIteration: + raise TypeError( + "reduce() of empty sequence with no initial value" + ) from None + else: + value = cast(T, initial) + + for element in it: + value = await function(value, element) + function_called = True + else: + raise TypeError("reduce() argument 2 must be an iterable or async iterable") + + # Make sure there is at least one checkpoint, even if an empty iterable and an + # initial value were given + if not function_called: + await checkpoint() + + return value diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/lowlevel.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/lowlevel.py new file mode 100644 index 0000000000000000000000000000000000000000..ffbb75a7079aa4b1a13318641c1e1299e7bc1827 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/lowlevel.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +__all__ = ( + "EventLoopToken", + "RunvarToken", + "RunVar", + "checkpoint", + "checkpoint_if_cancelled", + "cancel_shielded_checkpoint", + "current_token", +) + +import enum +from dataclasses import dataclass +from types import TracebackType +from typing import Any, Generic, Literal, TypeVar, final, overload +from weakref import WeakKeyDictionary + +from ._core._eventloop import get_async_backend +from .abc import AsyncBackend + +T = TypeVar("T") +D = TypeVar("D") + + +async def checkpoint() -> None: + """ + Check for cancellation and allow the scheduler to switch to another task. + + Equivalent to (but more efficient than):: + + await checkpoint_if_cancelled() + await cancel_shielded_checkpoint() + + .. versionadded:: 3.0 + + """ + await get_async_backend().checkpoint() + + +async def checkpoint_if_cancelled() -> None: + """ + Enter a checkpoint if the enclosing cancel scope has been cancelled. + + This does not allow the scheduler to switch to a different task. + + .. versionadded:: 3.0 + + """ + await get_async_backend().checkpoint_if_cancelled() + + +async def cancel_shielded_checkpoint() -> None: + """ + Allow the scheduler to switch to another task but without checking for cancellation. + + Equivalent to (but potentially more efficient than):: + + with CancelScope(shield=True): + await checkpoint() + + .. versionadded:: 3.0 + + """ + await get_async_backend().cancel_shielded_checkpoint() + + +@final +@dataclass(frozen=True, repr=False) +class EventLoopToken: + """ + An opaque object that holds a reference to an event loop. + + .. versionadded:: 4.11.0 + """ + + backend_class: type[AsyncBackend] + native_token: object + + +def current_token() -> EventLoopToken: + """ + Return a token object that can be used to call code in the current event loop from + another thread. + + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + .. versionadded:: 4.11.0 + + """ + backend_class = get_async_backend() + raw_token = backend_class.current_token() + return EventLoopToken(backend_class, raw_token) + + +_run_vars: WeakKeyDictionary[object, dict[RunVar[Any], Any]] = WeakKeyDictionary() + + +class _NoValueSet(enum.Enum): + NO_VALUE_SET = enum.auto() + + +class RunvarToken(Generic[T]): + __slots__ = "_var", "_value", "_redeemed" + + def __init__(self, var: RunVar[T], value: T | Literal[_NoValueSet.NO_VALUE_SET]): + self._var = var + self._value: T | Literal[_NoValueSet.NO_VALUE_SET] = value + self._redeemed = False + + def __enter__(self) -> RunvarToken[T]: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + self._var.reset(self) + + +class RunVar(Generic[T]): + """ + Like a :class:`~contextvars.ContextVar`, except scoped to the running event loop. + + Can be used as a context manager, Just like :class:`~contextvars.ContextVar`, that + will reset the variable to its previous value when the context block is exited. + """ + + __slots__ = "_name", "_default" + + NO_VALUE_SET: Literal[_NoValueSet.NO_VALUE_SET] = _NoValueSet.NO_VALUE_SET + + def __init__( + self, name: str, default: T | Literal[_NoValueSet.NO_VALUE_SET] = NO_VALUE_SET + ): + self._name = name + self._default = default + + @property + def _current_vars(self) -> dict[RunVar[T], T]: + native_token = current_token().native_token + try: + return _run_vars[native_token] + except KeyError: + run_vars = _run_vars[native_token] = {} + return run_vars + + @overload + def get(self, default: D) -> T | D: ... + + @overload + def get(self) -> T: ... + + def get( + self, default: D | Literal[_NoValueSet.NO_VALUE_SET] = NO_VALUE_SET + ) -> T | D: + try: + return self._current_vars[self] + except KeyError: + if default is not RunVar.NO_VALUE_SET: + return default + elif self._default is not RunVar.NO_VALUE_SET: + return self._default + + raise LookupError( + f'Run variable "{self._name}" has no value and no default set' + ) + + def set(self, value: T) -> RunvarToken[T]: + current_vars = self._current_vars + token = RunvarToken(self, current_vars.get(self, RunVar.NO_VALUE_SET)) + current_vars[self] = value + return token + + def reset(self, token: RunvarToken[T]) -> None: + if token._var is not self: + raise ValueError("This token does not belong to this RunVar") + + if token._redeemed: + raise ValueError("This token has already been used") + + if token._value is _NoValueSet.NO_VALUE_SET: + try: + del self._current_vars[self] + except KeyError: + pass + else: + self._current_vars[self] = token._value + + token._redeemed = True + + def __repr__(self) -> str: + return f"" diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/pytest_plugin.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/pytest_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..a5183f0c84590ca912ac08b61e9f9eb676fc01e3 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/pytest_plugin.py @@ -0,0 +1,363 @@ +from __future__ import annotations + +import dataclasses +import socket +import sys +from collections.abc import Callable, Generator, Iterator +from contextlib import ExitStack, contextmanager +from inspect import isasyncgenfunction, iscoroutinefunction, ismethod +from typing import Any, cast + +import pytest +from _pytest.fixtures import FuncFixtureInfo, SubRequest +from _pytest.outcomes import Exit +from _pytest.python import CallSpec2 +from _pytest.scope import Scope + +from . import get_available_backends +from ._core._eventloop import ( + current_async_library, + get_async_backend, + reset_current_async_library, + set_current_async_library, +) +from ._core._exceptions import iterate_exceptions +from .abc import TestRunner + +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + +_current_runner: TestRunner | None = None +_runner_stack: ExitStack | None = None +_runner_leases = 0 + + +def extract_backend_and_options(backend: object) -> tuple[str, dict[str, Any]]: + if isinstance(backend, str): + return backend, {} + elif isinstance(backend, tuple) and len(backend) == 2: + if isinstance(backend[0], str) and isinstance(backend[1], dict): + return cast(tuple[str, dict[str, Any]], backend) + + raise TypeError("anyio_backend must be either a string or tuple of (string, dict)") + + +@contextmanager +def get_runner( + backend_name: str, backend_options: dict[str, Any] +) -> Iterator[TestRunner]: + global _current_runner, _runner_leases, _runner_stack + if _current_runner is None: + asynclib = get_async_backend(backend_name) + _runner_stack = ExitStack() + if current_async_library() is None: + # Since we're in control of the event loop, we can cache the name of the + # async library + token = set_current_async_library(backend_name) + _runner_stack.callback(reset_current_async_library, token) + + backend_options = backend_options or {} + _current_runner = _runner_stack.enter_context( + asynclib.create_test_runner(backend_options) + ) + + _runner_leases += 1 + try: + yield _current_runner + finally: + _runner_leases -= 1 + if not _runner_leases: + assert _runner_stack is not None + _runner_stack.close() + _runner_stack = _current_runner = None + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addini( + "anyio_mode", + default="strict", + help='AnyIO plugin mode (either "strict" or "auto")', + ) + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", + "anyio: mark the (coroutine function) test to be run asynchronously via anyio.", + ) + if ( + config.getini("anyio_mode") == "auto" + and config.pluginmanager.has_plugin("asyncio") + and config.getini("asyncio_mode") == "auto" + ): + config.issue_config_time_warning( + pytest.PytestConfigWarning( + "AnyIO auto mode has been enabled together with pytest-asyncio auto " + "mode. This may cause unexpected behavior." + ), + 1, + ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_fixture_setup(fixturedef: Any, request: Any) -> Generator[Any]: + def wrapper(anyio_backend: Any, request: SubRequest, **kwargs: Any) -> Any: + # Rebind any fixture methods to the request instance + if ( + request.instance + and ismethod(func) + and type(func.__self__) is type(request.instance) + ): + local_func = func.__func__.__get__(request.instance) + else: + local_func = func + + backend_name, backend_options = extract_backend_and_options(anyio_backend) + if has_backend_arg: + kwargs["anyio_backend"] = anyio_backend + + if has_request_arg: + kwargs["request"] = request + + with get_runner(backend_name, backend_options) as runner: + if isasyncgenfunction(local_func): + yield from runner.run_asyncgen_fixture(local_func, kwargs) + else: + yield runner.run_fixture(local_func, kwargs) + + # Only apply this to coroutine functions and async generator functions in requests + # that involve the anyio_backend fixture + func = fixturedef.func + if isasyncgenfunction(func) or iscoroutinefunction(func): + if "anyio_backend" in request.fixturenames: + fixturedef.func = wrapper + original_argname = fixturedef.argnames + + if not (has_backend_arg := "anyio_backend" in fixturedef.argnames): + fixturedef.argnames += ("anyio_backend",) + + if not (has_request_arg := "request" in fixturedef.argnames): + fixturedef.argnames += ("request",) + + try: + return (yield) + finally: + fixturedef.func = func + fixturedef.argnames = original_argname + + return (yield) + + +@pytest.hookimpl(tryfirst=True) +def pytest_pycollect_makeitem( + collector: pytest.Module | pytest.Class, name: str, obj: object +) -> None: + if collector.istestfunction(obj, name): + inner_func = obj.hypothesis.inner_test if hasattr(obj, "hypothesis") else obj + if iscoroutinefunction(inner_func): + anyio_auto_mode = collector.config.getini("anyio_mode") == "auto" + marker = collector.get_closest_marker("anyio") + own_markers = getattr(obj, "pytestmark", ()) + if ( + anyio_auto_mode + or marker + or any(marker.name == "anyio" for marker in own_markers) + ): + pytest.mark.usefixtures("anyio_backend")(obj) + + +def pytest_collection_finish(session: pytest.Session) -> None: + for i, item in reversed(list(enumerate(session.items))): + if ( + isinstance(item, pytest.Function) + and iscoroutinefunction(item.function) + and item.get_closest_marker("anyio") is not None + and "anyio_backend" not in item.fixturenames + ): + new_items = [] + try: + cs_fields = {f.name for f in dataclasses.fields(CallSpec2)} + except TypeError: + cs_fields = set() + + for param_index, backend in enumerate(get_available_backends()): + if "_arg2scope" in cs_fields: # pytest >= 8 + callspec = CallSpec2( + params={"anyio_backend": backend}, + indices={"anyio_backend": param_index}, + _arg2scope={"anyio_backend": Scope.Module}, + _idlist=[backend], + marks=[], + ) + else: # pytest 7.x + callspec = CallSpec2( # type: ignore[call-arg] + funcargs={}, + params={"anyio_backend": backend}, + indices={"anyio_backend": param_index}, + arg2scope={"anyio_backend": Scope.Module}, + idlist=[backend], + marks=[], + ) + + fi = item._fixtureinfo + new_names_closure = list(fi.names_closure) + if "anyio_backend" not in new_names_closure: + new_names_closure.append("anyio_backend") + + new_fixtureinfo = FuncFixtureInfo( + argnames=fi.argnames, + initialnames=fi.initialnames, + names_closure=new_names_closure, + name2fixturedefs=fi.name2fixturedefs, + ) + new_item = pytest.Function.from_parent( + item.parent, + name=f"{item.originalname}[{backend}]", + callspec=callspec, + callobj=item.obj, + fixtureinfo=new_fixtureinfo, + keywords=item.keywords, + originalname=item.originalname, + ) + new_items.append(new_item) + + session.items[i : i + 1] = new_items + + +@pytest.hookimpl(tryfirst=True) +def pytest_pyfunc_call(pyfuncitem: Any) -> bool | None: + def run_with_hypothesis(**kwargs: Any) -> None: + with get_runner(backend_name, backend_options) as runner: + runner.run_test(original_func, kwargs) + + backend = pyfuncitem.funcargs.get("anyio_backend") + if backend: + backend_name, backend_options = extract_backend_and_options(backend) + + if hasattr(pyfuncitem.obj, "hypothesis"): + # Wrap the inner test function unless it's already wrapped + original_func = pyfuncitem.obj.hypothesis.inner_test + if original_func.__qualname__ != run_with_hypothesis.__qualname__: + if iscoroutinefunction(original_func): + pyfuncitem.obj.hypothesis.inner_test = run_with_hypothesis + + return None + + if iscoroutinefunction(pyfuncitem.obj): + funcargs = pyfuncitem.funcargs + testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} + with get_runner(backend_name, backend_options) as runner: + try: + runner.run_test(pyfuncitem.obj, testargs) + except ExceptionGroup as excgrp: + for exc in iterate_exceptions(excgrp): + if isinstance(exc, (Exit, KeyboardInterrupt, SystemExit)): + raise exc from excgrp + + raise + + return True + + return None + + +@pytest.fixture(scope="module", params=get_available_backends()) +def anyio_backend(request: Any) -> Any: + return request.param + + +@pytest.fixture +def anyio_backend_name(anyio_backend: Any) -> str: + if isinstance(anyio_backend, str): + return anyio_backend + else: + return anyio_backend[0] + + +@pytest.fixture +def anyio_backend_options(anyio_backend: Any) -> dict[str, Any]: + if isinstance(anyio_backend, str): + return {} + else: + return anyio_backend[1] + + +class FreePortFactory: + """ + Manages port generation based on specified socket kind, ensuring no duplicate + ports are generated. + + This class provides functionality for generating available free ports on the + system. It is initialized with a specific socket kind and can generate ports + for given address families while avoiding reuse of previously generated ports. + + Users should not instantiate this class directly, but use the + ``free_tcp_port_factory`` and ``free_udp_port_factory`` fixtures instead. For simple + uses cases, ``free_tcp_port`` and ``free_udp_port`` can be used instead. + """ + + def __init__(self, kind: socket.SocketKind) -> None: + self._kind = kind + self._generated = set[int]() + + @property + def kind(self) -> socket.SocketKind: + """ + The type of socket connection (e.g., :data:`~socket.SOCK_STREAM` or + :data:`~socket.SOCK_DGRAM`) used to bind for checking port availability + + """ + return self._kind + + def __call__(self, family: socket.AddressFamily | None = None) -> int: + """ + Return an unbound port for the given address family. + + :param family: if omitted, both IPv4 and IPv6 addresses will be tried + :return: a port number + + """ + if family is not None: + families = [family] + else: + families = [socket.AF_INET] + if socket.has_ipv6: + families.append(socket.AF_INET6) + + while True: + port = 0 + with ExitStack() as stack: + for family in families: + sock = stack.enter_context(socket.socket(family, self._kind)) + addr = "::1" if family == socket.AF_INET6 else "127.0.0.1" + try: + sock.bind((addr, port)) + except OSError: + break + + if not port: + port = sock.getsockname()[1] + else: + if port not in self._generated: + self._generated.add(port) + return port + + +@pytest.fixture(scope="session") +def free_tcp_port_factory() -> FreePortFactory: + return FreePortFactory(socket.SOCK_STREAM) + + +@pytest.fixture(scope="session") +def free_udp_port_factory() -> FreePortFactory: + return FreePortFactory(socket.SOCK_DGRAM) + + +@pytest.fixture +def free_tcp_port(free_tcp_port_factory: Callable[[], int]) -> int: + return free_tcp_port_factory() + + +@pytest.fixture +def free_udp_port(free_udp_port_factory: Callable[[], int]) -> int: + return free_udp_port_factory() diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/to_interpreter.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/to_interpreter.py new file mode 100644 index 0000000000000000000000000000000000000000..694dbe77bc8581032ee72316afe4e0590311ba00 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/to_interpreter.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +__all__ = ( + "run_sync", + "current_default_interpreter_limiter", +) + +import atexit +import os +import sys +from collections import deque +from collections.abc import Callable +from typing import Any, Final, TypeVar + +from . import current_time, to_thread +from ._core._exceptions import BrokenWorkerInterpreter +from ._core._synchronization import CapacityLimiter +from .lowlevel import RunVar + +if sys.version_info >= (3, 11): + from typing import TypeVarTuple, Unpack +else: + from typing_extensions import TypeVarTuple, Unpack + +if sys.version_info >= (3, 14): + from concurrent.interpreters import ExecutionFailed, create + + def _interp_call( + func: Callable[..., Any], args: tuple[Any, ...] + ) -> tuple[Any, bool]: + try: + retval = func(*args) + except BaseException as exc: + return exc, True + else: + return retval, False + + class _Worker: + last_used: float = 0 + + def __init__(self) -> None: + self._interpreter = create() + + def destroy(self) -> None: + self._interpreter.close() + + def call( + self, + func: Callable[..., T_Retval], + args: tuple[Any, ...], + ) -> T_Retval: + try: + res, is_exception = self._interpreter.call(_interp_call, func, args) + except ExecutionFailed as exc: + raise BrokenWorkerInterpreter(exc.excinfo) from exc + + if is_exception: + raise res + + return res +elif sys.version_info >= (3, 13): + import _interpqueues + import _interpreters + + UNBOUND: Final = 2 # I have no clue how this works, but it was used in the stdlib + FMT_UNPICKLED: Final = 0 + FMT_PICKLED: Final = 1 + QUEUE_PICKLE_ARGS: Final = (FMT_PICKLED, UNBOUND) + QUEUE_UNPICKLE_ARGS: Final = (FMT_UNPICKLED, UNBOUND) + + _run_func = compile( + """ +import _interpqueues +from _interpreters import NotShareableError +from pickle import loads, dumps, HIGHEST_PROTOCOL + +QUEUE_PICKLE_ARGS = (1, 2) +QUEUE_UNPICKLE_ARGS = (0, 2) + +item = _interpqueues.get(queue_id)[0] +try: + func, args = loads(item) + retval = func(*args) +except BaseException as exc: + is_exception = True + retval = exc +else: + is_exception = False + +try: + _interpqueues.put(queue_id, (retval, is_exception), *QUEUE_UNPICKLE_ARGS) +except NotShareableError: + retval = dumps(retval, HIGHEST_PROTOCOL) + _interpqueues.put(queue_id, (retval, is_exception), *QUEUE_PICKLE_ARGS) + """, + "", + "exec", + ) + + class _Worker: + last_used: float = 0 + + def __init__(self) -> None: + self._interpreter_id = _interpreters.create() + self._queue_id = _interpqueues.create(1, *QUEUE_UNPICKLE_ARGS) + _interpreters.set___main___attrs( + self._interpreter_id, {"queue_id": self._queue_id} + ) + + def destroy(self) -> None: + _interpqueues.destroy(self._queue_id) + _interpreters.destroy(self._interpreter_id) + + def call( + self, + func: Callable[..., T_Retval], + args: tuple[Any, ...], + ) -> T_Retval: + import pickle + + item = pickle.dumps((func, args), pickle.HIGHEST_PROTOCOL) + _interpqueues.put(self._queue_id, item, *QUEUE_PICKLE_ARGS) + exc_info = _interpreters.exec(self._interpreter_id, _run_func) + if exc_info: + raise BrokenWorkerInterpreter(exc_info) + + res = _interpqueues.get(self._queue_id) + (res, is_exception), fmt = res[:2] + if fmt == FMT_PICKLED: + res = pickle.loads(res) + + if is_exception: + raise res + + return res +else: + + class _Worker: + last_used: float = 0 + + def __init__(self) -> None: + raise RuntimeError("subinterpreters require at least Python 3.13") + + def call( + self, + func: Callable[..., T_Retval], + args: tuple[Any, ...], + ) -> T_Retval: + raise NotImplementedError + + def destroy(self) -> None: + pass + + +DEFAULT_CPU_COUNT: Final = 8 # this is just an arbitrarily selected value +MAX_WORKER_IDLE_TIME = ( + 30 # seconds a subinterpreter can be idle before becoming eligible for pruning +) + +T_Retval = TypeVar("T_Retval") +PosArgsT = TypeVarTuple("PosArgsT") + +_idle_workers = RunVar[deque[_Worker]]("_available_workers") +_default_interpreter_limiter = RunVar[CapacityLimiter]("_default_interpreter_limiter") + + +def _stop_workers(workers: deque[_Worker]) -> None: + for worker in workers: + worker.destroy() + + workers.clear() + + +async def run_sync( + func: Callable[[Unpack[PosArgsT]], T_Retval], + *args: Unpack[PosArgsT], + limiter: CapacityLimiter | None = None, +) -> T_Retval: + """ + Call the given function with the given arguments in a subinterpreter. + + .. warning:: On Python 3.13, the :mod:`concurrent.interpreters` module was not yet + available, so the code path for that Python version relies on an undocumented, + private API. As such, it is recommended to not rely on this function for anything + mission-critical on Python 3.13. + + :param func: a callable + :param args: the positional arguments for the callable + :param limiter: capacity limiter to use to limit the total number of subinterpreters + running (if omitted, the default limiter is used) + :return: the result of the call + :raises BrokenWorkerInterpreter: if there's an internal error in a subinterpreter + + """ + if limiter is None: + limiter = current_default_interpreter_limiter() + + try: + idle_workers = _idle_workers.get() + except LookupError: + idle_workers = deque() + _idle_workers.set(idle_workers) + atexit.register(_stop_workers, idle_workers) + + async with limiter: + try: + worker = idle_workers.pop() + except IndexError: + worker = _Worker() + + try: + return await to_thread.run_sync( + worker.call, + func, + args, + limiter=limiter, + ) + finally: + # Prune workers that have been idle for too long + now = current_time() + while idle_workers: + if now - idle_workers[0].last_used <= MAX_WORKER_IDLE_TIME: + break + + await to_thread.run_sync(idle_workers.popleft().destroy, limiter=limiter) + + worker.last_used = current_time() + idle_workers.append(worker) + + +def current_default_interpreter_limiter() -> CapacityLimiter: + """ + Return the capacity limiter used by default to limit the number of concurrently + running subinterpreters. + + Defaults to the number of CPU cores. + + :return: a capacity limiter object + + """ + try: + return _default_interpreter_limiter.get() + except LookupError: + limiter = CapacityLimiter(os.cpu_count() or DEFAULT_CPU_COUNT) + _default_interpreter_limiter.set(limiter) + return limiter diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/to_process.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/to_process.py new file mode 100644 index 0000000000000000000000000000000000000000..b289234ecfaafc1651dfa76e924291e5a5e9521e --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/to_process.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +__all__ = ( + "current_default_process_limiter", + "process_worker", + "run_sync", +) + +import os +import pickle +import subprocess +import sys +from collections import deque +from collections.abc import Callable +from importlib.util import module_from_spec, spec_from_file_location +from typing import TypeVar, cast + +from ._core._eventloop import current_time, get_async_backend, get_cancelled_exc_class +from ._core._exceptions import BrokenWorkerProcess +from ._core._subprocesses import open_process +from ._core._synchronization import CapacityLimiter +from ._core._tasks import CancelScope, fail_after +from .abc import ByteReceiveStream, ByteSendStream, Process +from .lowlevel import RunVar, checkpoint_if_cancelled +from .streams.buffered import BufferedByteReceiveStream + +if sys.version_info >= (3, 11): + from typing import TypeVarTuple, Unpack +else: + from typing_extensions import TypeVarTuple, Unpack + +WORKER_MAX_IDLE_TIME = 300 # 5 minutes + +T_Retval = TypeVar("T_Retval") +PosArgsT = TypeVarTuple("PosArgsT") + +_process_pool_workers: RunVar[set[Process]] = RunVar("_process_pool_workers") +_process_pool_idle_workers: RunVar[deque[tuple[Process, float]]] = RunVar( + "_process_pool_idle_workers" +) +_default_process_limiter: RunVar[CapacityLimiter] = RunVar("_default_process_limiter") + + +async def run_sync( # type: ignore[return] + func: Callable[[Unpack[PosArgsT]], T_Retval], + *args: Unpack[PosArgsT], + cancellable: bool = False, + limiter: CapacityLimiter | None = None, +) -> T_Retval: + """ + Call the given function with the given arguments in a worker process. + + If the ``cancellable`` option is enabled and the task waiting for its completion is + cancelled, the worker process running it will be abruptly terminated using SIGKILL + (or ``terminateProcess()`` on Windows). + + :param func: a callable + :param args: positional arguments for the callable + :param cancellable: ``True`` to allow cancellation of the operation while it's + running + :param limiter: capacity limiter to use to limit the total amount of processes + running (if omitted, the default limiter is used) + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + :return: an awaitable that yields the return value of the function. + + """ + + async def send_raw_command(pickled_cmd: bytes) -> object: + try: + await stdin.send(pickled_cmd) + response = await buffered.receive_until(b"\n", 50) + status, length = response.split(b" ") + if status not in (b"RETURN", b"EXCEPTION"): + raise RuntimeError( + f"Worker process returned unexpected response: {response!r}" + ) + + pickled_response = await buffered.receive_exactly(int(length)) + except BaseException as exc: + workers.discard(process) + try: + process.kill() + with CancelScope(shield=True): + await process.aclose() + except ProcessLookupError: + pass + + if isinstance(exc, get_cancelled_exc_class()): + raise + else: + raise BrokenWorkerProcess from exc + + retval = pickle.loads(pickled_response) + if status == b"EXCEPTION": + assert isinstance(retval, BaseException) + raise retval + else: + return retval + + # First pickle the request before trying to reserve a worker process + await checkpoint_if_cancelled() + request = pickle.dumps(("run", func, args), protocol=pickle.HIGHEST_PROTOCOL) + + # If this is the first run in this event loop thread, set up the necessary variables + try: + workers = _process_pool_workers.get() + idle_workers = _process_pool_idle_workers.get() + except LookupError: + workers = set() + idle_workers = deque() + _process_pool_workers.set(workers) + _process_pool_idle_workers.set(idle_workers) + get_async_backend().setup_process_pool_exit_at_shutdown(workers) + + async with limiter or current_default_process_limiter(): + # Pop processes from the pool (starting from the most recently used) until we + # find one that hasn't exited yet + process: Process + while idle_workers: + process, idle_since = idle_workers.pop() + if process.returncode is None: + stdin = cast(ByteSendStream, process.stdin) + buffered = BufferedByteReceiveStream( + cast(ByteReceiveStream, process.stdout) + ) + + # Prune any other workers that have been idle for WORKER_MAX_IDLE_TIME + # seconds or longer + now = current_time() + killed_processes: list[Process] = [] + while idle_workers: + if now - idle_workers[0][1] < WORKER_MAX_IDLE_TIME: + break + + process_to_kill, idle_since = idle_workers.popleft() + process_to_kill.kill() + workers.remove(process_to_kill) + killed_processes.append(process_to_kill) + + with CancelScope(shield=True): + for killed_process in killed_processes: + await killed_process.aclose() + + break + + workers.remove(process) + else: + command = [sys.executable, "-u", "-m", __name__] + process = await open_process( + command, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ) + try: + stdin = cast(ByteSendStream, process.stdin) + buffered = BufferedByteReceiveStream( + cast(ByteReceiveStream, process.stdout) + ) + with fail_after(20): + message = await buffered.receive(6) + + if message != b"READY\n": + raise BrokenWorkerProcess( + f"Worker process returned unexpected response: {message!r}" + ) + + main_module_path = getattr(sys.modules["__main__"], "__file__", None) + pickled = pickle.dumps( + ("init", sys.path, main_module_path), + protocol=pickle.HIGHEST_PROTOCOL, + ) + await send_raw_command(pickled) + except (BrokenWorkerProcess, get_cancelled_exc_class()): + raise + except BaseException as exc: + process.kill() + raise BrokenWorkerProcess( + "Error during worker process initialization" + ) from exc + + workers.add(process) + + with CancelScope(shield=not cancellable): + try: + return cast(T_Retval, await send_raw_command(request)) + finally: + if process in workers: + idle_workers.append((process, current_time())) + + +def current_default_process_limiter() -> CapacityLimiter: + """ + Return the capacity limiter that is used by default to limit the number of worker + processes. + + :return: a capacity limiter object + + """ + try: + return _default_process_limiter.get() + except LookupError: + limiter = CapacityLimiter(os.cpu_count() or 2) + _default_process_limiter.set(limiter) + return limiter + + +def process_worker() -> None: + # Redirect standard streams to os.devnull so that user code won't interfere with the + # parent-worker communication + stdin = sys.stdin + stdout = sys.stdout + sys.stdin = open(os.devnull) + sys.stdout = open(os.devnull, "w") + + stdout.buffer.write(b"READY\n") + while True: + retval = exception = None + try: + command, *args = pickle.load(stdin.buffer) + except EOFError: + return + except BaseException as exc: + exception = exc + else: + if command == "run": + func, args = args + try: + retval = func(*args) + except BaseException as exc: + exception = exc + elif command == "init": + main_module_path: str | None + sys.path, main_module_path = args + del sys.modules["__main__"] + if main_module_path and os.path.isfile(main_module_path): + # Load the parent's main module but as __mp_main__ instead of + # __main__ (like multiprocessing does) to avoid infinite recursion + try: + spec = spec_from_file_location("__mp_main__", main_module_path) + if spec and spec.loader: + main = module_from_spec(spec) + spec.loader.exec_module(main) + sys.modules["__main__"] = main + except BaseException as exc: + exception = exc + try: + if exception is not None: + status = b"EXCEPTION" + pickled = pickle.dumps(exception, pickle.HIGHEST_PROTOCOL) + else: + status = b"RETURN" + pickled = pickle.dumps(retval, pickle.HIGHEST_PROTOCOL) + except BaseException as exc: + exception = exc + status = b"EXCEPTION" + pickled = pickle.dumps(exc, pickle.HIGHEST_PROTOCOL) + + stdout.buffer.write(b"%s %d\n" % (status, len(pickled))) + stdout.buffer.write(pickled) + + # Respect SIGTERM + if isinstance(exception, SystemExit): + raise exception + + +if __name__ == "__main__": + process_worker() diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/to_thread.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/to_thread.py new file mode 100644 index 0000000000000000000000000000000000000000..83c79d1cd4029ccdaf23b22a9ad10074a0979ec2 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/anyio/to_thread.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +__all__ = ( + "run_sync", + "current_default_thread_limiter", +) + +import sys +from collections.abc import Callable +from typing import TypeVar +from warnings import warn + +from ._core._eventloop import get_async_backend +from .abc import CapacityLimiter + +if sys.version_info >= (3, 11): + from typing import TypeVarTuple, Unpack +else: + from typing_extensions import TypeVarTuple, Unpack + +T_Retval = TypeVar("T_Retval") +PosArgsT = TypeVarTuple("PosArgsT") + + +async def run_sync( + func: Callable[[Unpack[PosArgsT]], T_Retval], + *args: Unpack[PosArgsT], + abandon_on_cancel: bool = False, + cancellable: bool | None = None, + limiter: CapacityLimiter | None = None, +) -> T_Retval: + """ + Call the given function with the given arguments in a worker thread. + + If the ``abandon_on_cancel`` option is enabled and the task waiting for its + completion is cancelled, the thread will still run its course but its + return value (or any raised exception) will be ignored. + + :param func: a callable + :param args: positional arguments for the callable + :param abandon_on_cancel: ``True`` to abandon the thread (leaving it to run + unchecked on own) if the host task is cancelled, ``False`` to ignore + cancellations in the host task until the operation has completed in the worker + thread + :param cancellable: deprecated alias of ``abandon_on_cancel``; will override + ``abandon_on_cancel`` if both parameters are passed + :param limiter: capacity limiter to use to limit the total amount of threads running + (if omitted, the default limiter is used) + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + :return: an awaitable that yields the return value of the function. + + """ + if cancellable is not None: + abandon_on_cancel = cancellable + warn( + "The `cancellable=` keyword argument to `anyio.to_thread.run_sync` is " + "deprecated since AnyIO 4.1.0; use `abandon_on_cancel=` instead", + DeprecationWarning, + stacklevel=2, + ) + + return await get_async_backend().run_sync_in_worker_thread( + func, args, abandon_on_cancel=abandon_on_cancel, limiter=limiter + ) + + +def current_default_thread_limiter() -> CapacityLimiter: + """ + Return the capacity limiter that is used by default to limit the number of + concurrent threads. + + :return: a capacity limiter object + :raises NoEventLoopError: if no supported asynchronous event loop is running in the + current thread + + """ + return get_async_backend().current_default_thread_limiter() diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/whisper/configuration_whisper.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/whisper/configuration_whisper.py new file mode 100644 index 0000000000000000000000000000000000000000..26150b06f82f445c5117bdaf8419489d374ea260 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/whisper/configuration_whisper.py @@ -0,0 +1,167 @@ +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Whisper model configuration""" + +from huggingface_hub.dataclasses import strict + +from ...configuration_utils import PreTrainedConfig +from ...utils import auto_docstring + + +# fmt: off +NON_SPEECH_TOKENS = [ + 1, 2, 7, 8, 9, 10, 14, 25, + 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, + 63, 90, 91, 92, 93, 357, 366, 438, 532, 685, + 705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377, + 1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211, + 4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786, + 11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791, + 17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409, + 34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361 +] +NON_SPEECH_TOKENS_MULTI = [ + 1, 2, 7, 8, 9, 10, 14, 25, + 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, + 63, 90, 91, 92, 93, 359, 503, 522, 542, 873, + 893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627, + 3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647, + 7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793, + 14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, + 22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865, + 42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362 +] +# fmt: on + + +@auto_docstring(checkpoint="openai/whisper-tiny") +@strict +class WhisperConfig(PreTrainedConfig): + r""" + max_source_positions (`int`, *optional*, defaults to 1500): + The maximum sequence length of log-mel filter-bank features that this model might ever be used with. + max_target_positions (`int`, *optional*, defaults to 448): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + suppress_tokens (`list[int]`, *optional*): + A list containing the non-speech tokens that will be used by the logit processor in the `generate` + function. NON_SPEECH_TOKENS and NON_SPEECH_TOKENS_MULTI each correspond to the `english-only` and the + `multilingual` model. + begin_suppress_tokens (`list[int]`, *optional*, defaults to `[220,50256]`): + A list containing tokens that will be suppressed at the beginning of the sampling process. Initialized as + the token for `" "` (`blank_token_id`) and the `eos_token_id` + use_weighted_layer_sum (`bool`, *optional*, defaults to `False`): + Whether to use a weighted average of layer outputs with learned weights. Only relevant when using an + instance of [`WhisperForAudioClassification`]. + classifier_proj_size (`int`, *optional*, defaults to 256): + Dimensionality of the projection before token mean-pooling for classification. Only relevant when using an + instance of [`WhisperForAudioClassification`]. + apply_spec_augment (`bool`, *optional*, defaults to `False`): + Whether to apply *SpecAugment* data augmentation to the outputs of the feature encoder. For reference see + [SpecAugment: A Simple Data Augmentation Method for Automatic Speech + Recognition](https://huggingface.co/papers/1904.08779). + mask_time_prob (`float`, *optional*, defaults to 0.05): + Percentage (between 0 and 1) of all feature vectors along the time axis which will be masked. The masking + procedure generates `mask_time_prob*len(time_axis)/mask_time_length` independent masks over the axis. If + reasoning from the probability of each feature vector to be chosen as the start of the vector span to be + masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease the + actual percentage of masked vectors. This is only relevant if `apply_spec_augment == True`. + mask_time_length (`int`, *optional*, defaults to 10): + Length of vector span along the time axis. + mask_time_min_masks (`int`, *optional*, defaults to 2),: + The minimum number of masks of length `mask_feature_length` generated along the time axis, each time step, + irrespectively of `mask_feature_prob`. Only relevant if ''mask_time_prob*len(time_axis)/mask_time_length < + mask_time_min_masks'' + mask_feature_prob (`float`, *optional*, defaults to 0.0): + Percentage (between 0 and 1) of all feature vectors along the feature axis which will be masked. The + masking procedure generates `mask_feature_prob*len(feature_axis)/mask_time_length` independent masks over + the axis. If reasoning from the probability of each feature vector to be chosen as the start of the vector + span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that overlap + may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is + True`. + mask_feature_length (`int`, *optional*, defaults to 10): + Length of vector span along the feature axis. + mask_feature_min_masks (`int`, *optional*, defaults to 0): + The minimum number of masks of length `mask_feature_length` generated along the feature axis, each time + step, irrespectively of `mask_feature_prob`. Only relevant if + `mask_feature_prob*len(feature_axis)/mask_feature_length < mask_feature_min_masks`. + median_filter_width (`int`, *optional*, defaults to 7): + Width of the median filter used to smoothen to cross-attention outputs when computing token timestamps. + Should be an odd number. + + Example: + + ```python + >>> from transformers import WhisperConfig, WhisperModel + + >>> # Initializing a Whisper tiny style configuration + >>> configuration = WhisperConfig() + + >>> # Initializing a model (with random weights) from the tiny style configuration + >>> model = WhisperModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "whisper" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = { + "num_key_value_heads": "encoder_attention_heads", + "num_attention_heads": "encoder_attention_heads", + "hidden_size": "d_model", + "num_hidden_layers": "encoder_layers", + } + + vocab_size: int = 51865 + num_mel_bins: int = 80 + encoder_layers: int = 4 + encoder_attention_heads: int = 6 + decoder_layers: int = 4 + decoder_attention_heads: int = 6 + decoder_ffn_dim: int = 1536 + encoder_ffn_dim: int = 1536 + encoder_layerdrop: float | int = 0.0 + decoder_layerdrop: float | int = 0.0 + decoder_start_token_id: int = 50257 + use_cache: bool = True + is_encoder_decoder: bool = True + activation_function: str = "gelu" + d_model: int = 384 + dropout: float | int = 0.0 + attention_dropout: float | int = 0.0 + activation_dropout: float | int = 0.0 + init_std: float = 0.02 + scale_embedding: bool = False + max_source_positions: int = 1500 + max_target_positions: int = 448 + pad_token_id: int | None = 50256 + bos_token_id: int | None = 50256 + eos_token_id: int | list[int] | None = 50256 + suppress_tokens: list | None = None + begin_suppress_tokens: list[int] | tuple[int, ...] | None = (220, 50256) + use_weighted_layer_sum: bool = False + classifier_proj_size: int = 256 + apply_spec_augment: bool = False + mask_time_prob: float | int = 0.05 + mask_time_length: int = 10 + mask_time_min_masks: int = 2 + mask_feature_prob: float | int = 0.0 + mask_feature_length: int = 10 + mask_feature_min_masks: int = 0 + median_filter_width: int = 7 + tie_word_embeddings: bool = True + + +__all__ = ["WhisperConfig"] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/whisper/english_normalizer.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/whisper/english_normalizer.py new file mode 100644 index 0000000000000000000000000000000000000000..99441655067efe1b959426b6c6a255cd7dd326a9 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/whisper/english_normalizer.py @@ -0,0 +1,596 @@ +# Copyright 2022 The OpenAI team and The HuggingFace Team. All rights reserved. +# Most of the code is copy pasted from the original whisper repository +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import unicodedata +from collections.abc import Iterator +from fractions import Fraction +from re import Match + +import regex + + +# non-ASCII letters that are not separated by "NFKD" normalization +ADDITIONAL_DIACRITICS = { + "œ": "oe", + "Œ": "OE", + "ø": "o", + "Ø": "O", + "æ": "ae", + "Æ": "AE", + "ß": "ss", + "ẞ": "SS", + "đ": "d", + "Đ": "D", + "ð": "d", + "Ð": "D", + "þ": "th", + "Þ": "th", + "ł": "l", + "Ł": "L", +} + + +def remove_symbols_and_diacritics(s: str, keep=""): + """ + Replace any other markers, symbols, and punctuations with a space, and drop any diacritics (category 'Mn' and some + manual mappings) + """ + + def replace_character(char): + if char in keep: + return char + elif char in ADDITIONAL_DIACRITICS: + return ADDITIONAL_DIACRITICS[char] + + elif unicodedata.category(char) == "Mn": + return "" + + elif unicodedata.category(char)[0] in "MSP": + return " " + + return char + + return "".join(replace_character(c) for c in unicodedata.normalize("NFKD", s)) + + +def remove_symbols(s: str): + """ + Replace any other markers, symbols, punctuations with a space, keeping diacritics + """ + return "".join(" " if unicodedata.category(c)[0] in "MSP" else c for c in unicodedata.normalize("NFKC", s)) + + +class BasicTextNormalizer: + def __init__(self, remove_diacritics: bool = False, split_letters: bool = False): + self.clean = remove_symbols_and_diacritics if remove_diacritics else remove_symbols + self.split_letters = split_letters + + def __call__(self, s: str): + s = s.lower() + s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets + s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis + s = self.clean(s).lower() + + if self.split_letters: + s = " ".join(regex.findall(r"\X", s, regex.U)) + + s = re.sub(r"\s+", " ", s) # replace any successive whitespace characters with a space + + return s + + +class EnglishNumberNormalizer: + """ + Convert any spelled-out numbers into arabic numbers, while handling: + + - remove any commas + - keep the suffixes such as: `1960s`, `274th`, `32nd`, etc. + - spell out currency symbols after the number. e.g. `$20 million` -> `20000000 dollars` + - spell out `one` and `ones` + - interpret successive single-digit numbers as nominal: `one oh one` -> `101` + """ + + def __init__(self): + super().__init__() + + self.zeros = {"o", "oh", "zero"} + # fmt: off + self.ones = { + name: i + for i, name in enumerate( + ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen"], + start=1, + ) + } + # fmt: on + self.ones_plural = { + "sixes" if name == "six" else name + "s": (value, "s") for name, value in self.ones.items() + } + self.ones_ordinal = { + "zeroth": (0, "th"), + "first": (1, "st"), + "second": (2, "nd"), + "third": (3, "rd"), + "fifth": (5, "th"), + "twelfth": (12, "th"), + **{ + name + ("h" if name.endswith("t") else "th"): (value, "th") + for name, value in self.ones.items() + if value > 3 and value != 5 and value != 12 + }, + } + self.ones_suffixed = {**self.ones_plural, **self.ones_ordinal} + + self.tens = { + "twenty": 20, + "thirty": 30, + "forty": 40, + "fifty": 50, + "sixty": 60, + "seventy": 70, + "eighty": 80, + "ninety": 90, + } + self.tens_plural = {name.replace("y", "ies"): (value, "s") for name, value in self.tens.items()} + self.tens_ordinal = {name.replace("y", "ieth"): (value, "th") for name, value in self.tens.items()} + self.tens_suffixed = {**self.tens_plural, **self.tens_ordinal} + + self.multipliers = { + "hundred": 100, + "thousand": 1_000, + "million": 1_000_000, + "billion": 1_000_000_000, + "trillion": 1_000_000_000_000, + "quadrillion": 1_000_000_000_000_000, + "quintillion": 1_000_000_000_000_000_000, + "sextillion": 1_000_000_000_000_000_000_000, + "septillion": 1_000_000_000_000_000_000_000_000, + "octillion": 1_000_000_000_000_000_000_000_000_000, + "nonillion": 1_000_000_000_000_000_000_000_000_000_000, + "decillion": 1_000_000_000_000_000_000_000_000_000_000_000, + } + self.multipliers_plural = {name + "s": (value, "s") for name, value in self.multipliers.items()} + self.multipliers_ordinal = {name + "th": (value, "th") for name, value in self.multipliers.items()} + self.multipliers_suffixed = {**self.multipliers_plural, **self.multipliers_ordinal} + self.decimals = {*self.ones, *self.tens, *self.zeros} + + self.preceding_prefixers = { + "minus": "-", + "negative": "-", + "plus": "+", + "positive": "+", + } + self.following_prefixers = { + "pound": "£", + "pounds": "£", + "euro": "€", + "euros": "€", + "dollar": "$", + "dollars": "$", + "cent": "¢", + "cents": "¢", + } + self.prefixes = set(list(self.preceding_prefixers.values()) + list(self.following_prefixers.values())) + self.suffixers = { + "per": {"cent": "%"}, + "percent": "%", + } + self.specials = {"and", "double", "triple", "point"} + + self.words = { + key + for mapping in [ + self.zeros, + self.ones, + self.ones_suffixed, + self.tens, + self.tens_suffixed, + self.multipliers, + self.multipliers_suffixed, + self.preceding_prefixers, + self.following_prefixers, + self.suffixers, + self.specials, + ] + for key in mapping + } + self.literal_words = {"one", "ones"} + + def process_words(self, words: list[str]) -> Iterator[str]: + prefix: str | None = None + value: str | int | None = None + skip = False + + def to_fraction(s: str): + try: + return Fraction(s) + except ValueError: + return None + + def output(result: str | int): + nonlocal prefix, value + result = str(result) + if prefix is not None: + result = prefix + result + value = None + prefix = None + return result + + if len(words) == 0: + return + + for i, current in enumerate(words): + prev = words[i - 1] if i != 0 else None + next = words[i + 1] if i != len(words) - 1 else None + if skip: + skip = False + continue + + next_is_numeric = next is not None and re.match(r"^\d+(\.\d+)?$", next) + has_prefix = current[0] in self.prefixes + current_without_prefix = current[1:] if has_prefix else current + if re.match(r"^\d+(\.\d+)?$", current_without_prefix): + # arabic numbers (potentially with signs and fractions) + f = to_fraction(current_without_prefix) + if f is None: + raise ValueError("Converting the fraction failed") + + if value is not None: + if isinstance(value, str) and value.endswith("."): + # concatenate decimals / ip address components + value = str(value) + str(current) + continue + else: + yield output(value) + + prefix = current[0] if has_prefix else prefix + if f.denominator == 1: + value = f.numerator # store integers as int + else: + value = current_without_prefix + elif current not in self.words: + # non-numeric words + if value is not None: + yield output(value) + yield output(current) + elif current in self.zeros: + value = str(value or "") + "0" + elif current in self.ones: + ones = self.ones[current] + + if value is None: + value = ones + elif isinstance(value, str) or prev in self.ones: + if prev in self.tens and ones < 10: # replace the last zero with the digit + value = value[:-1] + str(ones) + else: + value = str(value) + str(ones) + elif ones < 10: + if value % 10 == 0: + value += ones + else: + value = str(value) + str(ones) + else: # eleven to nineteen + if value % 100 == 0: + value += ones + else: + value = str(value) + str(ones) + elif current in self.ones_suffixed: + # ordinal or cardinal; yield the number right away + ones, suffix = self.ones_suffixed[current] + if value is None: + yield output(str(ones) + suffix) + elif isinstance(value, str) or prev in self.ones: + if prev in self.tens and ones < 10: + yield output(value[:-1] + str(ones) + suffix) + else: + yield output(str(value) + str(ones) + suffix) + elif ones < 10: + if value % 10 == 0: + yield output(str(value + ones) + suffix) + else: + yield output(str(value) + str(ones) + suffix) + else: # eleven to nineteen + if value % 100 == 0: + yield output(str(value + ones) + suffix) + else: + yield output(str(value) + str(ones) + suffix) + value = None + elif current in self.tens: + tens = self.tens[current] + if value is None: + value = tens + elif isinstance(value, str): + value = str(value) + str(tens) + else: + if value % 100 == 0: + value += tens + else: + value = str(value) + str(tens) + elif current in self.tens_suffixed: + # ordinal or cardinal; yield the number right away + tens, suffix = self.tens_suffixed[current] + if value is None: + yield output(str(tens) + suffix) + elif isinstance(value, str): + yield output(str(value) + str(tens) + suffix) + else: + if value % 100 == 0: + yield output(str(value + tens) + suffix) + else: + yield output(str(value) + str(tens) + suffix) + elif current in self.multipliers: + multiplier = self.multipliers[current] + if value is None: + value = multiplier + elif isinstance(value, str) or value == 0: + f = to_fraction(value) + p = f * multiplier if f is not None else None + if f is not None and p.denominator == 1: + value = p.numerator + else: + yield output(value) + value = multiplier + else: + before = value // 1000 * 1000 + residual = value % 1000 + value = before + residual * multiplier + elif current in self.multipliers_suffixed: + multiplier, suffix = self.multipliers_suffixed[current] + if value is None: + yield output(str(multiplier) + suffix) + elif isinstance(value, str): + f = to_fraction(value) + p = f * multiplier if f is not None else None + if f is not None and p.denominator == 1: + yield output(str(p.numerator) + suffix) + else: + yield output(value) + yield output(str(multiplier) + suffix) + else: # int + before = value // 1000 * 1000 + residual = value % 1000 + value = before + residual * multiplier + yield output(str(value) + suffix) + value = None + elif current in self.preceding_prefixers: + # apply prefix (positive, minus, etc.) if it precedes a number + if value is not None: + yield output(value) + + if next in self.words or next_is_numeric: + prefix = self.preceding_prefixers[current] + else: + yield output(current) + elif current in self.following_prefixers: + # apply prefix (dollars, cents, etc.) only after a number + if value is not None: + prefix = self.following_prefixers[current] + yield output(value) + else: + yield output(current) + elif current in self.suffixers: + # apply suffix symbols (percent -> '%') + if value is not None: + suffix = self.suffixers[current] + if isinstance(suffix, dict): + if next in suffix: + yield output(str(value) + suffix[next]) + skip = True + else: + yield output(value) + yield output(current) + else: + yield output(str(value) + suffix) + else: + yield output(current) + elif current in self.specials: + if next not in self.words and not next_is_numeric: + # apply special handling only if the next word can be numeric + if value is not None: + yield output(value) + yield output(current) + elif current == "and": + # ignore "and" after hundreds, thousands, etc. + if prev not in self.multipliers: + if value is not None: + yield output(value) + yield output(current) + elif current == "double" or current == "triple": + if next in self.ones or next in self.zeros: + repeats = 2 if current == "double" else 3 + ones = self.ones.get(next, 0) + value = str(value or "") + str(ones) * repeats + skip = True + else: + if value is not None: + yield output(value) + yield output(current) + elif current == "point": + if next in self.decimals or next_is_numeric: + value = str(value or "") + "." + else: + # should all have been covered at this point + raise ValueError(f"Unexpected token: {current}") + else: + # all should have been covered at this point + raise ValueError(f"Unexpected token: {current}") + + if value is not None: + yield output(value) + + def preprocess(self, s: str): + # replace " and a half" with " point five" + results = [] + + segments = re.split(r"\band\s+a\s+half\b", s) + for i, segment in enumerate(segments): + if len(segment.strip()) == 0: + continue + if i == len(segments) - 1: + results.append(segment) + else: + results.append(segment) + last_word = segment.rsplit(maxsplit=2)[-1] + if last_word in self.decimals or last_word in self.multipliers: + results.append("point five") + else: + results.append("and a half") + + s = " ".join(results) + + # put a space at number/letter boundary + s = re.sub(r"([a-z])([0-9])", r"\1 \2", s) + s = re.sub(r"([0-9])([a-z])", r"\1 \2", s) + + # but remove spaces which could be a suffix + s = re.sub(r"([0-9])\s+(st|nd|rd|th|s)\b", r"\1\2", s) + + return s + + def postprocess(self, s: str): + def combine_cents(m: Match): + try: + currency = m.group(1) + integer = m.group(2) + cents = int(m.group(3)) + return f"{currency}{integer}.{cents:02d}" + except ValueError: + return m.string + + def extract_cents(m: Match): + try: + return f"¢{int(m.group(1))}" + except ValueError: + return m.string + + # apply currency postprocessing; "$2 and ¢7" -> "$2.07" + s = re.sub(r"([€£$])([0-9]+) (?:and )?¢([0-9]{1,2})\b", combine_cents, s) + s = re.sub(r"[€£$]0.([0-9]{1,2})\b", extract_cents, s) + + # write "one(s)" instead of "1(s)", just for the readability + s = re.sub(r"\b1(s?)\b", r"one\1", s) + + return s + + def __call__(self, s: str): + s = self.preprocess(s) + s = " ".join(word for word in self.process_words(s.split()) if word is not None) + s = self.postprocess(s) + + return s + + +class EnglishSpellingNormalizer: + """ + Applies British-American spelling mappings as listed in [1]. + + [1] https://www.tysto.com/uk-us-spelling-list.html + """ + + def __init__(self, english_spelling_mapping): + self.mapping = english_spelling_mapping + + def __call__(self, s: str): + return " ".join(self.mapping.get(word, word) for word in s.split()) + + +class EnglishTextNormalizer: + def __init__(self, english_spelling_mapping): + self.ignore_patterns = r"\b(hmm|mm|mhm|mmm|uh|um)\b" + self.replacers = { + # common contractions + r"\bwon't\b": "will not", + r"\bcan't\b": "can not", + r"\blet's\b": "let us", + r"\bain't\b": "aint", + r"\by'all\b": "you all", + r"\bwanna\b": "want to", + r"\bgotta\b": "got to", + r"\bgonna\b": "going to", + r"\bi'ma\b": "i am going to", + r"\bimma\b": "i am going to", + r"\bwoulda\b": "would have", + r"\bcoulda\b": "could have", + r"\bshoulda\b": "should have", + r"\bma'am\b": "madam", + # contractions in titles/prefixes + r"\bmr\b": "mister ", + r"\bmrs\b": "missus ", + r"\bst\b": "saint ", + r"\bdr\b": "doctor ", + r"\bprof\b": "professor ", + r"\bcapt\b": "captain ", + r"\bgov\b": "governor ", + r"\bald\b": "alderman ", + r"\bgen\b": "general ", + r"\bsen\b": "senator ", + r"\brep\b": "representative ", + r"\bpres\b": "president ", + r"\brev\b": "reverend ", + r"\bhon\b": "honorable ", + r"\basst\b": "assistant ", + r"\bassoc\b": "associate ", + r"\blt\b": "lieutenant ", + r"\bcol\b": "colonel ", + r"\bjr\b": "junior ", + r"\bsr\b": "senior ", + r"\besq\b": "esquire ", + # prefect tenses, ideally it should be any past participles, but it's harder.. + r"'d been\b": " had been", + r"'s been\b": " has been", + r"'d gone\b": " had gone", + r"'s gone\b": " has gone", + r"'d done\b": " had done", # "'s done" is ambiguous + r"'s got\b": " has got", + # general contractions + r"n't\b": " not", + r"'re\b": " are", + r"'s\b": " is", + r"'d\b": " would", + r"'ll\b": " will", + r"'t\b": " not", + r"'ve\b": " have", + r"'m\b": " am", + } + self.standardize_numbers = EnglishNumberNormalizer() + self.standardize_spellings = EnglishSpellingNormalizer(english_spelling_mapping) + + def __call__(self, s: str): + s = s.lower() + + s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets + s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis + s = re.sub(self.ignore_patterns, "", s) + s = re.sub(r"\s+'", "'", s) # standardize when there's a space before an apostrophe + + for pattern, replacement in self.replacers.items(): + s = re.sub(pattern, replacement, s) + + s = re.sub(r"(\d),(\d)", r"\1\2", s) # remove commas between digits + s = re.sub(r"\.([^0-9]|$)", r" \1", s) # remove periods not followed by numbers + s = remove_symbols_and_diacritics(s, keep=".%$¢€£") # keep some symbols for numerics + + s = self.standardize_numbers(s) + s = self.standardize_spellings(s) + + # now remove prefix/suffix symbols that are not preceded/followed by numbers + s = re.sub(r"[.$¢€£]([^0-9])", r" \1", s) + s = re.sub(r"([^0-9])%", r"\1 ", s) + + s = re.sub(r"\s+", " ", s) # replace any successive whitespace characters with a space + + return s diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/whisper/generation_whisper.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/whisper/generation_whisper.py new file mode 100644 index 0000000000000000000000000000000000000000..1f9c9843d34a3887c1f75cef092b53b8a8365f3a --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/whisper/generation_whisper.py @@ -0,0 +1,2073 @@ +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy +import math +import zlib +from collections.abc import Callable, Iterator + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from transformers.cache_utils import EncoderDecoderCache + +from ...generation import GenerationConfig, GenerationMixin +from ...generation.logits_process import ( + LogitsProcessorList, + SuppressTokensAtBeginLogitsProcessor, + SuppressTokensLogitsProcessor, + WhisperNoSpeechDetection, + WhisperTimeStampLogitsProcessor, +) +from ...generation.stopping_criteria import StoppingCriteriaList +from ...modeling_outputs import BaseModelOutput +from ...utils import logging +from .tokenization_whisper import TASK_IDS, TO_LANGUAGE_CODE + + +logger = logging.get_logger(__name__) + + +def _median_filter(inputs: torch.Tensor, filter_width: int) -> torch.Tensor: + """ + Applies a median filter of width `filter_width` along the last dimension of the input. + + The `inputs` tensor is assumed to be 3- or 4-dimensional. + """ + if filter_width <= 0 or filter_width % 2 != 1: + raise ValueError("`filter_width` should be an odd number") + + pad_width = filter_width // 2 + if inputs.shape[-1] <= pad_width: + return inputs + + # Pad the left and right edges. + inputs = nn.functional.pad(inputs, (pad_width, pad_width, 0, 0), mode="reflect") + + # sort() is faster than torch.median (https://github.com/pytorch/pytorch/issues/51450) + result = inputs.unfold(-1, filter_width, 1).sort()[0][..., pad_width] + return result + + +def _dynamic_time_warping(matrix: np.ndarray): + """ + Measures similarity between two temporal sequences: the input audio and the output tokens. Used to generate + token-level timestamps. + """ + output_length, input_length = matrix.shape + cost = np.ones((output_length + 1, input_length + 1), dtype=np.float32) * np.inf + trace = -np.ones((output_length + 1, input_length + 1), dtype=np.float32) + + cost[0, 0] = 0 + for j in range(1, input_length + 1): + for i in range(1, output_length + 1): + c0 = cost[i - 1, j - 1] + c1 = cost[i - 1, j] + c2 = cost[i, j - 1] + + if c0 < c1 and c0 < c2: + c, t = c0, 0 + elif c1 < c0 and c1 < c2: + c, t = c1, 1 + else: + c, t = c2, 2 + + cost[i, j] = matrix[i - 1, j - 1] + c + trace[i, j] = t + + # backtrace + i = trace.shape[0] - 1 + j = trace.shape[1] - 1 + trace[0, :] = 2 + trace[:, 0] = 1 + + text_indices = [] + time_indices = [] + while i > 0 or j > 0: + text_indices.append(i - 1) + time_indices.append(j - 1) + if trace[i, j] == 0: + i -= 1 + j -= 1 + elif trace[i, j] == 1: + i -= 1 + elif trace[i, j] == 2: + j -= 1 + else: + raise RuntimeError( + f"Internal error in dynamic time warping. Unexpected trace[{i}, {j}]. Please file a bug report." + ) + + text_indices = np.array(text_indices)[::-1] + time_indices = np.array(time_indices)[::-1] + return text_indices, time_indices + + +def _get_attr_from_logit_processors(logits_processor, logit_processor_class, attribute_name): + if logits_processor is not None: + logit_processor = next((cls for cls in logits_processor if isinstance(cls, logit_processor_class)), None) + if logit_processor: + return getattr(logit_processor, attribute_name, None) + return None + + +def _pad_to_max_length( + current_segments, + pad_token_id, + device, + padding_side="right", + padding="longest", + bos_token_tensor=None, + cut_off_length=None, + return_token_timestamps=False, + force_unique_generate_call=False, + skip_ending_double_timestamps=False, + timestamp_begin=None, +): + """ + skip_ending_double_timestamps: when the segment ended with two timestamp tokens, whether to ignore the last timestamp token + see https://github.com/huggingface/transformers/pull/35750 + + _pad_to_max_length is used in different contexts: + 1. At the end of generation: we need to keep both ending timestamp tokens in the segment (see https://github.com/huggingface/transformers/pull/34537). + 2. In the middle of generation, e.g. when condition_on_prev_tokens=True and we want to use the last generated tokens as decoder_input_ids: + we must skip one of the double ending timestamp tokens (see https://github.com/huggingface/transformers/pull/35750). + """ + max_total_length = 0 + sequences = [] + token_timestamps_list = [] + + if padding_side not in ["right", "left"]: + raise ValueError(f"`padding_side` must be either 'right' or 'left', not {padding_side}") + + if padding not in ["longest", "max_length"]: + raise ValueError(f"`padding` must be either 'longest' or 'max_length', not {padding}") + elif padding == "max_length" and cut_off_length is None: + raise ValueError("`cut_off_length` must be specified when `padding='max_length'`") + + if force_unique_generate_call: + sequences_list = [] + timestamps_list = [] + for segments in current_segments: + result = segments[0]["result"] + sequences_list.append(result if isinstance(result, torch.Tensor) else result["sequences"]) + if return_token_timestamps: + timestamps_list.append(result["token_timestamps"]) + + sequences = torch.stack(sequences_list, dim=0) + if return_token_timestamps: + token_timestamps = torch.stack(timestamps_list, dim=0) + return sequences, token_timestamps + return sequences + + for current_segment_list in current_segments: + if current_segment_list is not None and len([d["tokens"] for d in current_segment_list]) > 0: + sequences_list = [] + for d in current_segment_list: + if skip_ending_double_timestamps and len(d["tokens"]) > 2 and d["tokens"][-2] >= timestamp_begin: + # the segment finishes with two timestamp tokens + # we need to ignore the last timestamp token + # see https://github.com/huggingface/transformers/pull/34537 + sequences_list.append(d["tokens"][:-1]) + else: + sequences_list.append(d["tokens"]) + sequence = torch.cat(sequences_list, dim=-1) + + if return_token_timestamps: + token_timestamps = torch.cat( + [d["result"]["token_timestamps"][d["idxs"][0] : d["idxs"][1]] for d in current_segment_list], + dim=-1, + ) + + if cut_off_length is not None: + sequence = sequence[-cut_off_length:] + if return_token_timestamps: + token_timestamps = token_timestamps[-cut_off_length:] + + if bos_token_tensor is not None: + sequence = torch.cat([bos_token_tensor, sequence]) + if return_token_timestamps: + token_timestamps = torch.cat( + [torch.ones_like(bos_token_tensor, device=device) * 0.0, token_timestamps] + ) + sequences.append(sequence) + if return_token_timestamps: + token_timestamps_list.append(token_timestamps) + max_total_length = max(max_total_length, len(sequences[-1])) + elif bos_token_tensor is not None: + sequences.append(bos_token_tensor) + if return_token_timestamps: + token_timestamps_list.append(torch.ones_like(bos_token_tensor, device=device) * 0.0) + else: + sequences.append(torch.tensor([], device=device)) + if return_token_timestamps: + token_timestamps_list.append(torch.tensor([], device=device)) + + max_total_length = cut_off_length + 1 if padding == "max_length" else max_total_length + for i in range(len(current_segments)): + pad_length = max_total_length - len(sequences[i]) + pad = (0, pad_length) if padding_side == "right" else (pad_length, 0) + + sequences[i] = F.pad(sequences[i], pad=pad, value=pad_token_id) + if return_token_timestamps: + token_timestamps_list[i] = F.pad( + token_timestamps_list[i], + pad=pad, + value=token_timestamps_list[i][-1] if len(token_timestamps_list[i]) > 0 else 0.0, + ) + + sequences = torch.stack(sequences, dim=0) + + if return_token_timestamps: + token_timestamps = torch.stack(token_timestamps_list, dim=0) + return sequences, token_timestamps + else: + return sequences + + +class WhisperGenerationMixin(GenerationMixin): + def _extract_token_timestamps( + self, generate_outputs, alignment_heads, time_precision=0.02, num_frames=None, num_input_ids=None + ): + """ + Calculates token-level timestamps using the encoder-decoder cross-attentions and dynamic time-warping (DTW) to + map each output token to a position in the input audio. If `num_frames` is specified, the encoder-decoder + cross-attentions will be cropped before applying DTW. + + Returns: + tensor containing the timestamps in seconds for each predicted token + """ + # Create a list with `decoder_layers` elements, each a tensor of shape + # (batch size * num beams, attention_heads, output length, input length). + cross_attentions = [] + for i in range(self.config.decoder_layers): + cross_attentions.append(torch.cat([x[i] for x in generate_outputs.cross_attentions], dim=2)) + + # Select specific cross-attention layers and heads. This is a tensor + # of shape (batch size * num beams, num selected heads, output length, input length). + weights = torch.stack([cross_attentions[l][:, h] for l, h in alignment_heads]) + weights = weights.permute([1, 0, 2, 3]) + + weight_length = None + + if "beam_indices" in generate_outputs: + # If beam search was used, the sequence length of the outputs may not be the real sequence length: + # beam search may end up returning a sequence that finished a few steps earlier while decoding. + # In that case, the `cross_attentions` weights are too long and we have to make sure that they have + # the right `output_length` + + # get the real sequence length of the longest sequence, crop the beam_indices to the real length + weight_length = (generate_outputs.beam_indices != -1).sum(-1).max() + beam_indices = generate_outputs.beam_indices[:, :weight_length] + + # The first forward pass (prefill) may have processed more than one token and, therefore, contain + # cross-attention weights for several tokens. + # Let's unroll the first `beam_indices` accordingly, so we can use it to gather the weights. + if num_input_ids is not None and num_input_ids > 1: + # `-1`: `beam_indices` can be used as-is to gather the weights when `num_input_ids` is 1 + weight_length += num_input_ids - 1 + beam_indices_first_step_unrolled = ( + torch.ones(beam_indices.shape[0], num_input_ids - 1, device=beam_indices.device, dtype=torch.long) + * (beam_indices[:, 0:1]) + ) + unrolled_beam_indices = torch.cat([beam_indices_first_step_unrolled, beam_indices], dim=-1) + else: + unrolled_beam_indices = beam_indices + + # If beam index is still -1, it means that the associated token id is EOS + # We need to replace the index with 0 since index_select gives an error if any of the indexes is -1. + unrolled_beam_indices = unrolled_beam_indices.masked_fill(unrolled_beam_indices == -1, 0) + + # Select the cross attention from the right beam for each output sequence, up to the real sequence + # length (`weight_length`) + weights = torch.stack( + [ + torch.index_select(weights[:, :, i, :], dim=0, index=unrolled_beam_indices[:, i]) + for i in range(unrolled_beam_indices.shape[1]) + ], + dim=2, + ) + + # make sure timestamps are as long as weights + input_length = weight_length or cross_attentions[0].shape[2] + batch_size = generate_outputs.sequences.shape[0] + timestamps = torch.zeros( + (batch_size, input_length + 1), dtype=torch.float32, device=generate_outputs.sequences.device + ) + + if num_frames is not None: + # two cases: + # 1. num_frames is the same for each sample -> compute the DTW matrix for each sample in parallel + # 2. num_frames is different, compute the DTW matrix for each sample sequentially + + # we're using np.unique because num_frames can be int/list/tuple + if isinstance(num_frames, int): + weights = weights[..., : num_frames // 2] + + elif isinstance(num_frames, (list, tuple, np.ndarray)) and len(np.unique(num_frames)) == 1: + weights = weights[..., : num_frames[0] // 2] + + elif isinstance(num_frames, (torch.Tensor)) and len(torch.unique(num_frames)) == 1: + weights = weights[..., : num_frames[0] // 2] + + else: + # num_frames is of shape (batch_size,) whereas batch_size is truly batch_size*num_return_sequences + repeat_time = batch_size if isinstance(num_frames, int) else batch_size // len(num_frames) + num_frames = num_frames.cpu() if isinstance(num_frames, (torch.Tensor)) else num_frames + num_frames = np.repeat(num_frames, repeat_time) + + # let's ignore decoder_input_ids that can negatively impact the DTW while we know they have timestamps 0.0s + # (they are not taken into account for the DTW in OAI implementation) + if num_input_ids is not None: + weights = weights[:, :, num_input_ids:, :] + + # Since we ignore `decoder_input_ids` in the DTW and in the case where we generated only one token (for which we don't have cross attentions, see below comments), + # the DTW sequence length is 0 and we should return only 0.0s for the token timestamps + if weights.shape[2] == 0: + return timestamps + + if num_frames is None or isinstance(num_frames, int): + # Normalize and smoothen the weights. + std = torch.std(weights, dim=-2, keepdim=True, unbiased=False) + mean = torch.mean(weights, dim=-2, keepdim=True) + weights = (weights - mean) / std + weights = _median_filter(weights, self.config.median_filter_width) + + # Average the different cross-attention heads. + weights = weights.mean(dim=1) + + # Perform dynamic time warping on each element of the batch. + for batch_idx in range(batch_size): + if num_frames is not None and isinstance(num_frames, (tuple, list, np.ndarray, torch.Tensor)): + matrix = weights[batch_idx, ..., : num_frames[batch_idx] // 2] + + # Normalize and smoothen the weights. + std = torch.std(matrix, dim=-2, keepdim=True, unbiased=False) + mean = torch.mean(matrix, dim=-2, keepdim=True) + matrix = (matrix - mean) / std + matrix = _median_filter(matrix, self.config.median_filter_width) + + # Average the different cross-attention heads. + matrix = matrix.mean(dim=0) + else: + matrix = weights[batch_idx] + + text_indices, time_indices = _dynamic_time_warping(-matrix.cpu().double().numpy()) + jumps = np.pad(np.diff(text_indices), (1, 0), constant_values=1).astype(bool) + jump_times = time_indices[jumps] * time_precision + + # each predicted token has a corresponding timestamp, expect the eos token (or last predicted token) for which we don't retrieve cross attentions + # (indeed contrary to OAI that re-run a full forward to retrieve cross attentions for each token and therefore also the last one predicted, we retrieve + # cross attentions directly from the auto-regressive generation, so we don't have cross attentiosn for the token at the end of the sequence. Nevertheless, + # that is not important since we expect this last token to be the eos token) + # 1. for decoder_input_ids, we set the timestamps to 0.0 + # 2. for the eos token (or last predicted token), we simply duplicate the timestamp of the last non-eos token + timestamps[batch_idx] = torch.cat( + [torch.zeros(num_input_ids), torch.tensor(jump_times), torch.tensor([jump_times[-1]])] + ) + + return timestamps + + def generate( + self, + input_features: torch.Tensor | None = None, + generation_config: GenerationConfig | None = None, + logits_processor: LogitsProcessorList | None = None, + stopping_criteria: StoppingCriteriaList | None = None, + prefix_allowed_tokens_fn: Callable[[int, torch.Tensor], list[int]] | None = None, + synced_gpus: bool = False, + return_timestamps: bool | None = None, + task: str | None = None, + language: str | list[str] | None = None, + is_multilingual: bool | None = None, + prompt_ids: torch.Tensor | None = None, + prompt_condition_type: str | None = None, # first-segment, all-segments + condition_on_prev_tokens: bool | None = None, + temperature: float | tuple[float, ...] | None = None, + compression_ratio_threshold: float | None = None, + logprob_threshold: float | None = None, + no_speech_threshold: float | None = None, + num_segment_frames: int | None = None, + attention_mask: torch.Tensor | None = None, + time_precision: float = 0.02, + time_precision_features: float = 0.01, + return_token_timestamps: bool | None = None, + return_segments: bool = False, + return_dict_in_generate: bool | None = None, + force_unique_generate_call: bool | None = None, + monitor_progress: Callable[[torch.Tensor], None] | None = None, + **kwargs, + ): + """ + Transcribes or translates log-mel input features to a sequence of auto-regressively generated token ids. + + + + Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the + model's default generation configuration. You can override any `generation_config` by passing the corresponding + parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`. + + For an overview of generation strategies and code examples, check out the [following + guide](../generation_strategies). + + + + Parameters: + input_features (`torch.Tensor` of shape `(batch_size, feature_size, sequence_length)`, *optional*): + Float values of log-mel features extracted from the raw speech waveform. The raw speech waveform can be obtained by + loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, + *e.g.* via the torchcodec library (`pip install torchcodec`) or the soundfile library (`pip install soundfile`). + To prepare the array into `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel + features, padding and conversion into a tensor of type `torch.FloatTensor`. + See [`~WhisperFeatureExtractor.__call__`] for details. + generation_config ([`~generation.GenerationConfig`], *optional*): + The generation configuration to be used as base parametrization for the generation call. `**kwargs` + passed to generate matching the attributes of `generation_config` will override them. If + `generation_config` is not provided, the default will be used, which had the following loading + priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model + configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s + default values, whose documentation should be checked to parameterize generation. + logits_processor (`LogitsProcessorList`, *optional*): + Custom logits processors that complement the default logits processors built from arguments and + generation config. If a logit processor is passed that is already created with the arguments or a + generation config an error is thrown. This feature is intended for advanced users. + stopping_criteria (`StoppingCriteriaList`, *optional*): + Custom stopping criteria that complement the default stopping criteria built from arguments and a + generation config. If a stopping criteria is passed that is already created with the arguments or a + generation config an error is thrown. This feature is intended for advanced users. + prefix_allowed_tokens_fn (`Callable[[int, torch.Tensor], list[int]]`, *optional*): + If provided, this function constraints the beam search to allowed tokens only at each step. If not + provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and + `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned + on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful + for constrained generation conditioned on the prefix, as described in [Autoregressive Entity + Retrieval](https://huggingface.co/papers/2010.00904). + synced_gpus (`bool`, *optional*, defaults to `False`): + Whether to continue running the while loop until max_length (needed to avoid deadlocking with + `FullyShardedDataParallel` and DeepSpeed ZeRO Stage 3). + return_timestamps (`bool`, *optional*): + Whether to return the timestamps with the text. This enables the `WhisperTimestampsLogitsProcessor`. + For audios longer than 30 seconds, it is necessary to set `return_timestamps=True`. + task (`str`, *optional*): + Task to use for generation, either "translate" or "transcribe". + language (`str` or list of `str`, *optional*): + Language token to use for generation, can be either in the form of `<|en|>`, `en` or `english`. For + batched generation, a list of language tokens can be passed. You can find all the possible language + tokens in the `model.generation_config.lang_to_id` dictionary. + is_multilingual (`bool`, *optional*): + Whether or not the model is multilingual. + prompt_ids (`torch.Tensor`, *optional*): + Rank-1 tensor of token IDs created by passing text to [`~WhisperProcessor.get_prompt_ids`] that is + provided as a prompt to each chunk. This can be used to provide or "prompt-engineer" a context for + transcription, e.g. custom vocabularies or proper nouns to make it more likely to predict those words + correctly. It cannot be used in conjunction with `decoder_start_token_id` as it overwrites this value. + prompt_condition_type (`str`, *optional*): + Only relevant for long-form transcription. Condition type of `prompt_ids`. 'first-segment' means only the first segment is conditioned on `prompt_ids`. 'all-segments' means each segment is conditioned on `prompt_ids`. Make sure to enable `condition_on_prev_tokens` for 'all-segments'. + Defaults to 'first-segment'. For short-term transcription only 'first-segment' is possible. + condition_on_prev_tokens (`bool`, *optional*): + Only relevant for long-form transcription. Whether to condition each segment on the previous segment. + As shown in the [the Whisper paper](https://cdn.openai.com/papers/whisper.pdf), this can help to improve + performance. + temperature (`float` or list of `float`, *optional*): + The temperature to be used for generation. Passing a single `float` value and `do_sample=True` activates + generation using sampling. For long-form transcription, temperature fallback can be activated by passing + a list of float values such as (0.0, 0.2, 0.4, 0.6, 0.8, 1.0). As shown in the [the Whisper paper](https://cdn.openai.com/papers/whisper.pdf), this can help to improve + performance. + compression_ratio_threshold (`float`, *optional*): + Only relevant for long-form transcription. If defined, the zlib compression rate of each segment will be computed. If the compression rate of + a segment is higher than `compression_ratio_threshold`, temperature fallback is activated: the generated segment is discarded and the generation is + repeated using a higher temperature. The intuition behind this feature is that segments with very high compression rates + suffer from a lot of repetition. The unwanted repetition can be reduced by injecting more randomness by increasing the temperature. If `compression_ratio_threshold` is defined + make sure that `temperature` is a list of values. A common value for `compression_ratio_threshold` is 1.35. + As shown in the [the Whisper paper](https://cdn.openai.com/papers/whisper.pdf), this can help to improve + performance. + logprob_threshold (`float`, *optional*): + Only relevant for long-form transcription. If defined, the average log-probability of each segment will be computed. If the log-probability of + a given segment is lower than `logprob_threshold`, temperature fallback is activated: the generated segment is discarded and the generation is + repeated using a higher temperature. The intuition behind this feature is that segments of low log-probability + can be improved by injecting more randomness by increasing the temperature. If `logprob_threshold` is defined + make sure that `temperature` is a list of values. A common value for `logprob_threshold` is -1.0. + As shown in the [the Whisper paper](https://cdn.openai.com/papers/whisper.pdf), this can help to improve + performance. + no_speech_threshold (`float`, *optional*): + Only relevant for long-form transcription. If defined, the "no-speech" token combined with the `logprob_threshold` + is used to determine whether a segment contains only silence. In this case, the transcription for this segment + is skipped. + As shown in the [the Whisper paper](https://cdn.openai.com/papers/whisper.pdf), this can help to improve + performance. + num_segment_frames (`int`, *optional*): + The number of frames a single segment is made of. If not defined, `num_segment_frames` defaults to the model's stride + times the maximum input length. + attention_mask (`torch.Tensor`, *optional*): + `attention_mask` needs to be passed when doing long-form transcription using a batch size > 1. + time_precision (`int`, *optional*, defaults to 0.02): + The duration of output token in seconds. *E.g.* 0.02 means that a generated token on average accounts + for 20 ms. + time_precision_features (`int`, *optional*, defaults to 0.01): + The duration represented by a feature frame in seconds. + return_token_timestamps (`bool`, *optional*): + Whether to return token-level timestamps with the text. This can be used with or without the + `return_timestamps` option. To get word-level timestamps, use the tokenizer to group the tokens into + words. + return_segments (`bool`, *optional*, defaults to `False`): + Whether to additionally return a list of all segments. Note that this option can only be enabled + when doing long-form transcription. + return_dict_in_generate (`bool`, *optional*, defaults to `False`): + Whether or not to return a [`~utils.ModelOutput`] instead of just returning the generated tokens. + Note that when doing long-form transcription, `return_dict_in_generate` can only be enabled when + `return_segments` is set True. In this case the generation outputs of each segment is added to each + segment. + force_unique_generate_call (`bool`, *optional*): + Whether to force a unique call to the underlying GenerationMixin's [`~generation.GenerationMixin.generate`] method. This is useful for assisted decoding and testing purposes to ensure + that only one call to [`~generation.GenerationMixin.generate`] is made and therefore decoder input token ids and eos token ids are returned. + monitor_progress (`Callable[[torch.Tensor], None]`, *optional*): + If provided, this function can be called to report the progress of the audio transcription. The function + takes a tensor argument `p` of shape `(n, 2)`, where `n` is the batch size. `p[i, 0]` contains the + index of the audio frame that is currently being transcribed for batch item `i`. `p[i, 1]` contains + the total number of frames for batch item `i`. No return value is expected. + kwargs (`dict[str, Any]`, *optional*): + Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be + forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder + specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*. + Return: + [`~utils.ModelOutput`] or `dict[str, Any]` or `torch.LongTensor`: + + One of the following: + - [`~utils.ModelOutput`] when `return_dict_in_generate=True` and (`return_timestamps=False` or `force_unique_generate_call=True`), including the decoder input ids and end of sequence id. + - `dict[str, Any]` when (`return_dict_in_generate=True` and `return_timestamps=True`) or `return_segments=True` or `return_token_timestamps=True`. + - `torch.LongTensor` in all other cases, excluding the decoder input ids and end of sequence id. + + The possible [`~utils.ModelOutput`] types are: + - [`~generation.GenerateEncoderDecoderOutput`] + - [`~generation.GenerateBeamEncoderDecoderOutput`] + + `segments` is a list of lists (one list per batch element) of `segment`. + A `segment` is a dictionary with keys `start`, `end`, `tokens`, `idxs`, and `result`. + - `start`: the start timestamp of the segment. + - `end`: the end timestamp of the segment. + - `tokens`: the tokens of the segment, excluding the decoder input ids and end of sequence id. + - `idxs`: the start (included) and end (excluded) indices of the `tokens` of the segment in the underlying call to GenerationMixin's [`~generation.GenerationMixin.generate`] (present in `result`). + - `result`: the result of the underlying call to GenerationMixin's [`~generation.GenerationMixin.generate`]. + + When `return_timestamps=True`, `return_dict_in_generate=True` applies to each call of the underlying GenerationMixin's [`~generation.GenerationMixin.generate`], with outputs stored in `result` of each `segment`. + + Example: + + - *Longform transcription*: To transcribe or translate audios longer than 30 seconds, process the audio files without truncation and pass all mel features at once to generate. It is necessary to set `return_timestamps=True`. + Indeed, long-form transcription uses a sequential algorithm based on timestamps predictions, with heuristics like compression ratio threshold, log probability threshold and temperature fallback. This algorithm is described in the [the Whisper original paper](https://cdn.openai.com/papers/whisper.pdf), section *3.8. Long-form Transcription*. + + ```python + >>> import torch + >>> from transformers import AutoProcessor, WhisperForConditionalGeneration + >>> from datasets import load_dataset, Audio + + >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en") + >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en") + >>> model.cuda() # doctest: +IGNORE_RESULT + + >>> # load audios > 30 seconds + >>> ds = load_dataset("distil-whisper/meanwhile", "default")["test"] + >>> # resample to 16kHz + >>> ds = ds.cast_column("audio", Audio(sampling_rate=16000)) + >>> # take first 8 audios and retrieve array + >>> audio = ds[:8]["audio"] + >>> audio = [x["array"] for x in audio] + + >>> # make sure to NOT truncate the input audio, to return the `attention_mask` and to pad to the longest audio + >>> inputs = processor(audio, return_tensors="pt", truncation=False, padding="longest", return_attention_mask=True, sampling_rate=16_000) + >>> inputs = inputs.to("cuda", torch.float32) + + >>> # transcribe audio to ids + >>> generated_ids = model.generate(**inputs, return_timestamps=True) + + >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True) + >>> transcription[0] + " Folks, if you watch the show, you know, I spent a lot of time right over there. Patiently and astutely scrutinizing the boxwood and mahogany chest set of the day's biggest stories developing the central headline pawns, definitely maneuvering an oso topical night to F6, fainting a classic Sicilian, nade door variation on the news, all the while seeing eight moves deep and patiently marshalling the latest press releases into a fisher's shows in Lip Nitsky attack that culminates in the elegant lethal slow-played, all-passant checkmate that is my nightly monologue. But sometimes, sometimes, folks, I. CHEERING AND APPLAUSE Sometimes I startle away, cubside down in the monkey bars of a condemned playground on a super fun site. Get all hept up on goofballs. Rummage that were discarded tag bag of defective toys. Yank out a fist bowl of disembodied doll limbs, toss them on a stained kid's place mat from a defunct dennies. set up a table inside a rusty cargo container down by the Wharf and challenged toothless drifters to the godless bughouse blitz of tournament that is my segment. Meanwhile." + ``` + + The `monitor_progress` callback can be used to monitor the progress of the transcription: + ```python + >>> from tqdm import tqdm + + >>> # prepare inputs like above + + >>> # define a callback to monitor the progress of the transcription. + >>> with tqdm(desc="Progress") as pbar: + >>> def monitor_progress(p_batch): + >>> i = torch.argmax(p_batch[:, 1]) + >>> p = p_batch[i].detach().cpu() + >>> pbar.total = int(p[1]) + >>> pbar.n = int(p[0]) + >>> pbar.update() + + >>> # transcribe audio to ids + >>> generated_ids = model.generate(**inputs, return_timestamps=True, monitor_progress=monitor_progress) + + >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True) + >>> transcription[0] + Progress: 95%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 8497/8901 [00:04<00:00, 2052.79it/s] + " Folks, if you watch the show, you know, I spent a lot of time right over there. Patiently and astutely scrutinizing the boxwood and mahogany chest set of the day's biggest stories developing the central headline pawns, definitely maneuvering an oso topical night to F6, fainting a classic Sicilian, nade door variation on the news, all the while seeing eight moves deep and patiently marshalling the latest press releases into a fisher's shows in Lip Nitsky attack that culminates in the elegant lethal slow-played, all-passant checkmate that is my nightly monologue. But sometimes, sometimes, folks, I. CHEERING AND APPLAUSE Sometimes I startle away, cubside down in the monkey bars of a condemned playground on a super fun site. Get all hept up on goofballs. Rummage that were discarded tag bag of defective toys. Yank out a fist bowl of disembodied doll limbs, toss them on a stained kid's place mat from a defunct dennies. set up a table inside a rusty cargo container down by the Wharf and challenged toothless drifters to the godless bughouse blitz of tournament that is my segment. Meanwhile." + ``` + + - *Shortform transcription*: If passed mel input features are <= 30 seconds, there are two possibilities: + - `return_timestamps=False`: the whole audio will be transcribed with a single call to GenerationMixin's [`~generation.GenerationMixin.generate`]. + - `return_timestamps=True`: the audio will be transcribed using the same logic as long-form transcription. + + ```python + >>> import torch + >>> from transformers import AutoProcessor, WhisperForConditionalGeneration + >>> from datasets import load_dataset + + >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en") + >>> model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en") + + >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") + + >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt") + >>> input_features = inputs.input_features + + >>> generated_ids = model.generate(inputs=input_features) + + >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + >>> transcription + ' Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.' + ``` + + """ + # 1. prepare generation config + generation_config, kwargs = self._prepare_generation_config(generation_config, **kwargs) + + # 2. set global generate variables + input_stride = self.model.encoder.conv1.stride[0] * self.model.encoder.conv2.stride[0] + num_segment_frames = input_stride * self.config.max_source_positions + batch_size, total_input_frames = self._retrieve_total_input_frames( + input_features=input_features, input_stride=input_stride, kwargs=kwargs + ) + is_shortform = total_input_frames <= num_segment_frames + + # 3. Make sure generation config is correctly set + # Make sure the generation config is correctly set depending on whether timestamps are to be returned or not + return_dict_in_generate = self._set_return_outputs( + return_dict_in_generate=return_dict_in_generate, + return_token_timestamps=return_token_timestamps, + logprob_threshold=logprob_threshold, + generation_config=generation_config, + ) + timestamp_begin = self._set_return_timestamps( + return_timestamps=return_timestamps, is_shortform=is_shortform, generation_config=generation_config + ) + self._set_language_and_task( + language=language, task=task, is_multilingual=is_multilingual, generation_config=generation_config + ) + self._set_num_frames( + return_token_timestamps=return_token_timestamps, + generation_config=generation_config, + attention_mask=attention_mask, + kwargs=kwargs, + ) + self._set_thresholds_and_condition( + generation_config=generation_config, + logprob_threshold=logprob_threshold, + compression_ratio_threshold=compression_ratio_threshold, + no_speech_threshold=no_speech_threshold, + condition_on_prev_tokens=condition_on_prev_tokens, + ) + self._set_prompt_condition_type( + generation_config=generation_config, + prompt_condition_type=prompt_condition_type, + ) + + # pass self.config for backward compatibility + init_tokens = self._retrieve_init_tokens( + input_features, + batch_size=batch_size, + generation_config=generation_config, + config=self.config, + num_segment_frames=num_segment_frames, + kwargs=kwargs, + ) + # passing `decoder_input_ids` is deprecated - the only exception is for assisted generation + # where the input ids are handled explicitly by the generate method + self._check_decoder_input_ids(kwargs=kwargs) + # `output_attentions` is deprecated - we force eager attention if this feature is + # indirectly requested, e.g. through return_token_timestamps + if return_token_timestamps: + self.model.config._attn_implementation = "eager" + + # 3. Retrieve logits processors + device = kwargs["encoder_outputs"][0].device if "encoder_outputs" in kwargs else input_features.device + begin_index = init_tokens.shape[1] + num_beams = kwargs.get( + "num_beams", + generation_config.num_beams + if hasattr(generation_config, "num_beams") and generation_config.num_beams is not None + else 1, + ) + if "assistant_model" in kwargs: + # speculative decoding: the model should be able to return eos token + generation_config.begin_suppress_tokens = None + + logits_processor = self._retrieve_logit_processors( + generation_config=generation_config, + logits_processor=logits_processor, + begin_index=begin_index, # begin index is index of first generated decoder token + num_beams=num_beams, + device=device, + ) + + # 4 Set and retrieve global generation variables + self._set_condition_on_prev_tokens( + condition_on_prev_tokens=condition_on_prev_tokens, generation_config=generation_config + ) + + temperatures = [temperature] if not isinstance(temperature, (list, tuple)) else temperature + temperature = temperatures[0] + + max_frames, seek = self._retrieve_max_frames_and_seek( + batch_size=batch_size, + attention_mask=attention_mask, + total_input_frames=total_input_frames, + is_shortform=is_shortform, + ) + + # 5 Prepare running variables, list for generation + num_return_sequences = generation_config.num_return_sequences + ( + batch_idx_map, + cur_bsz, + input_features, + seek, + max_frames, + init_tokens, + do_condition_on_prev_tokens, + ) = self._expand_variables_for_generation( + input_features=input_features, + seek=seek, + max_frames=max_frames, + init_tokens=init_tokens, + batch_size=batch_size, + condition_on_prev_tokens=condition_on_prev_tokens, + generation_config=generation_config, + ) + + current_segments = self._prepare_segments( + prompt_ids=prompt_ids, + batch_size=cur_bsz, + generation_config=generation_config, + ) + # 5bis speculative decoding: ensure the assistant model does only one call to generate and therefore returns decoder input token ids and eos token id + # we set a flag in the generation config to force the model to make only one call to generate and return the decoder input token ids and eos token id + if "assistant_model" in kwargs: + assistant_model = kwargs["assistant_model"] + assistant_model.generation_config.force_unique_generate_call = True + + if force_unique_generate_call is None: + if hasattr(generation_config, "force_unique_generate_call"): + force_unique_generate_call = generation_config.force_unique_generate_call + elif hasattr(self.generation_config, "force_unique_generate_call"): + force_unique_generate_call = self.generation_config.force_unique_generate_call + else: + force_unique_generate_call = False + + # 6 Transcribe audio until we reach the end of all input audios + while (seek < max_frames).any(): + if monitor_progress is not None: + monitor_progress(torch.stack((seek, max_frames), dim=1)) + + # 6.1 NOTE: When in longform transcription mode and batch size > 1 we need to dynamically reduce the batch size during the loop + # in case one audio finished earlier than another one. Thus, we need to keep a table of "previous-index-2-current-index" in order + # to know which original audio is being decoded + # Set updated index map, duration of previously decoded chunks and number of max frames of current decoding chunk + input_features, cur_bsz, batch_idx_map = self._maybe_reduce_batch( + input_features=input_features, + seek=seek, + max_frames=max_frames, + cur_bsz=cur_bsz, + batch_idx_map=batch_idx_map, + ) + time_offset = ( + seek.to(torch.float32 if device.type == "mps" else torch.float64) * time_precision / input_stride + ) + seek_num_frames = (max_frames - seek).clamp(max=num_segment_frames) + + # 6.2 cut out next 30s segment from input features + segment_input = self._get_input_segment( + input_features=input_features, + seek=seek, + seek_num_frames=seek_num_frames, + num_segment_frames=num_segment_frames, + cur_bsz=cur_bsz, + batch_idx_map=batch_idx_map, + ) + + # 6.3 prepare decoder input ids + suppress_tokens = _get_attr_from_logit_processors( + logits_processor, SuppressTokensLogitsProcessor, "suppress_tokens" + ) + + decoder_input_ids, kwargs = self._prepare_decoder_input_ids( + cur_bsz=cur_bsz, + init_tokens=init_tokens, + current_segments=current_segments, + batch_idx_map=batch_idx_map, + do_condition_on_prev_tokens=do_condition_on_prev_tokens, + prompt_ids=prompt_ids, + generation_config=generation_config, + config=self.config, + device=init_tokens.device, + suppress_tokens=suppress_tokens, + timestamp_begin=timestamp_begin, + kwargs=kwargs, + ) + + # 6.4 set max new tokens or max length + self._set_max_new_tokens_and_length( + config=self.config, + decoder_input_ids=decoder_input_ids, + generation_config=generation_config, + ) + + # 6.5 Set current `begin_index` for all logit processors + if logits_processor is not None: + for proc in logits_processor: + if hasattr(proc, "set_begin_index"): + proc.set_begin_index(decoder_input_ids.shape[-1]) + + # 6.6 Run generate with fallback + ( + seek_sequences, + seek_outputs, + should_skip, + do_condition_on_prev_tokens, + model_output_type, + ) = self.generate_with_fallback( + segment_input=segment_input, + decoder_input_ids=decoder_input_ids, + cur_bsz=cur_bsz, + seek=seek, + batch_idx_map=batch_idx_map, + temperatures=temperatures, + generation_config=generation_config, + logits_processor=logits_processor, + stopping_criteria=stopping_criteria, + prefix_allowed_tokens_fn=prefix_allowed_tokens_fn, + synced_gpus=synced_gpus, + return_token_timestamps=return_token_timestamps, + do_condition_on_prev_tokens=do_condition_on_prev_tokens, + is_shortform=is_shortform, + batch_size=batch_size, + attention_mask=attention_mask, + kwargs=kwargs, + ) + + # 6.7 In every generated sequence, split by timestamp tokens and extract segments + for i, seek_sequence in enumerate(seek_sequences): + prev_i = batch_idx_map[i] + + if should_skip[i]: + seek[prev_i] += seek_num_frames[prev_i] + continue + + segments, segment_offset = self._retrieve_segment( + seek_sequence=seek_sequence, + seek_outputs=seek_outputs, + time_offset=time_offset, + timestamp_begin=timestamp_begin, + seek_num_frames=seek_num_frames, + time_precision=time_precision, + time_precision_features=time_precision_features, + input_stride=input_stride, + prev_idx=prev_i, + idx=i, + return_token_timestamps=return_token_timestamps, + decoder_input_ids=decoder_input_ids, + ) + + seek[prev_i] += segment_offset + + current_segments[prev_i] += segments + + if force_unique_generate_call: + break + + # 7. Once all segments are added to the list of all segments, called `current_segments`, we extract the predicted + # output tokens from the list of dicts. If we use batch size > 1, we make sure to pad the output + final_segments = ( + [x[1:] for x in current_segments] + if (prompt_ids is not None and generation_config.prompt_condition_type == "first-segment") + else current_segments + ) + + # if return_dict_in_generate=True and we forced a unique call to generate or return_timestamps=False, meaning we are sure only one call to generate has been made, + # -> we can return a ModelOutput + # otherwise, return_dict_in_generate is applied in the 'result' of each segment in final_segments + if ( + return_dict_in_generate + and generation_config.return_dict_in_generate + and (force_unique_generate_call or not return_timestamps) + ): + # only one call to generate_with_fallback, we can return a ModelOutput + outputs = self._stack_split_outputs(seek_outputs, model_output_type, self.device, kwargs) + if num_return_sequences > 1: + if hasattr(outputs, "encoder_attentions") and outputs.encoder_attentions is not None: + outputs.encoder_attentions = tuple( + outputs.encoder_attentions[i][::num_return_sequences] + for i in range(len(outputs.encoder_attentions)) + ) + if hasattr(outputs, "encoder_hidden_states") and outputs.encoder_hidden_states is not None: + outputs.encoder_hidden_states = tuple( + outputs.encoder_hidden_states[i][::num_return_sequences] + for i in range(len(outputs.encoder_hidden_states)) + ) + return outputs + + padded_outputs = _pad_to_max_length( + current_segments=final_segments, + pad_token_id=generation_config.pad_token_id, + device=self.device, + padding_side="right", + return_token_timestamps=return_token_timestamps, + force_unique_generate_call=force_unique_generate_call, + ) + + if return_dict_in_generate and generation_config.return_dict_in_generate: + logger.warning_once( + "You have passed `return_dict_in_generate=True` and `return_timestamps=True`, this automatically sets `return_segments=True` to access the results of the underlying calls to GenerationMixin's generate in the returned `segments`." + ) + return_segments = True + elif not return_segments and not return_token_timestamps: + return padded_outputs + + if return_token_timestamps: + sequences, token_timestamps = padded_outputs + outputs = { + "sequences": sequences, + "token_timestamps": token_timestamps, + } + else: + sequences = padded_outputs + outputs = { + "sequences": sequences, + } + + if return_segments: + outputs["segments"] = final_segments + + return outputs + + def generate_with_fallback( + self, + segment_input, + decoder_input_ids, + cur_bsz, + seek, + batch_idx_map, + temperatures, + generation_config, + logits_processor, + stopping_criteria, + prefix_allowed_tokens_fn, + synced_gpus, + return_token_timestamps, + do_condition_on_prev_tokens, + is_shortform, + batch_size, + attention_mask, + kwargs, + ): + kwargs = copy.copy(kwargs) + + # 6.6 Batch generate current chunk + seek_sequence_list = [None for _ in range(cur_bsz)] + seek_outputs_list = [None for _ in range(cur_bsz)] + needs_fallback = [False for _ in range(cur_bsz)] + should_skip = [False for _ in range(cur_bsz)] + fallback_index_map = list(range(cur_bsz)) + if generation_config.no_speech_threshold is not None: + self._setup_no_speech_detection(logits_processor, segment_input, decoder_input_ids, kwargs) + + for fallback_idx, temperature in enumerate(temperatures): + generation_config.do_sample = temperature is not None and temperature > 0.0 + generation_config.temperature = temperature if generation_config.do_sample else 1.0 + if generation_config.do_sample: + generation_config.num_beams = 1 + + generate_kwargs = copy.copy(kwargs) + for key in ["do_sample", "temperature", "num_beams"]: + if key in generate_kwargs: + del generate_kwargs[key] + + cur_bsz = decoder_input_ids.shape[0] + if generation_config.cache_implementation == "static" and cur_bsz < batch_size: + segment_input = F.pad(segment_input, (0, 0, 0, 0, 0, batch_size - cur_bsz), value=0) + decoder_input_ids = F.pad( + decoder_input_ids, (0, 0, 0, batch_size - cur_bsz), value=generation_config.pad_token_id + ) + if generate_kwargs.get("decoder_attention_mask") is not None: + generate_kwargs["decoder_attention_mask"] = F.pad( + generate_kwargs["decoder_attention_mask"], (0, 0, 0, batch_size - cur_bsz), value=True + ) + if generate_kwargs.get("encoder_outputs") is not None: + generate_kwargs["encoder_outputs"] = F.pad( + generate_kwargs["encoder_outputs"], (0, 0, 0, 0, 0, batch_size - cur_bsz), value=0 + ) + + seek_outputs = super().generate( + segment_input, + generation_config=generation_config, + logits_processor=logits_processor, + stopping_criteria=stopping_criteria, + prefix_allowed_tokens_fn=prefix_allowed_tokens_fn, + synced_gpus=synced_gpus, + decoder_input_ids=decoder_input_ids, + attention_mask=attention_mask, + **generate_kwargs, + ) + + model_output_type = type(seek_outputs) + + # post-process sequence tokens and outputs to be in list form + seek_sequences, seek_outputs = self._postprocess_outputs( + seek_outputs=seek_outputs, + decoder_input_ids=decoder_input_ids, + return_token_timestamps=return_token_timestamps, + generation_config=generation_config, + is_shortform=is_shortform, + seek=seek, + batch_idx_map=batch_idx_map, + ) + + if cur_bsz < batch_size: + seek_sequences = seek_sequences[:cur_bsz] + seek_outputs = seek_outputs[:cur_bsz] + + # 6.7 Extract cut sequences from every sequence and check if fallback should be applied + # Loop over each decoded audio individually as each decoding can be of a different length + new_fallback_index_map = [] + new_segment_input = [] + new_decoder_input_ids = [] + new_decoder_attention_mask = [] + + for i, seek_sequence in enumerate(seek_sequences): + # remove all padding tokens, except for the eos token + if seek_sequence[-1] == generation_config.pad_token_id: + num_paddings = (seek_sequence == generation_config.pad_token_id).sum() + if generation_config.pad_token_id == generation_config.eos_token_id: + # we do not remove the eos token id since it is needed for avg logprob calculation in _need_fallback + num_paddings -= 1 + if num_paddings != 0: + seek_sequence = seek_sequence[:-num_paddings] + + # check which sequences in batch need fallback & which should be skipped + needs_fallback[i], should_skip[i] = self._need_fallback( + seek_sequence, + seek_outputs, + i, + logits_processor, + generation_config, + self.config.vocab_size, + temperature, + ) + + # remove eos token + if seek_sequence[-1] == generation_config.eos_token_id: + seek_sequence = seek_sequence[:-1] + + seek_sequence_list[fallback_index_map[i]] = seek_sequence + seek_outputs_list[fallback_index_map[i]] = seek_outputs[i] + is_low_temperature = temperature is None or temperature < 0.5 + do_condition_on_prev_tokens[fallback_index_map[i]] = ( + generation_config.condition_on_prev_tokens and is_low_temperature + ) + + if needs_fallback[i]: + new_fallback_index_map.append(fallback_index_map[i]) + new_segment_input.append(segment_input[i]) + new_decoder_input_ids.append(decoder_input_ids[i]) + if "decoder_attention_mask" in kwargs: + new_decoder_attention_mask.append(kwargs["decoder_attention_mask"][i]) + + fallback_index_map = new_fallback_index_map + + # if no sequence needs to be run with temperature fallback, we're finished + if len(fallback_index_map) == 0 or fallback_idx == len(temperatures) - 1: + seek_sequences = seek_sequence_list + seek_outputs = seek_outputs_list + break + + # if we're still in the loop, make sure that decoder_input_ids and segment inputs are tensors + decoder_input_ids = torch.stack(new_decoder_input_ids) + segment_input = torch.stack(new_segment_input) + if "decoder_attention_mask" in kwargs: + kwargs["decoder_attention_mask"] = torch.stack(new_decoder_attention_mask) + + return seek_sequences, seek_outputs, should_skip, do_condition_on_prev_tokens, model_output_type + + @staticmethod + def _prepare_segments(prompt_ids, batch_size, generation_config): + if prompt_ids is not None and generation_config.prompt_condition_type == "first-segment": + prev_sot_token_id = getattr(generation_config, "prev_sot_token_id", None) + prompt_ids = prompt_ids[1:] if prompt_ids[0] == prev_sot_token_id else prompt_ids + current_segments = [[{"tokens": prompt_ids}] for _ in range(batch_size)] + else: + current_segments = [[] for _ in range(batch_size)] + + return current_segments + + def _postprocess_outputs( + self, + seek_outputs, + decoder_input_ids, + return_token_timestamps, + generation_config, + is_shortform, + seek, + batch_idx_map, + ): + # remove all previously passed decoder input ids + # should happen only if it is the first generated segment + start_idx = decoder_input_ids.shape[-1] + + if isinstance(seek_outputs, torch.Tensor): + return seek_outputs[:, start_idx:], seek_outputs + + if return_token_timestamps and hasattr(generation_config, "alignment_heads"): + num_frames = getattr(generation_config, "num_frames") + if num_frames is not None: + num_frames = num_frames - seek + num_frames = num_frames[batch_idx_map] + + seek_outputs["token_timestamps"] = self._extract_token_timestamps( + seek_outputs, + generation_config.alignment_heads, + num_frames=num_frames, + num_input_ids=decoder_input_ids.shape[-1], + ) + + def split_by_batch_index(values, key, batch_idx, is_shortform, beam_indices=None): + if beam_indices is not None and key == "scores": + return [v[beam_idx].cpu() for (v, beam_idx) in zip(values, beam_indices[batch_idx][: len(values)])] + if key in ["scores", "encoder_attentions", "encoder_hidden_states", "logits"]: + return [v[batch_idx].cpu() for v in values] + if key in ["decoder_attentions", "decoder_hidden_states", "cross_attentions"]: + return tuple(tuple(w[batch_idx][None].cpu() for w in v) for v in values) + elif key == "past_key_values": + if not is_shortform: + # we don't save `past_key_values` as this is too costly for longform + return None + all_past_key_values = [] + for layer_idx in range(self.config.decoder_layers): + layer_cache = ( + values.self_attention_cache.layers[layer_idx].keys[batch_idx][None].cpu(), + values.self_attention_cache.layers[layer_idx].values[batch_idx][None].cpu(), + values.cross_attention_cache.layers[layer_idx].keys[batch_idx][None].cpu(), + values.cross_attention_cache.layers[layer_idx].values[batch_idx][None].cpu(), + ) + all_past_key_values.append(layer_cache) + return EncoderDecoderCache(all_past_key_values) + + return values[batch_idx].cpu() + + sequence_tokens = seek_outputs["sequences"][:, start_idx:] + seek_outputs = [ + { + k: split_by_batch_index(v, k, i, is_shortform, beam_indices=seek_outputs.get("beam_indices")) + for k, v in seek_outputs.items() + } + for i in range(sequence_tokens.shape[0]) + ] + + return sequence_tokens, seek_outputs + + def _stack_split_outputs(self, seek_outputs, model_output_type, device, kwargs): + # Stack back seek_outputs tensors after splitting them with the split_by_batch_index method + outputs = {} + for key in seek_outputs[0]: + if key in ["sequences", "beam_indices", "token_timestamps"]: + outputs[key] = torch.stack([v[key] for v in seek_outputs], dim=0).to(device) + elif key in ["scores", "encoder_attentions", "encoder_hidden_states", "logits"]: + outputs[key] = tuple( + torch.stack([v[key][i] for v in seek_outputs]).to(device) for i in range(len(seek_outputs[0][key])) + ) + elif key == "sequences_scores": + outputs[key] = torch.stack([v[key] for v in seek_outputs], dim=0).to(device) + elif key in ["decoder_attentions", "decoder_hidden_states", "cross_attentions"]: + outputs[key] = tuple( + tuple( + torch.stack([v[key][i][j] for v in seek_outputs]).squeeze(1).to(device) + for j in range(len(seek_outputs[0][key][0])) + ) + for i in range(len(seek_outputs[0][key])) + ) + elif key == "past_key_values": + if seek_outputs[0][key] is not None: + all_past_key_values = [] + for layer_idx in range(len(seek_outputs[0][key])): + self_attention_k, self_attention_v, cross_attention_k, cross_attention_v = ( + torch.stack( + [ + getattr(getattr(sub_output[key], sub_cache).layers[layer_idx], sub_key) + for sub_output in seek_outputs + ] + ) + .squeeze(1) + .to(device) + for sub_cache in ["self_attention_cache", "cross_attention_cache"] + for sub_key in ["keys", "values"] + ) + all_past_key_values.append( + (self_attention_k, self_attention_v, cross_attention_k, cross_attention_v) + ) + outputs[key] = EncoderDecoderCache(tuple(all_past_key_values)) + else: + outputs[key] = None + + token_timestamps = outputs.get("token_timestamps") + if token_timestamps is not None: + model_output_type = dict + + return model_output_type(**outputs) + + def _need_fallback( + self, + seek_sequence, + seek_outputs, + index, + logits_processor, + generation_config, + vocab_size, + temperature, + ): + needs_fallback = False + should_skip = False + if generation_config.compression_ratio_threshold is not None: + compression_ratio = self._retrieve_compression_ratio(seek_sequence, vocab_size) + + if compression_ratio > generation_config.compression_ratio_threshold: + needs_fallback = True + + if generation_config.logprob_threshold is not None: + if hasattr(seek_outputs[0], "sequences_scores"): + logprobs = [s["sequences_scores"] for s in seek_outputs][index] + else: + scores = seek_outputs[index]["scores"] + logprobs = self._retrieve_avg_logprobs( + scores, + seek_sequence, + temperature, + ) + + if logprobs < generation_config.logprob_threshold: + needs_fallback = True + + if generation_config.no_speech_threshold is not None: + no_speech_prob = _get_attr_from_logit_processors( + logits_processor, WhisperNoSpeechDetection, "no_speech_prob" + ) + + if ( + logprobs < generation_config.logprob_threshold + and no_speech_prob[index] > generation_config.no_speech_threshold + ): + needs_fallback = False + should_skip = True + + return needs_fallback, should_skip + + def _expand_variables_for_generation( + self, input_features, seek, max_frames, init_tokens, batch_size, condition_on_prev_tokens, generation_config + ): + if generation_config.num_return_sequences is not None and generation_config.num_return_sequences > 1: + batch_idx_map = list(range(batch_size * generation_config.num_return_sequences)) + cur_bsz = len(batch_idx_map) + do_condition_on_prev_tokens = [condition_on_prev_tokens for _ in range(len(batch_idx_map))] + input_features = input_features.repeat_interleave(generation_config.num_return_sequences, dim=0) + seek = seek.repeat_interleave(generation_config.num_return_sequences, dim=0) + max_frames = max_frames.repeat_interleave(generation_config.num_return_sequences, dim=0) + init_tokens = init_tokens.repeat_interleave(generation_config.num_return_sequences, dim=0) + generation_config.num_return_sequences = 1 + else: + cur_bsz = batch_size + batch_idx_map = list(range(cur_bsz)) + do_condition_on_prev_tokens = [condition_on_prev_tokens for _ in range(cur_bsz)] + + return ( + batch_idx_map, + cur_bsz, + input_features, + seek, + max_frames, + init_tokens, + do_condition_on_prev_tokens, + ) + + @staticmethod + def _setup_no_speech_detection(logits_processor, segment_input, decoder_input_ids, kwargs): + set_inputs = _get_attr_from_logit_processors(logits_processor, WhisperNoSpeechDetection, "set_inputs") + extra_kwargs = {k: v for k, v in kwargs.items() if torch.is_tensor(v)} + set_inputs({"inputs": segment_input, "input_ids": decoder_input_ids, **extra_kwargs}) + + @staticmethod + def _retrieve_total_input_frames(input_features, input_stride, kwargs): + if input_features is not None: + return input_features.shape[0], input_features.shape[-1] + + if "encoder_outputs" in kwargs: + encoder_outputs_shape = ( + kwargs["encoder_outputs"][0].shape + if isinstance(kwargs["encoder_outputs"], BaseModelOutput) + else kwargs["encoder_outputs"].shape + ) + return encoder_outputs_shape[0], encoder_outputs_shape[1] * input_stride + + raise ValueError("Make sure to provide either `input_features` or `encoder_outputs` to `generate`.") + + @staticmethod + def _maybe_warn_unused_inputs( + condition_on_prev_tokens, + temperature, + compression_ratio_threshold, + logprob_threshold, + no_speech_threshold, + total_input_frames, + ): + warning_prefix = ( + f"Audio input consists of only {total_input_frames}. " + "Short-form transcription is activated." + "{}, but will be ignored." + ) + if condition_on_prev_tokens is not None: + logger.warning(warning_prefix.format(f"condition_on_prev_tokens is set to {condition_on_prev_tokens}")) + + if compression_ratio_threshold is not None: + logger.warning( + warning_prefix.format(f"compression_ratio_threshold is set to {compression_ratio_threshold}") + ) + + if logprob_threshold is not None: + logger.warning(warning_prefix.format(f"logprob_threshold is set to {logprob_threshold}")) + + if no_speech_threshold is not None: + logger.warning(warning_prefix.format(f"no_speech_threshold is set to {no_speech_threshold}")) + + @staticmethod + def _set_return_outputs(return_dict_in_generate, return_token_timestamps, logprob_threshold, generation_config): + if return_dict_in_generate is None: + return_dict_in_generate = generation_config.return_dict_in_generate + else: + generation_config.return_dict_in_generate = return_dict_in_generate + + generation_config.return_token_timestamps = return_token_timestamps + if return_token_timestamps: + generation_config.return_dict_in_generate = True + generation_config.output_attentions = True + generation_config.output_scores = True + + if logprob_threshold is not None: + generation_config.return_dict_in_generate = True + generation_config.output_scores = True + + return return_dict_in_generate + + def _set_return_timestamps(self, return_timestamps, is_shortform, generation_config): + if return_timestamps is None and hasattr(generation_config, "return_timestamps"): + return_timestamps = generation_config.return_timestamps + + if not is_shortform: + if return_timestamps is False: + raise ValueError( + "You have passed more than 3000 mel input features (> 30 seconds) which automatically " + "enables long-form generation which requires the model to predict timestamp tokens. Please " + "either pass `return_timestamps=True` or make sure to pass no more than 3000 mel input features." + ) + + logger.info("Setting `return_timestamps=True` for long-form generation.") + return_timestamps = True + + if return_timestamps and not hasattr(generation_config, "no_timestamps_token_id"): + raise ValueError( + "You are trying to return timestamps, but the generation config is not properly set. " + "Make sure to initialize the generation config with the correct attributes that are needed such as " + "`no_timestamps_token_id`. For more details on how to generate the approtiate config, refer to " + "https://github.com/huggingface/transformers/issues/21878#issuecomment-1451902363" + ) + + generation_config.return_timestamps = return_timestamps + + if hasattr(generation_config, "no_timestamps_token_id"): + timestamp_begin = generation_config.no_timestamps_token_id + 1 + else: + # BC for models missing the `no_timestamps_token_id` in the generation config when generating short-form + # with no timestamps. We set the timestamp begin token larger than the vocab size, such that the + # timestamp condition is never met in the decoding loop + timestamp_begin = self.config.vocab_size + 1 + + return timestamp_begin + + @staticmethod + def _set_language_and_task(language, task, is_multilingual, generation_config): + if is_multilingual is not None: + if not hasattr(generation_config, "is_multilingual"): + raise ValueError( + "The generation config is outdated and is thus not compatible with the `is_multilingual` argument " + "to `generate`. Please update the generation config as per the instructions " + "https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224" + ) + generation_config.is_multilingual = is_multilingual + + if hasattr(generation_config, "is_multilingual") and not generation_config.is_multilingual: + if task is not None or language is not None: + raise ValueError( + "Cannot specify `task` or `language` for an English-only model. If the model is intended to be " + "multilingual, pass `is_multilingual=True` to generate, or update the generation config." + ) + + if language is not None: + if not hasattr(generation_config, "lang_to_id"): + raise ValueError( + "The generation config is outdated and is thus not compatible with the `language` argument " + "to `generate`. Please update the generation config as per the instructions " + "https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224" + ) + generation_config.language = language + + if task is not None: + if not hasattr(generation_config, "task_to_id"): + raise ValueError( + "The generation config is outdated and is thus not compatible with the `task` argument " + "to `generate`. Please update the generation config as per the instructions " + "https://github.com/huggingface/transformers/issues/25084#issuecomment-1664398224" + ) + generation_config.task = task + + def _retrieve_init_tokens(self, input_features, batch_size, generation_config, config, num_segment_frames, kwargs): + def replace_or_add(lst: list[int], num: int, itr: Iterator[int]): + """short function to replace num with a itr in lst""" + found = any(i in lst for i in itr) + if found: + lst = [num if i in itr else i for i in lst] + else: + lst.append(num) + return lst + + def language_to_id(language: str) -> int: + language = language.lower() + if language in generation_config.lang_to_id: + language_token = language + elif language in TO_LANGUAGE_CODE: + language_token = f"<|{TO_LANGUAGE_CODE[language]}|>" + elif language in TO_LANGUAGE_CODE.values(): + language_token = f"<|{language}|>" + else: + is_language_code = len(language) == 2 + raise ValueError( + f"Unsupported language: {language}. Language should be one of:" + f" {list(TO_LANGUAGE_CODE.values()) if is_language_code else list(TO_LANGUAGE_CODE.keys())}." + ) + if language_token not in generation_config.lang_to_id: + raise ValueError( + f"{language_token} is not supported by this specific model as it is not in the " + "`generation_config.lang_to_id`. (You should just add it to the generation config)" + ) + + return generation_config.lang_to_id[language_token] + + task = getattr(generation_config, "task", None) + language = getattr(generation_config, "language", None) + init_tokens = [generation_config.decoder_start_token_id] + + # TL;DR we silently ignore `forced_decoder_ids` (old flag) when `task` or `language` (new flags) are set. + # `forced_decoder_ids` is an old generation config attribute that is now deprecated in favor of `task` and + # `language` (see https://github.com/huggingface/transformers/pull/28687). Nevertheless, keep in mind that + # the original checkpoints all contain this attribute, and thus we should maintain backwards compatibility. + if task is None and language is None: + forced_decoder_ids = getattr(generation_config, "forced_decoder_ids", None) + # fallback: check the model config for forced_decoder_ids + if forced_decoder_ids is None and getattr(config, "forced_decoder_ids", None) is not None: + forced_decoder_ids = config.forced_decoder_ids + + if forced_decoder_ids is not None: + logger.warning_once( + "Using custom `forced_decoder_ids` from the (generation) config. This is deprecated in favor of " + "the `task` and `language` flags/config options." + ) + + if forced_decoder_ids is not None and forced_decoder_ids[0][1] is None: + logger.warning_once( + "Transcription using a multilingual Whisper will default to language detection followed by " + "transcription instead of translation to English. This might be a breaking change for your " + "use case. If you want to instead always translate your audio to English, make sure to pass " + "`language='en'`. See https://github.com/huggingface/transformers/pull/28687 for more details." + ) + + if forced_decoder_ids is not None and forced_decoder_ids[0][0] == 1: + i = 1 + while len(forced_decoder_ids) > 0 and forced_decoder_ids[0][0] == i: + init_tokens += [forced_decoder_ids[0][1]] + forced_decoder_ids = forced_decoder_ids[1:] + i += 1 + + if len(forced_decoder_ids) > 0: + raise ValueError( + f"You are using token ids in `forced_decoder_ids` that do not seem to correctly follow " + f"the prompt pattern of Whisper. Make sure that {forced_decoder_ids} has an entry for all " + f"indices >= 1 and < {forced_decoder_ids[0][0]}.", + ) + + is_lang_id_undefined = len(init_tokens) <= 1 or (len(init_tokens) > 1 and init_tokens[1] is None) + + # Make sure language is a list of strings of the correct length + if isinstance(language, (list, tuple)): + if any(l is None for l in language): + raise TypeError( + "Expected `language` to be `None`, a single string (e.g. `'en'`), or a list of strings with " + "length equal to the batch size (e.g. `('en', 'fr')` for a batch size of 2). Got a list " + "containing `None`." + ) + if len(language) != batch_size: + raise ValueError( + "When passing a list of languages, the length of the list must match the batch size. " + f"Expected length of {batch_size}, but got {len(language)} languages." + ) + languages = language + elif language is None: + # Language will be detected for each item in batch + languages = [None] * batch_size + else: + languages = [language] # Use a length-1 list now, broadcast later + + # Separate init_tokens for each language + init_tokens = [copy.copy(init_tokens) for _ in languages] + + # Update init_tokens with languages + lang_ids = None + if language is not None: + lang_ids = [language_to_id(l) for l in languages] + elif hasattr(generation_config, "lang_to_id") and is_lang_id_undefined: + # language is not defined or intentionally set to `None` to trigger language detection + lang_ids = self.detect_language( + input_features=input_features, + encoder_outputs=kwargs.get("encoder_outputs", None), + generation_config=generation_config, + num_segment_frames=num_segment_frames, + ).tolist() + if lang_ids is not None: + # append or replace lang_ids to init_tokens + for i in range(len(init_tokens)): + if len(init_tokens[i]) > 1: + init_tokens[i][1] = lang_ids[i] + else: + init_tokens[i].append(lang_ids[i]) + del languages + + # Update init_tokens with task + for i in range(len(init_tokens)): + if task is not None: + if task in TASK_IDS: + init_tokens[i].append(generation_config.task_to_id[generation_config.task]) + task_id = generation_config.task_to_id[generation_config.task] + + # if task is defined it'll overwrite task ids that might have already been defined via the generation_config + replace_or_add(init_tokens[i], task_id, generation_config.task_to_id.values()) + else: + raise ValueError(f"The `{task}` task is not supported. The task should be one of `{TASK_IDS}`") + elif language is not None and hasattr(generation_config, "task_to_id"): + # if language is defined, but no task id is in `init_tokens`, default to transcribe + if not any(ti in init_tokens[i] for ti in generation_config.task_to_id.values()): + init_tokens[i].append(generation_config.task_to_id["transcribe"]) + + if ( + not generation_config.return_timestamps + and hasattr(generation_config, "no_timestamps_token_id") + and init_tokens[i][-1] != generation_config.no_timestamps_token_id + ): + init_tokens[i].append(generation_config.no_timestamps_token_id) + elif ( + generation_config.return_timestamps and init_tokens[i][-1] == generation_config.no_timestamps_token_id + ): + logger.info( + "<|notimestamps|> prompt token is removed from generation_config since `return_timestamps` is set to `'True'`." + ) + init_tokens[i] = init_tokens[i][:-1] + + # let's make sure we don't pass `None` tokens as prompt tokens + init_tokens[i] = [t for t in init_tokens[i] if t is not None] + + return torch.as_tensor(init_tokens, dtype=torch.long, device=self.device).expand(batch_size, -1) + + def detect_language( + self, + input_features: torch.FloatTensor | None = None, + encoder_outputs: torch.FloatTensor | BaseModelOutput | None = None, + generation_config: GenerationConfig | None = None, + num_segment_frames: int = 3000, + ) -> torch.Tensor: + """ + Detects language from log-mel input features or encoder_outputs + + Parameters: + input_features (`torch.Tensor` of shape `(batch_size, feature_size, sequence_length)`, *optional*): + Float values of log-mel features extracted from the raw speech waveform. The raw speech waveform can be obtained by + loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via + the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the + [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a + tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] for details. + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. + generation_config (`~generation.GenerationConfig`, *optional*): + The generation configuration to be used as base parametrization for the generation call. `**kwargs` + passed to generate matching the attributes of `generation_config` will override them. If + `generation_config` is not provided, the default will be used, which had the following loading + priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model + configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s + default values, whose documentation should be checked to parameterize generation. + num_segment_frames (`int`, *optional*, defaults to 3000): + The number of log-mel frames the model expects + + Return: + A `torch.LongTensor` representing the detected language ids. + """ + if input_features is None and encoder_outputs is None: + raise ValueError("You have to specify either `input_features` or `encoder_outputs`") + elif input_features is not None and encoder_outputs is not None: + raise ValueError("Make sure to specify only one of `input_features` or `encoder_outputs` - not both!") + elif input_features is not None: + inputs = {"input_features": input_features[:, :, :num_segment_frames]} + batch_size = input_features.shape[0] + elif encoder_outputs is not None: + inputs = {"encoder_outputs": encoder_outputs} + batch_size = ( + encoder_outputs[0].shape[0] if isinstance(encoder_outputs, BaseModelOutput) else encoder_outputs[0] + ) + + generation_config = generation_config or self.generation_config + decoder_input_ids = ( + torch.ones((batch_size, 1), device=self.device, dtype=torch.long) + * generation_config.decoder_start_token_id + ) + + with torch.no_grad(): + logits = self(**inputs, decoder_input_ids=decoder_input_ids, use_cache=False).logits[:, -1] + + non_lang_mask = torch.ones_like(logits[0], dtype=torch.bool) + non_lang_mask[list(generation_config.lang_to_id.values())] = False + + logits[:, non_lang_mask] = -np.inf + + lang_ids = logits.argmax(-1) + + return lang_ids + + @staticmethod + def _check_decoder_input_ids(kwargs): + decoder_input_ids = kwargs.get("decoder_input_ids", None) + assistant_model = kwargs.get("assistant_model", None) + if decoder_input_ids is not None and assistant_model is not None: + raise ValueError( + "Passing `decoder_input_ids` is deprecated. Consider passing `prompt_ids` instead.", + ) + + @staticmethod + def _set_num_frames(return_token_timestamps, generation_config, attention_mask, kwargs): + if return_token_timestamps: + if getattr(generation_config, "task", None) == "translate": + logger.warning("Token-level timestamps may not be reliable for task 'translate'.") + if not hasattr(generation_config, "alignment_heads"): + raise ValueError( + "Model generation config has no `alignment_heads`, token-level timestamps not available. " + "See https://gist.github.com/hollance/42e32852f24243b748ae6bc1f985b13a on how to add this property to the generation config." + ) + if attention_mask is not None: + generation_config.num_frames = attention_mask.sum(-1).cpu() + else: + logger.warning_once( + "When setting `return_token_timestamps` to `True`, make sure to pass an `attention_mask` to get precise token-level timestamps. You can retrieve the `attention_mask` by doing `processor(audio, ..., return_attention_mask=True)` " + ) + generation_config.num_frames = None + + @staticmethod + def _set_thresholds_and_condition( + generation_config, + logprob_threshold, + compression_ratio_threshold, + no_speech_threshold, + condition_on_prev_tokens, + ): + generation_config.logprob_threshold = ( + logprob_threshold + if logprob_threshold is not None + else getattr(generation_config, "logprob_threshold", None) + ) + generation_config.compression_ratio_threshold = ( + compression_ratio_threshold + if compression_ratio_threshold is not None + else getattr(generation_config, "compression_ratio_threshold", None) + ) + generation_config.no_speech_threshold = ( + no_speech_threshold + if no_speech_threshold is not None + else getattr(generation_config, "no_speech_threshold", None) + ) + generation_config.condition_on_prev_tokens = ( + condition_on_prev_tokens + if condition_on_prev_tokens is not None + else getattr(generation_config, "condition_on_prev_tokens", None) + ) + + @staticmethod + def _set_prompt_condition_type(generation_config, prompt_condition_type): + allowed_cond_types = ["first-segment", "all-segments"] + + # default to "first-segment" + prompt_condition_type = prompt_condition_type or allowed_cond_types[0] + + if prompt_condition_type not in allowed_cond_types: + raise ValueError( + f"`prompt_condition_type={prompt_condition_type} does not exist. Make sure to set `prompt_condition_type` to one of {', '.join(allowed_cond_types)}" + ) + + if generation_config.condition_on_prev_tokens is not True and prompt_condition_type == "all-segments": + raise ValueError( + "Make sure to set `condition_on_prev_tokens=True` when setting `prompt_condition_type='all-segments'`." + ) + + generation_config.prompt_condition_type = prompt_condition_type + + @staticmethod + def _set_condition_on_prev_tokens(condition_on_prev_tokens, generation_config): + condition_on_prev_tokens = ( + condition_on_prev_tokens + if condition_on_prev_tokens is not None + else getattr(generation_config, "condition_on_prev_tokens", False) + ) + generation_config.condition_on_prev_tokens = condition_on_prev_tokens + + @staticmethod + def _retrieve_max_frames_and_seek(batch_size, attention_mask, total_input_frames, is_shortform): + if batch_size > 1 and not is_shortform and attention_mask is None: + raise ValueError( + "When doing batched long-form audio transcription, make sure to pass an `attention_mask`. You can retrieve the `attention_mask` by doing `processor(audio, ..., return_attention_mask=True)` " + ) + elif batch_size > 1 and not is_shortform: + max_frames = attention_mask.sum(-1).cpu().to(torch.long) + seek = torch.zeros((batch_size,), dtype=torch.long) + else: + max_frames = torch.ones((batch_size,), dtype=torch.long) * total_input_frames + seek = torch.zeros((batch_size,), dtype=torch.long) + + return max_frames, seek + + def _retrieve_logit_processors(self, generation_config, logits_processor, begin_index, num_beams, device): + if generation_config.return_timestamps is True: + timestamp_processor = WhisperTimeStampLogitsProcessor(generation_config, begin_index=begin_index) + logits_processor = ( + [timestamp_processor] if logits_processor is None else [timestamp_processor] + logits_processor + ) + + if generation_config.suppress_tokens is not None: + suppress_tokens_processor = SuppressTokensLogitsProcessor(generation_config.suppress_tokens, device=device) + logits_processor = ( + [suppress_tokens_processor] + if logits_processor is None + else [suppress_tokens_processor] + logits_processor + ) + generation_config.suppress_tokens = None + + if generation_config.begin_suppress_tokens is not None: + begin_suppress_processor = SuppressTokensAtBeginLogitsProcessor( + generation_config.begin_suppress_tokens, begin_index=begin_index, device=device + ) + logits_processor = ( + [begin_suppress_processor] + if logits_processor is None + else [begin_suppress_processor] + logits_processor + ) + generation_config.begin_suppress_tokens = None + + if generation_config.no_speech_threshold is not None: + no_speech_detector = WhisperNoSpeechDetection( + no_speech_token=generation_config.no_timestamps_token_id - 1, + begin_index=begin_index, + scores_is_logprobs=num_beams > 1, + ) + logits_processor = ( + [no_speech_detector] if logits_processor is None else [no_speech_detector] + logits_processor + ) + no_speech_detector.set_model(self) + + return logits_processor + + @staticmethod + def _maybe_reduce_batch(input_features, seek, max_frames, cur_bsz, batch_idx_map): + prev_bsz = cur_bsz + new_batch_idx_map = [] + for i in range(prev_bsz): + prev_i = batch_idx_map[i] + if seek[prev_i] >= max_frames[prev_i]: + cut_index = i + (cur_bsz - prev_bsz) + cur_bsz -= 1 + input_features = torch.cat([input_features[:cut_index], input_features[cut_index + 1 :]], dim=0) + else: + # cut out index that goes away + new_batch_idx_map.append(prev_i) + + return input_features, cur_bsz, new_batch_idx_map + + @staticmethod + def _get_input_segment(input_features, seek, seek_num_frames, num_segment_frames, cur_bsz, batch_idx_map): + if input_features is None: + return None + + segment_input = [] + for i in range(cur_bsz): + prev_i = batch_idx_map[i] + segment_input_slice = input_features[i : i + 1, :, seek[prev_i] : seek[prev_i] + seek_num_frames[prev_i]] + + if segment_input_slice.shape[-1] < num_segment_frames: + # pad to 3000 if necessary + segment_input_slice = F.pad( + segment_input_slice, pad=(0, num_segment_frames - segment_input_slice.shape[-1]) + ) + + segment_input.append(segment_input_slice) + + segment_input = torch.cat(segment_input, dim=0) + + return segment_input + + @staticmethod + def _prepare_decoder_input_ids( + cur_bsz, + init_tokens, + current_segments, + batch_idx_map, + do_condition_on_prev_tokens, + prompt_ids, + generation_config, + config, + device, + suppress_tokens, + timestamp_begin, + kwargs, + ): + if "decoder_input_ids" in kwargs: + decoder_input_ids = kwargs.pop("decoder_input_ids") + + return decoder_input_ids, kwargs + + cut_off_length = config.max_target_positions // 2 - 1 + + decoder_input_ids = init_tokens[batch_idx_map] + + prev_start_of_text = getattr(generation_config, "prev_sot_token_id", None) + if prev_start_of_text is None: + if suppress_tokens is not None and len(suppress_tokens) >= 2: + prev_start_of_text = suppress_tokens[-2] + else: + prev_start_of_text = None + + if any(do_condition_on_prev_tokens) and len(current_segments[0]) > 0: + # according to https://github.com/openai/whisper/blob/e58f28804528831904c3b6f2c0e473f346223433/whisper/decoding.py#L609 + active_segments = [current_segments[i] if do_condition_on_prev_tokens[i] else None for i in batch_idx_map] + + if prompt_ids is not None and generation_config.prompt_condition_type == "all-segments": + prev_ids = prompt_ids + else: + one_tensor = torch.ones((cur_bsz, 1), device=device, dtype=torch.long) + prev_ids = prev_start_of_text * one_tensor[0] if prev_start_of_text is not None else None + + padding = "max_length" if generation_config.cache_implementation == "static" else "longest" + + prev_tokens = _pad_to_max_length( + active_segments, + generation_config.pad_token_id, + device=device, + padding_side="left", + padding=padding, + bos_token_tensor=prev_ids, + cut_off_length=cut_off_length, + skip_ending_double_timestamps=True, + timestamp_begin=timestamp_begin, + ) + decoder_input_ids = torch.cat([prev_tokens, decoder_input_ids], dim=-1) + + kwargs["decoder_attention_mask"] = decoder_input_ids != generation_config.pad_token_id + elif prompt_ids is not None: + prev_tokens = prompt_ids[None].repeat(decoder_input_ids.shape[0], 1) + decoder_input_ids = torch.cat([prev_tokens, decoder_input_ids], dim=-1) + # make sure `"decoder_attention_mask"` is not passed to forward + kwargs.pop("decoder_attention_mask", None) + else: + # make sure `"decoder_attention_mask"` is not passed to forward + kwargs.pop("decoder_attention_mask", None) + + return decoder_input_ids, kwargs + + def _set_max_new_tokens_and_length(self, config, decoder_input_ids, generation_config): + max_new_tokens = generation_config.max_new_tokens if generation_config.max_new_tokens is not None else 0 + if max_new_tokens + decoder_input_ids.shape[-1] > self.config.max_target_positions: + raise ValueError( + f"The length of `decoder_input_ids`, including special start tokens, prompt tokens, and previous tokens, is {decoder_input_ids.shape[-1]}, " + f" and `max_new_tokens` is {max_new_tokens}. Thus, the combined length of " + f"`decoder_input_ids` and `max_new_tokens` is: {max_new_tokens + decoder_input_ids.shape[-1]}. This exceeds the " + f"`max_target_positions` of the Whisper model: {self.config.max_target_positions}. " + "You should either reduce the length of your prompt, or reduce the value of `max_new_tokens`, " + f"so that their combined length is less than {self.config.max_target_positions}." + ) + + num_initial_tokens = min(config.max_target_positions // 2 - 1, decoder_input_ids.shape[-1] - 1) + + # Make sure we don't get larger than `max_length` + if generation_config.max_length is not None and generation_config.max_new_tokens is None: + max_length = min(generation_config.max_length + num_initial_tokens, config.max_target_positions) + logger.info( + f"Increase max_length from {generation_config.max_length} to {max_length} since input is conditioned on previous segment." + ) + elif ( + generation_config.max_new_tokens is not None + and generation_config.max_new_tokens + decoder_input_ids.shape[-1] > config.max_target_positions + ): + max_new_tokens = config.max_target_positions - decoder_input_ids.shape[-1] + generation_config.max_new_tokens = max_new_tokens + + @staticmethod + def _retrieve_compression_ratio(tokens, vocab_size): + """Compute byte length of zlib compressed token bytes vs. byte length of raw token bytes""" + length = int(math.log2(vocab_size) / 8) + 1 + token_bytes = b"".join([t.to_bytes(length, "little") for t in tokens.tolist()]) + compression_ratio = len(token_bytes) / len(zlib.compress(token_bytes)) + + return compression_ratio + + @staticmethod + def _retrieve_avg_logprobs(scores, tokens, temperature): + rescale_temperature = temperature if temperature > 0.0 else 1 + scores = torch.stack(scores).to(tokens.device) + + if scores.shape[0] > tokens.shape[0]: + scores = scores[: tokens.shape[0]] + else: + tokens = tokens[-scores.shape[0] :] + + logprobs = F.log_softmax((scores * rescale_temperature).float(), dim=-1).to(scores.dtype) + + # retrieve logprob of selected tokens and sum + # don't remove the eos token logprob! it counts in avg_logprob calculation in the original implementation + sum_logprobs = sum(logprobs[i][tokens[i]] for i in range(logprobs.shape[0])) + + avg_logprobs = sum_logprobs / len(tokens) + return avg_logprobs + + @staticmethod + def _retrieve_segment( + seek_sequence, + seek_outputs, + time_offset, + timestamp_begin, + seek_num_frames, + time_precision, + time_precision_features, + input_stride, + prev_idx, + idx, + return_token_timestamps, + decoder_input_ids, + ): + # find the predicted "end of segment" predictions of Whisper + # "end of segment" predictions occur whenever Whisper predicts a timestamp token + timestamp_tokens: torch.Tensor = seek_sequence.ge(timestamp_begin) + single_timestamp_ending = timestamp_tokens[-2:].tolist() == [False, True] + timestamp_segment_indices = torch.where(timestamp_tokens[:-1] & timestamp_tokens[1:])[0] + timestamp_segment_indices.add_(1) + token_timestamps = seek_outputs[idx]["token_timestamps"] if return_token_timestamps else [] + idx_offset = decoder_input_ids.shape[-1] + device = seek_sequence.device + + # If whisper predicted a "end of segment" via a timestep token, let's go ever each + # "end of segment" prediction and slice the decoding into segments accordingly + if len(timestamp_segment_indices) > 0: + # if the output contains two consecutive timestamp tokens + slices = timestamp_segment_indices.tolist() + segments = [] + if single_timestamp_ending: + slices.append(len(seek_sequence)) + else: + # we want to include the last timestamp token in the last segment to know it was no single ending + slices[-1] += 1 + + last_slice = 0 + # Add each segment to list of all segments + for i, current_slice in enumerate(slices): + is_last_slice = i == len(slices) - 1 + sliced_tokens = seek_sequence[last_slice:current_slice] + start_timestamp_pos = sliced_tokens[0] - timestamp_begin + idx_sliced_tokens = -1 if not is_last_slice or single_timestamp_ending else -2 + end_timestamp_pos = sliced_tokens[idx_sliced_tokens] - timestamp_begin + segments.append( + { + "start": time_offset[prev_idx] + + start_timestamp_pos.to(torch.float32 if device.type == "mps" else torch.float64) + * time_precision, + "end": time_offset[prev_idx] + + end_timestamp_pos.to(torch.float32 if device.type == "mps" else torch.float64) + * time_precision, + "tokens": sliced_tokens, + "idxs": (idx_offset + last_slice, idx_offset + current_slice), + "result": seek_outputs[idx], + } + ) + if return_token_timestamps: + segments[-1]["token_timestamps"] = ( + token_timestamps[idx_offset + last_slice : idx_offset + current_slice] + time_offset[prev_idx] + ) + last_slice = current_slice + + if single_timestamp_ending: + # single timestamp at the end means no speech after the last timestamp. + segment_offset = seek_num_frames[prev_idx] + else: + # otherwise, ignore the unfinished segment and seek to the last timestamp + # here we throw away all predictions after the last predicted "end of segment" + # since we are cutting right in the middle of an audio + last_timestamp_pos = seek_sequence[last_slice - 2].item() - timestamp_begin + segment_offset = last_timestamp_pos * input_stride + else: + # If whisper does not predict any "end of segment" token, then + # the whole decoding is considered a segment and we add it to the list of segments + timestamps = seek_sequence[timestamp_tokens.nonzero().flatten()] + last_timestamp_pos = int(seek_num_frames[prev_idx] * time_precision_features / time_precision) + if timestamps.numel() > 0 and timestamps[-1] != timestamp_begin: + # no consecutive timestamps but it has a timestamp; use the last one. + last_timestamp_pos = (timestamps[-1] - timestamp_begin).to( + torch.float32 if device.type == "mps" else torch.float64 + ) + segments = [ + { + "start": time_offset[prev_idx], + "end": time_offset[prev_idx] + last_timestamp_pos * time_precision, + "tokens": seek_sequence, + "idxs": (idx_offset, idx_offset + len(seek_sequence)), + "result": seek_outputs[idx], + } + ] + if return_token_timestamps: + segments[-1]["token_timestamps"] = ( + token_timestamps[idx_offset : idx_offset + len(seek_sequence)] + time_offset[prev_idx] + ) + segment_offset = seek_num_frames[prev_idx] + + return segments, segment_offset diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/audits/owt_ultraclean10k_row_audit/summary.txt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/audits/owt_ultraclean10k_row_audit/summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..5faa27d78d4c5bbab2a8e52a41c10d724c7144a1 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/audits/owt_ultraclean10k_row_audit/summary.txt @@ -0,0 +1,155 @@ +cache=cache/owt_t5_stream_pack1023_ultraclean_probe80k_rowvalid_10k_seed20260527_appendeos1.pt +shape=(10000, 1024) audited_rows=10000 source=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext + +score: mean=0.3208 p50=0.24 p90=0.6619 p99=1.084 max=1.609 +flag_count: mean=0.5033 p50=0 p90=1 p99=2 max=4 +unique: mean=452.5 p50=456 p90=493 p99=522 max=575 +top_frac: mean=0.04649 p50=0.0459 p90=0.05566 p99=0.06738 max=0.0791 +max_run: mean=1.419 p50=1 p90=2 p99=3 max=7 +bigram_count: mean=3.315 p50=0 p90=10 p99=15 max=27 +trigram_count: mean=0.5936 p50=0 p90=0 p99=12 max=24 +word_count: mean=699.2 p50=701 p90=747 p99=786 max=845 +sentence_count: mean=32.58 p50=32 p90=42 p99=55 max=96 +alpha_frac: mean=0.9531 p50=0.955 p90=0.9658 p99=0.9738 max=0.9902 +punct_frac: mean=0.03229 p50=0.03133 p90=0.04128 p99=0.05411 max=0.1134 +code_symbol_frac: mean=0.00182 p50=0.001563 p90=0.0028 p99=0.009539 max=0.02947 +non_ascii_frac: mean=0.005314 p50=0.00464 p90=0.01107 p99=0.01762 max=0.03831 +stop_frac: mean=0.3791 p50=0.3789 p90=0.4178 p99=0.4567 max=0.4976 +internal_eos_count: mean=1.149 p50=1 p90=2 p99=3 max=4 +line_count: mean=1 p50=1 p90=1 p99=1 max=1 +short_line_frac: mean=0 p50=0 p90=0 p99=0 max=0 + +flag_counts: + bigram: 3442/10000 (34.42%) + trigram: 605/10000 (6.05%) + unk: 519/10000 (5.19%) + top_token: 445/10000 (4.45%) + run: 9/10000 (0.09%) + low_stop: 9/10000 (0.09%) + non_ascii: 4/10000 (0.04%) + +top_tokens_all: + 0.04133 423170 3 + 0.03649 373630 8 the + 0.03331 341070 6 , + 0.03187 326307 5 . + 0.02573 263476 7 s + 0.01981 202887 9 a + 0.01980 202730 12 to + 0.01815 185825 13 of + 0.01685 172539 11 and + 0.01329 136117 16 in + 0.00872 89305 24 that + 0.00811 83078 22 ’ + 0.00752 77047 19 is + 0.00751 76871 18 - + 0.00653 66907 21 for + 0.00571 58477 31 ' + 0.00562 57548 30 on + 0.00546 55881 17 t + 0.00525 53802 15 e + 0.00501 51300 34 it + 0.00471 48255 28 with + 0.00465 47642 26 d + 0.00454 46459 37 The + 0.00450 46046 47 was + 0.00408 41808 38 as + 0.00394 40346 27 I + 0.00384 39298 36 be + 0.00383 39251 105 “ + 0.00355 36301 33 are + 0.00337 34494 43 have + 0.00330 33785 44 at + 0.00321 32872 88 he + 0.00305 31188 57 by + 0.00302 30973 45 from + 0.00298 30506 53 ing + 0.00279 28570 65 has + 0.00275 28201 48 this + 0.00275 28112 96 " + 0.00274 28090 59 not + 0.00274 28080 243 said + 0.00266 27234 10 : + 0.00262 26795 25 you + 0.00259 26554 46 an + 0.00252 25779 112 his + 0.00230 23522 79 they + 0.00222 22698 29 n + 0.00218 22294 32 o + 0.00215 22001 42 or + 0.00214 21929 56 will + 0.00213 21859 41 ( + 0.00210 21490 1 + 0.00208 21259 68 but + 0.00203 20805 62 we + 0.00198 20252 70 their + 0.00197 20159 23 i + 0.00196 20115 63 y + 0.00194 19815 113 who + 0.00183 18738 72 more + 0.00177 18169 81 about + 0.00175 17879 54 can + +first_tokens: + 0.04450 445 3 + 0.03880 388 8 the + 0.03380 338 6 , + 0.02950 295 5 . + 0.02540 254 7 s + 0.02340 234 12 to + 0.02050 205 13 of + 0.02030 203 9 a + 0.01670 167 11 and + 0.01300 130 16 in + 0.00980 98 22 ’ + 0.00810 81 24 that + 0.00680 68 17 t + 0.00640 64 21 for + 0.00640 64 18 - + 0.00620 62 31 ' + 0.00600 60 19 is + 0.00520 52 15 e + 0.00480 48 47 was + 0.00460 46 26 d + 0.00460 46 30 on + 0.00410 41 38 as + 0.00410 41 43 have + 0.00400 40 34 it + 0.00390 39 27 I + 0.00370 37 48 this + 0.00360 36 59 not + 0.00360 36 44 at + 0.00350 35 37 The + 0.00350 35 105 “ + 0.00340 34 33 are + 0.00340 34 65 has + 0.00340 34 36 be + 0.00340 34 28 with + 0.00340 34 96 " + 0.00340 34 25 you + 0.00320 32 53 ing + 0.00310 31 57 by + 0.00310 31 88 he + 0.00290 29 29 n + +worst_indices: + idx=8304 score=1.6087 flags=top_token=0.076:.|bigram=12:? Robinson|trigram=12:? Robinson: preview='s been fun. ESPN.com: As a basketball player, how important are your socks? Robinson: Oh, man, it's big-time. Trust me. I've been around where you have uncomfortable socks, and they just don't feel right. It's like, 'Ugh,' it's like a drag + idx=9444 score=1.5775 flags=top_token=0.074:|bigram=12:- Ya|trigram=12:- Ya preview=rashant Bhushan. They were treated unfairly," she said. Arvind Kejriwal has, meanwhile, returned to the National Council meeting venue in Kapashera, say reports. Arvind and his gang are a bunch of uncouth goons - Shazia Ilmi They (Yogendra + idx=2840 score=1.5587 flags=top_token=0.061:,|run=6:a|bigram=21:. SH|trigram=21:. SHAME preview=allowed Sleep is not allowed The crowd will be encouraged to chant “SHAME. SHAME. SHAME. SHAME” for the entire round Admission would be free and open, allowing fresh hecklers to fill in and yell throughout the 41-hour process In hour five, + idx=4393 score=1.4962 flags=top_token=0.072:the|bigram=17:Hong Kong|trigram=11:in Hong Kong preview=by the National People’s Congress Standing Committee on the broad outlines of policy changes envisioned for the 2017 election of the next chief executive of Hong Kong. In his speech, he said giving people in Hong Kong an unfettered choice i + idx=4831 score=1.4837 flags=top_token=0.068:,|bigram=14:You are|trigram=12:. You are preview=k in the movie, All the King’s Men (based on the eponymous Pulitzer Prize-winning novel by Robert Penn Warren) is apropos the treatment of the supporters of Donald Trump by the reigning tastemakers of the conservative movement. An updated v + idx=7195 score=1.4837 flags=top_token=0.068:the|bigram=13:the playoff|trigram=12:the playoffs preview=something we’d like to change. We invite the rest of the political media to follow our lead and reject tired anti-Hillary narratives. Stop rewarding conservative Clinton-bashers who constructed a toxic framework for Hillary coverage decades + idx=8417 score=1.4752 flags=top_token=0.079:the|bigram=12:of the|trigram=8:’s Department preview=the error three days before the parliamentary record was corrected on 4 June. Labor has been highly critical of the government for waiting until the final day of a parliamentary sitting fortnight to retract the previous claims, and argues t + idx=6276 score=1.4712 flags=unk=1|top_token=0.064:the|bigram=14:of the|trigram=12:Hatch Act preview=and defining prohibited political activity to include all "activity directed toward the success or failure of a political party, candidate for partisan political office, or partisan political group." Did Director Comey's letter to Congress + idx=2214 score=1.4337 flags=top_token=0.068:the|bigram=12:s Canada|trigram=11:Parks Canada preview=a.m. on Saturday, the moment that the shootings began in the Aurora theater a year earlier. Scheduled participants include a wounded survivor of the Aurora shootings, parents whose adult children were killed in the theater and a woman whose + idx=6972 score=1.4337 flags=top_token=0.068:the|bigram=27:the loop|trigram=11:of the loop preview=included a death. It takes a week or two for blood tests to reveal whether alcohol was involved in a crash, so for the time being, the driver was sent home pending charges. Troy police said the couple in the Mazda attended the Dream Cruise. + idx=8139 score=1.4056 flags=top_token=0.063:.|bigram=14:. Water|trigram=14:Mr. Water preview=says Mr. McMullen, who provided The Globe with a May, 2015, document signed by Mr. Watermulder, releasing his former employer from claims of wrongful dismissal in exchange for severance pay. Mr. Watermulder says he was not fired and that he + idx=3650 score=1.4025 flags=top_token=0.066:|bigram=14:Miss Joseph|trigram=11:Mr Cund preview='Yes.' The ex-model girlfriend of a superrich financier was refused a £4million payout because his advisers suspected she was being 'greedy and undeserving'. Nigerian born Tina Chantale Joseph, 51, lost a High Court legal battle over th + idx=1079 score=1.3747 flags=top_token=0.062:|bigram=18:at the|trigram=16:the rim preview=the way here is USC. The same team we previously saw sitting in eighth place in block percentage and third in block percentage at the rim. Meanwhile, Colorado is in second with just the seventh best block percentage overall and at the rim. + idx=7582 score=1.3494 flags=unk=1|top_token=0.069:|bigram=10:the Red|trigram=10:the Reds preview=was good enough to hold on to victory, but this team needs to go for the jugular when it has a lead instead of desperately retreating inwards to try and simply eke out a win. utilized an ultra-conservative game plan that almost backfired on + idx=4692 score=1.3312 flags=top_token=0.061:|bigram=8:of Bor|trigram=13:a burg|low_stop=0.171 preview=s Borrelia burgdorferi spirochetes in 4 weeks after ceftriaxone treatment in C3H/He mice. J Infect Dis 2007;195:1489–1496. Yrjanainen H, Hytonen J, Hartiala P, Oksi J, Vijanen MK. Persistence of borrelial DNA in the joints of Borrelia burgd + idx=5891 score=1.3056 flags=unk=1|top_token=0.063:the|bigram=11:Democratic Party|trigram=10:the Democratic Party preview=to 46%. This is the largest percentage saying so since November 1994, after the party's losses in that year's midterm elections. Most major demographic and attitudinal subgroups show at least a slight uptick since 2008 in perceptions that t + idx=3012 score=1.2994 flags=top_token=0.069:|bigram=10:ex tape|trigram=10:sex tape preview=performance amid a watershed result for the team. MICHAEL COX’S ASSESSMENT: Francis Coquelin has unquestionably been the revelation of Arsenal’s season, returning from a loan spell at Charlton to become a fixture in the first team. Having i + idx=7972 score=1.2869 flags=top_token=0.065:|bigram=14:in one|trigram=9:a hole preview=unemployed men (currently 1.5 million and a rate of 75.4% as opposed to 1 million and a rate of 65.4% women)to get onto the employment ladder (and women of course). So why are there are special schemes for female entrepreneurs and why is th + idx=184 score=1.2806 flags=unk=1|bigram=15:of the|trigram=12:the Alam preview=was imprisoned for his involvement.[18] Travis was commissioned as a lieutenant colonel of the Legion of Cavalry and became the chief recruiting officer for a new regular Texian army.[18] Governor Henry Smith ordered Travis to raise a compa + idx=4154 score=1.2775 flags=bigram=15:( ISBN|trigram=15:( ISBN preview=video is from CNN's Newsroom, broadcast on November 21. Bob the Angry Flower Author(s) Stephen Notley Website http://www.angryflower.com/ Current status/schedule Weekly Launch date 1992 Genre(s) Comedy Bob the Angry Flower is a webcomic