diff --git a/LTA_openwebtext_dualt/logs/elf_lm1b_t5small_elfb_len128_4gpu_smoke20_20260513.log b/LTA_openwebtext_dualt/logs/elf_lm1b_t5small_elfb_len128_4gpu_smoke20_20260513.log new file mode 100644 index 0000000000000000000000000000000000000000..f7c948ca042bdaf0b1b3e08069ae86d439792db7 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/elf_lm1b_t5small_elfb_len128_4gpu_smoke20_20260513.log @@ -0,0 +1,34 @@ +You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 +You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 +You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 +You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 +[elf-lm1b] encoder=/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small enc_dim=512 vocab=32100 +[elf-lm1b] batch=16 world=4 grad_accum=8 gbs~=512 +/usr/local/lib/python3.12/dist-packages/apex/_autocast_utils.py:26: FutureWarning: `torch.cuda.amp.autocast_mode._cast(value, dtype)` is deprecated. Please use `torch.amp.autocast_mode._cast(value, 'cuda', dtype)` instead. + return torch.cuda.amp.autocast_mode._cast(args, torch.get_autocast_gpu_dtype()) +/usr/local/lib/python3.12/dist-packages/apex/_autocast_utils.py:26: FutureWarning: `torch.cuda.amp.autocast_mode._cast(value, dtype)` is deprecated. Please use `torch.amp.autocast_mode._cast(value, 'cuda', dtype)` instead. + return torch.cuda.amp.autocast_mode._cast(args, torch.get_autocast_gpu_dtype()) +/usr/local/lib/python3.12/dist-packages/apex/_autocast_utils.py:26: FutureWarning: `torch.cuda.amp.autocast_mode._cast(value, dtype)` is deprecated. Please use `torch.amp.autocast_mode._cast(value, 'cuda', dtype)` instead. + return torch.cuda.amp.autocast_mode._cast(args, torch.get_autocast_gpu_dtype()) +/usr/local/lib/python3.12/dist-packages/apex/_autocast_utils.py:26: FutureWarning: `torch.cuda.amp.autocast_mode._cast(value, dtype)` is deprecated. Please use `torch.amp.autocast_mode._cast(value, 'cuda', dtype)` instead. + return torch.cuda.amp.autocast_mode._cast(args, torch.get_autocast_gpu_dtype()) +[2026-05-13 16:31:30] step=1 elapsed=1.2s lr=4.000000e-04 loss=3.0106 l2=1.7109 ce=1.2996 decoder_frac=0.125 t=0.314 tokens=595 +[2026-05-13 16:31:30] step=2 elapsed=0.4s lr=6.000000e-04 loss=2.7744 l2=1.4773 ce=1.2972 decoder_frac=0.125 t=0.284 tokens=554 +[2026-05-13 16:31:31] step=3 elapsed=0.4s lr=8.000000e-04 loss=3.8823 l2=1.2978 ce=2.5844 decoder_frac=0.250 t=0.389 tokens=561 +[2026-05-13 16:31:31] step=4 elapsed=0.4s lr=1.000000e-03 loss=2.8143 l2=1.5264 ce=1.2878 decoder_frac=0.125 t=0.295 tokens=565 +[2026-05-13 16:31:32] step=5 elapsed=0.4s lr=1.000000e-03 loss=3.7896 l2=1.2306 ce=2.5590 decoder_frac=0.250 t=0.381 tokens=530 +[2026-05-13 16:31:32] step=6 elapsed=0.4s lr=1.000000e-03 loss=2.7638 l2=1.4906 ce=1.2732 decoder_frac=0.125 t=0.303 tokens=525 +[2026-05-13 16:31:33] step=7 elapsed=0.4s lr=1.000000e-03 loss=1.7470 l2=1.7470 ce=0.0000 decoder_frac=0.000 t=0.211 tokens=550 +[2026-05-13 16:31:33] step=8 elapsed=0.4s lr=1.000000e-03 loss=3.9152 l2=1.3883 ce=2.5269 decoder_frac=0.250 t=0.412 tokens=560 +[2026-05-13 16:31:33] step=9 elapsed=0.4s lr=1.000000e-03 loss=3.7428 l2=1.2287 ce=2.5141 decoder_frac=0.250 t=0.395 tokens=576 +[2026-05-13 16:31:34] step=10 elapsed=0.4s lr=1.000000e-03 loss=2.8288 l2=1.5745 ce=1.2543 decoder_frac=0.125 t=0.315 tokens=532 +[2026-05-13 16:31:34] step=11 elapsed=0.4s lr=1.000000e-03 loss=3.7346 l2=1.2263 ce=2.5083 decoder_frac=0.250 t=0.401 tokens=585 +[2026-05-13 16:31:35] step=12 elapsed=0.4s lr=1.000000e-03 loss=3.8412 l2=1.3451 ce=2.4961 decoder_frac=0.250 t=0.407 tokens=550 +[2026-05-13 16:31:35] step=13 elapsed=0.4s lr=1.000000e-03 loss=3.8036 l2=1.3080 ce=2.4955 decoder_frac=0.250 t=0.409 tokens=590 +[2026-05-13 16:31:35] step=14 elapsed=0.4s lr=1.000000e-03 loss=2.7358 l2=1.4966 ce=1.2392 decoder_frac=0.125 t=0.302 tokens=540 +[2026-05-13 16:31:36] step=15 elapsed=0.4s lr=1.000000e-03 loss=4.7200 l2=1.0126 ce=3.7075 decoder_frac=0.375 t=0.494 tokens=529 +[2026-05-13 16:31:36] step=16 elapsed=0.4s lr=1.000000e-03 loss=3.7269 l2=1.2624 ce=2.4645 decoder_frac=0.250 t=0.408 tokens=569 +[2026-05-13 16:31:37] step=17 elapsed=0.4s lr=1.000000e-03 loss=2.7924 l2=1.5645 ce=1.2279 decoder_frac=0.125 t=0.313 tokens=557 +[2026-05-13 16:31:37] step=18 elapsed=0.4s lr=1.000000e-03 loss=2.7998 l2=1.5674 ce=1.2323 decoder_frac=0.125 t=0.324 tokens=540 +[2026-05-13 16:31:38] step=19 elapsed=0.4s lr=1.000000e-03 loss=1.8059 l2=1.8059 ce=0.0000 decoder_frac=0.000 t=0.231 tokens=541 +[2026-05-13 16:31:38] step=20 elapsed=0.4s lr=1.000000e-03 loss=3.7640 l2=1.3074 ce=2.4566 decoder_frac=0.250 t=0.416 tokens=569 diff --git a/LTA_openwebtext_dualt/logs/infer_owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large_resume_20260520_201156.log b/LTA_openwebtext_dualt/logs/infer_owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large_resume_20260520_201156.log new file mode 100644 index 0000000000000000000000000000000000000000..bc8259bebdc4235dff8432e38a34a7055d5daf50 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/infer_owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large_resume_20260520_201156.log @@ -0,0 +1,87 @@ +[sweep] run=runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817 +[skip] step=20359 existing=docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step20359_steps128_c1024_t1p45.jsonl +[infer] step=40718 out=docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step40718_steps128_c1024_t1p45.jsonl +[ckpt] runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817/step_0040718.pt step=40718 +[decode-base] n=8 max_len=1024 steps=128 model_t=post +[decode-time] schedule=linear s=[0.0,0.25] gumbel=(2.2,0.8) force_final=True t0=0.000000 t_mid=0.500000 t_end=1.000000 dt_mean=0.007812 dt_max=0.007812 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 1/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 2/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 3/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 5/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 6/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 7/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8 +[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817/step_0040718.pt", "step": 40718, "decode": {"steps": 128, "model_t_mode": "post", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.0078125, 0.015625, 0.0234375, 0.03125, 0.0390625, 0.046875, 0.0546875, 0.0625, 0.0703125, 0.078125, 0.0859375, 0.09375, 0.1015625, 0.109375, 0.1171875, 0.125, 0.1328125, 0.140625, 0.1484375, 0.15625, 0.1640625, 0.171875, 0.1796875, 0.1875, 0.1953125, 0.203125, 0.2109375, 0.21875, 0.2265625, 0.234375, 0.2421875, 0.25, 0.2578125, 0.265625, 0.2734375, 0.28125, 0.2890625, 0.296875, 0.3046875, 0.3125, 0.3203125, 0.328125, 0.3359375, 0.34375, 0.3515625, 0.359375, 0.3671875, 0.375, 0.3828125, 0.390625, 0.3984375, 0.40625, 0.4140625, 0.421875, 0.4296875, 0.4375, 0.4453125, 0.453125, 0.4609375, 0.46875, 0.4765625, 0.484375, 0.4921875, 0.5, 0.5078125, 0.515625, 0.5234375, 0.53125, 0.5390625, 0.546875, 0.5546875, 0.5625, 0.5703125, 0.578125, 0.5859375, 0.59375, 0.6015625, 0.609375, 0.6171875, 0.625, 0.6328125, 0.640625, 0.6484375, 0.65625, 0.6640625, 0.671875, 0.6796875, 0.6875, 0.6953125, 0.703125, 0.7109375, 0.71875, 0.7265625, 0.734375, 0.7421875, 0.75, 0.7578125, 0.765625, 0.7734375, 0.78125, 0.7890625, 0.796875, 0.8046875, 0.8125, 0.8203125, 0.828125, 0.8359375, 0.84375, 0.8515625, 0.859375, 0.8671875, 0.875, 0.8828125, 0.890625, 0.8984375, 0.90625, 0.9140625, 0.921875, 0.9296875, 0.9375, 0.9453125, 0.953125, 0.9609375, 0.96875, 0.9765625, 0.984375, 0.9921875, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "state", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 1024.0, "target_prob": 1.0, "endpoint_temp": 1.45, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260520}, "raw_genppl": {"ppl": 1.002838985729012, "nll_per_token": 0.0028349634200807603, "tokens": 504, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 1.002838985729012, "nll_per_token": 0.0028349634200807603, "tokens": 504, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 0.0, "unique_tokens": 1, "token_count": 8192, "distinct_1": 0.0001220703125, "distinct_2": 0.00012218963831867058, "top_token_mass": 1.0}} +[done] docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step40718_steps128_c1024_t1p45.jsonl +[infer] step=61077 out=docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step61077_steps128_c1024_t1p45.jsonl +[ckpt] runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817/step_0061077.pt step=61077 +[decode-base] n=8 max_len=1024 steps=128 model_t=post +[decode-time] schedule=linear s=[0.0,0.25] gumbel=(2.2,0.8) force_final=True t0=0.000000 t_mid=0.500000 t_end=1.000000 dt_mean=0.007812 dt_max=0.007812 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 1/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 2/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 3/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 5/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 6/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 7/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8 +[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817/step_0061077.pt", "step": 61077, "decode": {"steps": 128, "model_t_mode": "post", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.0078125, 0.015625, 0.0234375, 0.03125, 0.0390625, 0.046875, 0.0546875, 0.0625, 0.0703125, 0.078125, 0.0859375, 0.09375, 0.1015625, 0.109375, 0.1171875, 0.125, 0.1328125, 0.140625, 0.1484375, 0.15625, 0.1640625, 0.171875, 0.1796875, 0.1875, 0.1953125, 0.203125, 0.2109375, 0.21875, 0.2265625, 0.234375, 0.2421875, 0.25, 0.2578125, 0.265625, 0.2734375, 0.28125, 0.2890625, 0.296875, 0.3046875, 0.3125, 0.3203125, 0.328125, 0.3359375, 0.34375, 0.3515625, 0.359375, 0.3671875, 0.375, 0.3828125, 0.390625, 0.3984375, 0.40625, 0.4140625, 0.421875, 0.4296875, 0.4375, 0.4453125, 0.453125, 0.4609375, 0.46875, 0.4765625, 0.484375, 0.4921875, 0.5, 0.5078125, 0.515625, 0.5234375, 0.53125, 0.5390625, 0.546875, 0.5546875, 0.5625, 0.5703125, 0.578125, 0.5859375, 0.59375, 0.6015625, 0.609375, 0.6171875, 0.625, 0.6328125, 0.640625, 0.6484375, 0.65625, 0.6640625, 0.671875, 0.6796875, 0.6875, 0.6953125, 0.703125, 0.7109375, 0.71875, 0.7265625, 0.734375, 0.7421875, 0.75, 0.7578125, 0.765625, 0.7734375, 0.78125, 0.7890625, 0.796875, 0.8046875, 0.8125, 0.8203125, 0.828125, 0.8359375, 0.84375, 0.8515625, 0.859375, 0.8671875, 0.875, 0.8828125, 0.890625, 0.8984375, 0.90625, 0.9140625, 0.921875, 0.9296875, 0.9375, 0.9453125, 0.953125, 0.9609375, 0.96875, 0.9765625, 0.984375, 0.9921875, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "state", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 1024.0, "target_prob": 1.0, "endpoint_temp": 1.45, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260520}, "raw_genppl": {"ppl": 105.9955547549316, "nll_per_token": 4.663397156958487, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 105.9955547549316, "nll_per_token": 4.663397156958487, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 2.40693613511493, "unique_tokens": 143, "token_count": 8192, "distinct_1": 0.0174560546875, "distinct_2": 0.10117302052785923, "top_token_mass": 0.2647705078125}} +[done] docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step61077_steps128_c1024_t1p45.jsonl +[infer] step=81436 out=docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step81436_steps128_c1024_t1p45.jsonl +[ckpt] runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817/step_0081436.pt step=81436 +[decode-base] n=8 max_len=1024 steps=128 model_t=post +[decode-time] schedule=linear s=[0.0,0.25] gumbel=(2.2,0.8) force_final=True t0=0.000000 t_mid=0.500000 t_end=1.000000 dt_mean=0.007812 dt_max=0.007812 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 1/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 2/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 3/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 5/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 6/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 7/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8 +[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817/step_0081436.pt", "step": 81436, "decode": {"steps": 128, "model_t_mode": "post", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.0078125, 0.015625, 0.0234375, 0.03125, 0.0390625, 0.046875, 0.0546875, 0.0625, 0.0703125, 0.078125, 0.0859375, 0.09375, 0.1015625, 0.109375, 0.1171875, 0.125, 0.1328125, 0.140625, 0.1484375, 0.15625, 0.1640625, 0.171875, 0.1796875, 0.1875, 0.1953125, 0.203125, 0.2109375, 0.21875, 0.2265625, 0.234375, 0.2421875, 0.25, 0.2578125, 0.265625, 0.2734375, 0.28125, 0.2890625, 0.296875, 0.3046875, 0.3125, 0.3203125, 0.328125, 0.3359375, 0.34375, 0.3515625, 0.359375, 0.3671875, 0.375, 0.3828125, 0.390625, 0.3984375, 0.40625, 0.4140625, 0.421875, 0.4296875, 0.4375, 0.4453125, 0.453125, 0.4609375, 0.46875, 0.4765625, 0.484375, 0.4921875, 0.5, 0.5078125, 0.515625, 0.5234375, 0.53125, 0.5390625, 0.546875, 0.5546875, 0.5625, 0.5703125, 0.578125, 0.5859375, 0.59375, 0.6015625, 0.609375, 0.6171875, 0.625, 0.6328125, 0.640625, 0.6484375, 0.65625, 0.6640625, 0.671875, 0.6796875, 0.6875, 0.6953125, 0.703125, 0.7109375, 0.71875, 0.7265625, 0.734375, 0.7421875, 0.75, 0.7578125, 0.765625, 0.7734375, 0.78125, 0.7890625, 0.796875, 0.8046875, 0.8125, 0.8203125, 0.828125, 0.8359375, 0.84375, 0.8515625, 0.859375, 0.8671875, 0.875, 0.8828125, 0.890625, 0.8984375, 0.90625, 0.9140625, 0.921875, 0.9296875, 0.9375, 0.9453125, 0.953125, 0.9609375, 0.96875, 0.9765625, 0.984375, 0.9921875, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "state", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 1024.0, "target_prob": 1.0, "endpoint_temp": 1.45, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260520}, "raw_genppl": {"ppl": 20.025109430324235, "nll_per_token": 2.9969869576248467, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 20.02894006760397, "nll_per_token": 2.9971782310336246, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 1.6417594596790073, "unique_tokens": 63, "token_count": 8192, "distinct_1": 0.0076904296875, "distinct_2": 0.056329423264907134, "top_token_mass": 0.4775390625}} +[done] docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step81436_steps128_c1024_t1p45.jsonl +[infer] step=101795 out=docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step101795_steps128_c1024_t1p45.jsonl +[ckpt] runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817/step_0101795.pt step=101795 +[decode-base] n=8 max_len=1024 steps=128 model_t=post +[decode-time] schedule=linear s=[0.0,0.25] gumbel=(2.2,0.8) force_final=True t0=0.000000 t_mid=0.500000 t_end=1.000000 dt_mean=0.007812 dt_max=0.007812 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 1/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 2/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 3/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 5/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 6/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 7/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8 +[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817/step_0101795.pt", "step": 101795, "decode": {"steps": 128, "model_t_mode": "post", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.0078125, 0.015625, 0.0234375, 0.03125, 0.0390625, 0.046875, 0.0546875, 0.0625, 0.0703125, 0.078125, 0.0859375, 0.09375, 0.1015625, 0.109375, 0.1171875, 0.125, 0.1328125, 0.140625, 0.1484375, 0.15625, 0.1640625, 0.171875, 0.1796875, 0.1875, 0.1953125, 0.203125, 0.2109375, 0.21875, 0.2265625, 0.234375, 0.2421875, 0.25, 0.2578125, 0.265625, 0.2734375, 0.28125, 0.2890625, 0.296875, 0.3046875, 0.3125, 0.3203125, 0.328125, 0.3359375, 0.34375, 0.3515625, 0.359375, 0.3671875, 0.375, 0.3828125, 0.390625, 0.3984375, 0.40625, 0.4140625, 0.421875, 0.4296875, 0.4375, 0.4453125, 0.453125, 0.4609375, 0.46875, 0.4765625, 0.484375, 0.4921875, 0.5, 0.5078125, 0.515625, 0.5234375, 0.53125, 0.5390625, 0.546875, 0.5546875, 0.5625, 0.5703125, 0.578125, 0.5859375, 0.59375, 0.6015625, 0.609375, 0.6171875, 0.625, 0.6328125, 0.640625, 0.6484375, 0.65625, 0.6640625, 0.671875, 0.6796875, 0.6875, 0.6953125, 0.703125, 0.7109375, 0.71875, 0.7265625, 0.734375, 0.7421875, 0.75, 0.7578125, 0.765625, 0.7734375, 0.78125, 0.7890625, 0.796875, 0.8046875, 0.8125, 0.8203125, 0.828125, 0.8359375, 0.84375, 0.8515625, 0.859375, 0.8671875, 0.875, 0.8828125, 0.890625, 0.8984375, 0.90625, 0.9140625, 0.921875, 0.9296875, 0.9375, 0.9453125, 0.953125, 0.9609375, 0.96875, 0.9765625, 0.984375, 0.9921875, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "state", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 1024.0, "target_prob": 1.0, "endpoint_temp": 1.45, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260520}, "raw_genppl": {"ppl": 13.46160283327985, "nll_per_token": 2.599841398351333, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 6.77012163217799, "nll_per_token": 1.9125190531110905, "tokens": 2022, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 0.7844788420542315, "unique_tokens": 9, "token_count": 8192, "distinct_1": 0.0010986328125, "distinct_2": 0.0039100684261974585, "top_token_mass": 0.6846923828125}} +[done] docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step101795_steps128_c1024_t1p45.jsonl +[infer] step=122154 out=docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step122154_steps128_c1024_t1p45.jsonl +[ckpt] runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817/step_0122154.pt step=122154 +[decode-base] n=8 max_len=1024 steps=128 model_t=post +[decode-time] schedule=linear s=[0.0,0.25] gumbel=(2.2,0.8) force_final=True t0=0.000000 t_mid=0.500000 t_end=1.000000 dt_mean=0.007812 dt_max=0.007812 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 1/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 2/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 3/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 5/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 6/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 7/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8 +[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817/step_0122154.pt", "step": 122154, "decode": {"steps": 128, "model_t_mode": "post", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.0078125, 0.015625, 0.0234375, 0.03125, 0.0390625, 0.046875, 0.0546875, 0.0625, 0.0703125, 0.078125, 0.0859375, 0.09375, 0.1015625, 0.109375, 0.1171875, 0.125, 0.1328125, 0.140625, 0.1484375, 0.15625, 0.1640625, 0.171875, 0.1796875, 0.1875, 0.1953125, 0.203125, 0.2109375, 0.21875, 0.2265625, 0.234375, 0.2421875, 0.25, 0.2578125, 0.265625, 0.2734375, 0.28125, 0.2890625, 0.296875, 0.3046875, 0.3125, 0.3203125, 0.328125, 0.3359375, 0.34375, 0.3515625, 0.359375, 0.3671875, 0.375, 0.3828125, 0.390625, 0.3984375, 0.40625, 0.4140625, 0.421875, 0.4296875, 0.4375, 0.4453125, 0.453125, 0.4609375, 0.46875, 0.4765625, 0.484375, 0.4921875, 0.5, 0.5078125, 0.515625, 0.5234375, 0.53125, 0.5390625, 0.546875, 0.5546875, 0.5625, 0.5703125, 0.578125, 0.5859375, 0.59375, 0.6015625, 0.609375, 0.6171875, 0.625, 0.6328125, 0.640625, 0.6484375, 0.65625, 0.6640625, 0.671875, 0.6796875, 0.6875, 0.6953125, 0.703125, 0.7109375, 0.71875, 0.7265625, 0.734375, 0.7421875, 0.75, 0.7578125, 0.765625, 0.7734375, 0.78125, 0.7890625, 0.796875, 0.8046875, 0.8125, 0.8203125, 0.828125, 0.8359375, 0.84375, 0.8515625, 0.859375, 0.8671875, 0.875, 0.8828125, 0.890625, 0.8984375, 0.90625, 0.9140625, 0.921875, 0.9296875, 0.9375, 0.9453125, 0.953125, 0.9609375, 0.96875, 0.9765625, 0.984375, 0.9921875, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "state", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 1024.0, "target_prob": 1.0, "endpoint_temp": 1.45, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260520}, "raw_genppl": {"ppl": 1.0, "nll_per_token": 0.0, "tokens": 0, "kept_samples": 0, "total_samples": 0, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 1.0, "nll_per_token": 0.0, "tokens": 0, "kept_samples": 0, "total_samples": 0, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 0.0, "unique_tokens": 1, "token_count": 8192, "distinct_1": 0.0001220703125, "distinct_2": 0.00012218963831867058, "top_token_mass": 1.0}} +[done] docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step122154_steps128_c1024_t1p45.jsonl +[infer] step=142513 out=docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step142513_steps128_c1024_t1p45.jsonl +[ckpt] runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817/step_0142513.pt step=142513 +[decode-base] n=8 max_len=1024 steps=128 model_t=post +[decode-time] schedule=linear s=[0.0,0.25] gumbel=(2.2,0.8) force_final=True t0=0.000000 t_mid=0.500000 t_end=1.000000 dt_mean=0.007812 dt_max=0.007812 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 1/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 2/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 3/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 5/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 6/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 7/8 +[decode] temp=1.45 final=state rule=dual_line_resample support=1 semantic=1 anchor=state cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8 +[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v8192_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_uniformt_hardce_mask0p1-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260519_201817/step_0142513.pt", "step": 142513, "decode": {"steps": 128, "model_t_mode": "post", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.0078125, 0.015625, 0.0234375, 0.03125, 0.0390625, 0.046875, 0.0546875, 0.0625, 0.0703125, 0.078125, 0.0859375, 0.09375, 0.1015625, 0.109375, 0.1171875, 0.125, 0.1328125, 0.140625, 0.1484375, 0.15625, 0.1640625, 0.171875, 0.1796875, 0.1875, 0.1953125, 0.203125, 0.2109375, 0.21875, 0.2265625, 0.234375, 0.2421875, 0.25, 0.2578125, 0.265625, 0.2734375, 0.28125, 0.2890625, 0.296875, 0.3046875, 0.3125, 0.3203125, 0.328125, 0.3359375, 0.34375, 0.3515625, 0.359375, 0.3671875, 0.375, 0.3828125, 0.390625, 0.3984375, 0.40625, 0.4140625, 0.421875, 0.4296875, 0.4375, 0.4453125, 0.453125, 0.4609375, 0.46875, 0.4765625, 0.484375, 0.4921875, 0.5, 0.5078125, 0.515625, 0.5234375, 0.53125, 0.5390625, 0.546875, 0.5546875, 0.5625, 0.5703125, 0.578125, 0.5859375, 0.59375, 0.6015625, 0.609375, 0.6171875, 0.625, 0.6328125, 0.640625, 0.6484375, 0.65625, 0.6640625, 0.671875, 0.6796875, 0.6875, 0.6953125, 0.703125, 0.7109375, 0.71875, 0.7265625, 0.734375, 0.7421875, 0.75, 0.7578125, 0.765625, 0.7734375, 0.78125, 0.7890625, 0.796875, 0.8046875, 0.8125, 0.8203125, 0.828125, 0.8359375, 0.84375, 0.8515625, 0.859375, 0.8671875, 0.875, 0.8828125, 0.890625, 0.8984375, 0.90625, 0.9140625, 0.921875, 0.9296875, 0.9375, 0.9453125, 0.953125, 0.9609375, 0.96875, 0.9765625, 0.984375, 0.9921875, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "state", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 1024.0, "target_prob": 1.0, "endpoint_temp": 1.45, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260520}, "raw_genppl": {"ppl": 5.245929944176233, "nll_per_token": 1.657452527214499, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 5.293541278301558, "nll_per_token": 1.666487450693168, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 1.2322328503813487, "unique_tokens": 55, "token_count": 8192, "distinct_1": 0.0067138671875, "distinct_2": 0.02761485826001955, "top_token_mass": 0.4080810546875}} +[done] docs/lta_samples/metrics_20260520/owt_compact_v8192_ckpt_sweep_steps128_c1024_t1p45_n8_large/step142513_steps128_c1024_t1p45.jsonl +step raw stripped entropy unique top_mass diff --git a/LTA_openwebtext_dualt/logs/lta_owt_bert_absrope_time0_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_1m_save10k_20260526.log b/LTA_openwebtext_dualt/logs/lta_owt_bert_absrope_time0_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_1m_save10k_20260526.log new file mode 100644 index 0000000000000000000000000000000000000000..79e277f86e2ebf70f3cc3720efe8522689d300af --- /dev/null +++ b/LTA_openwebtext_dualt/logs/lta_owt_bert_absrope_time0_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_1m_save10k_20260526.log @@ -0,0 +1,3185 @@ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:16:27.295435104 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank2]: model = EndpointPredictor( +[rank2]: ^^^^^^^^^^^^^^^^^^ +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank2]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank2]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:16:27.743000 10344 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10433 closing signal SIGTERM +W0526 02:16:27.744000 10344 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10434 closing signal SIGTERM +W0526 02:16:27.744000 10344 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10435 closing signal SIGTERM +W0526 02:16:27.745000 10344 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10436 closing signal SIGTERM +W0526 02:16:27.745000 10344 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10438 closing signal SIGTERM +W0526 02:16:27.745000 10344 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10439 closing signal SIGTERM +E0526 02:16:28.073000 10344 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10432) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-26_02:16:27 + host : t-20260526101508-xcqln-worker-1.t-20260526101508-xcqln-worker.mlplatform-customtask.svc.cluster.local + rank : 5 (local_rank: 5) + exitcode : 1 (pid: 10437) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:16:27 + host : t-20260526101508-xcqln-worker-1.t-20260526101508-xcqln-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10432) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:16:28.219016324 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:16:28.644000 10342 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10431 closing signal SIGTERM +W0526 02:16:28.644000 10342 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10432 closing signal SIGTERM +W0526 02:16:28.645000 10342 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10433 closing signal SIGTERM +W0526 02:16:28.645000 10342 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10434 closing signal SIGTERM +W0526 02:16:28.646000 10342 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10435 closing signal SIGTERM +W0526 02:16:28.646000 10342 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10436 closing signal SIGTERM +W0526 02:16:28.646000 10342 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10437 closing signal SIGTERM +E0526 02:16:29.024000 10342 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10430) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:16:28 + host : t-20260526101508-xcqln-worker-0.t-20260526101508-xcqln-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10430) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank2]: model = EndpointPredictor( +[rank2]: ^^^^^^^^^^^^^^^^^^ +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank2]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank2]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:18:55.255661930 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0526 02:18:55.553000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10483 closing signal SIGTERM +W0526 02:18:55.554000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10485 closing signal SIGTERM +W0526 02:18:55.554000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10487 closing signal SIGTERM +W0526 02:18:55.554000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10488 closing signal SIGTERM +W0526 02:18:55.555000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10489 closing signal SIGTERM +E0526 02:18:55.832000 10394 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10482) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-26_02:18:55 + host : t-20260526101645-sqkht-worker-1.t-20260526101645-sqkht-worker.mlplatform-customtask.svc.cluster.local + rank : 2 (local_rank: 2) + exitcode : 1 (pid: 10484) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2026-05-26_02:18:55 + host : t-20260526101645-sqkht-worker-1.t-20260526101645-sqkht-worker.mlplatform-customtask.svc.cluster.local + rank : 4 (local_rank: 4) + exitcode : 1 (pid: 10486) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:18:55 + host : t-20260526101645-sqkht-worker-1.t-20260526101645-sqkht-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10482) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:18:56.664643426 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank2]: model = EndpointPredictor( +[rank2]: ^^^^^^^^^^^^^^^^^^ +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank2]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank2]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:18:57.018000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10479 closing signal SIGTERM +W0526 02:18:57.018000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10480 closing signal SIGTERM +W0526 02:18:57.019000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10481 closing signal SIGTERM +W0526 02:18:57.019000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10482 closing signal SIGTERM +W0526 02:18:57.019000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10483 closing signal SIGTERM +W0526 02:18:57.020000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10484 closing signal SIGTERM +W0526 02:18:57.020000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10485 closing signal SIGTERM +E0526 02:18:57.385000 10390 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10478) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:18:57 + host : t-20260526101645-sqkht-worker-0.t-20260526101645-sqkht-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10478) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:21:07.025222798 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:21:07.247000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10483 closing signal SIGTERM +W0526 02:21:07.248000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10484 closing signal SIGTERM +W0526 02:21:07.248000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10485 closing signal SIGTERM +W0526 02:21:07.248000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10486 closing signal SIGTERM +W0526 02:21:07.249000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10487 closing signal SIGTERM +W0526 02:21:07.249000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10488 closing signal SIGTERM +W0526 02:21:07.250000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10489 closing signal SIGTERM +E0526 02:21:07.592000 10394 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10482) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:21:07 + host : t-20260526101912-nghf9-worker-1.t-20260526101912-nghf9-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10482) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:21:08.146755728 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank2]: model = EndpointPredictor( +[rank2]: ^^^^^^^^^^^^^^^^^^ +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank2]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank2]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:21:08.506000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10479 closing signal SIGTERM +W0526 02:21:08.507000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10480 closing signal SIGTERM +W0526 02:21:08.507000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10481 closing signal SIGTERM +W0526 02:21:08.507000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10482 closing signal SIGTERM +W0526 02:21:08.508000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10483 closing signal SIGTERM +W0526 02:21:08.508000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10484 closing signal SIGTERM +W0526 02:21:08.509000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10485 closing signal SIGTERM +E0526 02:21:08.850000 10390 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10478) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:21:08 + host : t-20260526101912-nghf9-worker-0.t-20260526101912-nghf9-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10478) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:23:28.441506247 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank2]: model = EndpointPredictor( +[rank2]: ^^^^^^^^^^^^^^^^^^ +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank2]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank2]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:23:28.784000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10479 closing signal SIGTERM +W0526 02:23:28.784000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10480 closing signal SIGTERM +W0526 02:23:28.785000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10481 closing signal SIGTERM +W0526 02:23:28.786000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10482 closing signal SIGTERM +W0526 02:23:28.786000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10483 closing signal SIGTERM +W0526 02:23:28.786000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10484 closing signal SIGTERM +W0526 02:23:28.787000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10485 closing signal SIGTERM +E0526 02:23:29.146000 10390 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10478) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:23:28 + host : t-20260526102124-wc9zp-worker-0.t-20260526102124-wc9zp-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10478) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:23:30.231388018 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank2]: model = EndpointPredictor( +[rank2]: ^^^^^^^^^^^^^^^^^^ +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank2]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank2]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:23:30.473000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10483 closing signal SIGTERM +W0526 02:23:30.474000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10484 closing signal SIGTERM +W0526 02:23:30.474000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10485 closing signal SIGTERM +W0526 02:23:30.475000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10486 closing signal SIGTERM +W0526 02:23:30.475000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10487 closing signal SIGTERM +W0526 02:23:30.475000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10488 closing signal SIGTERM +W0526 02:23:30.476000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10489 closing signal SIGTERM +E0526 02:23:30.868000 10394 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10482) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:23:30 + host : t-20260526102124-wc9zp-worker-1.t-20260526102124-wc9zp-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10482) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:25:55.817377691 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank2]: model = EndpointPredictor( +[rank2]: ^^^^^^^^^^^^^^^^^^ +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank2]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank2]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:25:56.136000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10479 closing signal SIGTERM +W0526 02:25:56.136000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10480 closing signal SIGTERM +W0526 02:25:56.137000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10481 closing signal SIGTERM +W0526 02:25:56.137000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10482 closing signal SIGTERM +W0526 02:25:56.138000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10483 closing signal SIGTERM +W0526 02:25:56.138000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10484 closing signal SIGTERM +W0526 02:25:56.138000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10485 closing signal SIGTERM +E0526 02:25:56.481000 10390 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10478) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:25:56 + host : t-20260526102345-xkqcz-worker-0.t-20260526102345-xkqcz-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10478) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:25:56.709664531 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank2]: model = EndpointPredictor( +[rank2]: ^^^^^^^^^^^^^^^^^^ +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank2]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank2]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:25:56.977000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10483 closing signal SIGTERM +W0526 02:25:56.978000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10484 closing signal SIGTERM +W0526 02:25:56.978000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10485 closing signal SIGTERM +W0526 02:25:56.979000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10486 closing signal SIGTERM +W0526 02:25:56.979000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10487 closing signal SIGTERM +W0526 02:25:56.980000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10488 closing signal SIGTERM +W0526 02:25:56.980000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10489 closing signal SIGTERM +E0526 02:25:57.339000 10394 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10482) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:25:56 + host : t-20260526102345-xkqcz-worker-1.t-20260526102345-xkqcz-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10482) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:28:22.448284962 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:28:22.620000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10483 closing signal SIGTERM +W0526 02:28:22.621000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10484 closing signal SIGTERM +W0526 02:28:22.622000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10485 closing signal SIGTERM +W0526 02:28:22.622000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10486 closing signal SIGTERM +W0526 02:28:22.622000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10487 closing signal SIGTERM +W0526 02:28:22.623000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10488 closing signal SIGTERM +W0526 02:28:22.623000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10489 closing signal SIGTERM +E0526 02:28:23.001000 10394 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10482) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:28:22 + host : t-20260526102613-b79k9-worker-1.t-20260526102613-b79k9-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10482) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:28:23.665138947 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank2]: model = EndpointPredictor( +[rank2]: ^^^^^^^^^^^^^^^^^^ +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank2]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank2]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:28:23.938000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10478 closing signal SIGTERM +W0526 02:28:23.938000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10479 closing signal SIGTERM +W0526 02:28:23.939000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10480 closing signal SIGTERM +W0526 02:28:23.939000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10481 closing signal SIGTERM +W0526 02:28:23.940000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10482 closing signal SIGTERM +W0526 02:28:23.940000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10483 closing signal SIGTERM +W0526 02:28:23.940000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10485 closing signal SIGTERM +E0526 02:28:24.281000 10390 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 6 (pid: 10484) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:28:23 + host : t-20260526102613-b79k9-worker-0.t-20260526102613-b79k9-worker.mlplatform-customtask.svc.cluster.local + rank : 6 (local_rank: 6) + exitcode : 1 (pid: 10484) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:30:29.710018069 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:30:29.823000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10479 closing signal SIGTERM +W0526 02:30:29.824000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10480 closing signal SIGTERM +W0526 02:30:29.824000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10481 closing signal SIGTERM +W0526 02:30:29.825000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10482 closing signal SIGTERM +W0526 02:30:29.825000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10483 closing signal SIGTERM +W0526 02:30:29.826000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10484 closing signal SIGTERM +W0526 02:30:29.826000 10390 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10485 closing signal SIGTERM +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:30:29.968561783 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank2]: model = EndpointPredictor( +[rank2]: ^^^^^^^^^^^^^^^^^^ +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank2]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank2]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:30:30.148000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10483 closing signal SIGTERM +W0526 02:30:30.149000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10484 closing signal SIGTERM +W0526 02:30:30.150000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10485 closing signal SIGTERM +W0526 02:30:30.150000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10486 closing signal SIGTERM +W0526 02:30:30.150000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10487 closing signal SIGTERM +W0526 02:30:30.151000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10488 closing signal SIGTERM +W0526 02:30:30.151000 10394 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10489 closing signal SIGTERM +E0526 02:30:30.204000 10390 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10478) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:30:29 + host : t-20260526102839-2t2ct-worker-0.t-20260526102839-2t2ct-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10478) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +E0526 02:30:30.529000 10394 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10482) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:30:30 + host : t-20260526102839-2t2ct-worker-1.t-20260526102839-2t2ct-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10482) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:31:49.872464473 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank2]: model = EndpointPredictor( +[rank2]: ^^^^^^^^^^^^^^^^^^ +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank2]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank2]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +W0526 02:31:50.081000 10270 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10359 closing signal SIGTERM +W0526 02:31:50.082000 10270 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10360 closing signal SIGTERM +W0526 02:31:50.082000 10270 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10361 closing signal SIGTERM +W0526 02:31:50.083000 10270 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10362 closing signal SIGTERM +W0526 02:31:50.083000 10270 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10363 closing signal SIGTERM +W0526 02:31:50.083000 10270 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10364 closing signal SIGTERM +W0526 02:31:50.084000 10270 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10365 closing signal SIGTERM +E0526 02:31:50.462000 10270 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10358) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:31:50 + host : t-20260526103049-vglbz-worker-0.t-20260526103049-vglbz-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10358) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[rank0]: Traceback (most recent call last): +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank0]: main() +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank0]: model = EndpointPredictor( +[rank0]: ^^^^^^^^^^^^^^^^^^ +[rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank0]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank0]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank1]: Traceback (most recent call last): +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank1]: main() +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank1]: model = EndpointPredictor( +[rank1]: ^^^^^^^^^^^^^^^^^^ +[rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank1]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank1]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank6]: Traceback (most recent call last): +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank6]: main() +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank6]: model = EndpointPredictor( +[rank6]: ^^^^^^^^^^^^^^^^^^ +[rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank6]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank6]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank5]: Traceback (most recent call last): +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank5]: main() +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank5]: model = EndpointPredictor( +[rank5]: ^^^^^^^^^^^^^^^^^^ +[rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank5]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank5]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank7]: Traceback (most recent call last): +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank7]: main() +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank7]: model = EndpointPredictor( +[rank7]: ^^^^^^^^^^^^^^^^^^ +[rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank7]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank7]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank3]: Traceback (most recent call last): +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank3]: main() +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank3]: model = EndpointPredictor( +[rank3]: ^^^^^^^^^^^^^^^^^^ +[rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank3]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank3]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank2]: Traceback (most recent call last): +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank2]: main() +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank2]: model = EndpointPredictor( +[rank2]: ^^^^^^^^^^^^^^^^^^ +[rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank2]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank2]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank4]: Traceback (most recent call last): +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 3143, in +[rank4]: main() +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 2503, in main +[rank4]: model = EndpointPredictor( +[rank4]: ^^^^^^^^^^^^^^^^^^ +[rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/flowtext_lab/model.py", line 580, in __init__ +[rank4]: raise ValueError("ddit_elf requires elf_num_time_tokens > 0") +[rank4]: ValueError: ddit_elf requires elf_num_time_tokens > 0 +[rank0]:[W526 02:33:53.083526617 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0526 02:33:53.322000 10318 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10408 closing signal SIGTERM +W0526 02:33:53.323000 10318 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10409 closing signal SIGTERM +W0526 02:33:53.323000 10318 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10410 closing signal SIGTERM +W0526 02:33:53.323000 10318 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10411 closing signal SIGTERM +W0526 02:33:53.324000 10318 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 10413 closing signal SIGTERM +E0526 02:33:53.588000 10318 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 10406) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-05-26_02:33:53 + host : t-20260526103206-7zvk9-worker-0.t-20260526103206-7zvk9-worker.mlplatform-customtask.svc.cluster.local + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 10407) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +[2]: + time : 2026-05-26_02:33:53 + host : t-20260526103206-7zvk9-worker-0.t-20260526103206-7zvk9-worker.mlplatform-customtask.svc.cluster.local + rank : 6 (local_rank: 6) + exitcode : 1 (pid: 10412) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-26_02:33:53 + host : t-20260526103206-7zvk9-worker-0.t-20260526103206-7zvk9-worker.mlplatform-customtask.svc.cluster.local + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 10406) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +t-20260526104746-rhql7-worker-0:10430:10430 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10430:10430 [0] NCCL INFO Bootstrap: Using eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10430:10430 [0] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-0:10430:10430 [0] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-0:10430:10430 [0] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-0:10435:10435 [5] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-0:10435:10435 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10435:10435 [5] NCCL INFO Bootstrap: Using eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10435:10435 [5] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-0:10435:10435 [5] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-0:10432:10432 [2] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-0:10432:10432 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10432:10432 [2] NCCL INFO Bootstrap: Using eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10432:10432 [2] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-0:10437:10437 [7] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-0:10437:10437 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10432:10432 [2] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-0:10437:10437 [7] NCCL INFO Bootstrap: Using eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10437:10437 [7] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-0:10437:10437 [7] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-0:10436:10436 [6] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-0:10436:10436 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10431:10431 [1] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-0:10431:10431 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10436:10436 [6] NCCL INFO Bootstrap: Using eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10431:10431 [1] NCCL INFO Bootstrap: Using eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10431:10431 [1] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-0:10436:10436 [6] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-0:10431:10431 [1] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-0:10434:10434 [4] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-0:10434:10434 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10434:10434 [4] NCCL INFO Bootstrap: Using eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10434:10434 [4] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-0:10436:10436 [6] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-0:10434:10434 [4] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-0:10433:10433 [3] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-0:10433:10433 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10433:10433 [3] NCCL INFO Bootstrap: Using eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10433:10433 [3] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-0:10433:10433 [3] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO ncclCommInitRankConfig comm 0xa3c9480 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x93ebe2bd4ad77d47 - Init START +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.64.85<0> +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO ncclCommInitRankConfig comm 0xb5df1a0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x93ebe2bd4ad77d47 - Init START +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO ncclCommInitRankConfig comm 0xaabc160 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x93ebe2bd4ad77d47 - Init START +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO ncclCommInitRankConfig comm 0xa9921f0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x93ebe2bd4ad77d47 - Init START +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO ncclCommInitRankConfig comm 0xa7320e0 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x93ebe2bd4ad77d47 - Init START +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO ncclCommInitRankConfig comm 0x9e7d540 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x93ebe2bd4ad77d47 - Init START +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO ncclCommInitRankConfig comm 0xa4e9b20 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x93ebe2bd4ad77d47 - Init START +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO ncclCommInitRankConfig comm 0xaf78ec0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x93ebe2bd4ad77d47 - Init START +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO Bootstrap timings total 0.404038 (create 0.000022, send 0.000072, recv 0.401086, ring 0.000622, delay 0.000001) +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO Bootstrap timings total 0.001067 (create 0.000022, send 0.000082, recv 0.000042, ring 0.000112, delay 0.000001) +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Bootstrap timings total 1.109738 (create 0.000026, send 0.000075, recv 1.101790, ring 0.007496, delay 0.000001) +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO Bootstrap timings total 0.156584 (create 0.000020, send 0.000068, recv 0.000071, ring 0.002560, delay 0.000001) +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO Bootstrap timings total 0.000550 (create 0.000021, send 0.000075, recv 0.000068, ring 0.000124, delay 0.000000) +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO Bootstrap timings total 0.003052 (create 0.000022, send 0.000093, recv 0.000077, ring 0.002572, delay 0.000000) +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO Bootstrap timings total 0.008047 (create 0.000021, send 0.000078, recv 0.000122, ring 0.007549, delay 0.000000) +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO Bootstrap timings total 0.048627 (create 0.000025, send 0.000070, recv 0.048124, ring 0.000109, delay 0.000001) +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO MNNVL busId 0x75020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO MNNVL busId 0x6b020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO MNNVL busId 0x73020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO MNNVL busId 0x6f020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO MNNVL busId 0x71020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO MNNVL busId 0x67020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO MNNVL busId 0x65040 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO MNNVL busId 0x69020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO Setting affinity for GPU 6 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO NVLS multicast support is available on dev 6 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Setting affinity for GPU 0 to 03ffffff,ffffffff,ffffffff +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO Setting affinity for GPU 5 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO NVLS multicast support is available on dev 0 +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO NVLS multicast support is available on dev 5 +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO Setting affinity for GPU 4 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO Setting affinity for GPU 3 to 03ffffff,ffffffff,ffffffff +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO NVLS multicast support is available on dev 3 +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO Setting affinity for GPU 1 to 03ffffff,ffffffff,ffffffff +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO NVLS multicast support is available on dev 1 +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO NVLS multicast support is available on dev 4 +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO Setting affinity for GPU 2 to 03ffffff,ffffffff,ffffffff +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO NVLS multicast support is available on dev 2 +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO Setting affinity for GPU 7 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO NVLS multicast support is available on dev 7 +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO comm 0x9e7d540 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0 +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO comm 0xb5df1a0 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0 +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO comm 0xa4e9b20 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0 +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO comm 0xa9921f0 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0 +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO comm 0xaf78ec0 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0 +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO comm 0xaabc160 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0 +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO comm 0xa7320e0 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO comm 0xa3c9480 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1 +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2 +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0 +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4 +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3 +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Check P2P Type intraNodeP2pSupport 1 directMode 0 +t-20260526104746-rhql7-worker-0:10430:10582 [0] NCCL INFO [Proxy Service] Device 0 CPU core 2 +t-20260526104746-rhql7-worker-0:10430:10584 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 4 +t-20260526104746-rhql7-worker-0:10432:10583 [2] NCCL INFO [Proxy Service] Device 2 CPU core 64 +t-20260526104746-rhql7-worker-0:10432:10585 [2] NCCL INFO [Proxy Service UDS] Device 2 CPU core 66 +t-20260526104746-rhql7-worker-0:10431:10586 [1] NCCL INFO [Proxy Service] Device 1 CPU core 10 +t-20260526104746-rhql7-worker-0:10431:10587 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 14 +t-20260526104746-rhql7-worker-0:10437:10588 [7] NCCL INFO [Proxy Service] Device 7 CPU core 124 +t-20260526104746-rhql7-worker-0:10437:10589 [7] NCCL INFO [Proxy Service UDS] Device 7 CPU core 126 +t-20260526104746-rhql7-worker-0:10435:10590 [5] NCCL INFO [Proxy Service] Device 5 CPU core 92 +t-20260526104746-rhql7-worker-0:10435:10591 [5] NCCL INFO [Proxy Service UDS] Device 5 CPU core 94 +t-20260526104746-rhql7-worker-0:10434:10592 [4] NCCL INFO [Proxy Service] Device 4 CPU core 94 +t-20260526104746-rhql7-worker-0:10434:10593 [4] NCCL INFO [Proxy Service UDS] Device 4 CPU core 92 +t-20260526104746-rhql7-worker-0:10436:10594 [6] NCCL INFO [Proxy Service] Device 6 CPU core 98 +t-20260526104746-rhql7-worker-0:10436:10595 [6] NCCL INFO [Proxy Service UDS] Device 6 CPU core 100 +t-20260526104746-rhql7-worker-0:10433:10596 [3] NCCL INFO [Proxy Service] Device 3 CPU core 2 +t-20260526104746-rhql7-worker-0:10433:10597 [3] NCCL INFO [Proxy Service UDS] Device 3 CPU core 5 +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO CC Off, workFifoBytes 1048576 +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO ncclCommInitRankConfig comm 0xaabc160 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x93ebe2bd4ad77d47 - Init COMPLETE +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO ncclCommInitRankConfig comm 0x9e7d540 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x93ebe2bd4ad77d47 - Init COMPLETE +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO ncclCommInitRankConfig comm 0xb5df1a0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x93ebe2bd4ad77d47 - Init COMPLETE +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO ncclCommInitRankConfig comm 0xa9921f0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x93ebe2bd4ad77d47 - Init COMPLETE +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-0:10437:10513 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 2.51 (kernels 0.25, alloc 0.83, bootstrap 0.16, allgathers 0.00, topo 0.61, graphs 0.01, connections 0.60, rest 0.06) +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO ncclCommInitRankConfig comm 0xa4e9b20 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x93ebe2bd4ad77d47 - Init COMPLETE +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-0:10436:10515 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 2.49 (kernels 0.34, alloc 0.87, bootstrap 0.00, allgathers 0.01, topo 0.60, graphs 0.01, connections 0.59, rest 0.06) +t-20260526104746-rhql7-worker-0:10432:10512 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 2.52 (kernels 0.30, alloc 0.89, bootstrap 0.05, allgathers 0.01, topo 0.60, graphs 0.01, connections 0.61, rest 0.05) +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-0:10434:10516 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 2.48 (kernels 0.30, alloc 0.91, bootstrap 0.00, allgathers 0.00, topo 0.60, graphs 0.01, connections 0.60, rest 0.06) +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO ncclCommInitRankConfig comm 0xa7320e0 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x93ebe2bd4ad77d47 - Init COMPLETE +t-20260526104746-rhql7-worker-0:10435:10503 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 2.68 (kernels 0.32, alloc 0.68, bootstrap 0.40, allgathers 0.00, topo 0.60, graphs 0.01, connections 0.60, rest 0.06) +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO ncclCommInitRankConfig comm 0xa3c9480 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x93ebe2bd4ad77d47 - Init COMPLETE +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO ncclCommInitRankConfig comm 0xaf78ec0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x93ebe2bd4ad77d47 - Init COMPLETE +t-20260526104746-rhql7-worker-0:10431:10514 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 2.49 (kernels 0.29, alloc 0.91, bootstrap 0.01, allgathers 0.01, topo 0.60, graphs 0.01, connections 0.62, rest 0.03) +t-20260526104746-rhql7-worker-0:10433:10517 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 2.48 (kernels 0.29, alloc 0.91, bootstrap 0.00, allgathers 0.00, topo 0.60, graphs 0.01, connections 0.59, rest 0.06) +t-20260526104746-rhql7-worker-0:10430:10502 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 2.87 (kernels 0.20, alloc 0.28, bootstrap 1.11, allgathers 0.01, topo 0.60, graphs 0.01, connections 0.61, rest 0.05) +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-0:10436:10603 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-0:10432:10601 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-0:10431:10600 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-0:10430:10604 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-0:10437:10598 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-0:10434:10599 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-0:10435:10602 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-0:10433:10605 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 8, + "samples": "wrapped_stream", + "vocab_size": 30522, + "tokenizer_vocab_size": 30522, + "save_dir": "runs/lta_owt_bert_absrope_time0_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_1m_save10k_20260526", + "max_len": 1024, + "effective_model_max_len": 1024, + "batch_size": 32, + "grad_accum": 2, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "adamw", + "epochs": 0.0, + "steps_per_epoch": 0, + "total_steps": 1000000, + "warmup_steps": 2500, + "warmup_epochs": -1.0, + "min_lr": 6e-05, + "weight_decay": 0.0, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "", + "muon_param_count": 0, + "muon_adam_param_count": 0, + "muon_param_names": [], + "muon_adam_param_names": [], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.0, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.0, + "ema_decay": 0.0, + "ema_start_step": 0, + "model_type": "ddit_elf", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 0, + "elf_num_model_mode_tokens": 0, + "abs_pos_embed": true, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "t_sampling_gumbel_loc": 2.2, + "t_sampling_gumbel_scale": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "block_ar_two_stream": false, + "block_ar_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "dirichlet_support_t_curve": "linear", + "dirichlet_support_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_gold_prob_floor": 0.0, + "categorical_gold_prob_ceil": 1.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4,8,16,32,64", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_steps_min": -1, + "rollout_train_infer_steps": 64, + "rollout_train_time_mode": "fixed_steps", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.125, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_rule": "flowmap", + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "rollout_train_keep_grad": false, + "rollout_train_sync_t": false, + "rollout_train_state_mix_mode": "final", + "rollout_train_state_mix_alpha": 0.5, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 2, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": false, + "owt_chunk_cache_dir": "", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +t-20260526104746-rhql7-worker-0:10430:10654 [0] NCCL INFO NVLS comm 0xa3c9480 headRank 0 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-0:10433:10711 [3] NCCL INFO NVLS comm 0xaf78ec0 headRank 3 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-0:10436:10712 [6] NCCL INFO NVLS comm 0x9e7d540 headRank 6 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-0:10432:10713 [2] NCCL INFO NVLS comm 0xa9921f0 headRank 2 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-0:10437:10715 [7] NCCL INFO NVLS comm 0xaabc160 headRank 7 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-0:10434:10716 [4] NCCL INFO NVLS comm 0xa4e9b20 headRank 4 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-0:10435:10714 [5] NCCL INFO NVLS comm 0xb5df1a0 headRank 5 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-0:10431:10717 [1] NCCL INFO NVLS comm 0xa7320e0 headRank 1 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-1:10434:10434 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10434:10434 [0] NCCL INFO Bootstrap: Using eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10434:10434 [0] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-1:10434:10434 [0] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-1:10434:10434 [0] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-1:10440:10440 [6] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-1:10440:10440 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10440:10440 [6] NCCL INFO Bootstrap: Using eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10440:10440 [6] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-1:10440:10440 [6] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-1:10439:10439 [5] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-1:10439:10439 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10439:10439 [5] NCCL INFO Bootstrap: Using eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10439:10439 [5] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-1:10439:10439 [5] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10437:10437 [3] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-1:10437:10437 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10437:10437 [3] NCCL INFO Bootstrap: Using eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10437:10437 [3] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-1:10437:10437 [3] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-1:10436:10436 [2] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-1:10436:10436 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10436:10436 [2] NCCL INFO Bootstrap: Using eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10436:10436 [2] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-1:10436:10436 [2] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-1:10435:10435 [1] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-1:10435:10435 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10435:10435 [1] NCCL INFO Bootstrap: Using eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10435:10435 [1] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-1:10435:10435 [1] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-1:10441:10441 [7] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-1:10441:10441 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10441:10441 [7] NCCL INFO Bootstrap: Using eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10441:10441 [7] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-1:10441:10441 [7] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-1:10438:10438 [4] NCCL INFO cudaDriverVersion 12080 +t-20260526104746-rhql7-worker-1:10438:10438 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10438:10438 [4] NCCL INFO Bootstrap: Using eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10438:10438 [4] NCCL INFO NCCL version 2.25.1+cuda12.8 +t-20260526104746-rhql7-worker-1:10438:10438 [4] NCCL INFO Comm config Blocking set to 1 +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO ncclCommInitRankConfig comm 0xa1e8fc0 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0xde76458c60378bf2 - Init START +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO ncclCommInitRankConfig comm 0x9bef870 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0xde76458c60378bf2 - Init START +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9) +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9) +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO P2P plugin v9 IBext_v9 +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1 +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1. +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.4<0> +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so. +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO Using network IBext_v9 +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO ncclCommInitRankConfig comm 0xb69b2b0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0xde76458c60378bf2 - Init START +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO ncclCommInitRankConfig comm 0xaadb320 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0xde76458c60378bf2 - Init START +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO ncclCommInitRankConfig comm 0xa2762c0 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0xde76458c60378bf2 - Init START +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO ncclCommInitRankConfig comm 0xa953430 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0xde76458c60378bf2 - Init START +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO ncclCommInitRankConfig comm 0xa6cd270 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0xde76458c60378bf2 - Init START +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO ncclCommInitRankConfig comm 0x9eff180 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0xde76458c60378bf2 - Init START +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO RAS client listening socket at ::1<28028> +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO Bootstrap timings total 0.020517 (create 0.000023, send 0.000068, recv 0.020024, ring 0.000093, delay 0.000001) +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO Bootstrap timings total 0.001106 (create 0.000016, send 0.000045, recv 0.000102, ring 0.000133, delay 0.000000) +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO Bootstrap timings total 0.000568 (create 0.000022, send 0.000079, recv 0.000078, ring 0.000107, delay 0.000001) +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO Bootstrap timings total 0.598983 (create 0.000020, send 0.000070, recv 0.000092, ring 0.000616, delay 0.000001) +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO Bootstrap timings total 0.010436 (create 0.000023, send 0.000072, recv 0.000131, ring 0.009884, delay 0.000001) +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Bootstrap timings total 0.716095 (create 0.000026, send 0.000076, recv 0.709916, ring 0.005744, delay 0.000001) +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO Bootstrap timings total 0.007401 (create 0.000024, send 0.000072, recv 0.000120, ring 0.005755, delay 0.000001) +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO Bootstrap timings total 0.701322 (create 0.000026, send 0.000073, recv 0.690994, ring 0.009882, delay 0.000000) +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO MNNVL busId 0x6b020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO MNNVL busId 0x67020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO MNNVL busId 0x71020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO MNNVL busId 0x69020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO MNNVL busId 0x65040 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO MNNVL busId 0x75020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO MNNVL busId 0x6f020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO MNNVL busId 0x73020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0 +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO Setting affinity for GPU 2 to 03ffffff,ffffffff,ffffffff +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO NVLS multicast support is available on dev 2 +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO Setting affinity for GPU 1 to 03ffffff,ffffffff,ffffffff +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO Setting affinity for GPU 5 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO NVLS multicast support is available on dev 5 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Setting affinity for GPU 0 to 03ffffff,ffffffff,ffffffff +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO NVLS multicast support is available on dev 0 +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO Setting affinity for GPU 4 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO Setting affinity for GPU 7 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO NVLS multicast support is available on dev 4 +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO NVLS multicast support is available on dev 7 +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO NVLS multicast support is available on dev 1 +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO Setting affinity for GPU 3 to 03ffffff,ffffffff,ffffffff +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO Setting affinity for GPU 6 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000 +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO NVLS multicast support is available on dev 3 +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO NVLS multicast support is available on dev 6 +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO comm 0x9bef870 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0 +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO comm 0xb69b2b0 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0 +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO comm 0xa2762c0 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0 +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO comm 0xa6cd270 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0 +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO comm 0x9eff180 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0 +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO comm 0xa953430 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0 +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO comm 0xaadb320 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO comm 0xa1e8fc0 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4 +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6 +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3 +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1 +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO P2P Chunksize set to 524288 +t-20260526104746-rhql7-worker-1:10439:10586 [5] NCCL INFO [Proxy Service] Device 5 CPU core 104 +t-20260526104746-rhql7-worker-1:10439:10587 [5] NCCL INFO [Proxy Service UDS] Device 5 CPU core 107 +t-20260526104746-rhql7-worker-1:10441:10588 [7] NCCL INFO [Proxy Service] Device 7 CPU core 142 +t-20260526104746-rhql7-worker-1:10441:10589 [7] NCCL INFO [Proxy Service UDS] Device 7 CPU core 144 +t-20260526104746-rhql7-worker-1:10440:10590 [6] NCCL INFO [Proxy Service] Device 6 CPU core 164 +t-20260526104746-rhql7-worker-1:10440:10591 [6] NCCL INFO [Proxy Service UDS] Device 6 CPU core 166 +t-20260526104746-rhql7-worker-1:10437:10592 [3] NCCL INFO [Proxy Service] Device 3 CPU core 20 +t-20260526104746-rhql7-worker-1:10437:10593 [3] NCCL INFO [Proxy Service UDS] Device 3 CPU core 22 +t-20260526104746-rhql7-worker-1:10438:10594 [4] NCCL INFO [Proxy Service] Device 4 CPU core 92 +t-20260526104746-rhql7-worker-1:10438:10595 [4] NCCL INFO [Proxy Service UDS] Device 4 CPU core 94 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Check P2P Type intraNodeP2pSupport 1 directMode 0 +t-20260526104746-rhql7-worker-1:10434:10597 [0] NCCL INFO [Proxy Service] Device 0 CPU core 20 +t-20260526104746-rhql7-worker-1:10435:10598 [1] NCCL INFO [Proxy Service] Device 1 CPU core 26 +t-20260526104746-rhql7-worker-1:10436:10596 [2] NCCL INFO [Proxy Service] Device 2 CPU core 2 +t-20260526104746-rhql7-worker-1:10434:10600 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 22 +t-20260526104746-rhql7-worker-1:10436:10599 [2] NCCL INFO [Proxy Service UDS] Device 2 CPU core 4 +t-20260526104746-rhql7-worker-1:10435:10601 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 29 +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO CC Off, workFifoBytes 1048576 +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512 +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol. +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO ncclCommInitRankConfig comm 0xa6cd270 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0xde76458c60378bf2 - Init COMPLETE +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO ncclCommInitRankConfig comm 0x9bef870 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0xde76458c60378bf2 - Init COMPLETE +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO ncclCommInitRankConfig comm 0xa1e8fc0 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0xde76458c60378bf2 - Init COMPLETE +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol. +t-20260526104746-rhql7-worker-1:10438:10527 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 2.00 (kernels 0.36, alloc 0.66, bootstrap 0.00, allgathers 0.00, topo 0.54, graphs 0.01, connections 0.39, rest 0.03) +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead. +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO ncclCommInitRankConfig comm 0xb69b2b0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0xde76458c60378bf2 - Init COMPLETE +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO ncclCommInitRankConfig comm 0x9eff180 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0xde76458c60378bf2 - Init COMPLETE +t-20260526104746-rhql7-worker-1:10440:10507 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 2.24 (kernels 0.20, alloc 0.37, bootstrap 0.70, allgathers 0.00, topo 0.54, graphs 0.01, connections 0.41, rest 0.02) +t-20260526104746-rhql7-worker-1:10434:10506 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 2.24 (kernels 0.20, alloc 0.36, bootstrap 0.72, allgathers 0.00, topo 0.53, graphs 0.01, connections 0.39, rest 0.03) +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO ncclCommInitRankConfig comm 0xaadb320 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0xde76458c60378bf2 - Init COMPLETE +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO ncclCommInitRankConfig comm 0xa953430 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0xde76458c60378bf2 - Init COMPLETE +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO ncclCommInitRankConfig comm 0xa2762c0 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0xde76458c60378bf2 - Init COMPLETE +t-20260526104746-rhql7-worker-1:10437:10509 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 2.04 (kernels 0.41, alloc 0.66, bootstrap 0.00, allgathers 0.00, topo 0.54, graphs 0.01, connections 0.39, rest 0.03) +t-20260526104746-rhql7-worker-1:10439:10508 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 2.15 (kernels 0.28, alloc 0.30, bootstrap 0.60, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.41, rest 0.01) +t-20260526104746-rhql7-worker-1:10436:10514 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 2.03 (kernels 0.38, alloc 0.66, bootstrap 0.02, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.39, rest 0.03) +t-20260526104746-rhql7-worker-1:10435:10517 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 2.02 (kernels 0.38, alloc 0.67, bootstrap 0.01, allgathers 0.00, topo 0.53, graphs 0.01, connections 0.39, rest 0.03) +t-20260526104746-rhql7-worker-1:10441:10518 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 2.02 (kernels 0.37, alloc 0.67, bootstrap 0.01, allgathers 0.00, topo 0.54, graphs 0.01, connections 0.41, rest 0.01) +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM +t-20260526104746-rhql7-worker-1:10440:10607 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-1:10441:10603 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-1:10439:10604 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-1:10438:10606 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-1:10436:10602 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-1:10437:10609 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-1:10434:10608 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +t-20260526104746-rhql7-worker-1:10435:10605 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 8, + "samples": "wrapped_stream", + "vocab_size": 30522, + "tokenizer_vocab_size": 30522, + "save_dir": "runs/lta_owt_bert_absrope_time0_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_1m_save10k_20260526", + "max_len": 1024, + "effective_model_max_len": 1024, + "batch_size": 32, + "grad_accum": 2, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "adamw", + "epochs": 0.0, + "steps_per_epoch": 0, + "total_steps": 1000000, + "warmup_steps": 2500, + "warmup_epochs": -1.0, + "min_lr": 6e-05, + "weight_decay": 0.0, + "output_weight_decay": -1.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "adam_eps": 1e-08, + "muon_impl": "legacy", + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "muon_nesterov": false, + "muon_width_scale": false, + "muon_grouping": "", + "muon_param_count": 0, + "muon_adam_param_count": 0, + "muon_param_names": [], + "muon_adam_param_names": [], + "muon_effective_nesterov": false, + "muon_effective_width_scale": false, + "muon_effective_weight_decay": 0.0, + "muon_adam_fallback_nesterov": false, + "muon_adam_fallback_weight_decay": 0.0, + "ema_decay": 0.0, + "ema_start_step": 0, + "model_type": "ddit_elf", + "ddit_mlp_type": "gelu", + "elf_num_time_tokens": 0, + "elf_num_model_mode_tokens": 0, + "abs_pos_embed": true, + "qk_norm": true, + "output_bias": false, + "output_init_std": -1.0, + "norm_type": "rmsnorm", + "target_loss": "hard_ce", + "linear_soft_target_power": 1.0, + "linear_soft_target_min_conf": 0.0, + "linear_soft_target_max_conf": 1.0, + "t_sampling_mode": "uniform", + "t_sampling_power": 1.0, + "t_sampling_eps": 0.0001, + "t_sampling_logit_mean": -1.5, + "t_sampling_logit_std": 0.8, + "t_sampling_gumbel_loc": 2.2, + "t_sampling_gumbel_scale": 0.8, + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "block_ar_two_stream": false, + "block_ar_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "dirichlet_support_t_curve": "linear", + "dirichlet_support_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "categorical_wrong_basin_token_ids": "", + "categorical_wrong_basin_prob": 0.0, + "categorical_wrong_unigram_prob": 0.0, + "categorical_wrong_uniform_prob": 0.0, + "categorical_wrong_prob_floor": 0.0, + "categorical_gold_prob_floor": 0.0, + "categorical_gold_prob_ceil": 1.0, + "categorical_wrong_corpus_unigram_path": "", + "categorical_wrong_corpus_unigram_alpha": 1.0, + "categorical_wrong_basin_shared_prob": 0.0, + "categorical_wrong_unigram_shared_prob": 0.0, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4,8,16,32,64", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_steps_min": -1, + "rollout_train_infer_steps": 64, + "rollout_train_time_mode": "fixed_steps", + "rollout_train_s_dist": "uniform", + "rollout_train_s_min_frac": 0.0, + "rollout_train_s_max_frac": 0.125, + "rollout_train_s_beta_alpha": 2.0, + "rollout_train_s_beta_beta": 6.0, + "rollout_train_temp": 1.0, + "rollout_train_max_gamma": 1.0, + "rollout_train_rule": "flowmap", + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "rollout_train_keep_grad": false, + "rollout_train_sync_t": false, + "rollout_train_state_mix_mode": "final", + "rollout_train_state_mix_alpha": 0.5, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 2, + "full_train_stats": false, + "tokenized_hf": false, + "tokenized_pad_token": "pad", + "elf_conditional_hf": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": false, + "owt_chunk_cache_dir": "", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 0, + "latest_every": 1000, + "resume_path": "" +} +t-20260526104746-rhql7-worker-1:10434:10658 [0] NCCL INFO NVLS comm 0xa1e8fc0 headRank 0 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-1:10439:10715 [5] NCCL INFO NVLS comm 0xb69b2b0 headRank 5 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-1:10440:10716 [6] NCCL INFO NVLS comm 0x9bef870 headRank 6 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-1:10441:10717 [7] NCCL INFO NVLS comm 0xa2762c0 headRank 7 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-1:10438:10718 [4] NCCL INFO NVLS comm 0xa6cd270 headRank 4 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-1:10436:10719 [2] NCCL INFO NVLS comm 0xaadb320 headRank 2 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-1:10435:10720 [1] NCCL INFO NVLS comm 0xa953430 headRank 1 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +t-20260526104746-rhql7-worker-1:10437:10721 [3] NCCL INFO NVLS comm 0x9eff180 headRank 3 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456 +step=100 micro_steps=200 elapsed=66.4s lr=1.212000e-05 loss=10.1117 loss_recon=10.1117 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0161 corrupt_frac=1.0000 acc_corrupt=0.0161 loss_corrupt=10.1117 wrong_frac=0.4989 init_acc_corrupt=0.5011 acc_corrupt_t_0p0_0p2=0.0144 corrupt_frac_t_0p0_0p2=0.1984 acc_corrupt_t_0p2_0p4=0.0151 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.0167 corrupt_frac_t_0p4_0p6=0.1984 acc_corrupt_t_0p6_0p8=0.0172 corrupt_frac_t_0p6_0p8=0.1994 acc_corrupt_t_0p8_1p0=0.0170 corrupt_frac_t_0p8_1p0=0.2028 out_w_norm=0.9256 out_g_norm=1.7109 loss_all=9.6917 init_gold_top10=0.4889 init_gold_top100=0.4907 +step=100 micro_steps=200 elapsed=65.8s lr=1.212000e-05 loss=10.1117 loss_recon=10.1117 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0161 corrupt_frac=1.0000 acc_corrupt=0.0161 loss_corrupt=10.1117 wrong_frac=0.4989 init_acc_corrupt=0.5011 acc_corrupt_t_0p0_0p2=0.0145 corrupt_frac_t_0p0_0p2=0.1984 acc_corrupt_t_0p2_0p4=0.0151 corrupt_frac_t_0p2_0p4=0.2030 acc_corrupt_t_0p4_0p6=0.0167 corrupt_frac_t_0p4_0p6=0.1984 acc_corrupt_t_0p6_0p8=0.0171 corrupt_frac_t_0p6_0p8=0.1994 acc_corrupt_t_0p8_1p0=0.0170 corrupt_frac_t_0p8_1p0=0.2028 out_w_norm=0.9256 out_g_norm=1.7109 loss_all=9.6917 init_gold_top10=0.4889 init_gold_top100=0.4907 +step=200 micro_steps=400 elapsed=64.8s lr=2.412000e-05 loss=9.0635 loss_recon=9.0635 loss_meanflow=0.0000 mean_model_t=0.4984 mean_corrupt_t=0.4984 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0195 corrupt_frac=1.0000 acc_corrupt=0.0195 loss_corrupt=9.0635 wrong_frac=0.5019 init_acc_corrupt=0.4981 acc_corrupt_t_0p0_0p2=0.0190 corrupt_frac_t_0p0_0p2=0.1978 acc_corrupt_t_0p2_0p4=0.0196 corrupt_frac_t_0p2_0p4=0.2028 acc_corrupt_t_0p4_0p6=0.0195 corrupt_frac_t_0p4_0p6=0.2062 acc_corrupt_t_0p6_0p8=0.0196 corrupt_frac_t_0p6_0p8=0.1952 acc_corrupt_t_0p8_1p0=0.0200 corrupt_frac_t_0p8_1p0=0.1980 out_w_norm=5.7896 out_g_norm=1.8405 loss_all=8.4297 init_gold_top10=0.5044 init_gold_top100=0.5059 +step=200 micro_steps=400 elapsed=64.5s lr=2.412000e-05 loss=9.0635 loss_recon=9.0635 loss_meanflow=0.0000 mean_model_t=0.4984 mean_corrupt_t=0.4984 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0195 corrupt_frac=1.0000 acc_corrupt=0.0195 loss_corrupt=9.0635 wrong_frac=0.5019 init_acc_corrupt=0.4981 acc_corrupt_t_0p0_0p2=0.0190 corrupt_frac_t_0p0_0p2=0.1978 acc_corrupt_t_0p2_0p4=0.0196 corrupt_frac_t_0p2_0p4=0.2028 acc_corrupt_t_0p4_0p6=0.0196 corrupt_frac_t_0p4_0p6=0.2062 acc_corrupt_t_0p6_0p8=0.0196 corrupt_frac_t_0p6_0p8=0.1952 acc_corrupt_t_0p8_1p0=0.0200 corrupt_frac_t_0p8_1p0=0.1980 out_w_norm=5.7896 out_g_norm=1.8405 loss_all=8.4297 init_gold_top10=0.5044 init_gold_top100=0.5059 +step=300 micro_steps=600 elapsed=64.8s lr=3.612000e-05 loss=7.8879 loss_recon=7.8879 loss_meanflow=0.0000 mean_model_t=0.5071 mean_corrupt_t=0.5071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0399 corrupt_frac=1.0000 acc_corrupt=0.0399 loss_corrupt=7.8879 wrong_frac=0.4929 init_acc_corrupt=0.5071 acc_corrupt_t_0p0_0p2=0.0398 corrupt_frac_t_0p0_0p2=0.1891 acc_corrupt_t_0p2_0p4=0.0403 corrupt_frac_t_0p2_0p4=0.2003 acc_corrupt_t_0p4_0p6=0.0397 corrupt_frac_t_0p4_0p6=0.2123 acc_corrupt_t_0p6_0p8=0.0397 corrupt_frac_t_0p6_0p8=0.1945 acc_corrupt_t_0p8_1p0=0.0400 corrupt_frac_t_0p8_1p0=0.2056 out_w_norm=12.6164 out_g_norm=1.6228 loss_all=7.4679 init_gold_top10=0.5252 init_gold_top100=0.5266 +step=300 micro_steps=600 elapsed=64.3s lr=3.612000e-05 loss=7.8879 loss_recon=7.8879 loss_meanflow=0.0000 mean_model_t=0.5071 mean_corrupt_t=0.5071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0399 corrupt_frac=1.0000 acc_corrupt=0.0399 loss_corrupt=7.8879 wrong_frac=0.4929 init_acc_corrupt=0.5071 acc_corrupt_t_0p0_0p2=0.0398 corrupt_frac_t_0p0_0p2=0.1891 acc_corrupt_t_0p2_0p4=0.0403 corrupt_frac_t_0p2_0p4=0.2003 acc_corrupt_t_0p4_0p6=0.0397 corrupt_frac_t_0p4_0p6=0.2123 acc_corrupt_t_0p6_0p8=0.0397 corrupt_frac_t_0p6_0p8=0.1945 acc_corrupt_t_0p8_1p0=0.0400 corrupt_frac_t_0p8_1p0=0.2056 out_w_norm=12.6164 out_g_norm=1.6228 loss_all=7.4679 init_gold_top10=0.5252 init_gold_top100=0.5266 +step=400 micro_steps=800 elapsed=65.0s lr=4.812000e-05 loss=7.3268 loss_recon=7.3268 loss_meanflow=0.0000 mean_model_t=0.5094 mean_corrupt_t=0.5094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0427 corrupt_frac=1.0000 acc_corrupt=0.0427 loss_corrupt=7.3268 wrong_frac=0.4905 init_acc_corrupt=0.5096 acc_corrupt_t_0p0_0p2=0.0429 corrupt_frac_t_0p0_0p2=0.1906 acc_corrupt_t_0p2_0p4=0.0430 corrupt_frac_t_0p2_0p4=0.1928 acc_corrupt_t_0p4_0p6=0.0428 corrupt_frac_t_0p4_0p6=0.2041 acc_corrupt_t_0p6_0p8=0.0425 corrupt_frac_t_0p6_0p8=0.2069 acc_corrupt_t_0p8_1p0=0.0425 corrupt_frac_t_0p8_1p0=0.2066 out_w_norm=18.5947 out_g_norm=0.4529 loss_all=7.2815 init_gold_top10=0.5397 init_gold_top100=0.5413 +step=400 micro_steps=800 elapsed=64.3s lr=4.812000e-05 loss=7.3268 loss_recon=7.3268 loss_meanflow=0.0000 mean_model_t=0.5094 mean_corrupt_t=0.5094 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0427 corrupt_frac=1.0000 acc_corrupt=0.0427 loss_corrupt=7.3268 wrong_frac=0.4905 init_acc_corrupt=0.5096 acc_corrupt_t_0p0_0p2=0.0429 corrupt_frac_t_0p0_0p2=0.1906 acc_corrupt_t_0p2_0p4=0.0430 corrupt_frac_t_0p2_0p4=0.1928 acc_corrupt_t_0p4_0p6=0.0428 corrupt_frac_t_0p4_0p6=0.2041 acc_corrupt_t_0p6_0p8=0.0425 corrupt_frac_t_0p6_0p8=0.2069 acc_corrupt_t_0p8_1p0=0.0425 corrupt_frac_t_0p8_1p0=0.2066 out_w_norm=18.5945 out_g_norm=0.4530 loss_all=7.2814 init_gold_top10=0.5397 init_gold_top100=0.5413 +step=500 micro_steps=1000 elapsed=65.0s lr=6.012000e-05 loss=7.1275 loss_recon=7.1275 loss_meanflow=0.0000 mean_model_t=0.5016 mean_corrupt_t=0.5016 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0438 corrupt_frac=1.0000 acc_corrupt=0.0438 loss_corrupt=7.1275 wrong_frac=0.4984 init_acc_corrupt=0.5016 acc_corrupt_t_0p0_0p2=0.0431 corrupt_frac_t_0p0_0p2=0.2003 acc_corrupt_t_0p2_0p4=0.0436 corrupt_frac_t_0p2_0p4=0.1969 acc_corrupt_t_0p4_0p6=0.0441 corrupt_frac_t_0p4_0p6=0.1997 acc_corrupt_t_0p6_0p8=0.0438 corrupt_frac_t_0p6_0p8=0.2050 acc_corrupt_t_0p8_1p0=0.0442 corrupt_frac_t_0p8_1p0=0.1981 out_w_norm=22.1581 out_g_norm=0.3456 loss_all=7.0255 init_gold_top10=0.4441 init_gold_top100=0.4460 +step=500 micro_steps=1000 elapsed=64.3s lr=6.012000e-05 loss=7.1275 loss_recon=7.1275 loss_meanflow=0.0000 mean_model_t=0.5016 mean_corrupt_t=0.5016 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0438 corrupt_frac=1.0000 acc_corrupt=0.0438 loss_corrupt=7.1275 wrong_frac=0.4984 init_acc_corrupt=0.5016 acc_corrupt_t_0p0_0p2=0.0430 corrupt_frac_t_0p0_0p2=0.2003 acc_corrupt_t_0p2_0p4=0.0436 corrupt_frac_t_0p2_0p4=0.1969 acc_corrupt_t_0p4_0p6=0.0441 corrupt_frac_t_0p4_0p6=0.1997 acc_corrupt_t_0p6_0p8=0.0438 corrupt_frac_t_0p6_0p8=0.2050 acc_corrupt_t_0p8_1p0=0.0442 corrupt_frac_t_0p8_1p0=0.1981 out_w_norm=22.1576 out_g_norm=0.3449 loss_all=7.0211 init_gold_top10=0.4441 init_gold_top100=0.4460 +step=600 micro_steps=1200 elapsed=65.1s lr=7.212000e-05 loss=6.8668 loss_recon=6.8668 loss_meanflow=0.0000 mean_model_t=0.4978 mean_corrupt_t=0.4978 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0434 corrupt_frac=1.0000 acc_corrupt=0.0434 loss_corrupt=6.8668 wrong_frac=0.5022 init_acc_corrupt=0.4978 acc_corrupt_t_0p0_0p2=0.0431 corrupt_frac_t_0p0_0p2=0.2013 acc_corrupt_t_0p2_0p4=0.0435 corrupt_frac_t_0p2_0p4=0.2042 acc_corrupt_t_0p4_0p6=0.0430 corrupt_frac_t_0p4_0p6=0.1994 acc_corrupt_t_0p6_0p8=0.0432 corrupt_frac_t_0p6_0p8=0.1986 acc_corrupt_t_0p8_1p0=0.0439 corrupt_frac_t_0p8_1p0=0.1984 out_w_norm=26.1727 out_g_norm=0.2767 loss_all=6.6506 init_gold_top10=0.4997 init_gold_top100=0.5011 +step=600 micro_steps=1200 elapsed=64.5s lr=7.212000e-05 loss=6.8747 loss_recon=6.8747 loss_meanflow=0.0000 mean_model_t=0.4978 mean_corrupt_t=0.4978 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0433 corrupt_frac=1.0000 acc_corrupt=0.0433 loss_corrupt=6.8747 wrong_frac=0.5022 init_acc_corrupt=0.4978 acc_corrupt_t_0p0_0p2=0.0430 corrupt_frac_t_0p0_0p2=0.2013 acc_corrupt_t_0p2_0p4=0.0435 corrupt_frac_t_0p2_0p4=0.2042 acc_corrupt_t_0p4_0p6=0.0430 corrupt_frac_t_0p4_0p6=0.1994 acc_corrupt_t_0p6_0p8=0.0431 corrupt_frac_t_0p6_0p8=0.1986 acc_corrupt_t_0p8_1p0=0.0438 corrupt_frac_t_0p8_1p0=0.1984 out_w_norm=26.1406 out_g_norm=0.2735 loss_all=6.6612 init_gold_top10=0.4997 init_gold_top100=0.5011 +step=700 micro_steps=1400 elapsed=64.9s lr=8.412000e-05 loss=6.4931 loss_recon=6.4931 loss_meanflow=0.0000 mean_model_t=0.5025 mean_corrupt_t=0.5025 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0557 corrupt_frac=1.0000 acc_corrupt=0.0557 loss_corrupt=6.4931 wrong_frac=0.4974 init_acc_corrupt=0.5026 acc_corrupt_t_0p0_0p2=0.0454 corrupt_frac_t_0p0_0p2=0.1967 acc_corrupt_t_0p2_0p4=0.0513 corrupt_frac_t_0p2_0p4=0.2005 acc_corrupt_t_0p4_0p6=0.0547 corrupt_frac_t_0p4_0p6=0.1959 acc_corrupt_t_0p6_0p8=0.0601 corrupt_frac_t_0p6_0p8=0.2062 acc_corrupt_t_0p8_1p0=0.0668 corrupt_frac_t_0p8_1p0=0.2006 out_w_norm=31.0223 out_g_norm=0.3256 loss_all=6.0750 init_gold_top10=0.5452 init_gold_top100=0.5464 +step=700 micro_steps=1400 elapsed=64.6s lr=8.412000e-05 loss=6.4842 loss_recon=6.4842 loss_meanflow=0.0000 mean_model_t=0.5025 mean_corrupt_t=0.5025 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0562 corrupt_frac=1.0000 acc_corrupt=0.0562 loss_corrupt=6.4842 wrong_frac=0.4974 init_acc_corrupt=0.5026 acc_corrupt_t_0p0_0p2=0.0458 corrupt_frac_t_0p0_0p2=0.1967 acc_corrupt_t_0p2_0p4=0.0516 corrupt_frac_t_0p2_0p4=0.2005 acc_corrupt_t_0p4_0p6=0.0552 corrupt_frac_t_0p4_0p6=0.1959 acc_corrupt_t_0p6_0p8=0.0609 corrupt_frac_t_0p6_0p8=0.2062 acc_corrupt_t_0p8_1p0=0.0672 corrupt_frac_t_0p8_1p0=0.2006 out_w_norm=31.0037 out_g_norm=0.3259 loss_all=6.0442 init_gold_top10=0.5452 init_gold_top100=0.5464 +step=800 micro_steps=1600 elapsed=64.9s lr=9.612000e-05 loss=6.0366 loss_recon=6.0366 loss_meanflow=0.0000 mean_model_t=0.4962 mean_corrupt_t=0.4962 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1256 corrupt_frac=1.0000 acc_corrupt=0.1256 loss_corrupt=6.0366 wrong_frac=0.5040 init_acc_corrupt=0.4961 acc_corrupt_t_0p0_0p2=0.0579 corrupt_frac_t_0p0_0p2=0.2053 acc_corrupt_t_0p2_0p4=0.0915 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.1256 corrupt_frac_t_0p4_0p6=0.2031 acc_corrupt_t_0p6_0p8=0.1619 corrupt_frac_t_0p6_0p8=0.1939 acc_corrupt_t_0p8_1p0=0.1945 corrupt_frac_t_0p8_1p0=0.1980 out_w_norm=35.7814 out_g_norm=0.2793 loss_all=5.6355 init_gold_top10=0.5271 init_gold_top100=0.5285 +step=800 micro_steps=1600 elapsed=65.0s lr=9.612000e-05 loss=6.0074 loss_recon=6.0074 loss_meanflow=0.0000 mean_model_t=0.4962 mean_corrupt_t=0.4962 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1283 corrupt_frac=1.0000 acc_corrupt=0.1283 loss_corrupt=6.0074 wrong_frac=0.5040 init_acc_corrupt=0.4961 acc_corrupt_t_0p0_0p2=0.0581 corrupt_frac_t_0p0_0p2=0.2053 acc_corrupt_t_0p2_0p4=0.0927 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.1282 corrupt_frac_t_0p4_0p6=0.2031 acc_corrupt_t_0p6_0p8=0.1661 corrupt_frac_t_0p6_0p8=0.1939 acc_corrupt_t_0p8_1p0=0.2003 corrupt_frac_t_0p8_1p0=0.1980 out_w_norm=36.0009 out_g_norm=0.2876 loss_all=5.6325 init_gold_top10=0.5271 init_gold_top100=0.5285 +step=900 micro_steps=1800 elapsed=64.8s lr=1.081200e-04 loss=5.5245 loss_recon=5.5245 loss_meanflow=0.0000 mean_model_t=0.5067 mean_corrupt_t=0.5067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2288 corrupt_frac=1.0000 acc_corrupt=0.2288 loss_corrupt=5.5245 wrong_frac=0.4934 init_acc_corrupt=0.5066 acc_corrupt_t_0p0_0p2=0.0717 corrupt_frac_t_0p0_0p2=0.1952 acc_corrupt_t_0p2_0p4=0.1513 corrupt_frac_t_0p2_0p4=0.1983 acc_corrupt_t_0p4_0p6=0.2248 corrupt_frac_t_0p4_0p6=0.1966 acc_corrupt_t_0p6_0p8=0.3028 corrupt_frac_t_0p6_0p8=0.1998 acc_corrupt_t_0p8_1p0=0.3791 corrupt_frac_t_0p8_1p0=0.2120 out_w_norm=40.7402 out_g_norm=0.2514 loss_all=5.2338 init_gold_top10=0.5161 init_gold_top100=0.5174 +step=900 micro_steps=1800 elapsed=65.0s lr=1.081200e-04 loss=5.4992 loss_recon=5.4992 loss_meanflow=0.0000 mean_model_t=0.5067 mean_corrupt_t=0.5067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2392 corrupt_frac=1.0000 acc_corrupt=0.2392 loss_corrupt=5.4992 wrong_frac=0.4934 init_acc_corrupt=0.5066 acc_corrupt_t_0p0_0p2=0.0729 corrupt_frac_t_0p0_0p2=0.1952 acc_corrupt_t_0p2_0p4=0.1574 corrupt_frac_t_0p2_0p4=0.1983 acc_corrupt_t_0p4_0p6=0.2349 corrupt_frac_t_0p4_0p6=0.1966 acc_corrupt_t_0p6_0p8=0.3180 corrupt_frac_t_0p6_0p8=0.1998 acc_corrupt_t_0p8_1p0=0.3979 corrupt_frac_t_0p8_1p0=0.2120 out_w_norm=40.8976 out_g_norm=0.2490 loss_all=5.2238 init_gold_top10=0.5161 init_gold_top100=0.5174 +step=1000 micro_steps=2000 elapsed=64.7s lr=1.201200e-04 loss=5.1974 loss_recon=5.1974 loss_meanflow=0.0000 mean_model_t=0.4976 mean_corrupt_t=0.4976 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2848 corrupt_frac=1.0000 acc_corrupt=0.2848 loss_corrupt=5.1974 wrong_frac=0.5023 init_acc_corrupt=0.4977 acc_corrupt_t_0p0_0p2=0.0829 corrupt_frac_t_0p0_0p2=0.2011 acc_corrupt_t_0p2_0p4=0.1861 corrupt_frac_t_0p2_0p4=0.1958 acc_corrupt_t_0p4_0p6=0.2854 corrupt_frac_t_0p4_0p6=0.2048 acc_corrupt_t_0p6_0p8=0.3864 corrupt_frac_t_0p6_0p8=0.2065 acc_corrupt_t_0p8_1p0=0.4865 corrupt_frac_t_0p8_1p0=0.1938 out_w_norm=45.5001 out_g_norm=0.1774 loss_all=5.0559 init_gold_top10=0.5296 init_gold_top100=0.5307 +step=1000 micro_steps=2000 elapsed=64.8s lr=1.201200e-04 loss=5.1666 loss_recon=5.1666 loss_meanflow=0.0000 mean_model_t=0.4976 mean_corrupt_t=0.4976 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2952 corrupt_frac=1.0000 acc_corrupt=0.2952 loss_corrupt=5.1666 wrong_frac=0.5023 init_acc_corrupt=0.4977 acc_corrupt_t_0p0_0p2=0.0848 corrupt_frac_t_0p0_0p2=0.2011 acc_corrupt_t_0p2_0p4=0.1929 corrupt_frac_t_0p2_0p4=0.1958 acc_corrupt_t_0p4_0p6=0.2959 corrupt_frac_t_0p4_0p6=0.2048 acc_corrupt_t_0p6_0p8=0.4007 corrupt_frac_t_0p6_0p8=0.2065 acc_corrupt_t_0p8_1p0=0.5051 corrupt_frac_t_0p8_1p0=0.1938 out_w_norm=45.6769 out_g_norm=0.1756 loss_all=5.0324 init_gold_top10=0.5296 init_gold_top100=0.5307 +step=1100 micro_steps=2200 elapsed=66.1s lr=1.321200e-04 loss=4.9271 loss_recon=4.9271 loss_meanflow=0.0000 mean_model_t=0.4933 mean_corrupt_t=0.4933 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3331 corrupt_frac=1.0000 acc_corrupt=0.3331 loss_corrupt=4.9271 wrong_frac=0.5066 init_acc_corrupt=0.4935 acc_corrupt_t_0p0_0p2=0.0926 corrupt_frac_t_0p0_0p2=0.2034 acc_corrupt_t_0p2_0p4=0.2155 corrupt_frac_t_0p2_0p4=0.2067 acc_corrupt_t_0p4_0p6=0.3378 corrupt_frac_t_0p4_0p6=0.2034 acc_corrupt_t_0p6_0p8=0.4621 corrupt_frac_t_0p6_0p8=0.1933 acc_corrupt_t_0p8_1p0=0.5782 corrupt_frac_t_0p8_1p0=0.1931 out_w_norm=50.0510 out_g_norm=0.1557 loss_all=4.7649 init_gold_top10=0.4917 init_gold_top100=0.4931 +step=1100 micro_steps=2200 elapsed=67.5s lr=1.321200e-04 loss=4.8810 loss_recon=4.8810 loss_meanflow=0.0000 mean_model_t=0.4933 mean_corrupt_t=0.4933 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3440 corrupt_frac=1.0000 acc_corrupt=0.3440 loss_corrupt=4.8810 wrong_frac=0.5066 init_acc_corrupt=0.4935 acc_corrupt_t_0p0_0p2=0.0954 corrupt_frac_t_0p0_0p2=0.2034 acc_corrupt_t_0p2_0p4=0.2224 corrupt_frac_t_0p2_0p4=0.2067 acc_corrupt_t_0p4_0p6=0.3491 corrupt_frac_t_0p4_0p6=0.2034 acc_corrupt_t_0p6_0p8=0.4767 corrupt_frac_t_0p6_0p8=0.1933 acc_corrupt_t_0p8_1p0=0.5977 corrupt_frac_t_0p8_1p0=0.1931 out_w_norm=50.4181 out_g_norm=0.1474 loss_all=4.7416 init_gold_top10=0.4917 init_gold_top100=0.4931 +step=1200 micro_steps=2400 elapsed=64.6s lr=1.441200e-04 loss=4.6250 loss_recon=4.6250 loss_meanflow=0.0000 mean_model_t=0.5022 mean_corrupt_t=0.5022 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3817 corrupt_frac=1.0000 acc_corrupt=0.3817 loss_corrupt=4.6250 wrong_frac=0.4980 init_acc_corrupt=0.5020 acc_corrupt_t_0p0_0p2=0.1001 corrupt_frac_t_0p0_0p2=0.1931 acc_corrupt_t_0p2_0p4=0.2433 corrupt_frac_t_0p2_0p4=0.2072 acc_corrupt_t_0p4_0p6=0.3812 corrupt_frac_t_0p4_0p6=0.2022 acc_corrupt_t_0p6_0p8=0.5187 corrupt_frac_t_0p6_0p8=0.1928 acc_corrupt_t_0p8_1p0=0.6590 corrupt_frac_t_0p8_1p0=0.2047 out_w_norm=55.2050 out_g_norm=0.1330 loss_all=4.5543 init_gold_top10=0.5266 init_gold_top100=0.5279 +step=1200 micro_steps=2400 elapsed=66.0s lr=1.441200e-04 loss=4.6014 loss_recon=4.6014 loss_meanflow=0.0000 mean_model_t=0.5022 mean_corrupt_t=0.5022 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3849 corrupt_frac=1.0000 acc_corrupt=0.3849 loss_corrupt=4.6014 wrong_frac=0.4980 init_acc_corrupt=0.5020 acc_corrupt_t_0p0_0p2=0.1007 corrupt_frac_t_0p0_0p2=0.1931 acc_corrupt_t_0p2_0p4=0.2451 corrupt_frac_t_0p2_0p4=0.2072 acc_corrupt_t_0p4_0p6=0.3840 corrupt_frac_t_0p4_0p6=0.2022 acc_corrupt_t_0p6_0p8=0.5232 corrupt_frac_t_0p6_0p8=0.1928 acc_corrupt_t_0p8_1p0=0.6652 corrupt_frac_t_0p8_1p0=0.2047 out_w_norm=55.6135 out_g_norm=0.1313 loss_all=4.5200 init_gold_top10=0.5266 init_gold_top100=0.5279 +step=1300 micro_steps=2600 elapsed=64.6s lr=1.561200e-04 loss=4.4670 loss_recon=4.4670 loss_meanflow=0.0000 mean_model_t=0.5004 mean_corrupt_t=0.5004 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4041 corrupt_frac=1.0000 acc_corrupt=0.4041 loss_corrupt=4.4670 wrong_frac=0.4993 init_acc_corrupt=0.5007 acc_corrupt_t_0p0_0p2=0.1066 corrupt_frac_t_0p0_0p2=0.2023 acc_corrupt_t_0p2_0p4=0.2586 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.4049 corrupt_frac_t_0p4_0p6=0.1953 acc_corrupt_t_0p6_0p8=0.5534 corrupt_frac_t_0p6_0p8=0.2028 acc_corrupt_t_0p8_1p0=0.6987 corrupt_frac_t_0p8_1p0=0.1998 out_w_norm=60.4622 out_g_norm=0.1241 loss_all=4.3938 init_gold_top10=0.5177 init_gold_top100=0.5192 +step=1300 micro_steps=2600 elapsed=66.1s lr=1.561200e-04 loss=4.4463 loss_recon=4.4463 loss_meanflow=0.0000 mean_model_t=0.5004 mean_corrupt_t=0.5004 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4079 corrupt_frac=1.0000 acc_corrupt=0.4079 loss_corrupt=4.4463 wrong_frac=0.4993 init_acc_corrupt=0.5007 acc_corrupt_t_0p0_0p2=0.1071 corrupt_frac_t_0p0_0p2=0.2023 acc_corrupt_t_0p2_0p4=0.2596 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.4081 corrupt_frac_t_0p4_0p6=0.1953 acc_corrupt_t_0p6_0p8=0.5588 corrupt_frac_t_0p6_0p8=0.2028 acc_corrupt_t_0p8_1p0=0.7072 corrupt_frac_t_0p8_1p0=0.1998 out_w_norm=60.9285 out_g_norm=0.1227 loss_all=4.3528 init_gold_top10=0.5177 init_gold_top100=0.5192 +step=1400 micro_steps=2800 elapsed=64.4s lr=1.681200e-04 loss=4.2799 loss_recon=4.2799 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4329 corrupt_frac=1.0000 acc_corrupt=0.4329 loss_corrupt=4.2799 wrong_frac=0.4985 init_acc_corrupt=0.5015 acc_corrupt_t_0p0_0p2=0.1104 corrupt_frac_t_0p0_0p2=0.1983 acc_corrupt_t_0p2_0p4=0.2713 corrupt_frac_t_0p2_0p4=0.2005 acc_corrupt_t_0p4_0p6=0.4327 corrupt_frac_t_0p4_0p6=0.1998 acc_corrupt_t_0p6_0p8=0.5942 corrupt_frac_t_0p6_0p8=0.1981 acc_corrupt_t_0p8_1p0=0.7498 corrupt_frac_t_0p8_1p0=0.2033 out_w_norm=65.7236 out_g_norm=0.1235 loss_all=3.9681 init_gold_top10=0.5500 init_gold_top100=0.5510 +step=1400 micro_steps=2800 elapsed=66.0s lr=1.681200e-04 loss=4.2543 loss_recon=4.2543 loss_meanflow=0.0000 mean_model_t=0.5015 mean_corrupt_t=0.5015 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4393 corrupt_frac=1.0000 acc_corrupt=0.4393 loss_corrupt=4.2543 wrong_frac=0.4985 init_acc_corrupt=0.5015 acc_corrupt_t_0p0_0p2=0.1116 corrupt_frac_t_0p0_0p2=0.1983 acc_corrupt_t_0p2_0p4=0.2745 corrupt_frac_t_0p2_0p4=0.2005 acc_corrupt_t_0p4_0p6=0.4388 corrupt_frac_t_0p4_0p6=0.1998 acc_corrupt_t_0p6_0p8=0.6032 corrupt_frac_t_0p6_0p8=0.1981 acc_corrupt_t_0p8_1p0=0.7622 corrupt_frac_t_0p8_1p0=0.2033 out_w_norm=66.2689 out_g_norm=0.1241 loss_all=3.9344 init_gold_top10=0.5500 init_gold_top100=0.5510 +step=1500 micro_steps=3000 elapsed=64.8s lr=1.801200e-04 loss=4.1385 loss_recon=4.1385 loss_meanflow=0.0000 mean_model_t=0.4976 mean_corrupt_t=0.4976 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4611 corrupt_frac=1.0000 acc_corrupt=0.4611 loss_corrupt=4.1385 wrong_frac=0.5026 init_acc_corrupt=0.4974 acc_corrupt_t_0p0_0p2=0.1145 corrupt_frac_t_0p0_0p2=0.2023 acc_corrupt_t_0p2_0p4=0.2892 corrupt_frac_t_0p2_0p4=0.1936 acc_corrupt_t_0p4_0p6=0.4639 corrupt_frac_t_0p4_0p6=0.2120 acc_corrupt_t_0p6_0p8=0.6356 corrupt_frac_t_0p6_0p8=0.1961 acc_corrupt_t_0p8_1p0=0.8110 corrupt_frac_t_0p8_1p0=0.1959 out_w_norm=71.6063 out_g_norm=0.1228 loss_all=4.4546 init_gold_top10=0.4184 init_gold_top100=0.4200 +step=1500 micro_steps=3000 elapsed=66.0s lr=1.801200e-04 loss=4.1241 loss_recon=4.1241 loss_meanflow=0.0000 mean_model_t=0.4976 mean_corrupt_t=0.4976 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4634 corrupt_frac=1.0000 acc_corrupt=0.4634 loss_corrupt=4.1241 wrong_frac=0.5026 init_acc_corrupt=0.4974 acc_corrupt_t_0p0_0p2=0.1156 corrupt_frac_t_0p0_0p2=0.2023 acc_corrupt_t_0p2_0p4=0.2904 corrupt_frac_t_0p2_0p4=0.1936 acc_corrupt_t_0p4_0p6=0.4659 corrupt_frac_t_0p4_0p6=0.2120 acc_corrupt_t_0p6_0p8=0.6387 corrupt_frac_t_0p6_0p8=0.1961 acc_corrupt_t_0p8_1p0=0.8153 corrupt_frac_t_0p8_1p0=0.1959 out_w_norm=72.2266 out_g_norm=0.1204 loss_all=4.4559 init_gold_top10=0.4184 init_gold_top100=0.4200 +step=1600 micro_steps=3200 elapsed=64.3s lr=1.921200e-04 loss=3.9734 loss_recon=3.9734 loss_meanflow=0.0000 mean_model_t=0.5000 mean_corrupt_t=0.5000 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4838 corrupt_frac=1.0000 acc_corrupt=0.4838 loss_corrupt=3.9734 wrong_frac=0.5002 init_acc_corrupt=0.4999 acc_corrupt_t_0p0_0p2=0.1209 corrupt_frac_t_0p0_0p2=0.1978 acc_corrupt_t_0p2_0p4=0.3083 corrupt_frac_t_0p2_0p4=0.1984 acc_corrupt_t_0p4_0p6=0.4866 corrupt_frac_t_0p4_0p6=0.2092 acc_corrupt_t_0p6_0p8=0.6627 corrupt_frac_t_0p6_0p8=0.1966 acc_corrupt_t_0p8_1p0=0.8415 corrupt_frac_t_0p8_1p0=0.1980 out_w_norm=77.8529 out_g_norm=0.1016 loss_all=4.0314 init_gold_top10=0.5082 init_gold_top100=0.5099 +step=1600 micro_steps=3200 elapsed=65.6s lr=1.921200e-04 loss=3.9785 loss_recon=3.9785 loss_meanflow=0.0000 mean_model_t=0.5000 mean_corrupt_t=0.5000 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4834 corrupt_frac=1.0000 acc_corrupt=0.4834 loss_corrupt=3.9785 wrong_frac=0.5002 init_acc_corrupt=0.4999 acc_corrupt_t_0p0_0p2=0.1215 corrupt_frac_t_0p0_0p2=0.1978 acc_corrupt_t_0p2_0p4=0.3072 corrupt_frac_t_0p2_0p4=0.1984 acc_corrupt_t_0p4_0p6=0.4852 corrupt_frac_t_0p4_0p6=0.2092 acc_corrupt_t_0p6_0p8=0.6619 corrupt_frac_t_0p6_0p8=0.1966 acc_corrupt_t_0p8_1p0=0.8423 corrupt_frac_t_0p8_1p0=0.1980 out_w_norm=78.3280 out_g_norm=0.1022 loss_all=4.0445 init_gold_top10=0.5082 init_gold_top100=0.5099 +step=1700 micro_steps=3400 elapsed=64.5s lr=2.041200e-04 loss=3.8451 loss_recon=3.8451 loss_meanflow=0.0000 mean_model_t=0.4954 mean_corrupt_t=0.4954 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4937 corrupt_frac=1.0000 acc_corrupt=0.4937 loss_corrupt=3.8451 wrong_frac=0.5047 init_acc_corrupt=0.4954 acc_corrupt_t_0p0_0p2=0.1217 corrupt_frac_t_0p0_0p2=0.2028 acc_corrupt_t_0p2_0p4=0.3160 corrupt_frac_t_0p2_0p4=0.2104 acc_corrupt_t_0p4_0p6=0.5056 corrupt_frac_t_0p4_0p6=0.1967 acc_corrupt_t_0p6_0p8=0.6849 corrupt_frac_t_0p6_0p8=0.1916 acc_corrupt_t_0p8_1p0=0.8596 corrupt_frac_t_0p8_1p0=0.2006 out_w_norm=84.0527 out_g_norm=0.1027 loss_all=2.9728 init_gold_top10=0.6013 init_gold_top100=0.6025 +step=1700 micro_steps=3400 elapsed=64.8s lr=2.041200e-04 loss=3.8490 loss_recon=3.8490 loss_meanflow=0.0000 mean_model_t=0.4954 mean_corrupt_t=0.4954 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4936 corrupt_frac=1.0000 acc_corrupt=0.4936 loss_corrupt=3.8490 wrong_frac=0.5047 init_acc_corrupt=0.4954 acc_corrupt_t_0p0_0p2=0.1218 corrupt_frac_t_0p0_0p2=0.2028 acc_corrupt_t_0p2_0p4=0.3152 corrupt_frac_t_0p2_0p4=0.2104 acc_corrupt_t_0p4_0p6=0.5052 corrupt_frac_t_0p4_0p6=0.1967 acc_corrupt_t_0p6_0p8=0.6849 corrupt_frac_t_0p6_0p8=0.1916 acc_corrupt_t_0p8_1p0=0.8607 corrupt_frac_t_0p8_1p0=0.2006 out_w_norm=84.4468 out_g_norm=0.1023 loss_all=2.9924 init_gold_top10=0.6013 init_gold_top100=0.6025 +step=1800 micro_steps=3600 elapsed=64.7s lr=2.161200e-04 loss=3.7246 loss_recon=3.7246 loss_meanflow=0.0000 mean_model_t=0.4968 mean_corrupt_t=0.4968 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5046 corrupt_frac=1.0000 acc_corrupt=0.5046 loss_corrupt=3.7246 wrong_frac=0.5034 init_acc_corrupt=0.4966 acc_corrupt_t_0p0_0p2=0.1260 corrupt_frac_t_0p0_0p2=0.2016 acc_corrupt_t_0p2_0p4=0.3285 corrupt_frac_t_0p2_0p4=0.2011 acc_corrupt_t_0p4_0p6=0.5128 corrupt_frac_t_0p4_0p6=0.2061 acc_corrupt_t_0p6_0p8=0.6973 corrupt_frac_t_0p6_0p8=0.1947 acc_corrupt_t_0p8_1p0=0.8734 corrupt_frac_t_0p8_1p0=0.1976 out_w_norm=89.9215 out_g_norm=0.1001 loss_all=3.3731 init_gold_top10=0.5267 init_gold_top100=0.5280 +step=1800 micro_steps=3600 elapsed=65.2s lr=2.161200e-04 loss=3.7217 loss_recon=3.7217 loss_meanflow=0.0000 mean_model_t=0.4968 mean_corrupt_t=0.4968 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5053 corrupt_frac=1.0000 acc_corrupt=0.5053 loss_corrupt=3.7217 wrong_frac=0.5034 init_acc_corrupt=0.4966 acc_corrupt_t_0p0_0p2=0.1264 corrupt_frac_t_0p0_0p2=0.2016 acc_corrupt_t_0p2_0p4=0.3293 corrupt_frac_t_0p2_0p4=0.2011 acc_corrupt_t_0p4_0p6=0.5132 corrupt_frac_t_0p4_0p6=0.2061 acc_corrupt_t_0p6_0p8=0.6981 corrupt_frac_t_0p6_0p8=0.1947 acc_corrupt_t_0p8_1p0=0.8744 corrupt_frac_t_0p8_1p0=0.1976 out_w_norm=90.3153 out_g_norm=0.1001 loss_all=3.3715 init_gold_top10=0.5267 init_gold_top100=0.5280 +step=1900 micro_steps=3800 elapsed=64.4s lr=2.281200e-04 loss=3.6328 loss_recon=3.6328 loss_meanflow=0.0000 mean_model_t=0.4980 mean_corrupt_t=0.4980 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5130 corrupt_frac=1.0000 acc_corrupt=0.5130 loss_corrupt=3.6328 wrong_frac=0.5019 init_acc_corrupt=0.4981 acc_corrupt_t_0p0_0p2=0.1290 corrupt_frac_t_0p0_0p2=0.2003 acc_corrupt_t_0p2_0p4=0.3301 corrupt_frac_t_0p2_0p4=0.1995 acc_corrupt_t_0p4_0p6=0.5245 corrupt_frac_t_0p4_0p6=0.2102 acc_corrupt_t_0p6_0p8=0.7080 corrupt_frac_t_0p6_0p8=0.1988 acc_corrupt_t_0p8_1p0=0.8849 corrupt_frac_t_0p8_1p0=0.1933 out_w_norm=95.3994 out_g_norm=0.0986 loss_all=3.7482 init_gold_top10=0.4784 init_gold_top100=0.4807 +step=1900 micro_steps=3800 elapsed=64.8s lr=2.281200e-04 loss=3.6245 loss_recon=3.6245 loss_meanflow=0.0000 mean_model_t=0.4980 mean_corrupt_t=0.4980 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5145 corrupt_frac=1.0000 acc_corrupt=0.5145 loss_corrupt=3.6245 wrong_frac=0.5019 init_acc_corrupt=0.4981 acc_corrupt_t_0p0_0p2=0.1299 corrupt_frac_t_0p0_0p2=0.2003 acc_corrupt_t_0p2_0p4=0.3311 corrupt_frac_t_0p2_0p4=0.1995 acc_corrupt_t_0p4_0p6=0.5259 corrupt_frac_t_0p4_0p6=0.2102 acc_corrupt_t_0p6_0p8=0.7099 corrupt_frac_t_0p6_0p8=0.1988 acc_corrupt_t_0p8_1p0=0.8868 corrupt_frac_t_0p8_1p0=0.1933 out_w_norm=95.8142 out_g_norm=0.0982 loss_all=3.7395 init_gold_top10=0.4784 init_gold_top100=0.4807 +step=2000 micro_steps=4000 elapsed=64.3s lr=2.401200e-04 loss=3.5262 loss_recon=3.5262 loss_meanflow=0.0000 mean_model_t=0.5049 mean_corrupt_t=0.5049 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5243 corrupt_frac=1.0000 acc_corrupt=0.5243 loss_corrupt=3.5262 wrong_frac=0.4953 init_acc_corrupt=0.5047 acc_corrupt_t_0p0_0p2=0.1282 corrupt_frac_t_0p0_0p2=0.1909 acc_corrupt_t_0p2_0p4=0.3324 corrupt_frac_t_0p2_0p4=0.1986 acc_corrupt_t_0p4_0p6=0.5305 corrupt_frac_t_0p4_0p6=0.2073 acc_corrupt_t_0p6_0p8=0.7143 corrupt_frac_t_0p6_0p8=0.1988 acc_corrupt_t_0p8_1p0=0.8896 corrupt_frac_t_0p8_1p0=0.2044 out_w_norm=100.5291 out_g_norm=0.0979 loss_all=3.0783 init_gold_top10=0.5582 init_gold_top100=0.5592 +step=2000 micro_steps=4000 elapsed=64.8s lr=2.401200e-04 loss=3.5171 loss_recon=3.5171 loss_meanflow=0.0000 mean_model_t=0.5049 mean_corrupt_t=0.5049 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5256 corrupt_frac=1.0000 acc_corrupt=0.5256 loss_corrupt=3.5171 wrong_frac=0.4953 init_acc_corrupt=0.5047 acc_corrupt_t_0p0_0p2=0.1290 corrupt_frac_t_0p0_0p2=0.1909 acc_corrupt_t_0p2_0p4=0.3338 corrupt_frac_t_0p2_0p4=0.1986 acc_corrupt_t_0p4_0p6=0.5316 corrupt_frac_t_0p4_0p6=0.2073 acc_corrupt_t_0p6_0p8=0.7162 corrupt_frac_t_0p6_0p8=0.1988 acc_corrupt_t_0p8_1p0=0.8911 corrupt_frac_t_0p8_1p0=0.2044 out_w_norm=100.9775 out_g_norm=0.0980 loss_all=3.0756 init_gold_top10=0.5582 init_gold_top100=0.5592 +step=2100 micro_steps=4200 elapsed=65.5s lr=2.521200e-04 loss=3.4985 loss_recon=3.4985 loss_meanflow=0.0000 mean_model_t=0.4991 mean_corrupt_t=0.4991 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5251 corrupt_frac=1.0000 acc_corrupt=0.5251 loss_corrupt=3.4985 wrong_frac=0.5009 init_acc_corrupt=0.4991 acc_corrupt_t_0p0_0p2=0.1319 corrupt_frac_t_0p0_0p2=0.2059 acc_corrupt_t_0p2_0p4=0.3398 corrupt_frac_t_0p2_0p4=0.1939 acc_corrupt_t_0p4_0p6=0.5359 corrupt_frac_t_0p4_0p6=0.1934 acc_corrupt_t_0p6_0p8=0.7196 corrupt_frac_t_0p6_0p8=0.2064 acc_corrupt_t_0p8_1p0=0.8980 corrupt_frac_t_0p8_1p0=0.2003 out_w_norm=105.3674 out_g_norm=0.0874 loss_all=3.7083 init_gold_top10=0.4629 init_gold_top100=0.4644 +step=2100 micro_steps=4200 elapsed=66.6s lr=2.521200e-04 loss=3.5275 loss_recon=3.5275 loss_meanflow=0.0000 mean_model_t=0.4991 mean_corrupt_t=0.4991 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5210 corrupt_frac=1.0000 acc_corrupt=0.5210 loss_corrupt=3.5275 wrong_frac=0.5009 init_acc_corrupt=0.4991 acc_corrupt_t_0p0_0p2=0.1309 corrupt_frac_t_0p0_0p2=0.2059 acc_corrupt_t_0p2_0p4=0.3375 corrupt_frac_t_0p2_0p4=0.1939 acc_corrupt_t_0p4_0p6=0.5321 corrupt_frac_t_0p4_0p6=0.1934 acc_corrupt_t_0p6_0p8=0.7150 corrupt_frac_t_0p6_0p8=0.2064 acc_corrupt_t_0p8_1p0=0.8893 corrupt_frac_t_0p8_1p0=0.2003 out_w_norm=105.6610 out_g_norm=0.1045 loss_all=3.7159 init_gold_top10=0.4629 init_gold_top100=0.4644 +step=2200 micro_steps=4400 elapsed=64.7s lr=2.641200e-04 loss=3.4298 loss_recon=3.4298 loss_meanflow=0.0000 mean_model_t=0.5016 mean_corrupt_t=0.5016 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5320 corrupt_frac=1.0000 acc_corrupt=0.5320 loss_corrupt=3.4298 wrong_frac=0.4981 init_acc_corrupt=0.5019 acc_corrupt_t_0p0_0p2=0.1322 corrupt_frac_t_0p0_0p2=0.1966 acc_corrupt_t_0p2_0p4=0.3433 corrupt_frac_t_0p2_0p4=0.1991 acc_corrupt_t_0p4_0p6=0.5444 corrupt_frac_t_0p4_0p6=0.2058 acc_corrupt_t_0p6_0p8=0.7299 corrupt_frac_t_0p6_0p8=0.1974 acc_corrupt_t_0p8_1p0=0.9016 corrupt_frac_t_0p8_1p0=0.2022 out_w_norm=109.9628 out_g_norm=0.0846 loss_all=3.9325 init_gold_top10=0.4346 init_gold_top100=0.4363 +step=2200 micro_steps=4400 elapsed=64.7s lr=2.641200e-04 loss=3.4351 loss_recon=3.4351 loss_meanflow=0.0000 mean_model_t=0.5016 mean_corrupt_t=0.5016 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5314 corrupt_frac=1.0000 acc_corrupt=0.5314 loss_corrupt=3.4351 wrong_frac=0.4981 init_acc_corrupt=0.5019 acc_corrupt_t_0p0_0p2=0.1318 corrupt_frac_t_0p0_0p2=0.1966 acc_corrupt_t_0p2_0p4=0.3427 corrupt_frac_t_0p2_0p4=0.1991 acc_corrupt_t_0p4_0p6=0.5438 corrupt_frac_t_0p4_0p6=0.2058 acc_corrupt_t_0p6_0p8=0.7294 corrupt_frac_t_0p6_0p8=0.1974 acc_corrupt_t_0p8_1p0=0.9007 corrupt_frac_t_0p8_1p0=0.2022 out_w_norm=110.0622 out_g_norm=0.0850 loss_all=3.9540 init_gold_top10=0.4346 init_gold_top100=0.4363 +step=2300 micro_steps=4600 elapsed=64.4s lr=2.761200e-04 loss=3.3634 loss_recon=3.3634 loss_meanflow=0.0000 mean_model_t=0.5065 mean_corrupt_t=0.5065 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5383 corrupt_frac=1.0000 acc_corrupt=0.5383 loss_corrupt=3.3634 wrong_frac=0.4937 init_acc_corrupt=0.5063 acc_corrupt_t_0p0_0p2=0.1342 corrupt_frac_t_0p0_0p2=0.1934 acc_corrupt_t_0p2_0p4=0.3466 corrupt_frac_t_0p2_0p4=0.1977 acc_corrupt_t_0p4_0p6=0.5476 corrupt_frac_t_0p4_0p6=0.2045 acc_corrupt_t_0p6_0p8=0.7321 corrupt_frac_t_0p6_0p8=0.1963 acc_corrupt_t_0p8_1p0=0.9043 corrupt_frac_t_0p8_1p0=0.2081 out_w_norm=114.3054 out_g_norm=0.0895 loss_all=3.2297 init_gold_top10=0.5361 init_gold_top100=0.5376 +step=2300 micro_steps=4600 elapsed=64.8s lr=2.761200e-04 loss=3.3651 loss_recon=3.3651 loss_meanflow=0.0000 mean_model_t=0.5065 mean_corrupt_t=0.5065 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5380 corrupt_frac=1.0000 acc_corrupt=0.5380 loss_corrupt=3.3651 wrong_frac=0.4937 init_acc_corrupt=0.5063 acc_corrupt_t_0p0_0p2=0.1346 corrupt_frac_t_0p0_0p2=0.1934 acc_corrupt_t_0p2_0p4=0.3464 corrupt_frac_t_0p2_0p4=0.1977 acc_corrupt_t_0p4_0p6=0.5473 corrupt_frac_t_0p4_0p6=0.2045 acc_corrupt_t_0p6_0p8=0.7313 corrupt_frac_t_0p6_0p8=0.1963 acc_corrupt_t_0p8_1p0=0.9036 corrupt_frac_t_0p8_1p0=0.2081 out_w_norm=114.3982 out_g_norm=0.0880 loss_all=3.2243 init_gold_top10=0.5361 init_gold_top100=0.5376 +step=2400 micro_steps=4800 elapsed=65.2s lr=2.881200e-04 loss=3.3766 loss_recon=3.3766 loss_meanflow=0.0000 mean_model_t=0.4993 mean_corrupt_t=0.4993 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5343 corrupt_frac=1.0000 acc_corrupt=0.5343 loss_corrupt=3.3766 wrong_frac=0.5006 init_acc_corrupt=0.4995 acc_corrupt_t_0p0_0p2=0.1358 corrupt_frac_t_0p0_0p2=0.1988 acc_corrupt_t_0p2_0p4=0.3466 corrupt_frac_t_0p2_0p4=0.2008 acc_corrupt_t_0p4_0p6=0.5475 corrupt_frac_t_0p4_0p6=0.2008 acc_corrupt_t_0p6_0p8=0.7364 corrupt_frac_t_0p6_0p8=0.2035 acc_corrupt_t_0p8_1p0=0.9043 corrupt_frac_t_0p8_1p0=0.1981 out_w_norm=118.4513 out_g_norm=0.0867 loss_all=3.4436 init_gold_top10=0.4753 init_gold_top100=0.4774 +step=2400 micro_steps=4800 elapsed=64.8s lr=2.881200e-04 loss=3.3795 loss_recon=3.3795 loss_meanflow=0.0000 mean_model_t=0.4993 mean_corrupt_t=0.4993 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5339 corrupt_frac=1.0000 acc_corrupt=0.5339 loss_corrupt=3.3795 wrong_frac=0.5006 init_acc_corrupt=0.4995 acc_corrupt_t_0p0_0p2=0.1359 corrupt_frac_t_0p0_0p2=0.1988 acc_corrupt_t_0p2_0p4=0.3465 corrupt_frac_t_0p2_0p4=0.2008 acc_corrupt_t_0p4_0p6=0.5470 corrupt_frac_t_0p4_0p6=0.2008 acc_corrupt_t_0p6_0p8=0.7358 corrupt_frac_t_0p6_0p8=0.2035 acc_corrupt_t_0p8_1p0=0.9035 corrupt_frac_t_0p8_1p0=0.1981 out_w_norm=118.5529 out_g_norm=0.0861 loss_all=3.4340 init_gold_top10=0.4753 init_gold_top100=0.4774 +step=2500 micro_steps=5000 elapsed=64.6s lr=3.000000e-04 loss=3.3247 loss_recon=3.3247 loss_meanflow=0.0000 mean_model_t=0.4989 mean_corrupt_t=0.4989 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5387 corrupt_frac=1.0000 acc_corrupt=0.5387 loss_corrupt=3.3247 wrong_frac=0.5009 init_acc_corrupt=0.4991 acc_corrupt_t_0p0_0p2=0.1381 corrupt_frac_t_0p0_0p2=0.1992 acc_corrupt_t_0p2_0p4=0.3551 corrupt_frac_t_0p2_0p4=0.2009 acc_corrupt_t_0p4_0p6=0.5537 corrupt_frac_t_0p4_0p6=0.2049 acc_corrupt_t_0p6_0p8=0.7436 corrupt_frac_t_0p6_0p8=0.2005 acc_corrupt_t_0p8_1p0=0.9099 corrupt_frac_t_0p8_1p0=0.1955 out_w_norm=122.4103 out_g_norm=0.0885 loss_all=3.6761 init_gold_top10=0.4644 init_gold_top100=0.4657 +step=2500 micro_steps=5000 elapsed=64.9s lr=3.000000e-04 loss=3.3268 loss_recon=3.3268 loss_meanflow=0.0000 mean_model_t=0.4989 mean_corrupt_t=0.4989 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5385 corrupt_frac=1.0000 acc_corrupt=0.5385 loss_corrupt=3.3268 wrong_frac=0.5009 init_acc_corrupt=0.4991 acc_corrupt_t_0p0_0p2=0.1385 corrupt_frac_t_0p0_0p2=0.1992 acc_corrupt_t_0p2_0p4=0.3550 corrupt_frac_t_0p2_0p4=0.2009 acc_corrupt_t_0p4_0p6=0.5535 corrupt_frac_t_0p4_0p6=0.2049 acc_corrupt_t_0p6_0p8=0.7428 corrupt_frac_t_0p6_0p8=0.2005 acc_corrupt_t_0p8_1p0=0.9094 corrupt_frac_t_0p8_1p0=0.1955 out_w_norm=122.5221 out_g_norm=0.0885 loss_all=3.6677 init_gold_top10=0.4644 init_gold_top100=0.4657 +step=2600 micro_steps=5200 elapsed=64.7s lr=3.000000e-04 loss=3.2811 loss_recon=3.2811 loss_meanflow=0.0000 mean_model_t=0.5031 mean_corrupt_t=0.5031 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5440 corrupt_frac=1.0000 acc_corrupt=0.5440 loss_corrupt=3.2811 wrong_frac=0.4968 init_acc_corrupt=0.5032 acc_corrupt_t_0p0_0p2=0.1347 corrupt_frac_t_0p0_0p2=0.1919 acc_corrupt_t_0p2_0p4=0.3556 corrupt_frac_t_0p2_0p4=0.2072 acc_corrupt_t_0p4_0p6=0.5570 corrupt_frac_t_0p4_0p6=0.1989 acc_corrupt_t_0p6_0p8=0.7425 corrupt_frac_t_0p6_0p8=0.1945 acc_corrupt_t_0p8_1p0=0.9086 corrupt_frac_t_0p8_1p0=0.2084 out_w_norm=126.1883 out_g_norm=0.0880 loss_all=2.4266 init_gold_top10=0.6222 init_gold_top100=0.6234 +step=2600 micro_steps=5200 elapsed=64.2s lr=3.000000e-04 loss=3.2819 loss_recon=3.2819 loss_meanflow=0.0000 mean_model_t=0.5031 mean_corrupt_t=0.5031 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5437 corrupt_frac=1.0000 acc_corrupt=0.5437 loss_corrupt=3.2819 wrong_frac=0.4968 init_acc_corrupt=0.5032 acc_corrupt_t_0p0_0p2=0.1346 corrupt_frac_t_0p0_0p2=0.1919 acc_corrupt_t_0p2_0p4=0.3553 corrupt_frac_t_0p2_0p4=0.2072 acc_corrupt_t_0p4_0p6=0.5568 corrupt_frac_t_0p4_0p6=0.1989 acc_corrupt_t_0p6_0p8=0.7419 corrupt_frac_t_0p6_0p8=0.1945 acc_corrupt_t_0p8_1p0=0.9081 corrupt_frac_t_0p8_1p0=0.2084 out_w_norm=126.2888 out_g_norm=0.0872 loss_all=2.4375 init_gold_top10=0.6222 init_gold_top100=0.6234 +step=2700 micro_steps=5400 elapsed=65.2s lr=3.000000e-04 loss=3.3425 loss_recon=3.3425 loss_meanflow=0.0000 mean_model_t=0.4926 mean_corrupt_t=0.4926 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5348 corrupt_frac=1.0000 acc_corrupt=0.5348 loss_corrupt=3.3425 wrong_frac=0.5073 init_acc_corrupt=0.4927 acc_corrupt_t_0p0_0p2=0.1343 corrupt_frac_t_0p0_0p2=0.2130 acc_corrupt_t_0p2_0p4=0.3579 corrupt_frac_t_0p2_0p4=0.1950 acc_corrupt_t_0p4_0p6=0.5606 corrupt_frac_t_0p4_0p6=0.2061 acc_corrupt_t_0p8_1p0=0.9164 corrupt_frac_t_0p8_1p0=0.1939 out_w_norm=129.6275 out_g_norm=0.0931 acc_corrupt_t_0p6_0p8=0.7455 corrupt_frac_t_0p6_0p8=0.1930 loss_all=3.6756 init_gold_top10=0.4436 init_gold_top100=0.4449 +step=2700 micro_steps=5400 elapsed=64.3s lr=3.000000e-04 loss=3.3476 loss_recon=3.3476 loss_meanflow=0.0000 mean_model_t=0.4926 mean_corrupt_t=0.4926 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5340 corrupt_frac=1.0000 acc_corrupt=0.5340 loss_corrupt=3.3476 wrong_frac=0.5073 init_acc_corrupt=0.4927 acc_corrupt_t_0p0_0p2=0.1342 corrupt_frac_t_0p0_0p2=0.2130 acc_corrupt_t_0p2_0p4=0.3571 corrupt_frac_t_0p2_0p4=0.1950 acc_corrupt_t_0p4_0p6=0.5597 corrupt_frac_t_0p4_0p6=0.2061 acc_corrupt_t_0p8_1p0=0.9158 corrupt_frac_t_0p8_1p0=0.1939 out_w_norm=129.7760 out_g_norm=0.0921 acc_corrupt_t_0p6_0p8=0.7440 corrupt_frac_t_0p6_0p8=0.1930 loss_all=3.6916 init_gold_top10=0.4436 init_gold_top100=0.4449 diff --git a/LTA_openwebtext_dualt/logs/lta_owt_gpt2cached_len1024_ddit768x12_elfopt_only_muon_ema_gbs512_8gpu_1m_20260513_032325.log b/LTA_openwebtext_dualt/logs/lta_owt_gpt2cached_len1024_ddit768x12_elfopt_only_muon_ema_gbs512_8gpu_1m_20260513_032325.log new file mode 100644 index 0000000000000000000000000000000000000000..d265a05696dc5c5fb63ccb1c54ec4d6ab5425b97 --- /dev/null +++ b/LTA_openwebtext_dualt/logs/lta_owt_gpt2cached_len1024_ddit768x12_elfopt_only_muon_ema_gbs512_8gpu_1m_20260513_032325.log @@ -0,0 +1,127 @@ +{ + "device": "cuda:0", + "rank": 0, + "world_size": 1, + "samples": "wrapped_stream", + "vocab_size": 50257, + "tokenizer_vocab_size": 50257, + "save_dir": "runs/lta_owt_gpt2cached_len1024_ddit768x12_elfopt_only_muon_ema_gbs512_8gpu_1m_20260513_032325", + "batch_size": 32, + "grad_accum": 16, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "adamw", + "warmup_steps": 2000, + "min_lr": 6e-05, + "weight_decay": 0.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "adam_eps": 1e-08, + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "ema_decay": 0.0, + "ema_start_step": 0, + "model_type": "ddit", + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": 0.0, + "corrupt_max_t": 1.0, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4,8,16,32,64", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "target_loss": "hard_ce", + "meanflow_weight": 0.0, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 2, + "full_train_stats": false, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": false, + "owt_chunk_cache_dir": "", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 1, + "latest_every": 5000, + "resume_path": "" +} +Traceback (most recent call last): + File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 1235, in + main() + File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 1114, in main + bridge = make_bridge( + ^^^^^^^^^^^^ + File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 619, in make_bridge + return make_dirichlet_bridge_batch( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: make_dirichlet_bridge_batch() got an unexpected keyword argument 'categorical_wrong_from_batch_valid_tokens' +E0513 03:23:37.001000 470107 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 470197) of binary: /usr/bin/python +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-05-13_03:23:37 + host : localhost + rank : 0 (local_rank: 0) + exitcode : 1 (pid: 470197) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/LTA_openwebtext_dualt/logs/owt_dirichlet_len1024_Cv_to_2v_gumbel_sde_watch/infer_lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525_step_0001000.log b/LTA_openwebtext_dualt/logs/owt_dirichlet_len1024_Cv_to_2v_gumbel_sde_watch/infer_lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525_step_0001000.log new file mode 100644 index 0000000000000000000000000000000000000000..d5045b2b5b5daed4d2d16fe3602cc8248f7274cc --- /dev/null +++ b/LTA_openwebtext_dualt/logs/owt_dirichlet_len1024_Cv_to_2v_gumbel_sde_watch/infer_lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525_step_0001000.log @@ -0,0 +1,136 @@ +[watch-gumbel] 2026-05-25_16:35:54 infer runs/lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525/step_0001000.pt -> docs/lta_samples/metrics_20260525/owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0.95_tau1.0_to_0.2_blend_c32100_64200_n128/lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525/step_0001000 +[load] runs/lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525/step_0001000.pt +[ckpt] step=1000 +[sde] generated 2/128 +[sde] generated 4/128 +[sde] generated 6/128 +[sde] generated 8/128 +[sde] generated 10/128 +[sde] generated 12/128 +[sde] generated 14/128 +[sde] generated 16/128 +[sde] generated 18/128 +[sde] generated 20/128 +[sde] generated 22/128 +[sde] generated 24/128 +[sde] generated 26/128 +[sde] generated 28/128 +[sde] generated 30/128 +[sde] generated 32/128 +[sde] generated 34/128 +[sde] generated 36/128 +[sde] generated 38/128 +[sde] generated 40/128 +[sde] generated 42/128 +[sde] generated 44/128 +[sde] generated 46/128 +[sde] generated 48/128 +[sde] generated 50/128 +[sde] generated 52/128 +[sde] generated 54/128 +[sde] generated 56/128 +[sde] generated 58/128 +[sde] generated 60/128 +[sde] generated 62/128 +[sde] generated 64/128 +[sde] generated 66/128 +[sde] generated 68/128 +[sde] generated 70/128 +[sde] generated 72/128 +[sde] generated 74/128 +[sde] generated 76/128 +[sde] generated 78/128 +[sde] generated 80/128 +[sde] generated 82/128 +[sde] generated 84/128 +[sde] generated 86/128 +[sde] generated 88/128 +[sde] generated 90/128 +[sde] generated 92/128 +[sde] generated 94/128 +[sde] generated 96/128 +[sde] generated 98/128 +[sde] generated 100/128 +[sde] generated 102/128 +[sde] generated 104/128 +[sde] generated 106/128 +[sde] generated 108/128 +[sde] generated 110/128 +[sde] generated 112/128 +[sde] generated 114/128 +[sde] generated 116/128 +[sde] generated 118/128 +[sde] generated 120/128 +[sde] generated 122/128 +[sde] generated 124/128 +[sde] generated 126/128 +[sde] generated 128/128 +[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard +[summary] { + "type": "summary", + "checkpoint": "runs/lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525/step_0001000.pt", + "step": 1000, + "decode": { + "decode_rule": "dirichlet_resample_sde", + "steps": 128, + "model_t_mode": "support_t", + "mean_mode": "endpoint_only", + "anchor_gamma": 1.0, + "endpoint_floor": 0.0, + "concentration_min": 32100.0, + "concentration_max": 64200.0, + "endpoint_temp": 1.45, + "endpoint_temp_start": null, + "endpoint_temp_end": null, + "endpoint_projection": "gumbel_softmax", + "endpoint_top_k": 0, + "endpoint_top_p": 0.95, + "gumbel_tau_start": 1.0, + "gumbel_tau_end": 0.2, + "gumbel_noise_scale_start": 1.0, + "gumbel_noise_scale_end": 1.0, + "ban_special_tokens": false, + "banned_endpoint_ids": [], + "support_power": 1.0, + "semantic_power": 1.0, + "noise_init": "dirichlet", + "noise_sigma": -1.0, + "noise_dirichlet_concentration": 32100.0, + "sde_resample": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 3.0, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.0, + "final_from": "blend_0.5", + "n_samples": 128, + "seed": 20260524 + }, + "raw_genppl": { + "ppl": 538.7763431682185, + "nll_per_token": 6.2893005370545065, + "tokens": 111239, + "kept_samples": 128, + "total_samples": 128, + "empty_rate": 0.0, + "skipped_samples": 0 + }, + "stripped_genppl": { + "ppl": 804.1208604588479, + "nll_per_token": 6.689749581835244, + "tokens": 96338, + "kept_samples": 128, + "total_samples": 128, + "empty_rate": 0.0, + "skipped_samples": 0 + }, + "diversity": { + "sample_entropy": 3.6305307938962628, + "unique_tokens": 12697, + "token_count": 131072, + "distinct_1": 0.09687042236328125, + "distinct_2": 0.38274376832844575, + "top_token_mass": 0.277618408203125 + } +} +[done] docs/lta_samples/metrics_20260525/owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0.95_tau1.0_to_0.2_blend_c32100_64200_n128/lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b32_8gpu_20k_save1k_gumbelwatch_20260525/step_0001000/sde_steps128_samples128_scored.jsonl +[watch-gumbel] 2026-05-25_16:42:26 done step_0001000 diff --git a/LTA_openwebtext_dualt/logs/owt_dirichlet_len1024_Cv_to_2v_gumbel_sde_watch/processed_lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525_steps128_c32100_64200_gumbel_t1p45_n128.txt b/LTA_openwebtext_dualt/logs/owt_dirichlet_len1024_Cv_to_2v_gumbel_sde_watch/processed_lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525_steps128_c32100_64200_gumbel_t1p45_n128.txt new file mode 100644 index 0000000000000000000000000000000000000000..b74a2259f988f92f5c304085b431ddaab55a0cad --- /dev/null +++ b/LTA_openwebtext_dualt/logs/owt_dirichlet_len1024_Cv_to_2v_gumbel_sde_watch/processed_lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525_steps128_c32100_64200_gumbel_t1p45_n128.txt @@ -0,0 +1,2 @@ +runs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525/step_0010000.pt +runs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525/step_0020000.pt diff --git a/LTA_openwebtext_dualt/logs/owt_from_lm1b_c1024_4gpu/lta_owt_distilbert_len1024_init_lm1b1m_posemb_repeat_fully_c1024_adamw_gbs512_4gpu_20k_20260514_120045.log b/LTA_openwebtext_dualt/logs/owt_from_lm1b_c1024_4gpu/lta_owt_distilbert_len1024_init_lm1b1m_posemb_repeat_fully_c1024_adamw_gbs512_4gpu_20k_20260514_120045.log new file mode 100644 index 0000000000000000000000000000000000000000..d14d7f96ac99f3861189f91d1e7a2664c5d7365d --- /dev/null +++ b/LTA_openwebtext_dualt/logs/owt_from_lm1b_c1024_4gpu/lta_owt_distilbert_len1024_init_lm1b1m_posemb_repeat_fully_c1024_adamw_gbs512_4gpu_20k_20260514_120045.log @@ -0,0 +1,214 @@ +initialized_model_from=runs/lta_lm1b_dirichlet_categorical_fullvocab_c1024_fullycoupled_flmpack_onehot_hardce_ddit_small_len128_gbs512_8gpu_1m_nw0/step_1000000.pt start_step=1 +{ + "device": "cuda:0", + "rank": 0, + "world_size": 4, + "samples": "wrapped_stream", + "vocab_size": 30522, + "tokenizer_vocab_size": 30522, + "save_dir": "runs_transfer/lta_owt_distilbert_len1024_init_lm1b1m_posemb_repeat_fully_c1024_adamw_gbs512_4gpu_20k_20260514_120045", + "batch_size": 16, + "grad_accum": 8, + "effective_batch_size": 512, + "global_batch_size": 512, + "lr_schedule": "constant_warmup", + "optimizer": "adamw", + "warmup_steps": 2500, + "min_lr": 6e-05, + "weight_decay": 0.0, + "adamw_param_groups": "nanogpt", + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "adam_eps": 1e-08, + "muon_momentum": 0.95, + "muon_ns_steps": 5, + "muon_update_scale": 1.0, + "ema_decay": 0.0, + "ema_start_step": 0, + "model_type": "ddit", + "dual_t": true, + "corrupt_t_mode": "same", + "corrupt_min_t": null, + "corrupt_max_t": null, + "prefix_block_prob": 0.0, + "prefix_block_len": 128, + "mask_ratio_floor_schedule": "none", + "dirichlet_endpoint_mode": "categorical_dual_t", + "dirichlet_semantic_t_mode": "same", + "dirichlet_semantic_t_value": 0.0, + "dirichlet_semantic_t_curve": "linear", + "dirichlet_semantic_t_power": 1.0, + "endpoint_sequence_random_prob_alpha": 0.0, + "categorical_wrong_from_full_vocab": true, + "categorical_wrong_from_batch_valid_tokens": false, + "mask_mixture_original_prob": 0.0, + "mask_mixture_lowk_prob": 0.0, + "mask_mixture_lowcorrupt_prob": 0.0, + "mask_mixture_block_prob": 0.0, + "mask_mixture_all_prob": 0.0, + "mask_mixture_lowk_clean_tokens": "1,2,4,8,16,32,64", + "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64", + "mask_mixture_block_tokens": "64,128", + "simplex_bridge_sampler": "dirichlet", + "logistic_normal_sigma_min": 0.18, + "logistic_normal_sigma_max": 2.2, + "logistic_normal_tau_min": 0.65, + "logistic_normal_tau_max": 1.15, + "torch_compile": false, + "compile_mode": "max-autotune", + "state_format": "prob", + "target_loss": "hard_ce", + "meanflow_weight": 0.0, + "rollout_train_prob": 0.0, + "rollout_train_steps": 1, + "rollout_train_infer_steps": 64, + "rollout_train_temp": 1.45, + "rollout_train_max_gamma": 1.0, + "rollout_train_corrupt_only": true, + "rollout_train_samplewise": false, + "rollout_train_compute_always": false, + "bridge_noise_init": "logistic_normal", + "noise_sigma": -1.0, + "allow_tf32": true, + "activation_checkpointing": false, + "activation_checkpoint_interval": 1, + "activation_checkpoint_scope": "block", + "ddp_static_graph": false, + "ddp_gradient_as_bucket_view": true, + "blocking_data_transfer": false, + "dataloader_prefetch_factor": 4, + "full_train_stats": false, + "record_pad_truncate": false, + "record_add_eos": false, + "record_add_special_tokens": false, + "record_pad_token": "pad", + "record_shuffle_buffer": 10000, + "wrap": true, + "wrap_mode": "stream", + "wrap_record_buffer_size": 200, + "owt_cached_chunks": false, + "owt_chunk_cache_dir": "", + "owt_chunk_cache_rebuild": false, + "owt_chunk_cache_write_batch": 4096, + "owt_exact_repeat_per_chunk": 0, + "online_chunk_shuffle": false, + "online_chunk_shuffle_buffer": 10000, + "openwebtext_split": "train_minus_100k", + "detokenizer": "auto", + "resolved_detokenizer": null, + "num_workers": 4, + "latest_every": 500, + "resume_path": "" +} +step=50 micro_steps=400 elapsed=116.6s lr=6.120000e-06 loss=5.3339 loss_recon=5.3339 loss_meanflow=0.0000 mean_model_t=0.4751 mean_corrupt_t=0.4751 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.4602 out_g_norm=0.6815 acc_all=0.7375 acc_corrupt=0.4786 corrupt_frac=0.4672 loss_all=2.2783 loss_corrupt=4.1785 acc_corrupt_t_0p0_0p2=0.0980 corrupt_frac_t_0p0_0p2=0.1932 acc_corrupt_t_0p2_0p4=0.3492 corrupt_frac_t_0p2_0p4=0.2118 acc_corrupt_t_0p4_0p6=0.4777 corrupt_frac_t_0p4_0p6=0.2694 acc_corrupt_t_0p6_0p8=0.7390 corrupt_frac_t_0p6_0p8=0.1662 acc_corrupt_t_0p8_1p0=0.8419 corrupt_frac_t_0p8_1p0=0.1595 wrong_frac=0.5099 init_acc_corrupt=0.4611 init_gold_top10=0.4852 init_gold_top100=0.4888 +step=100 micro_steps=800 elapsed=126.7s lr=1.212000e-05 loss=3.8276 loss_recon=3.8276 loss_meanflow=0.0000 mean_model_t=0.5233 mean_corrupt_t=0.5233 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.4635 out_g_norm=0.0751 acc_all=0.6972 acc_corrupt=0.5113 corrupt_frac=0.5995 loss_all=2.4146 loss_corrupt=3.7195 acc_corrupt_t_0p0_0p2=0.1217 corrupt_frac_t_0p0_0p2=0.1698 acc_corrupt_t_0p2_0p4=0.3249 corrupt_frac_t_0p2_0p4=0.2843 acc_corrupt_t_0p4_0p6=0.5761 corrupt_frac_t_0p4_0p6=0.1986 acc_corrupt_t_0p6_0p8=0.7320 corrupt_frac_t_0p6_0p8=0.1850 acc_corrupt_t_0p8_1p0=0.9147 corrupt_frac_t_0p8_1p0=0.1623 wrong_frac=0.4992 init_acc_corrupt=0.4556 init_gold_top10=0.4936 init_gold_top100=0.5009 +step=150 micro_steps=1200 elapsed=127.8s lr=1.812000e-05 loss=3.6168 loss_recon=3.6168 loss_meanflow=0.0000 mean_model_t=0.5071 mean_corrupt_t=0.5071 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.4640 out_g_norm=0.0317 acc_all=0.7546 acc_corrupt=0.4738 corrupt_frac=0.4382 loss_all=1.8648 loss_corrupt=3.8321 acc_corrupt_t_0p0_0p2=0.1047 corrupt_frac_t_0p0_0p2=0.2834 acc_corrupt_t_0p2_0p4=0.2506 corrupt_frac_t_0p2_0p4=0.1623 acc_corrupt_t_0p4_0p6=0.5184 corrupt_frac_t_0p4_0p6=0.1400 acc_corrupt_t_0p6_0p8=0.7277 corrupt_frac_t_0p6_0p8=0.2260 acc_corrupt_t_0p8_1p0=0.8839 corrupt_frac_t_0p8_1p0=0.1883 wrong_frac=0.5518 init_acc_corrupt=0.4089 init_gold_top10=0.4396 init_gold_top100=0.4465 +step=200 micro_steps=1600 elapsed=128.5s lr=2.412000e-05 loss=3.5417 loss_recon=3.5417 loss_meanflow=0.0000 mean_model_t=0.4927 mean_corrupt_t=0.4927 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.4318 out_g_norm=0.0377 acc_all=0.7870 acc_corrupt=0.6161 corrupt_frac=0.5179 loss_all=1.6049 loss_corrupt=2.7910 acc_corrupt_t_0p0_0p2=0.1794 corrupt_frac_t_0p0_0p2=0.1511 acc_corrupt_t_0p2_0p4=0.3535 corrupt_frac_t_0p2_0p4=0.1710 acc_corrupt_t_0p6_0p8=0.7178 corrupt_frac_t_0p6_0p8=0.3984 acc_corrupt_t_0p8_1p0=0.8676 corrupt_frac_t_0p8_1p0=0.2795 wrong_frac=0.4097 init_acc_corrupt=0.5576 init_gold_top10=0.5852 init_gold_top100=0.5900 +step=250 micro_steps=2000 elapsed=128.2s lr=3.012000e-05 loss=3.3704 loss_recon=3.3704 loss_meanflow=0.0000 mean_model_t=0.4893 mean_corrupt_t=0.4893 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.3537 out_g_norm=0.0438 acc_all=0.7093 acc_corrupt=0.5355 corrupt_frac=0.5968 loss_all=2.0977 loss_corrupt=3.2917 acc_corrupt_t_0p0_0p2=0.1426 corrupt_frac_t_0p0_0p2=0.1958 acc_corrupt_t_0p2_0p4=0.3444 corrupt_frac_t_0p2_0p4=0.1321 acc_corrupt_t_0p4_0p6=0.5655 corrupt_frac_t_0p4_0p6=0.3194 acc_corrupt_t_0p6_0p8=0.7412 corrupt_frac_t_0p6_0p8=0.2502 acc_corrupt_t_0p8_1p0=0.9371 corrupt_frac_t_0p8_1p0=0.1025 wrong_frac=0.4984 init_acc_corrupt=0.4681 init_gold_top10=0.4957 init_gold_top100=0.5027 +step=300 micro_steps=2400 elapsed=128.2s lr=3.612000e-05 loss=3.2718 loss_recon=3.2718 loss_meanflow=0.0000 mean_model_t=0.4885 mean_corrupt_t=0.4885 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.2330 out_g_norm=0.0485 acc_all=0.6102 acc_corrupt=0.4058 corrupt_frac=0.6276 loss_all=2.7787 loss_corrupt=4.1649 acc_corrupt_t_0p0_0p2=0.1212 corrupt_frac_t_0p0_0p2=0.3121 acc_corrupt_t_0p2_0p4=0.3580 corrupt_frac_t_0p2_0p4=0.3434 acc_corrupt_t_0p4_0p6=0.5175 corrupt_frac_t_0p4_0p6=0.1254 acc_corrupt_t_0p6_0p8=0.8011 corrupt_frac_t_0p6_0p8=0.1594 acc_corrupt_t_0p8_1p0=0.8780 corrupt_frac_t_0p8_1p0=0.0598 wrong_frac=0.6378 init_acc_corrupt=0.3071 init_gold_top10=0.3533 init_gold_top100=0.3624 +step=350 micro_steps=2800 elapsed=128.9s lr=4.212000e-05 loss=3.0572 loss_recon=3.0572 loss_meanflow=0.0000 mean_model_t=0.5056 mean_corrupt_t=0.5056 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.0927 out_g_norm=0.0585 acc_all=0.7524 acc_corrupt=0.5071 corrupt_frac=0.4842 loss_all=1.7742 loss_corrupt=3.4789 acc_corrupt_t_0p0_0p2=0.1040 corrupt_frac_t_0p0_0p2=0.2085 acc_corrupt_t_0p2_0p4=0.3355 corrupt_frac_t_0p2_0p4=0.2972 acc_corrupt_t_0p4_0p6=0.5518 corrupt_frac_t_0p4_0p6=0.1046 acc_corrupt_t_0p6_0p8=0.7827 corrupt_frac_t_0p6_0p8=0.1775 acc_corrupt_t_0p8_1p0=0.8913 corrupt_frac_t_0p8_1p0=0.2122 wrong_frac=0.5311 init_acc_corrupt=0.4258 init_gold_top10=0.4624 init_gold_top100=0.4701 +step=400 micro_steps=3200 elapsed=128.3s lr=4.812000e-05 loss=2.9771 loss_recon=2.9771 loss_meanflow=0.0000 mean_model_t=0.5016 mean_corrupt_t=0.5016 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9699 out_g_norm=0.0609 acc_all=0.6943 acc_corrupt=0.5593 corrupt_frac=0.6827 loss_all=2.1328 loss_corrupt=3.0564 acc_corrupt_t_0p0_0p2=0.1100 corrupt_frac_t_0p0_0p2=0.1984 acc_corrupt_t_0p2_0p4=0.2757 corrupt_frac_t_0p2_0p4=0.2037 acc_corrupt_t_0p4_0p6=0.6867 corrupt_frac_t_0p4_0p6=0.0890 acc_corrupt_t_0p6_0p8=0.7985 corrupt_frac_t_0p6_0p8=0.3500 acc_corrupt_t_0p8_1p0=0.8858 corrupt_frac_t_0p8_1p0=0.1589 wrong_frac=0.4867 init_acc_corrupt=0.4782 init_gold_top10=0.5059 init_gold_top100=0.5117 +step=450 micro_steps=3600 elapsed=128.4s lr=5.412000e-05 loss=2.9486 loss_recon=2.9486 loss_meanflow=0.0000 mean_model_t=0.5026 mean_corrupt_t=0.5026 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9136 out_g_norm=0.0455 acc_all=0.7720 acc_corrupt=0.5810 corrupt_frac=0.5333 loss_all=1.5517 loss_corrupt=2.8319 acc_corrupt_t_0p0_0p2=0.1067 corrupt_frac_t_0p0_0p2=0.2069 acc_corrupt_t_0p2_0p4=0.4462 corrupt_frac_t_0p2_0p4=0.0298 acc_corrupt_t_0p4_0p6=0.6173 corrupt_frac_t_0p4_0p6=0.3867 acc_corrupt_t_0p6_0p8=0.7514 corrupt_frac_t_0p6_0p8=0.2440 acc_corrupt_t_0p8_1p0=0.9318 corrupt_frac_t_0p8_1p0=0.1326 wrong_frac=0.4705 init_acc_corrupt=0.5163 init_gold_top10=0.5245 init_gold_top100=0.5291 +step=500 micro_steps=4000 elapsed=128.5s lr=6.012000e-05 loss=3.0142 loss_recon=3.0142 loss_meanflow=0.0000 mean_model_t=0.4955 mean_corrupt_t=0.4955 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9002 out_g_norm=0.0361 acc_all=0.8072 acc_corrupt=0.6482 corrupt_frac=0.5325 loss_all=1.1912 loss_corrupt=2.1426 acc_corrupt_t_0p0_0p2=0.1981 corrupt_frac_t_0p0_0p2=0.1667 acc_corrupt_t_0p2_0p4=0.4098 corrupt_frac_t_0p2_0p4=0.1664 acc_corrupt_t_0p4_0p6=0.6542 corrupt_frac_t_0p4_0p6=0.0825 acc_corrupt_t_0p6_0p8=0.7455 corrupt_frac_t_0p6_0p8=0.2396 acc_corrupt_t_0p8_1p0=0.9119 corrupt_frac_t_0p8_1p0=0.3448 wrong_frac=0.4191 init_acc_corrupt=0.5616 init_gold_top10=0.5769 init_gold_top100=0.5794 +step=550 micro_steps=4400 elapsed=132.2s lr=6.612000e-05 loss=2.7808 loss_recon=2.7808 loss_meanflow=0.0000 mean_model_t=0.5161 mean_corrupt_t=0.5161 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.8984 out_g_norm=0.0314 acc_all=0.7191 acc_corrupt=0.4947 corrupt_frac=0.5416 loss_all=1.9256 loss_corrupt=3.4242 acc_corrupt_t_0p0_0p2=0.1651 corrupt_frac_t_0p0_0p2=0.2641 acc_corrupt_t_0p2_0p4=0.3936 corrupt_frac_t_0p2_0p4=0.2055 acc_corrupt_t_0p4_0p6=0.5540 corrupt_frac_t_0p4_0p6=0.2130 acc_corrupt_t_0p6_0p8=0.7080 corrupt_frac_t_0p6_0p8=0.2037 acc_corrupt_t_0p8_1p0=0.9504 corrupt_frac_t_0p8_1p0=0.1136 wrong_frac=0.5721 init_acc_corrupt=0.3973 init_gold_top10=0.4195 init_gold_top100=0.4274 +step=600 micro_steps=4800 elapsed=134.5s lr=7.212000e-05 loss=2.8882 loss_recon=2.8882 loss_meanflow=0.0000 mean_model_t=0.5044 mean_corrupt_t=0.5044 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9031 out_g_norm=0.0293 acc_all=0.7741 acc_corrupt=0.5030 corrupt_frac=0.4395 loss_all=1.5307 loss_corrupt=3.3578 acc_corrupt_t_0p0_0p2=0.1740 corrupt_frac_t_0p0_0p2=0.3209 acc_corrupt_t_0p2_0p4=0.4874 corrupt_frac_t_0p2_0p4=0.1103 acc_corrupt_t_0p4_0p6=0.5979 corrupt_frac_t_0p4_0p6=0.3454 acc_corrupt_t_0p6_0p8=0.8113 corrupt_frac_t_0p6_0p8=0.0912 acc_corrupt_t_0p8_1p0=0.8540 corrupt_frac_t_0p8_1p0=0.1322 wrong_frac=0.5721 init_acc_corrupt=0.3966 init_gold_top10=0.4213 init_gold_top100=0.4265 +step=650 micro_steps=5200 elapsed=135.0s lr=7.812000e-05 loss=2.8381 loss_recon=2.8381 loss_meanflow=0.0000 mean_model_t=0.4989 mean_corrupt_t=0.4989 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9124 out_g_norm=0.0258 acc_all=0.7846 acc_corrupt=0.6305 corrupt_frac=0.5763 loss_all=1.3745 loss_corrupt=2.3498 acc_corrupt_t_0p0_0p2=0.1394 corrupt_frac_t_0p0_0p2=0.1512 acc_corrupt_t_0p2_0p4=0.3558 corrupt_frac_t_0p2_0p4=0.0566 acc_corrupt_t_0p4_0p6=0.5369 corrupt_frac_t_0p4_0p6=0.2681 acc_corrupt_t_0p6_0p8=0.8018 corrupt_frac_t_0p6_0p8=0.3376 acc_corrupt_t_0p8_1p0=0.9364 corrupt_frac_t_0p8_1p0=0.1865 wrong_frac=0.4169 init_acc_corrupt=0.5614 init_gold_top10=0.5769 init_gold_top100=0.5823 +step=700 micro_steps=5600 elapsed=135.4s lr=8.412000e-05 loss=2.8914 loss_recon=2.8914 loss_meanflow=0.0000 mean_model_t=0.5073 mean_corrupt_t=0.5073 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9220 out_g_norm=0.0255 acc_all=0.7479 acc_corrupt=0.5593 corrupt_frac=0.5667 loss_all=1.6396 loss_corrupt=2.8295 acc_corrupt_t_0p0_0p2=0.1840 corrupt_frac_t_0p0_0p2=0.1692 acc_corrupt_t_0p2_0p4=0.4156 corrupt_frac_t_0p2_0p4=0.2674 acc_corrupt_t_0p4_0p6=0.6588 corrupt_frac_t_0p4_0p6=0.3012 acc_corrupt_t_0p6_0p8=0.7360 corrupt_frac_t_0p6_0p8=0.1207 acc_corrupt_t_0p8_1p0=0.9177 corrupt_frac_t_0p8_1p0=0.1414 wrong_frac=0.5222 init_acc_corrupt=0.4458 init_gold_top10=0.4727 init_gold_top100=0.4783 +step=750 micro_steps=6000 elapsed=135.4s lr=9.012000e-05 loss=2.8874 loss_recon=2.8874 loss_meanflow=0.0000 mean_model_t=0.4925 mean_corrupt_t=0.4925 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9276 out_g_norm=0.0243 acc_all=0.7809 acc_corrupt=0.6402 corrupt_frac=0.6071 loss_all=1.4569 loss_corrupt=2.3815 acc_corrupt_t_0p0_0p2=0.3356 corrupt_frac_t_0p0_0p2=0.0440 acc_corrupt_t_0p2_0p4=0.3152 corrupt_frac_t_0p2_0p4=0.1923 acc_corrupt_t_0p4_0p6=0.6658 corrupt_frac_t_0p4_0p6=0.2599 acc_corrupt_t_0p6_0p8=0.7486 corrupt_frac_t_0p6_0p8=0.3908 acc_corrupt_t_0p8_1p0=0.8780 corrupt_frac_t_0p8_1p0=0.1129 wrong_frac=0.4332 init_acc_corrupt=0.5396 init_gold_top10=0.5647 init_gold_top100=0.5691 +step=800 micro_steps=6400 elapsed=135.5s lr=9.612000e-05 loss=2.8735 loss_recon=2.8735 loss_meanflow=0.0000 mean_model_t=0.4870 mean_corrupt_t=0.4870 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9336 out_g_norm=0.0234 acc_all=0.7418 acc_corrupt=0.5373 corrupt_frac=0.5493 loss_all=1.6838 loss_corrupt=2.9801 acc_corrupt_t_0p0_0p2=0.2055 corrupt_frac_t_0p0_0p2=0.2411 acc_corrupt_t_0p2_0p4=0.4128 corrupt_frac_t_0p2_0p4=0.2937 acc_corrupt_t_0p4_0p6=0.6371 corrupt_frac_t_0p4_0p6=0.1001 acc_corrupt_t_0p6_0p8=0.7959 corrupt_frac_t_0p6_0p8=0.3027 acc_corrupt_t_0p8_1p0=0.9911 corrupt_frac_t_0p8_1p0=0.0623 wrong_frac=0.5511 init_acc_corrupt=0.4212 init_gold_top10=0.4455 init_gold_top100=0.4483 +step=850 micro_steps=6800 elapsed=135.2s lr=1.021200e-04 loss=2.7085 loss_recon=2.7085 loss_meanflow=0.0000 mean_model_t=0.5129 mean_corrupt_t=0.5129 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9404 out_g_norm=0.0226 acc_all=0.7911 acc_corrupt=0.6168 corrupt_frac=0.5336 loss_all=1.3531 loss_corrupt=2.4234 acc_corrupt_t_0p0_0p2=0.2197 corrupt_frac_t_0p0_0p2=0.2306 acc_corrupt_t_0p2_0p4=0.3948 corrupt_frac_t_0p2_0p4=0.1283 acc_corrupt_t_0p4_0p6=0.6159 corrupt_frac_t_0p4_0p6=0.2695 acc_corrupt_t_0p6_0p8=0.8349 corrupt_frac_t_0p6_0p8=0.0603 acc_corrupt_t_0p8_1p0=0.9611 corrupt_frac_t_0p8_1p0=0.3113 wrong_frac=0.4631 init_acc_corrupt=0.5093 init_gold_top10=0.5320 init_gold_top100=0.5357 +step=900 micro_steps=7200 elapsed=135.8s lr=1.081200e-04 loss=2.6962 loss_recon=2.6962 loss_meanflow=0.0000 mean_model_t=0.5020 mean_corrupt_t=0.5020 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9474 out_g_norm=0.0237 acc_all=0.8228 acc_corrupt=0.6664 corrupt_frac=0.5297 loss_all=1.1702 loss_corrupt=2.1729 acc_corrupt_t_0p0_0p2=0.1707 corrupt_frac_t_0p0_0p2=0.0722 acc_corrupt_t_0p2_0p4=0.4048 corrupt_frac_t_0p2_0p4=0.1477 acc_corrupt_t_0p4_0p6=0.5952 corrupt_frac_t_0p4_0p6=0.2693 acc_corrupt_t_0p6_0p8=0.8028 corrupt_frac_t_0p6_0p8=0.3506 acc_corrupt_t_0p8_1p0=0.9525 corrupt_frac_t_0p8_1p0=0.1602 wrong_frac=0.4166 init_acc_corrupt=0.5634 init_gold_top10=0.5812 init_gold_top100=0.5843 +step=950 micro_steps=7600 elapsed=135.6s lr=1.141200e-04 loss=2.7332 loss_recon=2.7332 loss_meanflow=0.0000 mean_model_t=0.5095 mean_corrupt_t=0.5095 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9545 out_g_norm=0.0247 acc_all=0.6908 acc_corrupt=0.5107 corrupt_frac=0.6260 loss_all=2.0542 loss_corrupt=3.2336 acc_corrupt_t_0p0_0p2=0.1500 corrupt_frac_t_0p0_0p2=0.1423 acc_corrupt_t_0p2_0p4=0.3361 corrupt_frac_t_0p2_0p4=0.4134 acc_corrupt_t_0p4_0p6=0.5653 corrupt_frac_t_0p4_0p6=0.0500 acc_corrupt_t_0p6_0p8=0.8033 corrupt_frac_t_0p6_0p8=0.3162 acc_corrupt_t_0p8_1p0=0.8727 corrupt_frac_t_0p8_1p0=0.0781 wrong_frac=0.5552 init_acc_corrupt=0.3961 init_gold_top10=0.4404 init_gold_top100=0.4452 +step=1000 micro_steps=8000 elapsed=135.1s lr=1.201200e-04 loss=2.6267 loss_recon=2.6267 loss_meanflow=0.0000 mean_model_t=0.5180 mean_corrupt_t=0.5180 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9606 out_g_norm=0.0223 acc_all=0.8404 acc_corrupt=0.6390 corrupt_frac=0.4393 loss_all=0.9768 loss_corrupt=2.1930 acc_corrupt_t_0p0_0p2=0.0933 corrupt_frac_t_0p0_0p2=0.1326 acc_corrupt_t_0p2_0p4=0.3475 corrupt_frac_t_0p2_0p4=0.1371 acc_corrupt_t_0p4_0p6=0.6511 corrupt_frac_t_0p4_0p6=0.1338 acc_corrupt_t_0p6_0p8=0.7651 corrupt_frac_t_0p6_0p8=0.4016 acc_corrupt_t_0p8_1p0=0.9473 corrupt_frac_t_0p8_1p0=0.1949 wrong_frac=0.4384 init_acc_corrupt=0.5355 init_gold_top10=0.5551 init_gold_top100=0.5627 +step=1050 micro_steps=8400 elapsed=145.5s lr=1.261200e-04 loss=2.7071 loss_recon=2.7071 loss_meanflow=0.0000 mean_model_t=0.5050 mean_corrupt_t=0.5050 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9661 out_g_norm=0.0226 acc_all=0.7328 acc_corrupt=0.5157 corrupt_frac=0.5459 loss_all=1.7703 loss_corrupt=3.1741 acc_corrupt_t_0p0_0p2=0.1688 corrupt_frac_t_0p0_0p2=0.2947 acc_corrupt_t_0p2_0p4=0.3957 corrupt_frac_t_0p2_0p4=0.2461 acc_corrupt_t_0p4_0p6=0.6469 corrupt_frac_t_0p4_0p6=0.1488 acc_corrupt_t_0p6_0p8=0.7754 corrupt_frac_t_0p6_0p8=0.0926 acc_corrupt_t_0p8_1p0=0.9204 corrupt_frac_t_0p8_1p0=0.2178 wrong_frac=0.5662 init_acc_corrupt=0.3931 init_gold_top10=0.4261 init_gold_top100=0.4345 +step=1100 micro_steps=8800 elapsed=162.2s lr=1.321200e-04 loss=2.7524 loss_recon=2.7524 loss_meanflow=0.0000 mean_model_t=0.4982 mean_corrupt_t=0.4982 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9731 out_g_norm=0.0218 acc_all=0.7803 acc_corrupt=0.5817 corrupt_frac=0.5175 loss_all=1.3652 loss_corrupt=2.5602 acc_corrupt_t_0p0_0p2=0.2331 corrupt_frac_t_0p0_0p2=0.1812 acc_corrupt_t_0p2_0p4=0.3940 corrupt_frac_t_0p2_0p4=0.1380 acc_corrupt_t_0p4_0p6=0.5580 corrupt_frac_t_0p4_0p6=0.3418 acc_corrupt_t_0p6_0p8=0.8058 corrupt_frac_t_0p6_0p8=0.1707 acc_corrupt_t_0p8_1p0=0.9320 corrupt_frac_t_0p8_1p0=0.1683 wrong_frac=0.5316 init_acc_corrupt=0.4409 init_gold_top10=0.4631 init_gold_top100=0.4677 +step=1150 micro_steps=9200 elapsed=136.0s lr=1.381200e-04 loss=2.8048 loss_recon=2.8048 loss_meanflow=0.0000 mean_model_t=0.4824 mean_corrupt_t=0.4824 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9701 out_g_norm=0.0228 acc_all=0.7565 acc_corrupt=0.4408 corrupt_frac=0.4284 loss_all=1.5773 loss_corrupt=3.5431 acc_corrupt_t_0p0_0p2=0.2233 corrupt_frac_t_0p0_0p2=0.4390 acc_corrupt_t_0p2_0p4=0.4506 corrupt_frac_t_0p2_0p4=0.2609 acc_corrupt_t_0p4_0p6=0.6832 corrupt_frac_t_0p4_0p6=0.1785 acc_corrupt_t_0p6_0p8=0.7470 corrupt_frac_t_0p6_0p8=0.0710 acc_corrupt_t_0p8_1p0=0.9916 corrupt_frac_t_0p8_1p0=0.0507 wrong_frac=0.6622 init_acc_corrupt=0.2892 init_gold_top10=0.3260 init_gold_top100=0.3357 +step=1200 micro_steps=9600 elapsed=140.0s lr=1.441200e-04 loss=2.6216 loss_recon=2.6216 loss_meanflow=0.0000 mean_model_t=0.4964 mean_corrupt_t=0.4964 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9689 out_g_norm=0.0232 acc_all=0.7330 acc_corrupt=0.6004 corrupt_frac=0.6655 loss_all=1.7570 loss_corrupt=2.6081 acc_corrupt_t_0p0_0p2=0.1446 corrupt_frac_t_0p0_0p2=0.2227 acc_corrupt_t_0p2_0p4=0.4816 corrupt_frac_t_0p2_0p4=0.0897 acc_corrupt_t_0p4_0p6=0.5694 corrupt_frac_t_0p4_0p6=0.2364 acc_corrupt_t_0p6_0p8=0.7657 corrupt_frac_t_0p6_0p8=0.2505 acc_corrupt_t_0p8_1p0=0.9895 corrupt_frac_t_0p8_1p0=0.2007 wrong_frac=0.4577 init_acc_corrupt=0.5181 init_gold_top10=0.5346 init_gold_top100=0.5425 +step=1250 micro_steps=10000 elapsed=135.0s lr=1.501200e-04 loss=2.7294 loss_recon=2.7294 loss_meanflow=0.0000 mean_model_t=0.4896 mean_corrupt_t=0.4896 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9615 out_g_norm=0.0224 acc_all=0.8127 acc_corrupt=0.5698 corrupt_frac=0.4279 loss_all=1.1853 loss_corrupt=2.6737 acc_corrupt_t_0p0_0p2=0.1977 corrupt_frac_t_0p0_0p2=0.2100 acc_corrupt_t_0p2_0p4=0.4490 corrupt_frac_t_0p2_0p4=0.3018 acc_corrupt_t_0p4_0p6=0.5878 corrupt_frac_t_0p4_0p6=0.1471 acc_corrupt_t_0p6_0p8=0.8509 corrupt_frac_t_0p6_0p8=0.1799 acc_corrupt_t_0p8_1p0=0.9505 corrupt_frac_t_0p8_1p0=0.1613 wrong_frac=0.5296 init_acc_corrupt=0.4329 init_gold_top10=0.4628 init_gold_top100=0.4701 +step=1300 micro_steps=10400 elapsed=160.7s lr=1.561200e-04 loss=2.7575 loss_recon=2.7575 loss_meanflow=0.0000 mean_model_t=0.4882 mean_corrupt_t=0.4882 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9627 out_g_norm=0.0216 acc_all=0.6938 acc_corrupt=0.4641 corrupt_frac=0.5604 loss_all=2.0383 loss_corrupt=3.5487 acc_corrupt_t_0p0_0p2=0.1329 corrupt_frac_t_0p0_0p2=0.2442 acc_corrupt_t_0p2_0p4=0.3960 corrupt_frac_t_0p2_0p4=0.3685 acc_corrupt_t_0p4_0p6=0.5903 corrupt_frac_t_0p4_0p6=0.2111 acc_corrupt_t_0p6_0p8=0.8778 corrupt_frac_t_0p6_0p8=0.0838 acc_corrupt_t_0p8_1p0=0.9470 corrupt_frac_t_0p8_1p0=0.0925 wrong_frac=0.6210 init_acc_corrupt=0.3333 init_gold_top10=0.3736 init_gold_top100=0.3785 +step=1350 micro_steps=10800 elapsed=169.7s lr=1.621200e-04 loss=2.6077 loss_recon=2.6077 loss_meanflow=0.0000 mean_model_t=0.5072 mean_corrupt_t=0.5072 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9658 out_g_norm=0.0225 acc_all=0.7194 acc_corrupt=0.5666 corrupt_frac=0.6429 loss_all=1.8358 loss_corrupt=2.8055 acc_corrupt_t_0p0_0p2=0.2255 corrupt_frac_t_0p0_0p2=0.1861 acc_corrupt_t_0p2_0p4=0.2778 corrupt_frac_t_0p2_0p4=0.1476 acc_corrupt_t_0p4_0p6=0.5801 corrupt_frac_t_0p4_0p6=0.2512 acc_corrupt_t_0p6_0p8=0.7721 corrupt_frac_t_0p6_0p8=0.2786 acc_corrupt_t_0p8_1p0=0.8998 corrupt_frac_t_0p8_1p0=0.1364 wrong_frac=0.5107 init_acc_corrupt=0.4538 init_gold_top10=0.4852 init_gold_top100=0.4905 +step=1400 micro_steps=11200 elapsed=145.5s lr=1.681200e-04 loss=2.5496 loss_recon=2.5496 loss_meanflow=0.0000 mean_model_t=0.5090 mean_corrupt_t=0.5090 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9692 out_g_norm=0.0214 acc_all=0.7827 acc_corrupt=0.6595 corrupt_frac=0.6263 loss_all=1.3720 loss_corrupt=2.1213 acc_corrupt_t_0p0_0p2=0.2239 corrupt_frac_t_0p0_0p2=0.2686 acc_corrupt_t_0p2_0p4=0.7283 corrupt_frac_t_0p2_0p4=0.0990 acc_corrupt_t_0p4_0p6=0.6662 corrupt_frac_t_0p4_0p6=0.2333 acc_corrupt_t_0p6_0p8=0.8332 corrupt_frac_t_0p6_0p8=0.1069 acc_corrupt_t_0p8_1p0=0.9676 corrupt_frac_t_0p8_1p0=0.2922 wrong_frac=0.4665 init_acc_corrupt=0.5010 init_gold_top10=0.5258 init_gold_top100=0.5330 +step=1450 micro_steps=11600 elapsed=162.8s lr=1.741200e-04 loss=2.5789 loss_recon=2.5789 loss_meanflow=0.0000 mean_model_t=0.5100 mean_corrupt_t=0.5100 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9557 out_g_norm=0.0212 acc_all=0.8300 acc_corrupt=0.6879 corrupt_frac=0.5385 loss_all=1.0353 loss_corrupt=1.8735 acc_corrupt_t_0p0_0p2=0.3133 corrupt_frac_t_0p0_0p2=0.1418 acc_corrupt_t_0p2_0p4=0.4381 corrupt_frac_t_0p2_0p4=0.1420 acc_corrupt_t_0p4_0p6=0.6334 corrupt_frac_t_0p4_0p6=0.2022 acc_corrupt_t_0p6_0p8=0.7672 corrupt_frac_t_0p6_0p8=0.2128 acc_corrupt_t_0p8_1p0=0.9627 corrupt_frac_t_0p8_1p0=0.3012 wrong_frac=0.4038 init_acc_corrupt=0.5722 init_gold_top10=0.5926 init_gold_top100=0.5958 +step=1500 micro_steps=12000 elapsed=132.3s lr=1.801200e-04 loss=2.6958 loss_recon=2.6958 loss_meanflow=0.0000 mean_model_t=0.4929 mean_corrupt_t=0.4929 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9660 out_g_norm=0.0197 acc_all=0.7805 acc_corrupt=0.5887 corrupt_frac=0.5320 loss_all=1.3908 loss_corrupt=2.5809 acc_corrupt_t_0p0_0p2=0.2995 corrupt_frac_t_0p0_0p2=0.2326 acc_corrupt_t_0p2_0p4=0.4327 corrupt_frac_t_0p2_0p4=0.2277 acc_corrupt_t_0p4_0p6=0.6452 corrupt_frac_t_0p4_0p6=0.0427 acc_corrupt_t_0p6_0p8=0.7559 corrupt_frac_t_0p6_0p8=0.3708 acc_corrupt_t_0p8_1p0=0.8927 corrupt_frac_t_0p8_1p0=0.1262 wrong_frac=0.5092 init_acc_corrupt=0.4466 init_gold_top10=0.4857 init_gold_top100=0.4924 +step=1550 micro_steps=12400 elapsed=127.5s lr=1.861200e-04 loss=2.6536 loss_recon=2.6536 loss_meanflow=0.0000 mean_model_t=0.4986 mean_corrupt_t=0.4986 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9730 out_g_norm=0.0205 acc_all=0.7263 acc_corrupt=0.5079 corrupt_frac=0.5446 loss_all=1.8184 loss_corrupt=3.2197 acc_corrupt_t_0p0_0p2=0.1868 corrupt_frac_t_0p0_0p2=0.3995 acc_corrupt_t_0p2_0p4=0.4309 corrupt_frac_t_0p2_0p4=0.2052 acc_corrupt_t_0p6_0p8=0.7667 corrupt_frac_t_0p6_0p8=0.1609 acc_corrupt_t_0p8_1p0=0.9450 corrupt_frac_t_0p8_1p0=0.2343 wrong_frac=0.5705 init_acc_corrupt=0.3783 init_gold_top10=0.4186 init_gold_top100=0.4284 +step=1600 micro_steps=12800 elapsed=126.0s lr=1.921200e-04 loss=2.5511 loss_recon=2.5511 loss_meanflow=0.0000 mean_model_t=0.4957 mean_corrupt_t=0.4957 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9776 out_g_norm=0.0224 acc_all=0.7740 acc_corrupt=0.5456 corrupt_frac=0.4934 loss_all=1.4688 loss_corrupt=2.9088 acc_corrupt_t_0p0_0p2=0.2934 corrupt_frac_t_0p0_0p2=0.2209 acc_corrupt_t_0p2_0p4=0.3587 corrupt_frac_t_0p2_0p4=0.2425 acc_corrupt_t_0p4_0p6=0.5636 corrupt_frac_t_0p4_0p6=0.2919 acc_corrupt_t_0p8_1p0=0.9373 corrupt_frac_t_0p8_1p0=0.2447 wrong_frac=0.5529 init_acc_corrupt=0.3945 init_gold_top10=0.4406 init_gold_top100=0.4484 +step=1650 micro_steps=13200 elapsed=125.8s lr=1.981200e-04 loss=2.7809 loss_recon=2.7809 loss_meanflow=0.0000 mean_model_t=0.4841 mean_corrupt_t=0.4841 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9821 out_g_norm=0.0201 acc_all=0.8197 acc_corrupt=0.6836 corrupt_frac=0.5569 loss_all=1.1670 loss_corrupt=2.0003 acc_corrupt_t_0p0_0p2=0.2510 corrupt_frac_t_0p0_0p2=0.2161 acc_corrupt_t_0p2_0p4=0.3195 corrupt_frac_t_0p2_0p4=0.0292 acc_corrupt_t_0p4_0p6=0.5628 corrupt_frac_t_0p4_0p6=0.1657 acc_corrupt_t_0p6_0p8=0.7957 corrupt_frac_t_0p6_0p8=0.1727 acc_corrupt_t_0p8_1p0=0.9352 corrupt_frac_t_0p8_1p0=0.4163 wrong_frac=0.3989 init_acc_corrupt=0.5797 init_gold_top10=0.5957 init_gold_top100=0.6002 +step=1700 micro_steps=13600 elapsed=125.9s lr=2.041200e-04 loss=2.5079 loss_recon=2.5079 loss_meanflow=0.0000 mean_model_t=0.5132 mean_corrupt_t=0.5132 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1436.9901 out_g_norm=0.0190 acc_all=0.7779 acc_corrupt=0.6384 corrupt_frac=0.6069 loss_all=1.5086 loss_corrupt=2.4391 acc_corrupt_t_0p0_0p2=0.1793 corrupt_frac_t_0p0_0p2=0.1638 acc_corrupt_t_0p2_0p4=0.3501 corrupt_frac_t_0p2_0p4=0.1896 acc_corrupt_t_0p4_0p6=0.6352 corrupt_frac_t_0p4_0p6=0.0970 acc_corrupt_t_0p6_0p8=0.8059 corrupt_frac_t_0p6_0p8=0.1751 acc_corrupt_t_0p8_1p0=0.9076 corrupt_frac_t_0p8_1p0=0.3745 wrong_frac=0.4287 init_acc_corrupt=0.5396 init_gold_top10=0.5660 init_gold_top100=0.5713 +step=1750 micro_steps=14000 elapsed=126.1s lr=2.101200e-04 loss=2.4587 loss_recon=2.4587 loss_meanflow=0.0000 mean_model_t=0.5126 mean_corrupt_t=0.5126 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.0043 out_g_norm=0.0190 acc_all=0.8370 acc_corrupt=0.6146 corrupt_frac=0.4172 loss_all=1.0378 loss_corrupt=2.3970 acc_corrupt_t_0p0_0p2=0.3241 corrupt_frac_t_0p0_0p2=0.1264 acc_corrupt_t_0p2_0p4=0.4424 corrupt_frac_t_0p2_0p4=0.3062 acc_corrupt_t_0p4_0p6=0.5769 corrupt_frac_t_0p4_0p6=0.2054 acc_corrupt_t_0p6_0p8=0.7101 corrupt_frac_t_0p6_0p8=0.0797 acc_corrupt_t_0p8_1p0=0.9321 corrupt_frac_t_0p8_1p0=0.2822 wrong_frac=0.4919 init_acc_corrupt=0.4625 init_gold_top10=0.5033 init_gold_top100=0.5086 +step=1800 micro_steps=14400 elapsed=126.6s lr=2.161200e-04 loss=2.6668 loss_recon=2.6668 loss_meanflow=0.0000 mean_model_t=0.4903 mean_corrupt_t=0.4903 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.0161 out_g_norm=0.0189 acc_all=0.8640 acc_corrupt=0.8027 corrupt_frac=0.6876 loss_all=0.8615 loss_corrupt=1.2416 acc_corrupt_t_0p0_0p2=0.3652 corrupt_frac_t_0p0_0p2=0.0316 acc_corrupt_t_0p2_0p4=0.3643 corrupt_frac_t_0p2_0p4=0.1279 acc_corrupt_t_0p6_0p8=0.8018 corrupt_frac_t_0p6_0p8=0.2924 acc_corrupt_t_0p8_1p0=0.9307 corrupt_frac_t_0p8_1p0=0.5481 wrong_frac=0.2614 init_acc_corrupt=0.7203 init_gold_top10=0.7382 init_gold_top100=0.7390 +step=1850 micro_steps=14800 elapsed=126.6s lr=2.221200e-04 loss=2.6053 loss_recon=2.6053 loss_meanflow=0.0000 mean_model_t=0.4987 mean_corrupt_t=0.4987 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.0267 out_g_norm=0.0224 acc_all=0.6619 acc_corrupt=0.4466 corrupt_frac=0.5974 loss_all=2.3384 loss_corrupt=3.7637 acc_corrupt_t_0p0_0p2=0.1569 corrupt_frac_t_0p0_0p2=0.4018 acc_corrupt_t_0p2_0p4=0.4129 corrupt_frac_t_0p2_0p4=0.2054 acc_corrupt_t_0p4_0p6=0.5571 corrupt_frac_t_0p4_0p6=0.1190 acc_corrupt_t_0p6_0p8=0.7219 corrupt_frac_t_0p6_0p8=0.1154 acc_corrupt_t_0p8_1p0=0.9413 corrupt_frac_t_0p8_1p0=0.1585 wrong_frac=0.6146 init_acc_corrupt=0.3289 init_gold_top10=0.3739 init_gold_top100=0.3853 +step=1900 micro_steps=15200 elapsed=126.8s lr=2.281200e-04 loss=2.6507 loss_recon=2.6507 loss_meanflow=0.0000 mean_model_t=0.4989 mean_corrupt_t=0.4989 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.0399 out_g_norm=0.0205 acc_all=0.6990 acc_corrupt=0.5078 corrupt_frac=0.6080 loss_all=2.1180 loss_corrupt=3.4310 acc_corrupt_t_0p0_0p2=0.1824 corrupt_frac_t_0p0_0p2=0.3347 acc_corrupt_t_0p2_0p4=0.4260 corrupt_frac_t_0p2_0p4=0.1473 acc_corrupt_t_0p4_0p6=0.5835 corrupt_frac_t_0p4_0p6=0.2037 acc_corrupt_t_0p6_0p8=0.7832 corrupt_frac_t_0p6_0p8=0.1792 acc_corrupt_t_0p8_1p0=0.9235 corrupt_frac_t_0p8_1p0=0.1352 wrong_frac=0.5631 init_acc_corrupt=0.3962 init_gold_top10=0.4266 init_gold_top100=0.4361 +step=1950 micro_steps=15600 elapsed=127.1s lr=2.341200e-04 loss=2.6638 loss_recon=2.6638 loss_meanflow=0.0000 mean_model_t=0.4882 mean_corrupt_t=0.4882 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.0478 out_g_norm=0.0212 acc_all=0.5979 acc_corrupt=0.4502 corrupt_frac=0.5277 loss_all=2.5492 loss_corrupt=3.5023 acc_corrupt_t_0p0_0p2=0.0795 corrupt_frac_t_0p0_0p2=0.2066 acc_corrupt_t_0p2_0p4=0.1603 corrupt_frac_t_0p2_0p4=0.0967 acc_corrupt_t_0p4_0p6=0.5448 corrupt_frac_t_0p4_0p6=0.3265 acc_corrupt_t_0p6_0p8=0.7777 corrupt_frac_t_0p6_0p8=0.1170 acc_corrupt_t_0p8_1p0=0.5898 corrupt_frac_t_0p8_1p0=0.2532 wrong_frac=0.5185 init_acc_corrupt=0.4648 init_gold_top10=0.4757 init_gold_top100=0.4813 +step=2000 micro_steps=16000 elapsed=127.3s lr=2.401200e-04 loss=2.6270 loss_recon=2.6270 loss_meanflow=0.0000 mean_model_t=0.4961 mean_corrupt_t=0.4961 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.0617 out_g_norm=0.0203 acc_all=0.6990 acc_corrupt=0.4510 corrupt_frac=0.5439 loss_all=1.9510 loss_corrupt=3.5173 acc_corrupt_t_0p0_0p2=0.2108 corrupt_frac_t_0p0_0p2=0.2742 acc_corrupt_t_0p2_0p4=0.3763 corrupt_frac_t_0p2_0p4=0.4351 acc_corrupt_t_0p4_0p6=0.6029 corrupt_frac_t_0p4_0p6=0.1085 acc_corrupt_t_0p6_0p8=0.8790 corrupt_frac_t_0p6_0p8=0.0742 acc_corrupt_t_0p8_1p0=0.9148 corrupt_frac_t_0p8_1p0=0.1081 wrong_frac=0.6441 init_acc_corrupt=0.2901 init_gold_top10=0.3463 init_gold_top100=0.3555 +step=2050 micro_steps=16400 elapsed=130.6s lr=2.461200e-04 loss=2.5362 loss_recon=2.5362 loss_meanflow=0.0000 mean_model_t=0.5098 mean_corrupt_t=0.5098 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.0771 out_g_norm=0.0224 acc_all=0.7955 acc_corrupt=0.6389 corrupt_frac=0.5554 loss_all=1.2729 loss_corrupt=2.2235 acc_corrupt_t_0p0_0p2=0.1632 corrupt_frac_t_0p0_0p2=0.1582 acc_corrupt_t_0p2_0p4=0.4926 corrupt_frac_t_0p2_0p4=0.2222 acc_corrupt_t_0p4_0p6=0.6393 corrupt_frac_t_0p4_0p6=0.0710 acc_corrupt_t_0p6_0p8=0.8082 corrupt_frac_t_0p6_0p8=0.4400 acc_corrupt_t_0p8_1p0=0.9453 corrupt_frac_t_0p8_1p0=0.1086 wrong_frac=0.4577 init_acc_corrupt=0.5146 init_gold_top10=0.5396 init_gold_top100=0.5435 +step=2100 micro_steps=16800 elapsed=129.7s lr=2.521200e-04 loss=2.5877 loss_recon=2.5877 loss_meanflow=0.0000 mean_model_t=0.5055 mean_corrupt_t=0.5055 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.0957 out_g_norm=0.0188 acc_all=0.7486 acc_corrupt=0.5588 corrupt_frac=0.5657 loss_all=1.5975 loss_corrupt=2.7605 acc_corrupt_t_0p0_0p2=0.1941 corrupt_frac_t_0p0_0p2=0.2763 acc_corrupt_t_0p2_0p4=0.4395 corrupt_frac_t_0p2_0p4=0.2079 acc_corrupt_t_0p4_0p6=0.6538 corrupt_frac_t_0p4_0p6=0.0589 acc_corrupt_t_0p6_0p8=0.7708 corrupt_frac_t_0p6_0p8=0.3441 acc_corrupt_t_0p8_1p0=0.9761 corrupt_frac_t_0p8_1p0=0.1128 wrong_frac=0.5373 init_acc_corrupt=0.4245 init_gold_top10=0.4539 init_gold_top100=0.4622 +step=2150 micro_steps=17200 elapsed=189.8s lr=2.581200e-04 loss=2.6215 loss_recon=2.6215 loss_meanflow=0.0000 mean_model_t=0.4933 mean_corrupt_t=0.4933 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.1193 out_g_norm=0.0182 acc_all=0.8499 acc_corrupt=0.7113 corrupt_frac=0.5118 loss_all=0.9500 loss_corrupt=1.7832 acc_corrupt_t_0p0_0p2=0.3760 corrupt_frac_t_0p0_0p2=0.0625 acc_corrupt_t_0p2_0p4=0.4883 corrupt_frac_t_0p2_0p4=0.0972 acc_corrupt_t_0p4_0p6=0.6377 corrupt_frac_t_0p4_0p6=0.3970 acc_corrupt_t_0p6_0p8=0.8138 corrupt_frac_t_0p6_0p8=0.2056 acc_corrupt_t_0p8_1p0=0.9248 corrupt_frac_t_0p8_1p0=0.2378 wrong_frac=0.3896 init_acc_corrupt=0.6040 init_gold_top10=0.6088 init_gold_top100=0.6113 +step=2200 micro_steps=17600 elapsed=151.9s lr=2.641200e-04 loss=2.5421 loss_recon=2.5421 loss_meanflow=0.0000 mean_model_t=0.5027 mean_corrupt_t=0.5027 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.1430 out_g_norm=0.0190 acc_all=0.7783 acc_corrupt=0.6010 corrupt_frac=0.5510 loss_all=1.4238 loss_corrupt=2.5334 acc_corrupt_t_0p0_0p2=0.1998 corrupt_frac_t_0p0_0p2=0.2146 acc_corrupt_t_0p2_0p4=0.4345 corrupt_frac_t_0p2_0p4=0.1489 acc_corrupt_t_0p4_0p6=0.6114 corrupt_frac_t_0p4_0p6=0.2924 acc_corrupt_t_0p6_0p8=0.8280 corrupt_frac_t_0p6_0p8=0.0689 acc_corrupt_t_0p8_1p0=0.9360 corrupt_frac_t_0p8_1p0=0.2753 wrong_frac=0.4800 init_acc_corrupt=0.4895 init_gold_top10=0.5151 init_gold_top100=0.5214 +step=2250 micro_steps=18000 elapsed=127.3s lr=2.701200e-04 loss=2.4855 loss_recon=2.4855 loss_meanflow=0.0000 mean_model_t=0.5112 mean_corrupt_t=0.5112 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.1690 out_g_norm=0.0175 acc_all=0.8243 acc_corrupt=0.6237 corrupt_frac=0.4653 loss_all=1.0825 loss_corrupt=2.2848 acc_corrupt_t_0p0_0p2=0.2535 corrupt_frac_t_0p0_0p2=0.2312 acc_corrupt_t_0p2_0p4=0.5107 corrupt_frac_t_0p2_0p4=0.2517 acc_corrupt_t_0p4_0p6=0.6739 corrupt_frac_t_0p4_0p6=0.1267 acc_corrupt_t_0p6_0p8=0.8069 corrupt_frac_t_0p6_0p8=0.1284 acc_corrupt_t_0p8_1p0=0.9449 corrupt_frac_t_0p8_1p0=0.2619 wrong_frac=0.4915 init_acc_corrupt=0.4643 init_gold_top10=0.5029 init_gold_top100=0.5097 +step=2300 micro_steps=18400 elapsed=127.3s lr=2.761200e-04 loss=2.6499 loss_recon=2.6499 loss_meanflow=0.0000 mean_model_t=0.4984 mean_corrupt_t=0.4984 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.1925 out_g_norm=0.0173 acc_all=0.7237 acc_corrupt=0.5649 corrupt_frac=0.6260 loss_all=1.7937 loss_corrupt=2.7912 acc_corrupt_t_0p0_0p2=0.1972 corrupt_frac_t_0p0_0p2=0.2481 acc_corrupt_t_0p2_0p4=0.4177 corrupt_frac_t_0p2_0p4=0.3312 acc_corrupt_t_0p4_0p6=0.6667 corrupt_frac_t_0p4_0p6=0.0331 acc_corrupt_t_0p6_0p8=0.7989 corrupt_frac_t_0p6_0p8=0.0892 acc_corrupt_t_0p8_1p0=0.9526 corrupt_frac_t_0p8_1p0=0.2984 wrong_frac=0.5141 init_acc_corrupt=0.4426 init_gold_top10=0.4771 init_gold_top100=0.4852 +step=2350 micro_steps=18800 elapsed=137.3s lr=2.821200e-04 loss=2.6045 loss_recon=2.6045 loss_meanflow=0.0000 mean_model_t=0.5036 mean_corrupt_t=0.5036 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.2200 out_g_norm=0.0197 acc_all=0.8186 acc_corrupt=0.6806 corrupt_frac=0.5667 loss_all=1.1970 loss_corrupt=2.0978 acc_corrupt_t_0p0_0p2=0.1663 corrupt_frac_t_0p0_0p2=0.1081 acc_corrupt_t_0p2_0p4=0.4853 corrupt_frac_t_0p2_0p4=0.1323 acc_corrupt_t_0p4_0p6=0.6358 corrupt_frac_t_0p4_0p6=0.2824 acc_corrupt_t_0p6_0p8=0.8219 corrupt_frac_t_0p6_0p8=0.1917 acc_corrupt_t_0p8_1p0=0.9151 corrupt_frac_t_0p8_1p0=0.2855 wrong_frac=0.4029 init_acc_corrupt=0.5750 init_gold_top10=0.5958 init_gold_top100=0.5980 +step=2400 micro_steps=19200 elapsed=191.9s lr=2.881200e-04 loss=2.5857 loss_recon=2.5857 loss_meanflow=0.0000 mean_model_t=0.5045 mean_corrupt_t=0.5045 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.2521 out_g_norm=0.0172 acc_all=0.8042 acc_corrupt=0.6555 corrupt_frac=0.5606 loss_all=1.3039 loss_corrupt=2.2668 acc_corrupt_t_0p0_0p2=0.2120 corrupt_frac_t_0p0_0p2=0.2157 acc_corrupt_t_0p2_0p4=0.5235 corrupt_frac_t_0p2_0p4=0.1042 acc_corrupt_t_0p4_0p6=0.5674 corrupt_frac_t_0p4_0p6=0.1437 acc_corrupt_t_0p6_0p8=0.8107 corrupt_frac_t_0p6_0p8=0.2703 acc_corrupt_t_0p8_1p0=0.9566 corrupt_frac_t_0p8_1p0=0.2661 wrong_frac=0.4133 init_acc_corrupt=0.5522 init_gold_top10=0.5816 init_gold_top100=0.5875 +step=2450 micro_steps=19600 elapsed=166.9s lr=2.941200e-04 loss=2.6024 loss_recon=2.6024 loss_meanflow=0.0000 mean_model_t=0.4977 mean_corrupt_t=0.4977 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.2734 out_g_norm=0.0186 acc_all=0.8117 acc_corrupt=0.5566 corrupt_frac=0.4149 loss_all=1.2019 loss_corrupt=2.7634 acc_corrupt_t_0p0_0p2=0.2468 corrupt_frac_t_0p0_0p2=0.2319 acc_corrupt_t_0p2_0p4=0.4267 corrupt_frac_t_0p2_0p4=0.1896 acc_corrupt_t_0p4_0p6=0.6373 corrupt_frac_t_0p4_0p6=0.2556 acc_corrupt_t_0p6_0p8=0.7369 corrupt_frac_t_0p6_0p8=0.2500 acc_corrupt_t_0p8_1p0=0.9778 corrupt_frac_t_0p8_1p0=0.0730 wrong_frac=0.5642 init_acc_corrupt=0.3936 init_gold_top10=0.4255 init_gold_top100=0.4355 +step=2500 micro_steps=20000 elapsed=188.7s lr=3.000000e-04 loss=2.5994 loss_recon=2.5994 loss_meanflow=0.0000 mean_model_t=0.4970 mean_corrupt_t=0.4970 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.3060 out_g_norm=0.0186 acc_all=0.7613 acc_corrupt=0.5852 corrupt_frac=0.5687 loss_all=1.5017 loss_corrupt=2.5873 acc_corrupt_t_0p0_0p2=0.1967 corrupt_frac_t_0p0_0p2=0.2565 acc_corrupt_t_0p2_0p4=0.5517 corrupt_frac_t_0p2_0p4=0.0924 acc_corrupt_t_0p4_0p6=0.6318 corrupt_frac_t_0p4_0p6=0.2582 acc_corrupt_t_0p6_0p8=0.8096 corrupt_frac_t_0p6_0p8=0.3782 acc_corrupt_t_0p8_1p0=0.9853 corrupt_frac_t_0p8_1p0=0.0146 wrong_frac=0.5193 init_acc_corrupt=0.4512 init_gold_top10=0.4723 init_gold_top100=0.4807 +step=2550 micro_steps=20400 elapsed=132.6s lr=3.000000e-04 loss=2.5504 loss_recon=2.5504 loss_meanflow=0.0000 mean_model_t=0.4926 mean_corrupt_t=0.4926 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.3398 out_g_norm=0.0175 acc_all=0.8160 acc_corrupt=0.6625 corrupt_frac=0.5410 loss_all=1.1167 loss_corrupt=2.0379 acc_corrupt_t_0p0_0p2=0.0795 corrupt_frac_t_0p0_0p2=0.1022 acc_corrupt_t_0p2_0p4=0.5069 corrupt_frac_t_0p2_0p4=0.0245 acc_corrupt_t_0p4_0p6=0.5825 corrupt_frac_t_0p4_0p6=0.3515 acc_corrupt_t_0p6_0p8=0.8030 corrupt_frac_t_0p6_0p8=0.4049 acc_corrupt_t_0p8_1p0=0.9585 corrupt_frac_t_0p8_1p0=0.1169 wrong_frac=0.4369 init_acc_corrupt=0.5573 init_gold_top10=0.5614 init_gold_top100=0.5633 +step=2600 micro_steps=20800 elapsed=135.9s lr=3.000000e-04 loss=2.6894 loss_recon=2.6894 loss_meanflow=0.0000 mean_model_t=0.4852 mean_corrupt_t=0.4852 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.3696 out_g_norm=0.0182 acc_all=0.7764 acc_corrupt=0.5881 corrupt_frac=0.5316 loss_all=1.4135 loss_corrupt=2.5560 acc_corrupt_t_0p0_0p2=0.2508 corrupt_frac_t_0p0_0p2=0.2665 acc_corrupt_t_0p2_0p4=0.4863 corrupt_frac_t_0p2_0p4=0.1759 acc_corrupt_t_0p4_0p6=0.6473 corrupt_frac_t_0p4_0p6=0.2393 acc_corrupt_t_0p6_0p8=0.8050 corrupt_frac_t_0p6_0p8=0.1643 acc_corrupt_t_0p8_1p0=0.9650 corrupt_frac_t_0p8_1p0=0.1540 wrong_frac=0.5369 init_acc_corrupt=0.4250 init_gold_top10=0.4578 init_gold_top100=0.4631 +step=2650 micro_steps=21200 elapsed=128.1s lr=3.000000e-04 loss=2.4579 loss_recon=2.4579 loss_meanflow=0.0000 mean_model_t=0.5082 mean_corrupt_t=0.5082 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.3994 out_g_norm=0.0159 acc_all=0.7357 acc_corrupt=0.5233 corrupt_frac=0.5531 loss_all=1.7401 loss_corrupt=3.1175 acc_corrupt_t_0p0_0p2=0.1625 corrupt_frac_t_0p0_0p2=0.2499 acc_corrupt_t_0p2_0p4=0.4180 corrupt_frac_t_0p2_0p4=0.1769 acc_corrupt_t_0p4_0p6=0.6472 corrupt_frac_t_0p4_0p6=0.3006 acc_corrupt_t_0p6_0p8=0.7649 corrupt_frac_t_0p6_0p8=0.2347 acc_corrupt_t_0p8_1p0=0.9155 corrupt_frac_t_0p8_1p0=0.0379 wrong_frac=0.5721 init_acc_corrupt=0.3760 init_gold_top10=0.4209 init_gold_top100=0.4289 +step=2700 micro_steps=21600 elapsed=128.0s lr=3.000000e-04 loss=2.5789 loss_recon=2.5789 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.4361 out_g_norm=0.0157 acc_all=0.7776 acc_corrupt=0.6148 corrupt_frac=0.5754 loss_all=1.4402 loss_corrupt=2.4666 acc_corrupt_t_0p0_0p2=0.1722 corrupt_frac_t_0p0_0p2=0.1971 acc_corrupt_t_0p2_0p4=0.5234 corrupt_frac_t_0p2_0p4=0.1019 acc_corrupt_t_0p4_0p6=0.6153 corrupt_frac_t_0p4_0p6=0.3220 acc_corrupt_t_0p6_0p8=0.8310 corrupt_frac_t_0p6_0p8=0.2285 acc_corrupt_t_0p8_1p0=0.9267 corrupt_frac_t_0p8_1p0=0.1505 wrong_frac=0.4891 init_acc_corrupt=0.4898 init_gold_top10=0.5039 init_gold_top100=0.5117 +step=2750 micro_steps=22000 elapsed=175.8s lr=3.000000e-04 loss=2.5256 loss_recon=2.5256 loss_meanflow=0.0000 mean_model_t=0.4917 mean_corrupt_t=0.4917 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.4698 out_g_norm=0.0160 acc_all=0.7535 acc_corrupt=0.5414 corrupt_frac=0.5229 loss_all=1.5800 loss_corrupt=2.9021 acc_corrupt_t_0p0_0p2=0.1629 corrupt_frac_t_0p0_0p2=0.2930 acc_corrupt_t_0p2_0p4=0.4371 corrupt_frac_t_0p2_0p4=0.1335 acc_corrupt_t_0p4_0p6=0.6552 corrupt_frac_t_0p4_0p6=0.2658 acc_corrupt_t_0p6_0p8=0.8143 corrupt_frac_t_0p6_0p8=0.2432 acc_corrupt_t_0p8_1p0=0.9783 corrupt_frac_t_0p8_1p0=0.0645 wrong_frac=0.5627 init_acc_corrupt=0.4105 init_gold_top10=0.4299 init_gold_top100=0.4364 +step=2800 micro_steps=22400 elapsed=127.5s lr=3.000000e-04 loss=2.5223 loss_recon=2.5223 loss_meanflow=0.0000 mean_model_t=0.4977 mean_corrupt_t=0.4977 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.4937 out_g_norm=0.0166 acc_all=0.7708 acc_corrupt=0.6060 corrupt_frac=0.5757 loss_all=1.4794 loss_corrupt=2.5187 acc_corrupt_t_0p0_0p2=0.1794 corrupt_frac_t_0p0_0p2=0.2759 acc_corrupt_t_0p2_0p4=0.4118 corrupt_frac_t_0p2_0p4=0.0896 acc_corrupt_t_0p4_0p6=0.6746 corrupt_frac_t_0p4_0p6=0.2362 acc_corrupt_t_0p6_0p8=0.8468 corrupt_frac_t_0p6_0p8=0.1460 acc_corrupt_t_0p8_1p0=0.9378 corrupt_frac_t_0p8_1p0=0.2523 wrong_frac=0.4759 init_acc_corrupt=0.4946 init_gold_top10=0.5192 init_gold_top100=0.5246 +step=2850 micro_steps=22800 elapsed=127.5s lr=3.000000e-04 loss=2.4808 loss_recon=2.4808 loss_meanflow=0.0000 mean_model_t=0.5025 mean_corrupt_t=0.5025 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.5213 out_g_norm=0.0167 acc_all=0.7151 acc_corrupt=0.5529 corrupt_frac=0.6359 loss_all=1.8672 loss_corrupt=2.9033 acc_corrupt_t_0p0_0p2=0.2304 corrupt_frac_t_0p0_0p2=0.1346 acc_corrupt_t_0p2_0p4=0.3629 corrupt_frac_t_0p2_0p4=0.2748 acc_corrupt_t_0p4_0p6=0.5599 corrupt_frac_t_0p4_0p6=0.2340 acc_corrupt_t_0p6_0p8=0.7679 corrupt_frac_t_0p6_0p8=0.1985 acc_corrupt_t_0p8_1p0=0.8774 corrupt_frac_t_0p8_1p0=0.1581 wrong_frac=0.5388 init_acc_corrupt=0.4100 init_gold_top10=0.4572 init_gold_top100=0.4627 +step=2900 micro_steps=23200 elapsed=175.2s lr=3.000000e-04 loss=2.4895 loss_recon=2.4895 loss_meanflow=0.0000 mean_model_t=0.4913 mean_corrupt_t=0.4913 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.5564 out_g_norm=0.0155 acc_all=0.8445 acc_corrupt=0.6641 corrupt_frac=0.4583 loss_all=0.9392 loss_corrupt=1.9931 acc_corrupt_t_0p0_0p2=0.3475 corrupt_frac_t_0p0_0p2=0.1541 acc_corrupt_t_0p2_0p4=0.5428 corrupt_frac_t_0p2_0p4=0.1401 acc_corrupt_t_0p4_0p6=0.6843 corrupt_frac_t_0p4_0p6=0.2016 acc_corrupt_t_0p6_0p8=0.7364 corrupt_frac_t_0p6_0p8=0.3512 acc_corrupt_t_0p8_1p0=0.9017 corrupt_frac_t_0p8_1p0=0.1530 wrong_frac=0.4686 init_acc_corrupt=0.5106 init_gold_top10=0.5282 init_gold_top100=0.5327 +step=2950 micro_steps=23600 elapsed=128.0s lr=3.000000e-04 loss=2.4493 loss_recon=2.4493 loss_meanflow=0.0000 mean_model_t=0.5021 mean_corrupt_t=0.5021 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.5919 out_g_norm=0.0154 acc_all=0.8262 acc_corrupt=0.6643 corrupt_frac=0.5140 loss_all=1.1565 loss_corrupt=2.2020 acc_corrupt_t_0p0_0p2=0.3403 corrupt_frac_t_0p0_0p2=0.0621 acc_corrupt_t_0p2_0p4=0.3602 corrupt_frac_t_0p2_0p4=0.2921 acc_corrupt_t_0p4_0p6=0.6340 corrupt_frac_t_0p4_0p6=0.1891 acc_corrupt_t_0p6_0p8=0.8232 corrupt_frac_t_0p6_0p8=0.2096 acc_corrupt_t_0p8_1p0=0.9938 corrupt_frac_t_0p8_1p0=0.2471 wrong_frac=0.4202 init_acc_corrupt=0.5444 init_gold_top10=0.5761 init_gold_top100=0.5810 +step=3000 micro_steps=24000 elapsed=127.7s lr=3.000000e-04 loss=2.5211 loss_recon=2.5211 loss_meanflow=0.0000 mean_model_t=0.4971 mean_corrupt_t=0.4971 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.6226 out_g_norm=0.0149 acc_all=0.8489 acc_corrupt=0.7070 corrupt_frac=0.5139 loss_all=0.9723 loss_corrupt=1.8654 acc_corrupt_t_0p0_0p2=0.1772 corrupt_frac_t_0p0_0p2=0.1468 acc_corrupt_t_0p2_0p4=0.4983 corrupt_frac_t_0p2_0p4=0.0691 acc_corrupt_t_0p4_0p6=0.6519 corrupt_frac_t_0p4_0p6=0.2010 acc_corrupt_t_0p6_0p8=0.8450 corrupt_frac_t_0p6_0p8=0.3365 acc_corrupt_t_0p8_1p0=0.9374 corrupt_frac_t_0p8_1p0=0.2467 wrong_frac=0.3857 init_acc_corrupt=0.5893 init_gold_top10=0.6100 init_gold_top100=0.6150 +step=3050 micro_steps=24400 elapsed=130.7s lr=3.000000e-04 loss=2.5400 loss_recon=2.5400 loss_meanflow=0.0000 mean_model_t=0.5055 mean_corrupt_t=0.5055 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.6574 out_g_norm=0.0160 acc_all=0.8238 acc_corrupt=0.7101 corrupt_frac=0.6019 loss_all=1.0997 loss_corrupt=1.7888 acc_corrupt_t_0p0_0p2=0.3058 corrupt_frac_t_0p0_0p2=0.0245 acc_corrupt_t_0p2_0p4=0.3496 corrupt_frac_t_0p2_0p4=0.1679 acc_corrupt_t_0p4_0p6=0.6469 corrupt_frac_t_0p4_0p6=0.2430 acc_corrupt_t_0p6_0p8=0.8100 corrupt_frac_t_0p6_0p8=0.3075 acc_corrupt_t_0p8_1p0=0.9243 corrupt_frac_t_0p8_1p0=0.2571 wrong_frac=0.3796 init_acc_corrupt=0.6051 init_gold_top10=0.6190 init_gold_top100=0.6210 +step=3100 micro_steps=24800 elapsed=176.4s lr=3.000000e-04 loss=2.4574 loss_recon=2.4574 loss_meanflow=0.0000 mean_model_t=0.4958 mean_corrupt_t=0.4958 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.6912 out_g_norm=0.0153 acc_all=0.9191 acc_corrupt=0.8415 corrupt_frac=0.4999 loss_all=0.4614 loss_corrupt=0.8822 acc_corrupt_t_0p0_0p2=0.3605 corrupt_frac_t_0p0_0p2=0.0420 acc_corrupt_t_0p2_0p4=0.6080 corrupt_frac_t_0p2_0p4=0.0243 acc_corrupt_t_0p4_0p6=0.7467 corrupt_frac_t_0p4_0p6=0.0554 acc_corrupt_t_0p6_0p8=0.8406 corrupt_frac_t_0p6_0p8=0.5783 acc_corrupt_t_0p8_1p0=0.9471 corrupt_frac_t_0p8_1p0=0.3000 wrong_frac=0.2622 init_acc_corrupt=0.7337 init_gold_top10=0.7370 init_gold_top100=0.7375 +step=3150 micro_steps=25200 elapsed=127.5s lr=3.000000e-04 loss=2.4397 loss_recon=2.4397 loss_meanflow=0.0000 mean_model_t=0.5056 mean_corrupt_t=0.5056 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.7293 out_g_norm=0.0156 acc_all=0.8318 acc_corrupt=0.6930 corrupt_frac=0.5402 loss_all=1.0561 loss_corrupt=1.8968 acc_corrupt_t_0p0_0p2=0.2058 corrupt_frac_t_0p0_0p2=0.2778 acc_corrupt_t_0p4_0p6=0.6782 corrupt_frac_t_0p4_0p6=0.1471 acc_corrupt_t_0p6_0p8=0.8676 corrupt_frac_t_0p6_0p8=0.1442 acc_corrupt_t_0p8_1p0=0.9539 corrupt_frac_t_0p8_1p0=0.4309 wrong_frac=0.3904 init_acc_corrupt=0.5841 init_gold_top10=0.6021 init_gold_top100=0.6101 +step=3200 micro_steps=25600 elapsed=127.6s lr=3.000000e-04 loss=2.6145 loss_recon=2.6145 loss_meanflow=0.0000 mean_model_t=0.4877 mean_corrupt_t=0.4877 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.7679 out_g_norm=0.0149 acc_all=0.7285 acc_corrupt=0.5476 corrupt_frac=0.5986 loss_all=1.7034 loss_corrupt=2.8148 acc_corrupt_t_0p0_0p2=0.1863 corrupt_frac_t_0p0_0p2=0.1882 acc_corrupt_t_0p2_0p4=0.4024 corrupt_frac_t_0p2_0p4=0.3281 acc_corrupt_t_0p4_0p6=0.6057 corrupt_frac_t_0p4_0p6=0.0608 acc_corrupt_t_0p6_0p8=0.7999 corrupt_frac_t_0p6_0p8=0.3608 acc_corrupt_t_0p8_1p0=0.8867 corrupt_frac_t_0p8_1p0=0.0621 wrong_frac=0.5645 init_acc_corrupt=0.3863 init_gold_top10=0.4295 init_gold_top100=0.4357 +step=3250 micro_steps=26000 elapsed=175.3s lr=3.000000e-04 loss=2.5833 loss_recon=2.5833 loss_meanflow=0.0000 mean_model_t=0.4957 mean_corrupt_t=0.4957 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.8014 out_g_norm=0.0148 acc_all=0.7147 acc_corrupt=0.5662 corrupt_frac=0.6557 loss_all=1.8117 loss_corrupt=2.7388 acc_corrupt_t_0p0_0p2=0.1201 corrupt_frac_t_0p0_0p2=0.1147 acc_corrupt_t_0p2_0p4=0.3546 corrupt_frac_t_0p2_0p4=0.3685 acc_corrupt_t_0p4_0p6=0.6875 corrupt_frac_t_0p4_0p6=0.1323 acc_corrupt_t_0p6_0p8=0.7612 corrupt_frac_t_0p6_0p8=0.1719 acc_corrupt_t_0p8_1p0=0.9405 corrupt_frac_t_0p8_1p0=0.2126 wrong_frac=0.5173 init_acc_corrupt=0.4363 init_gold_top10=0.4753 init_gold_top100=0.4837 +step=3300 micro_steps=26400 elapsed=127.6s lr=3.000000e-04 loss=2.4928 loss_recon=2.4928 loss_meanflow=0.0000 mean_model_t=0.5074 mean_corrupt_t=0.5074 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.8278 out_g_norm=0.0144 acc_all=0.6918 acc_corrupt=0.5090 corrupt_frac=0.6082 loss_all=1.8383 loss_corrupt=2.8961 acc_corrupt_t_0p0_0p2=0.1773 corrupt_frac_t_0p0_0p2=0.4835 acc_corrupt_t_0p2_0p4=0.5258 corrupt_frac_t_0p2_0p4=0.0584 acc_corrupt_t_0p4_0p6=0.6334 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.7753 corrupt_frac_t_0p6_0p8=0.1537 acc_corrupt_t_0p8_1p0=0.9826 corrupt_frac_t_0p8_1p0=0.2310 wrong_frac=0.5950 init_acc_corrupt=0.3712 init_gold_top10=0.3926 init_gold_top100=0.4018 +step=3350 micro_steps=26800 elapsed=127.7s lr=3.000000e-04 loss=2.4775 loss_recon=2.4775 loss_meanflow=0.0000 mean_model_t=0.5006 mean_corrupt_t=0.5006 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.8604 out_g_norm=0.0155 acc_all=0.8364 acc_corrupt=0.6639 corrupt_frac=0.4865 loss_all=1.0054 loss_corrupt=2.0573 acc_corrupt_t_0p2_0p4=0.3736 corrupt_frac_t_0p2_0p4=0.1747 acc_corrupt_t_0p4_0p6=0.6234 corrupt_frac_t_0p4_0p6=0.5304 acc_corrupt_t_0p6_0p8=0.8596 corrupt_frac_t_0p6_0p8=0.1483 acc_corrupt_t_0p8_1p0=0.9581 corrupt_frac_t_0p8_1p0=0.1467 wrong_frac=0.4442 init_acc_corrupt=0.5365 init_gold_top10=0.5548 init_gold_top100=0.5575 +step=3400 micro_steps=27200 elapsed=127.8s lr=3.000000e-04 loss=2.5129 loss_recon=2.5129 loss_meanflow=0.0000 mean_model_t=0.4976 mean_corrupt_t=0.4976 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.8924 out_g_norm=0.0153 acc_all=0.8296 acc_corrupt=0.6626 corrupt_frac=0.4969 loss_all=1.0641 loss_corrupt=2.0784 acc_corrupt_t_0p0_0p2=0.3536 corrupt_frac_t_0p0_0p2=0.0709 acc_corrupt_t_0p2_0p4=0.3471 corrupt_frac_t_0p2_0p4=0.2056 acc_corrupt_t_0p4_0p6=0.6397 corrupt_frac_t_0p4_0p6=0.1670 acc_corrupt_t_0p6_0p8=0.7876 corrupt_frac_t_0p6_0p8=0.4036 acc_corrupt_t_0p8_1p0=0.9253 corrupt_frac_t_0p8_1p0=0.1529 wrong_frac=0.4425 init_acc_corrupt=0.5321 init_gold_top10=0.5549 init_gold_top100=0.5588 +step=3450 micro_steps=27600 elapsed=127.3s lr=3.000000e-04 loss=2.6037 loss_recon=2.6037 loss_meanflow=0.0000 mean_model_t=0.4975 mean_corrupt_t=0.4975 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.9291 out_g_norm=0.0141 acc_all=0.7703 acc_corrupt=0.6035 corrupt_frac=0.5694 loss_all=1.4758 loss_corrupt=2.5194 acc_corrupt_t_0p0_0p2=0.1498 corrupt_frac_t_0p0_0p2=0.2934 acc_corrupt_t_0p2_0p4=0.4774 corrupt_frac_t_0p2_0p4=0.0593 acc_corrupt_t_0p4_0p6=0.6851 corrupt_frac_t_0p4_0p6=0.1202 acc_corrupt_t_0p6_0p8=0.8229 corrupt_frac_t_0p6_0p8=0.3225 acc_corrupt_t_0p8_1p0=0.8968 corrupt_frac_t_0p8_1p0=0.2046 wrong_frac=0.4948 init_acc_corrupt=0.4857 init_gold_top10=0.4977 init_gold_top100=0.5043 +step=3500 micro_steps=28000 elapsed=127.8s lr=3.000000e-04 loss=2.4503 loss_recon=2.4503 loss_meanflow=0.0000 mean_model_t=0.5024 mean_corrupt_t=0.5024 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1437.9701 out_g_norm=0.0170 acc_all=0.6820 acc_corrupt=0.4972 corrupt_frac=0.6281 loss_all=1.9257 loss_corrupt=3.0231 acc_corrupt_t_0p0_0p2=0.2649 corrupt_frac_t_0p0_0p2=0.3137 acc_corrupt_t_0p2_0p4=0.4566 corrupt_frac_t_0p2_0p4=0.1656 acc_corrupt_t_0p4_0p6=0.6000 corrupt_frac_t_0p4_0p6=0.4062 acc_corrupt_t_0p6_0p8=0.7780 corrupt_frac_t_0p6_0p8=0.0884 acc_corrupt_t_0p8_1p0=0.9963 corrupt_frac_t_0p8_1p0=0.0260 wrong_frac=0.6489 init_acc_corrupt=0.3135 init_gold_top10=0.3424 init_gold_top100=0.3500 +step=3550 micro_steps=28400 elapsed=129.1s lr=3.000000e-04 loss=2.3950 loss_recon=2.3950 loss_meanflow=0.0000 mean_model_t=0.5046 mean_corrupt_t=0.5046 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1438.0044 out_g_norm=0.0150 acc_all=0.7578 acc_corrupt=0.6122 corrupt_frac=0.6200 loss_all=1.5253 loss_corrupt=2.4125 acc_corrupt_t_0p0_0p2=0.2389 corrupt_frac_t_0p0_0p2=0.2456 acc_corrupt_t_0p2_0p4=0.4583 corrupt_frac_t_0p2_0p4=0.0921 acc_corrupt_t_0p4_0p6=0.6300 corrupt_frac_t_0p4_0p6=0.2469 acc_corrupt_t_0p6_0p8=0.8142 corrupt_frac_t_0p6_0p8=0.2740 acc_corrupt_t_0p8_1p0=0.9387 corrupt_frac_t_0p8_1p0=0.1414 wrong_frac=0.4884 init_acc_corrupt=0.4836 init_gold_top10=0.5053 init_gold_top100=0.5121 +step=3600 micro_steps=28800 elapsed=191.8s lr=3.000000e-04 loss=2.4143 loss_recon=2.4143 loss_meanflow=0.0000 mean_model_t=0.5186 mean_corrupt_t=0.5186 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1438.0403 out_g_norm=0.0154 acc_all=0.6730 acc_corrupt=0.4876 corrupt_frac=0.6334 loss_all=2.0922 loss_corrupt=3.2643 acc_corrupt_t_0p0_0p2=0.1370 corrupt_frac_t_0p0_0p2=0.1934 acc_corrupt_t_0p2_0p4=0.3826 corrupt_frac_t_0p2_0p4=0.2418 acc_corrupt_t_0p4_0p6=0.6180 corrupt_frac_t_0p4_0p6=0.4735 acc_corrupt_t_0p6_0p8=0.8321 corrupt_frac_t_0p6_0p8=0.0913 wrong_frac=0.6238 init_acc_corrupt=0.3367 init_gold_top10=0.3711 init_gold_top100=0.3758 +step=3650 micro_steps=29200 elapsed=219.2s lr=3.000000e-04 loss=2.5185 loss_recon=2.5185 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1438.0772 out_g_norm=0.0149 acc_all=0.7538 acc_corrupt=0.5342 corrupt_frac=0.5274 loss_all=1.5719 loss_corrupt=2.9423 acc_corrupt_t_0p0_0p2=0.2060 corrupt_frac_t_0p0_0p2=0.3326 acc_corrupt_t_0p2_0p4=0.4605 corrupt_frac_t_0p2_0p4=0.1010 acc_corrupt_t_0p4_0p6=0.5873 corrupt_frac_t_0p4_0p6=0.2871 acc_corrupt_t_0p6_0p8=0.7724 corrupt_frac_t_0p6_0p8=0.0941 acc_corrupt_t_0p8_1p0=0.9606 corrupt_frac_t_0p8_1p0=0.1852 wrong_frac=0.5544 init_acc_corrupt=0.4079 init_gold_top10=0.4362 init_gold_top100=0.4462 +step=3700 micro_steps=29600 elapsed=179.9s lr=3.000000e-04 loss=2.4679 loss_recon=2.4679 loss_meanflow=0.0000 mean_model_t=0.5130 mean_corrupt_t=0.5130 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1438.1130 out_g_norm=0.0145 acc_all=0.8342 acc_corrupt=0.6832 corrupt_frac=0.5200 loss_all=1.0293 loss_corrupt=1.9436 acc_corrupt_t_0p0_0p2=0.1624 corrupt_frac_t_0p0_0p2=0.1684 acc_corrupt_t_0p4_0p6=0.6700 corrupt_frac_t_0p4_0p6=0.3038 acc_corrupt_t_0p6_0p8=0.7804 corrupt_frac_t_0p6_0p8=0.2646 acc_corrupt_t_0p8_1p0=0.9340 corrupt_frac_t_0p8_1p0=0.2632 wrong_frac=0.4094 init_acc_corrupt=0.5766 init_gold_top10=0.5862 init_gold_top100=0.5919 +step=3750 micro_steps=30000 elapsed=128.6s lr=3.000000e-04 loss=2.4525 loss_recon=2.4525 loss_meanflow=0.0000 mean_model_t=0.5041 mean_corrupt_t=0.5041 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1438.1452 out_g_norm=0.0137 acc_all=0.7866 acc_corrupt=0.5120 corrupt_frac=0.4318 loss_all=1.3797 loss_corrupt=3.1090 acc_corrupt_t_0p0_0p2=0.2392 corrupt_frac_t_0p0_0p2=0.2364 acc_corrupt_t_0p2_0p4=0.3762 corrupt_frac_t_0p2_0p4=0.3773 acc_corrupt_t_0p4_0p6=0.6735 corrupt_frac_t_0p4_0p6=0.0900 acc_corrupt_t_0p6_0p8=0.8067 corrupt_frac_t_0p6_0p8=0.1814 acc_corrupt_t_0p8_1p0=0.9274 corrupt_frac_t_0p8_1p0=0.1149 wrong_frac=0.6179 init_acc_corrupt=0.3264 init_gold_top10=0.3739 init_gold_top100=0.3822 +step=3800 micro_steps=30400 elapsed=128.3s lr=3.000000e-04 loss=2.4515 loss_recon=2.4515 loss_meanflow=0.0000 mean_model_t=0.5096 mean_corrupt_t=0.5096 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1438.1779 out_g_norm=0.0138 acc_all=0.8350 acc_corrupt=0.6069 corrupt_frac=0.4124 loss_all=0.9658 loss_corrupt=2.2753 acc_corrupt_t_0p0_0p2=0.4688 corrupt_frac_t_0p0_0p2=0.0142 acc_corrupt_t_0p2_0p4=0.4925 corrupt_frac_t_0p2_0p4=0.5935 acc_corrupt_t_0p4_0p6=0.6746 corrupt_frac_t_0p4_0p6=0.1178 acc_corrupt_t_0p6_0p8=0.8102 corrupt_frac_t_0p6_0p8=0.2402 acc_corrupt_t_0p8_1p0=0.9870 corrupt_frac_t_0p8_1p0=0.0342 wrong_frac=0.5426 init_acc_corrupt=0.4093 init_gold_top10=0.4550 init_gold_top100=0.4596 +step=3850 micro_steps=30800 elapsed=137.0s lr=3.000000e-04 loss=2.5001 loss_recon=2.5001 loss_meanflow=0.0000 mean_model_t=0.4848 mean_corrupt_t=0.4848 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1438.2111 out_g_norm=0.0139 acc_all=0.8232 acc_corrupt=0.6890 corrupt_frac=0.5645 loss_all=1.0883 loss_corrupt=1.8999 acc_corrupt_t_0p0_0p2=0.2523 corrupt_frac_t_0p0_0p2=0.0591 acc_corrupt_t_0p2_0p4=0.3512 corrupt_frac_t_0p2_0p4=0.1776 acc_corrupt_t_0p4_0p6=0.6347 corrupt_frac_t_0p4_0p6=0.2516 acc_corrupt_t_0p6_0p8=0.8263 corrupt_frac_t_0p6_0p8=0.2801 acc_corrupt_t_0p8_1p0=0.9528 corrupt_frac_t_0p8_1p0=0.2315 wrong_frac=0.4125 init_acc_corrupt=0.5681 init_gold_top10=0.5856 init_gold_top100=0.5876 +step=3900 micro_steps=31200 elapsed=275.9s lr=3.000000e-04 loss=2.5571 loss_recon=2.5571 loss_meanflow=0.0000 mean_model_t=0.4961 mean_corrupt_t=0.4961 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 out_w_norm=1438.2457 out_g_norm=0.0176 acc_all=0.8155 acc_corrupt=0.5732 corrupt_frac=0.4196 loss_all=1.1297 loss_corrupt=2.5263 acc_corrupt_t_0p0_0p2=0.2637 corrupt_frac_t_0p0_0p2=0.4127 acc_corrupt_t_0p2_0p4=0.5651 corrupt_frac_t_0p2_0p4=0.1341 acc_corrupt_t_0p4_0p6=0.6612 corrupt_frac_t_0p4_0p6=0.0447 acc_corrupt_t_0p6_0p8=0.8271 corrupt_frac_t_0p6_0p8=0.2600 acc_corrupt_t_0p8_1p0=0.9696 corrupt_frac_t_0p8_1p0=0.1485 wrong_frac=0.5870 init_acc_corrupt=0.3843 init_gold_top10=0.4018 init_gold_top100=0.4098 +W0514 15:05:53.653000 1231273 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0514 15:05:53.654000 1231273 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 1231277 closing signal SIGTERM +W0514 15:05:53.659000 1231273 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 1231278 closing signal SIGTERM +W0514 15:05:53.660000 1231273 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 1231279 closing signal SIGTERM +W0514 15:05:53.661000 1231273 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 1231280 closing signal SIGTERM +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in + main() + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1231273 got signal: 15 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/lib/libnpyrandom.a b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/lib/libnpyrandom.a new file mode 100644 index 0000000000000000000000000000000000000000..96bd444f59250fb0219138f3f5729972b78e2220 Binary files /dev/null and b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/random/lib/libnpyrandom.a differ diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/tests/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/tests/test_numpy_version.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/tests/test_numpy_version.py new file mode 100644 index 0000000000000000000000000000000000000000..61643426c8d757c8367dc7e8d19f6d4c106314a3 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/tests/test_numpy_version.py @@ -0,0 +1,41 @@ +""" +Check the numpy version is valid. + +Note that a development version is marked by the presence of 'dev0' or '+' +in the version string, all else is treated as a release. The version string +itself is set from the output of ``git describe`` which relies on tags. + +Examples +-------- + +Valid Development: 1.22.0.dev0 1.22.0.dev0+5-g7999db4df2 1.22.0+5-g7999db4df2 +Valid Release: 1.21.0.rc1, 1.21.0.b1, 1.21.0 +Invalid: 1.22.0.dev, 1.22.0.dev0-5-g7999db4dfB, 1.21.0.d1, 1.21.a + +Note that a release is determined by the version string, which in turn +is controlled by the result of the ``git describe`` command. +""" +import re + +import numpy as np +from numpy.testing import assert_ + + +def test_valid_numpy_version(): + # Verify that the numpy version is a valid one (no .post suffix or other + # nonsense). See gh-6431 for an issue caused by an invalid version. + version_pattern = r"^[0-9]+\.[0-9]+\.[0-9]+(a[0-9]|b[0-9]|rc[0-9])?" + dev_suffix = r"(\.dev[0-9]+(\+git[0-9]+\.[0-9a-f]+)?)?" + res = re.match(version_pattern + dev_suffix + '$', np.__version__) + + assert_(res is not None, np.__version__) + + +def test_short_version(): + # Check numpy.short_version actually exists + if np.version.release: + assert_(np.__version__ == np.version.short_version, + "short_version mismatch in release version") + else: + assert_(np.__version__.split("+")[0] == np.version.short_version, + "short_version mismatch in development version") diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/tests/test_public_api.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/tests/test_public_api.py new file mode 100644 index 0000000000000000000000000000000000000000..54bf3dacf9722004d51cb13d8b5dd7c1105a655a --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/tests/test_public_api.py @@ -0,0 +1,551 @@ +import sys +import sysconfig +import subprocess +import pkgutil +import types +import importlib +import warnings + +import numpy as np +import numpy +import pytest +from numpy.testing import IS_WASM + +try: + import ctypes +except ImportError: + ctypes = None + + +def check_dir(module, module_name=None): + """Returns a mapping of all objects with the wrong __module__ attribute.""" + if module_name is None: + module_name = module.__name__ + results = {} + for name in dir(module): + item = getattr(module, name) + if (hasattr(item, '__module__') and hasattr(item, '__name__') + and item.__module__ != module_name): + results[name] = item.__module__ + '.' + item.__name__ + return results + + +def test_numpy_namespace(): + # None of these objects are publicly documented to be part of the main + # NumPy namespace (some are useful though, others need to be cleaned up) + undocumented = { + '_add_newdoc_ufunc': 'numpy.core._multiarray_umath._add_newdoc_ufunc', + 'add_docstring': 'numpy.core._multiarray_umath.add_docstring', + 'add_newdoc': 'numpy.core.function_base.add_newdoc', + 'add_newdoc_ufunc': 'numpy.core._multiarray_umath._add_newdoc_ufunc', + 'byte_bounds': 'numpy.lib.utils.byte_bounds', + 'compare_chararrays': 'numpy.core._multiarray_umath.compare_chararrays', + 'deprecate': 'numpy.lib.utils.deprecate', + 'deprecate_with_doc': 'numpy.lib.utils.deprecate_with_doc', + 'disp': 'numpy.lib.function_base.disp', + 'fastCopyAndTranspose': 'numpy.core._multiarray_umath.fastCopyAndTranspose', + 'get_array_wrap': 'numpy.lib.shape_base.get_array_wrap', + 'get_include': 'numpy.lib.utils.get_include', + 'recfromcsv': 'numpy.lib.npyio.recfromcsv', + 'recfromtxt': 'numpy.lib.npyio.recfromtxt', + 'safe_eval': 'numpy.lib.utils.safe_eval', + 'set_string_function': 'numpy.core.arrayprint.set_string_function', + 'show_config': 'numpy.__config__.show', + 'show_runtime': 'numpy.lib.utils.show_runtime', + 'who': 'numpy.lib.utils.who', + } + # We override dir to not show these members + allowlist = undocumented + bad_results = check_dir(np) + # pytest gives better error messages with the builtin assert than with + # assert_equal + assert bad_results == allowlist + + +@pytest.mark.skipif(IS_WASM, reason="can't start subprocess") +@pytest.mark.parametrize('name', ['testing']) +def test_import_lazy_import(name): + """Make sure we can actually use the modules we lazy load. + + While not exported as part of the public API, it was accessible. With the + use of __getattr__ and __dir__, this isn't always true It can happen that + an infinite recursion may happen. + + This is the only way I found that would force the failure to appear on the + badly implemented code. + + We also test for the presence of the lazily imported modules in dir + + """ + exe = (sys.executable, '-c', "import numpy; numpy." + name) + result = subprocess.check_output(exe) + assert not result + + # Make sure they are still in the __dir__ + assert name in dir(np) + + +def test_dir_testing(): + """Assert that output of dir has only one "testing/tester" + attribute without duplicate""" + assert len(dir(np)) == len(set(dir(np))) + + +def test_numpy_linalg(): + bad_results = check_dir(np.linalg) + assert bad_results == {} + + +def test_numpy_fft(): + bad_results = check_dir(np.fft) + assert bad_results == {} + + +@pytest.mark.skipif(ctypes is None, + reason="ctypes not available in this python") +def test_NPY_NO_EXPORT(): + cdll = ctypes.CDLL(np.core._multiarray_tests.__file__) + # Make sure an arbitrary NPY_NO_EXPORT function is actually hidden + f = getattr(cdll, 'test_not_exported', None) + assert f is None, ("'test_not_exported' is mistakenly exported, " + "NPY_NO_EXPORT does not work") + + +# Historically NumPy has not used leading underscores for private submodules +# much. This has resulted in lots of things that look like public modules +# (i.e. things that can be imported as `import numpy.somesubmodule.somefile`), +# but were never intended to be public. The PUBLIC_MODULES list contains +# modules that are either public because they were meant to be, or because they +# contain public functions/objects that aren't present in any other namespace +# for whatever reason and therefore should be treated as public. +# +# The PRIVATE_BUT_PRESENT_MODULES list contains modules that look public (lack +# of underscores) but should not be used. For many of those modules the +# current status is fine. For others it may make sense to work on making them +# private, to clean up our public API and avoid confusion. +PUBLIC_MODULES = ['numpy.' + s for s in [ + "array_api", + "array_api.linalg", + "ctypeslib", + "doc", + "doc.constants", + "doc.ufuncs", + "dtypes", + "exceptions", + "f2py", + "fft", + "lib", + "lib.format", # was this meant to be public? + "lib.mixins", + "lib.recfunctions", + "lib.scimath", + "lib.stride_tricks", + "linalg", + "ma", + "ma.extras", + "ma.mrecords", + "matlib", + "polynomial", + "polynomial.chebyshev", + "polynomial.hermite", + "polynomial.hermite_e", + "polynomial.laguerre", + "polynomial.legendre", + "polynomial.polynomial", + "random", + "testing", + "testing.overrides", + "typing", + "typing.mypy_plugin", + "version" # Should be removed for NumPy 2.0 +]] +if sys.version_info < (3, 12): + PUBLIC_MODULES += [ + 'numpy.' + s for s in [ + "distutils", + "distutils.cpuinfo", + "distutils.exec_command", + "distutils.misc_util", + "distutils.log", + "distutils.system_info", + ] + ] + + + +PUBLIC_ALIASED_MODULES = [ + "numpy.char", + "numpy.emath", + "numpy.rec", +] + + +PRIVATE_BUT_PRESENT_MODULES = ['numpy.' + s for s in [ + "compat", + "compat.py3k", + "conftest", + "core", + "core.arrayprint", + "core.defchararray", + "core.einsumfunc", + "core.fromnumeric", + "core.function_base", + "core.getlimits", + "core.memmap", + "core.multiarray", + "core.numeric", + "core.numerictypes", + "core.overrides", + "core.records", + "core.shape_base", + "core.umath", + "f2py.auxfuncs", + "f2py.capi_maps", + "f2py.cb_rules", + "f2py.cfuncs", + "f2py.common_rules", + "f2py.crackfortran", + "f2py.diagnose", + "f2py.f2py2e", + "f2py.f90mod_rules", + "f2py.func2subr", + "f2py.rules", + "f2py.symbolic", + "f2py.use_rules", + "fft.helper", + "lib.arraypad", + "lib.arraysetops", + "lib.arrayterator", + "lib.function_base", + "lib.histograms", + "lib.index_tricks", + "lib.nanfunctions", + "lib.npyio", + "lib.polynomial", + "lib.shape_base", + "lib.twodim_base", + "lib.type_check", + "lib.ufunclike", + "lib.user_array", # note: not in np.lib, but probably should just be deleted + "lib.utils", + "linalg.lapack_lite", + "linalg.linalg", + "ma.core", + "ma.testutils", + "ma.timer_comparison", + "matrixlib", + "matrixlib.defmatrix", + "polynomial.polyutils", + "random.mtrand", + "random.bit_generator", + "testing.print_coercion_tables", +]] +if sys.version_info < (3, 12): + PRIVATE_BUT_PRESENT_MODULES += [ + 'numpy.' + s for s in [ + "distutils.armccompiler", + "distutils.fujitsuccompiler", + "distutils.ccompiler", + 'distutils.ccompiler_opt', + "distutils.command", + "distutils.command.autodist", + "distutils.command.bdist_rpm", + "distutils.command.build", + "distutils.command.build_clib", + "distutils.command.build_ext", + "distutils.command.build_py", + "distutils.command.build_scripts", + "distutils.command.build_src", + "distutils.command.config", + "distutils.command.config_compiler", + "distutils.command.develop", + "distutils.command.egg_info", + "distutils.command.install", + "distutils.command.install_clib", + "distutils.command.install_data", + "distutils.command.install_headers", + "distutils.command.sdist", + "distutils.conv_template", + "distutils.core", + "distutils.extension", + "distutils.fcompiler", + "distutils.fcompiler.absoft", + "distutils.fcompiler.arm", + "distutils.fcompiler.compaq", + "distutils.fcompiler.environment", + "distutils.fcompiler.g95", + "distutils.fcompiler.gnu", + "distutils.fcompiler.hpux", + "distutils.fcompiler.ibm", + "distutils.fcompiler.intel", + "distutils.fcompiler.lahey", + "distutils.fcompiler.mips", + "distutils.fcompiler.nag", + "distutils.fcompiler.none", + "distutils.fcompiler.pathf95", + "distutils.fcompiler.pg", + "distutils.fcompiler.nv", + "distutils.fcompiler.sun", + "distutils.fcompiler.vast", + "distutils.fcompiler.fujitsu", + "distutils.from_template", + "distutils.intelccompiler", + "distutils.lib2def", + "distutils.line_endings", + "distutils.mingw32ccompiler", + "distutils.msvccompiler", + "distutils.npy_pkg_config", + "distutils.numpy_distribution", + "distutils.pathccompiler", + "distutils.unixccompiler", + ] + ] + + +def is_unexpected(name): + """Check if this needs to be considered.""" + if '._' in name or '.tests' in name or '.setup' in name: + return False + + if name in PUBLIC_MODULES: + return False + + if name in PUBLIC_ALIASED_MODULES: + return False + + if name in PRIVATE_BUT_PRESENT_MODULES: + return False + + return True + + +# These are present in a directory with an __init__.py but cannot be imported +# code_generators/ isn't installed, but present for an inplace build +SKIP_LIST = [ + "numpy.core.code_generators", + "numpy.core.code_generators.genapi", + "numpy.core.code_generators.generate_umath", + "numpy.core.code_generators.ufunc_docstrings", + "numpy.core.code_generators.generate_numpy_api", + "numpy.core.code_generators.generate_ufunc_api", + "numpy.core.code_generators.numpy_api", + "numpy.core.code_generators.generate_umath_doc", + "numpy.core.code_generators.verify_c_api_version", + "numpy.core.cversions", + "numpy.core.generate_numpy_api", + "numpy.core.umath_tests", +] +if sys.version_info < (3, 12): + SKIP_LIST += ["numpy.distutils.msvc9compiler"] + + +# suppressing warnings from deprecated modules +@pytest.mark.filterwarnings("ignore:.*np.compat.*:DeprecationWarning") +def test_all_modules_are_expected(): + """ + Test that we don't add anything that looks like a new public module by + accident. Check is based on filenames. + """ + + modnames = [] + for _, modname, ispkg in pkgutil.walk_packages(path=np.__path__, + prefix=np.__name__ + '.', + onerror=None): + if is_unexpected(modname) and modname not in SKIP_LIST: + # We have a name that is new. If that's on purpose, add it to + # PUBLIC_MODULES. We don't expect to have to add anything to + # PRIVATE_BUT_PRESENT_MODULES. Use an underscore in the name! + modnames.append(modname) + + if modnames: + raise AssertionError(f'Found unexpected modules: {modnames}') + + +# Stuff that clearly shouldn't be in the API and is detected by the next test +# below +SKIP_LIST_2 = [ + 'numpy.math', + 'numpy.doc.constants.re', + 'numpy.doc.constants.textwrap', + 'numpy.lib.emath', + 'numpy.lib.math', + 'numpy.matlib.char', + 'numpy.matlib.rec', + 'numpy.matlib.emath', + 'numpy.matlib.exceptions', + 'numpy.matlib.math', + 'numpy.matlib.linalg', + 'numpy.matlib.fft', + 'numpy.matlib.random', + 'numpy.matlib.ctypeslib', + 'numpy.matlib.ma', +] +if sys.version_info < (3, 12): + SKIP_LIST_2 += [ + 'numpy.distutils.log.sys', + 'numpy.distutils.log.logging', + 'numpy.distutils.log.warnings', + ] + + +def test_all_modules_are_expected_2(): + """ + Method checking all objects. The pkgutil-based method in + `test_all_modules_are_expected` does not catch imports into a namespace, + only filenames. So this test is more thorough, and checks this like: + + import .lib.scimath as emath + + To check if something in a module is (effectively) public, one can check if + there's anything in that namespace that's a public function/object but is + not exposed in a higher-level namespace. For example for a `numpy.lib` + submodule:: + + mod = np.lib.mixins + for obj in mod.__all__: + if obj in np.__all__: + continue + elif obj in np.lib.__all__: + continue + + else: + print(obj) + + """ + + def find_unexpected_members(mod_name): + members = [] + module = importlib.import_module(mod_name) + if hasattr(module, '__all__'): + objnames = module.__all__ + else: + objnames = dir(module) + + for objname in objnames: + if not objname.startswith('_'): + fullobjname = mod_name + '.' + objname + if isinstance(getattr(module, objname), types.ModuleType): + if is_unexpected(fullobjname): + if fullobjname not in SKIP_LIST_2: + members.append(fullobjname) + + return members + + unexpected_members = find_unexpected_members("numpy") + for modname in PUBLIC_MODULES: + unexpected_members.extend(find_unexpected_members(modname)) + + if unexpected_members: + raise AssertionError("Found unexpected object(s) that look like " + "modules: {}".format(unexpected_members)) + + +def test_api_importable(): + """ + Check that all submodules listed higher up in this file can be imported + + Note that if a PRIVATE_BUT_PRESENT_MODULES entry goes missing, it may + simply need to be removed from the list (deprecation may or may not be + needed - apply common sense). + """ + def check_importable(module_name): + try: + importlib.import_module(module_name) + except (ImportError, AttributeError): + return False + + return True + + module_names = [] + for module_name in PUBLIC_MODULES: + if not check_importable(module_name): + module_names.append(module_name) + + if module_names: + raise AssertionError("Modules in the public API that cannot be " + "imported: {}".format(module_names)) + + for module_name in PUBLIC_ALIASED_MODULES: + try: + eval(module_name) + except AttributeError: + module_names.append(module_name) + + if module_names: + raise AssertionError("Modules in the public API that were not " + "found: {}".format(module_names)) + + with warnings.catch_warnings(record=True) as w: + warnings.filterwarnings('always', category=DeprecationWarning) + warnings.filterwarnings('always', category=ImportWarning) + for module_name in PRIVATE_BUT_PRESENT_MODULES: + if not check_importable(module_name): + module_names.append(module_name) + + if module_names: + raise AssertionError("Modules that are not really public but looked " + "public and can not be imported: " + "{}".format(module_names)) + + +@pytest.mark.xfail( + sysconfig.get_config_var("Py_DEBUG") not in (None, 0, "0"), + reason=( + "NumPy possibly built with `USE_DEBUG=True ./tools/travis-test.sh`, " + "which does not expose the `array_api` entry point. " + "See https://github.com/numpy/numpy/pull/19800" + ), +) +def test_array_api_entry_point(): + """ + Entry point for Array API implementation can be found with importlib and + returns the numpy.array_api namespace. + """ + # For a development install that did not go through meson-python, + # the entrypoint will not have been installed. So ensure this test fails + # only if numpy is inside site-packages. + numpy_in_sitepackages = sysconfig.get_path('platlib') in np.__file__ + + eps = importlib.metadata.entry_points() + try: + xp_eps = eps.select(group="array_api") + except AttributeError: + # The select interface for entry_points was introduced in py3.10, + # deprecating its dict interface. We fallback to dict keys for finding + # Array API entry points so that running this test in <=3.9 will + # still work - see https://github.com/numpy/numpy/pull/19800. + xp_eps = eps.get("array_api", []) + if len(xp_eps) == 0: + if numpy_in_sitepackages: + msg = "No entry points for 'array_api' found" + raise AssertionError(msg) from None + return + + try: + ep = next(ep for ep in xp_eps if ep.name == "numpy") + except StopIteration: + if numpy_in_sitepackages: + msg = "'numpy' not in array_api entry points" + raise AssertionError(msg) from None + return + + xp = ep.load() + msg = ( + f"numpy entry point value '{ep.value}' " + "does not point to our Array API implementation" + ) + assert xp is numpy.array_api, msg + + +@pytest.mark.parametrize("name", [ + 'ModuleDeprecationWarning', 'VisibleDeprecationWarning', + 'ComplexWarning', 'TooHardError', 'AxisError']) +def test_moved_exceptions(name): + # These were moved to the exceptions namespace, but currently still + # available + assert name in np.__all__ + assert name not in np.__dir__() + # Fetching works, but __module__ is set correctly: + assert getattr(np, name).__module__ == "numpy.exceptions" + assert name in np.exceptions.__all__ + getattr(np.exceptions, name) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/tests/test_scripts.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/tests/test_scripts.py new file mode 100644 index 0000000000000000000000000000000000000000..892c04eef0bed4b9d92408419c547f8258a005e3 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/numpy/tests/test_scripts.py @@ -0,0 +1,47 @@ +""" Test scripts + +Test that we can run executable scripts that have been installed with numpy. +""" +import sys +import os +import pytest +from os.path import join as pathjoin, isfile, dirname +import subprocess + +import numpy as np +from numpy.testing import assert_equal, IS_WASM + +is_inplace = isfile(pathjoin(dirname(np.__file__), '..', 'setup.py')) + + +def find_f2py_commands(): + if sys.platform == 'win32': + exe_dir = dirname(sys.executable) + if exe_dir.endswith('Scripts'): # virtualenv + return [os.path.join(exe_dir, 'f2py')] + else: + return [os.path.join(exe_dir, "Scripts", 'f2py')] + else: + # Three scripts are installed in Unix-like systems: + # 'f2py', 'f2py{major}', and 'f2py{major.minor}'. For example, + # if installed with python3.9 the scripts would be named + # 'f2py', 'f2py3', and 'f2py3.9'. + version = sys.version_info + major = str(version.major) + minor = str(version.minor) + return ['f2py', 'f2py' + major, 'f2py' + major + '.' + minor] + + +@pytest.mark.skipif(is_inplace, reason="Cannot test f2py command inplace") +@pytest.mark.xfail(reason="Test is unreliable") +@pytest.mark.parametrize('f2py_cmd', find_f2py_commands()) +def test_f2py(f2py_cmd): + # test that we can run f2py script + stdout = subprocess.check_output([f2py_cmd, '-v']) + assert_equal(stdout.strip(), np.__version__.encode('ascii')) + + +@pytest.mark.skipif(IS_WASM, reason="Cannot start subprocess") +def test_pep338(): + stdout = subprocess.check_output([sys.executable, '-mnumpy.f2py', '-v']) + assert_equal(stdout.strip(), np.__version__.encode('ascii')) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/superpoint/__init__.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/superpoint/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cf1b097542ee7509bf4e89e922b6df6ddd475034 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/superpoint/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule +from ...utils.import_utils import define_import_structure + + +if TYPE_CHECKING: + from .configuration_superpoint import * + from .image_processing_pil_superpoint import * + from .image_processing_superpoint import * + from .modeling_superpoint import * +else: + import sys + + _file = globals()["__file__"] + sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__) diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/superpoint/configuration_superpoint.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/superpoint/configuration_superpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..d3f0dfeb4fe63f1252c67dd174855825f5cd7e92 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/superpoint/configuration_superpoint.py @@ -0,0 +1,66 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from huggingface_hub.dataclasses import strict + +from ...configuration_utils import PreTrainedConfig +from ...utils import auto_docstring + + +@auto_docstring(checkpoint="magic-leap-community/superpoint") +@strict +class SuperPointConfig(PreTrainedConfig): + r""" + encoder_hidden_sizes (`List`, *optional*, defaults to `[64, 64, 128, 128]`): + The number of channels in each convolutional layer in the encoder. + keypoint_decoder_dim (`int`, *optional*, defaults to 65): + The output dimension of the keypoint decoder. + descriptor_decoder_dim (`int`, *optional*, defaults to 256): + The output dimension of the descriptor decoder. + keypoint_threshold (`float`, *optional*, defaults to 0.005): + The threshold to use for extracting keypoints. + max_keypoints (`int`, *optional*, defaults to -1): + The maximum number of keypoints to extract. If `-1`, will extract all keypoints. + nms_radius (`int`, *optional*, defaults to 4): + The radius for non-maximum suppression. + border_removal_distance (`int`, *optional*, defaults to 4): + The distance from the border to remove keypoints. + + Example: + ```python + >>> from transformers import SuperPointConfig, SuperPointForKeypointDetection + + >>> # Initializing a SuperPoint superpoint style configuration + >>> configuration = SuperPointConfig() + >>> # Initializing a model from the superpoint style configuration + >>> model = SuperPointForKeypointDetection(configuration) + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "superpoint" + + encoder_hidden_sizes: list[int] | tuple[int, ...] = (64, 64, 128, 128) + decoder_hidden_size: int = 256 + keypoint_decoder_dim: int = 65 + descriptor_decoder_dim: int = 256 + keypoint_threshold: float = 0.005 + max_keypoints: int = -1 + nms_radius: int = 4 + border_removal_distance: int = 4 + initializer_range: float = 0.02 + + +__all__ = ["SuperPointConfig"] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/superpoint/image_processing_superpoint.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/superpoint/image_processing_superpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..a0e9660b521653eade29bbea01b6f8189b22df38 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/superpoint/image_processing_superpoint.py @@ -0,0 +1,164 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Image processor class for SuperPoint.""" + +from typing import TYPE_CHECKING + +import torch + +from ...image_processing_backends import TorchvisionBackend +from ...image_processing_utils import BatchFeature +from ...image_transforms import group_images_by_shape, reorder_images +from ...image_utils import PILImageResampling, SizeDict +from ...processing_utils import ImagesKwargs, Unpack +from ...utils import TensorType, auto_docstring + + +if TYPE_CHECKING: + from .modeling_superpoint import SuperPointKeypointDescriptionOutput + +from torchvision.transforms.v2 import functional as tvF + + +class SuperPointImageProcessorKwargs(ImagesKwargs, total=False): + r""" + do_grayscale (`bool`, *optional*, defaults to `self.do_grayscale`): + Whether to convert the image to grayscale. Can be overridden by `do_grayscale` in the `preprocess` method. + """ + + do_grayscale: bool + + +def is_grayscale(image: "torch.Tensor") -> bool: + """Checks if an image is grayscale (all RGB channels are identical).""" + if image.ndim < 3 or image.shape[0 if image.ndim == 3 else 1] == 1: + return True + return torch.all(image[..., 0, :, :] == image[..., 1, :, :]) and torch.all( + image[..., 1, :, :] == image[..., 2, :, :] + ) + + +def convert_to_grayscale(image: "torch.Tensor") -> "torch.Tensor": + """ + Converts an image to grayscale format using the NTSC formula. Only support torch.Tensor. + + This function is supposed to return a 1-channel image, but it returns a 3-channel image with the same value in each + channel, because of an issue that is discussed in : + https://github.com/huggingface/transformers/pull/25786#issuecomment-1730176446 + + Args: + image (torch.Tensor): + The image to convert. + """ + if is_grayscale(image): + return image + return tvF.rgb_to_grayscale(image, num_output_channels=3) + + +@auto_docstring +class SuperPointImageProcessor(TorchvisionBackend): + valid_kwargs = SuperPointImageProcessorKwargs + resample = PILImageResampling.BILINEAR + size = {"height": 480, "width": 640} + default_to_square = False + do_resize = True + do_rescale = True + rescale_factor = 1 / 255 + do_normalize = None + do_grayscale = False + + def __init__(self, **kwargs: Unpack[SuperPointImageProcessorKwargs]): + super().__init__(**kwargs) + + def _preprocess( + self, + images: list["torch.Tensor"], + do_resize: bool, + size: SizeDict, + resample: "PILImageResampling | tvF.InterpolationMode | int | None", + do_rescale: bool, + rescale_factor: float, + disable_grouping: bool | None, + return_tensors: str | TensorType | None, + do_grayscale: bool = False, + **kwargs, + ) -> BatchFeature: + # Group images by size for batched processing + grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping) + processed_images_grouped = {} + for shape, stacked_images in grouped_images.items(): + # Apply grayscale conversion before resize (if requested) + if do_grayscale: + stacked_images = convert_to_grayscale(stacked_images) + # Resize + if do_resize: + stacked_images = self.resize(stacked_images, size=size, resample=resample) + # Rescale + if do_rescale: + stacked_images = self.rescale(stacked_images, rescale_factor) + processed_images_grouped[shape] = stacked_images + processed_images = reorder_images(processed_images_grouped, grouped_images_index) + return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors) + + def post_process_keypoint_detection( + self, outputs: "SuperPointKeypointDescriptionOutput", target_sizes: TensorType | list[tuple] + ) -> list[dict[str, "torch.Tensor"]]: + """ + Converts the raw output of [`SuperPointForKeypointDetection`] into lists of keypoints, scores and descriptors + with coordinates absolute to the original image sizes. + + Args: + outputs ([`SuperPointKeypointDescriptionOutput`]): + Raw outputs of the model containing keypoints in a relative (x, y) format, with scores and descriptors. + target_sizes (`torch.Tensor` or `list[tuple[int, int]]`): + Tensor of shape `(batch_size, 2)` or list of tuples (`tuple[int, int]`) containing the target size + `(height, width)` of each image in the batch. This must be the original + image size (before any processing). + Returns: + `list[Dict]`: A list of dictionaries, each dictionary containing the keypoints in absolute format according + to target_sizes, scores and descriptors for an image in the batch as predicted by the model. + """ + if len(outputs.mask) != len(target_sizes): + raise ValueError("Make sure that you pass in as many target sizes as the batch dimension of the mask") + + if isinstance(target_sizes, list): + image_sizes = torch.tensor(target_sizes, device=outputs.mask.device) + else: + if target_sizes.shape[1] != 2: + raise ValueError( + "Each element of target_sizes must contain the size (h, w) of each image of the batch" + ) + image_sizes = target_sizes + + # Flip the image sizes to (width, height) and convert keypoints to absolute coordinates + image_sizes = torch.flip(image_sizes, [1]) + masked_keypoints = outputs.keypoints * image_sizes[:, None] + + # Convert masked_keypoints to int + masked_keypoints = masked_keypoints.to(torch.int32) + + results = [] + for image_mask, keypoints, scores, descriptors in zip( + outputs.mask, masked_keypoints, outputs.scores, outputs.descriptors + ): + indices = torch.nonzero(image_mask).squeeze(1) + keypoints = keypoints[indices] + scores = scores[indices] + descriptors = descriptors[indices] + results.append({"keypoints": keypoints, "scores": scores, "descriptors": descriptors}) + + return results + + +__all__ = ["SuperPointImageProcessor"] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/superpoint/modeling_superpoint.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/superpoint/modeling_superpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..cef96725d4d27a90064203a3b473c53d8fe10caa --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/superpoint/modeling_superpoint.py @@ -0,0 +1,468 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch SuperPoint model.""" + +from dataclasses import dataclass + +import torch +from torch import nn + +from transformers import PreTrainedModel +from transformers.modeling_outputs import ( + BaseModelOutputWithNoAttention, +) +from transformers.models.superpoint.configuration_superpoint import SuperPointConfig + +from ...utils import ( + ModelOutput, + auto_docstring, + logging, +) + + +logger = logging.get_logger(__name__) + + +def remove_keypoints_from_borders( + keypoints: torch.Tensor, scores: torch.Tensor, border: int, height: int, width: int +) -> tuple[torch.Tensor, torch.Tensor]: + """Removes keypoints (and their associated scores) that are too close to the border""" + mask_h = (keypoints[:, 0] >= border) & (keypoints[:, 0] < (height - border)) + mask_w = (keypoints[:, 1] >= border) & (keypoints[:, 1] < (width - border)) + mask = mask_h & mask_w + return keypoints[mask], scores[mask] + + +def top_k_keypoints(keypoints: torch.Tensor, scores: torch.Tensor, k: int) -> tuple[torch.Tensor, torch.Tensor]: + """Keeps the k keypoints with highest score""" + if k >= len(keypoints): + return keypoints, scores + scores, indices = torch.topk(scores, k, dim=0) + return keypoints[indices], scores + + +def simple_nms(scores: torch.Tensor, nms_radius: int) -> torch.Tensor: + """Applies non-maximum suppression on scores""" + if nms_radius < 0: + raise ValueError("Expected positive values for nms_radius") + + def max_pool(x): + return nn.functional.max_pool2d(x, kernel_size=nms_radius * 2 + 1, stride=1, padding=nms_radius) + + zeros = torch.zeros_like(scores) + max_mask = scores == max_pool(scores) + for _ in range(2): + supp_mask = max_pool(max_mask.float()) > 0 + supp_scores = torch.where(supp_mask, zeros, scores) + new_max_mask = supp_scores == max_pool(supp_scores) + max_mask = max_mask | (new_max_mask & (~supp_mask)) + return torch.where(max_mask, scores, zeros) + + +@auto_docstring( + custom_intro=""" + Base class for outputs of image point description models. Due to the nature of keypoint detection, the number of + keypoints is not fixed and can vary from image to image, which makes batching non-trivial. In the batch of images, + the maximum number of keypoints is set as the dimension of the keypoints, scores and descriptors tensors. The mask + tensor is used to indicate which values in the keypoints, scores and descriptors tensors are keypoint information + and which are padding. + """ +) +@dataclass +class SuperPointKeypointDescriptionOutput(ModelOutput): + r""" + loss (`torch.FloatTensor` of shape `(1,)`, *optional*): + Loss computed during training. + keypoints (`torch.FloatTensor` of shape `(batch_size, num_keypoints, 2)`): + Relative (x, y) coordinates of predicted keypoints in a given image. + scores (`torch.FloatTensor` of shape `(batch_size, num_keypoints)`): + Scores of predicted keypoints. + descriptors (`torch.FloatTensor` of shape `(batch_size, num_keypoints, descriptor_size)`): + Descriptors of predicted keypoints. + mask (`torch.BoolTensor` of shape `(batch_size, num_keypoints)`): + Mask indicating which values in keypoints, scores and descriptors are keypoint information. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or + when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each stage) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states + (also called feature maps) of the model at the output of each stage. + """ + + loss: torch.FloatTensor | None = None + keypoints: torch.IntTensor | None = None + scores: torch.FloatTensor | None = None + descriptors: torch.FloatTensor | None = None + mask: torch.BoolTensor | None = None + hidden_states: tuple[torch.FloatTensor] | None = None + + +class SuperPointConvBlock(nn.Module): + def __init__( + self, config: SuperPointConfig, in_channels: int, out_channels: int, add_pooling: bool = False + ) -> None: + super().__init__() + self.conv_a = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + ) + self.conv_b = nn.Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + ) + self.relu = nn.ReLU(inplace=True) + self.pool = nn.MaxPool2d(kernel_size=2, stride=2) if add_pooling else None + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.relu(self.conv_a(hidden_states)) + hidden_states = self.relu(self.conv_b(hidden_states)) + if self.pool is not None: + hidden_states = self.pool(hidden_states) + return hidden_states + + +class SuperPointEncoder(nn.Module): + """ + SuperPoint encoder module. It is made of 4 convolutional layers with ReLU activation and max pooling, reducing the + dimensionality of the image. + """ + + def __init__(self, config: SuperPointConfig) -> None: + super().__init__() + # SuperPoint uses 1 channel images + self.input_dim = 1 + + conv_blocks = [] + conv_blocks.append( + SuperPointConvBlock(config, self.input_dim, config.encoder_hidden_sizes[0], add_pooling=True) + ) + for i in range(1, len(config.encoder_hidden_sizes) - 1): + conv_blocks.append( + SuperPointConvBlock( + config, config.encoder_hidden_sizes[i - 1], config.encoder_hidden_sizes[i], add_pooling=True + ) + ) + conv_blocks.append( + SuperPointConvBlock( + config, config.encoder_hidden_sizes[-2], config.encoder_hidden_sizes[-1], add_pooling=False + ) + ) + self.conv_blocks = nn.ModuleList(conv_blocks) + + def forward( + self, + input, + output_hidden_states: bool | None = False, + return_dict: bool | None = True, + ) -> tuple | BaseModelOutputWithNoAttention: + all_hidden_states = () if output_hidden_states else None + + for conv_block in self.conv_blocks: + input = conv_block(input) + if output_hidden_states: + all_hidden_states = all_hidden_states + (input,) + output = input + if not return_dict: + return tuple(v for v in [output, all_hidden_states] if v is not None) + + return BaseModelOutputWithNoAttention( + last_hidden_state=output, + hidden_states=all_hidden_states, + ) + + +class SuperPointInterestPointDecoder(nn.Module): + """ + The SuperPointInterestPointDecoder uses the output of the SuperPointEncoder to compute the keypoint with scores. + The scores are first computed by a convolutional layer, then a softmax is applied to get a probability distribution + over the 65 possible keypoint classes. The keypoints are then extracted from the scores by thresholding and + non-maximum suppression. Post-processing is then applied to remove keypoints too close to the image borders as well + as to keep only the k keypoints with highest score. + """ + + def __init__(self, config: SuperPointConfig) -> None: + super().__init__() + self.keypoint_threshold = config.keypoint_threshold + self.max_keypoints = config.max_keypoints + self.nms_radius = config.nms_radius + self.border_removal_distance = config.border_removal_distance + + self.relu = nn.ReLU(inplace=True) + self.pool = nn.MaxPool2d(kernel_size=2, stride=2) + self.conv_score_a = nn.Conv2d( + config.encoder_hidden_sizes[-1], + config.decoder_hidden_size, + kernel_size=3, + stride=1, + padding=1, + ) + self.conv_score_b = nn.Conv2d( + config.decoder_hidden_size, config.keypoint_decoder_dim, kernel_size=1, stride=1, padding=0 + ) + + def forward(self, encoded: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + scores = self._get_pixel_scores(encoded) + keypoints, scores = self._extract_keypoints(scores) + + return keypoints, scores + + def _get_pixel_scores(self, encoded: torch.Tensor) -> torch.Tensor: + """Based on the encoder output, compute the scores for each pixel of the image""" + scores = self.relu(self.conv_score_a(encoded)) + scores = self.conv_score_b(scores) + scores = nn.functional.softmax(scores, 1)[:, :-1] + batch_size, _, height, width = scores.shape + scores = scores.permute(0, 2, 3, 1).reshape(batch_size, height, width, 8, 8) + scores = scores.permute(0, 1, 3, 2, 4).reshape(batch_size, height * 8, width * 8) + scores = simple_nms(scores, self.nms_radius) + return scores + + def _extract_keypoints(self, scores: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """ + Based on their scores, extract the pixels that represent the keypoints that will be used for descriptors computation. + The keypoints are in the form of relative (x, y) coordinates. + """ + _, height, width = scores.shape + + # Threshold keypoints by score value + keypoints = torch.nonzero(scores[0] > self.keypoint_threshold) + scores = scores[0][tuple(keypoints.t())] + + # Discard keypoints near the image borders + keypoints, scores = remove_keypoints_from_borders( + keypoints, scores, self.border_removal_distance, height * 8, width * 8 + ) + + # Keep the k keypoints with highest score + if self.max_keypoints >= 0: + keypoints, scores = top_k_keypoints(keypoints, scores, self.max_keypoints) + + # Convert (y, x) to (x, y) + keypoints = torch.flip(keypoints, [1]).to(scores.dtype) + + return keypoints, scores + + +class SuperPointDescriptorDecoder(nn.Module): + """ + The SuperPointDescriptorDecoder uses the outputs of both the SuperPointEncoder and the + SuperPointInterestPointDecoder to compute the descriptors at the keypoints locations. + + The descriptors are first computed by a convolutional layer, then normalized to have a norm of 1. The descriptors + are then interpolated at the keypoints locations. + """ + + def __init__(self, config: SuperPointConfig) -> None: + super().__init__() + + self.relu = nn.ReLU(inplace=True) + self.pool = nn.MaxPool2d(kernel_size=2, stride=2) + self.conv_descriptor_a = nn.Conv2d( + config.encoder_hidden_sizes[-1], + config.decoder_hidden_size, + kernel_size=3, + stride=1, + padding=1, + ) + self.conv_descriptor_b = nn.Conv2d( + config.decoder_hidden_size, + config.descriptor_decoder_dim, + kernel_size=1, + stride=1, + padding=0, + ) + + def forward(self, encoded: torch.Tensor, keypoints: torch.Tensor) -> torch.Tensor: + """Based on the encoder output and the keypoints, compute the descriptors for each keypoint""" + descriptors = self.conv_descriptor_b(self.relu(self.conv_descriptor_a(encoded))) + descriptors = nn.functional.normalize(descriptors, p=2, dim=1) + + descriptors = self._sample_descriptors(keypoints[None], descriptors[0][None], 8)[0] + + # [descriptor_dim, num_keypoints] -> [num_keypoints, descriptor_dim] + descriptors = torch.transpose(descriptors, 0, 1) + + return descriptors + + @staticmethod + def _sample_descriptors(keypoints, descriptors, scale: int = 8) -> torch.Tensor: + """Interpolate descriptors at keypoint locations""" + batch_size, num_channels, height, width = descriptors.shape + keypoints = keypoints - scale / 2 + 0.5 + divisor = torch.tensor([[(width * scale - scale / 2 - 0.5), (height * scale - scale / 2 - 0.5)]]) + divisor = divisor.to(keypoints) + keypoints /= divisor + keypoints = keypoints * 2 - 1 # normalize to (-1, 1) + kwargs = {"align_corners": True} + # [batch_size, num_channels, num_keypoints, 2] -> [batch_size, num_channels, num_keypoints, 2] + keypoints = keypoints.view(batch_size, 1, -1, 2) + descriptors = nn.functional.grid_sample(descriptors, keypoints, mode="bilinear", **kwargs) + # [batch_size, descriptor_decoder_dim, num_channels, num_keypoints] -> [batch_size, descriptor_decoder_dim, num_keypoints] + descriptors = descriptors.reshape(batch_size, num_channels, -1) + descriptors = nn.functional.normalize(descriptors, p=2, dim=1) + return descriptors + + +@auto_docstring +class SuperPointPreTrainedModel(PreTrainedModel): + config: SuperPointConfig + base_model_prefix = "superpoint" + main_input_name = "pixel_values" + input_modalities = ("image",) + supports_gradient_checkpointing = False + + def extract_one_channel_pixel_values(self, pixel_values: torch.FloatTensor) -> torch.FloatTensor: + """ + Assuming pixel_values has shape (batch_size, 3, height, width), and that all channels values are the same, + extract the first channel value to get a tensor of shape (batch_size, 1, height, width) for SuperPoint. This is + a workaround for the issue discussed in : + https://github.com/huggingface/transformers/pull/25786#issuecomment-1730176446 + + Args: + pixel_values: torch.FloatTensor of shape (batch_size, 3, height, width) + + Returns: + pixel_values: torch.FloatTensor of shape (batch_size, 1, height, width) + + """ + return pixel_values[:, 0, :, :][:, None, :, :] + + +@auto_docstring( + custom_intro=""" + SuperPoint model outputting keypoints and descriptors. + """ +) +class SuperPointForKeypointDetection(SuperPointPreTrainedModel): + """ + SuperPoint model. It consists of a SuperPointEncoder, a SuperPointInterestPointDecoder and a + SuperPointDescriptorDecoder. SuperPoint was proposed in `SuperPoint: Self-Supervised Interest Point Detection and + Description `__ by Daniel DeTone, Tomasz Malisiewicz, and Andrew Rabinovich. It + is a fully convolutional neural network that extracts keypoints and descriptors from an image. It is trained in a + self-supervised manner, using a combination of a photometric loss and a loss based on the homographic adaptation of + keypoints. It is made of a convolutional encoder and two decoders: one for keypoints and one for descriptors. + """ + + def __init__(self, config: SuperPointConfig) -> None: + super().__init__(config) + + self.config = config + + self.encoder = SuperPointEncoder(config) + self.keypoint_decoder = SuperPointInterestPointDecoder(config) + self.descriptor_decoder = SuperPointDescriptorDecoder(config) + + self.post_init() + + @auto_docstring + def forward( + self, + pixel_values: torch.FloatTensor, + labels: torch.LongTensor | None = None, + output_hidden_states: bool | None = None, + return_dict: bool | None = None, + **kwargs, + ) -> tuple | SuperPointKeypointDescriptionOutput: + r""" + Examples: + + ```python + >>> from transformers import AutoImageProcessor, SuperPointForKeypointDetection + >>> import torch + >>> from PIL import Image + >>> import httpx + >>> from io import BytesIO + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> with httpx.stream("GET", url) as response: + ... image = Image.open(BytesIO(response.read())) + + >>> processor = AutoImageProcessor.from_pretrained("magic-leap-community/superpoint") + >>> model = SuperPointForKeypointDetection.from_pretrained("magic-leap-community/superpoint") + + >>> inputs = processor(image, return_tensors="pt") + >>> outputs = model(**inputs) + ```""" + loss = None + if labels is not None: + raise ValueError("SuperPoint does not support training for now.") + + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + pixel_values = self.extract_one_channel_pixel_values(pixel_values) + + batch_size, _, height, width = pixel_values.shape + + encoder_outputs = self.encoder( + pixel_values, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + last_hidden_state = encoder_outputs[0] + + list_keypoints_scores = [ + self.keypoint_decoder(last_hidden_state[None, ...]) for last_hidden_state in last_hidden_state + ] + + list_keypoints = [keypoints_scores[0] for keypoints_scores in list_keypoints_scores] + list_scores = [keypoints_scores[1] for keypoints_scores in list_keypoints_scores] + + list_descriptors = [ + self.descriptor_decoder(last_hidden_state[None, ...], keypoints[None, ...]) + for last_hidden_state, keypoints in zip(last_hidden_state, list_keypoints) + ] + + maximum_num_keypoints = max(keypoints.shape[0] for keypoints in list_keypoints) + + keypoints = torch.zeros((batch_size, maximum_num_keypoints, 2), device=pixel_values.device) + scores = torch.zeros((batch_size, maximum_num_keypoints), device=pixel_values.device) + descriptors = torch.zeros( + (batch_size, maximum_num_keypoints, self.config.descriptor_decoder_dim), + device=pixel_values.device, + ) + mask = torch.zeros((batch_size, maximum_num_keypoints), device=pixel_values.device, dtype=torch.int) + + for i, (_keypoints, _scores, _descriptors) in enumerate(zip(list_keypoints, list_scores, list_descriptors)): + keypoints[i, : _keypoints.shape[0]] = _keypoints + scores[i, : _scores.shape[0]] = _scores + descriptors[i, : _descriptors.shape[0]] = _descriptors + mask[i, : _scores.shape[0]] = 1 + + # Convert to relative coordinates + keypoints = keypoints / torch.tensor([width, height], device=keypoints.device) + + hidden_states = encoder_outputs[1] if output_hidden_states else None + if not return_dict: + return tuple(v for v in [loss, keypoints, scores, descriptors, mask, hidden_states] if v is not None) + + return SuperPointKeypointDescriptionOutput( + loss=loss, + keypoints=keypoints, + scores=scores, + descriptors=descriptors, + mask=mask, + hidden_states=hidden_states, + ) + + +__all__ = ["SuperPointForKeypointDetection", "SuperPointPreTrainedModel"] diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245/step_049000.pt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245/step_049000.pt new file mode 100644 index 0000000000000000000000000000000000000000..d58c024f2335c6f3b3fce10fa87b14d658bd743e --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245/step_049000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2226a2bd8e48428f4db4ace5cf66e7599ac8f1fddc1dab9cb1b3d02c26d1b0b +size 1031030114 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245/step_050000.pt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245/step_050000.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3525e4079c96d366773997b3536e77ada0d9cae --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245/step_050000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e78dbec6f6736c9b082d50541356c64ca9ebecdfcd44a804fec78214b04b4538 +size 1031030114 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245/step_262000.pt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245/step_262000.pt new file mode 100644 index 0000000000000000000000000000000000000000..81e5d0c7519659c14a14d9c279ee64f217945f35 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245/step_262000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9a2c4435e123f5672449243a3513dca3018c2789107eef4eae0637b52058e05 +size 1031030114 diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245/step_295000.pt b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245/step_295000.pt new file mode 100644 index 0000000000000000000000000000000000000000..193fcf3cd0f82bfc4e5cc3a5b5ad761004873d37 --- /dev/null +++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/owt_t5_elftokenized_full_len1024_C1_to_1024_pow1_d768_l12_h12_gbs512_8gpu_50ep_lr6e4_ema0p9999_elfopt_t5embed_unfixed_norm_stateprobadd_selfcond_ce_fast_20260606_144245/step_295000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55e15c16ae46b6b5b03ffc62b504002d23cdde13fcc009402a48e552475659d9 +size 1031030114