Add files using upload-large-folder tool

LTA_openwebtext_dualt/logs/build_owt_gpt2_len1024_cached_chunks_fast.log

@@ -0,0 +1,100 @@
+[cache] records=100000 chunks=108485 buffer_tokens=653 elapsed=141.8s recent=141.8s
+[cache] records=200000 chunks=218385 buffer_tokens=769 elapsed=291.6s recent=149.8s
+[cache] records=300000 chunks=327438 buffer_tokens=316 elapsed=439.4s recent=147.8s
+[cache] records=400000 chunks=436813 buffer_tokens=707 elapsed=588.4s recent=149.0s
+[cache] records=500000 chunks=544661 buffer_tokens=271 elapsed=735.3s recent=146.9s
+[cache] records=600000 chunks=652783 buffer_tokens=137 elapsed=882.9s recent=147.6s
+[cache] records=700000 chunks=761924 buffer_tokens=967 elapsed=1033.7s recent=150.9s
+[cache] records=800000 chunks=869711 buffer_tokens=865 elapsed=1181.7s recent=148.0s
+[cache] records=900000 chunks=978424 buffer_tokens=33 elapsed=1330.2s recent=148.5s
+[cache] records=1000000 chunks=1104688 buffer_tokens=596 elapsed=1503.6s recent=173.4s
+[cache] records=1100000 chunks=1213189 buffer_tokens=557 elapsed=1652.5s recent=148.9s
+[cache] records=1200000 chunks=1321827 buffer_tokens=796 elapsed=1799.1s recent=146.6s
+[cache] records=1300000 chunks=1429996 buffer_tokens=815 elapsed=1946.7s recent=147.5s
+[cache] records=1400000 chunks=1538134 buffer_tokens=601 elapsed=2094.5s recent=147.9s
+[cache] records=1500000 chunks=1647134 buffer_tokens=193 elapsed=2244.5s recent=150.0s
+[cache] records=1600000 chunks=1755751 buffer_tokens=62 elapsed=2394.0s recent=149.4s
+[cache] records=1700000 chunks=1865278 buffer_tokens=483 elapsed=2544.0s recent=150.1s
+[cache] records=1800000 chunks=1973006 buffer_tokens=453 elapsed=2692.5s recent=148.5s
+[cache] records=1900000 chunks=2081066 buffer_tokens=830 elapsed=2840.7s recent=148.1s
+[cache] records=2000000 chunks=2207191 buffer_tokens=545 elapsed=3011.7s recent=171.0s
+[cache] records=2100000 chunks=2316435 buffer_tokens=164 elapsed=3160.0s recent=148.3s
+[cache] records=2200000 chunks=2425486 buffer_tokens=111 elapsed=3308.1s recent=148.1s
+[cache] records=2300000 chunks=2534619 buffer_tokens=319 elapsed=3456.0s recent=148.0s
+[cache] records=2400000 chunks=2644127 buffer_tokens=375 elapsed=3604.4s recent=148.3s
+[cache] records=2500000 chunks=2751903 buffer_tokens=709 elapsed=3750.8s recent=146.4s
+[cache] records=2600000 chunks=2861117 buffer_tokens=983 elapsed=3898.9s recent=148.1s
+[cache] records=2700000 chunks=2968962 buffer_tokens=768 elapsed=4045.3s recent=146.4s
+[cache] records=2800000 chunks=3076780 buffer_tokens=114 elapsed=4191.5s recent=146.2s
+[cache] records=2900000 chunks=3203666 buffer_tokens=101 elapsed=4363.9s recent=172.4s
+[cache] records=3000000 chunks=3311828 buffer_tokens=634 elapsed=4511.5s recent=147.5s
+[cache] records=3100000 chunks=3419378 buffer_tokens=156 elapsed=4657.2s recent=145.8s
+[cache] records=3200000 chunks=3527966 buffer_tokens=367 elapsed=4805.4s recent=148.1s
+[cache] records=3300000 chunks=3637008 buffer_tokens=810 elapsed=4953.5s recent=148.2s
+[cache] records=3400000 chunks=3744903 buffer_tokens=142 elapsed=5100.9s recent=147.4s
+[cache] records=3500000 chunks=3853630 buffer_tokens=506 elapsed=5249.2s recent=148.3s
+[cache] records=3600000 chunks=3962366 buffer_tokens=775 elapsed=5396.6s recent=147.5s
+[cache] records=3700000 chunks=4070628 buffer_tokens=493 elapsed=5543.8s recent=147.1s
+[cache] records=3800000 chunks=4178761 buffer_tokens=792 elapsed=5691.1s recent=147.3s
+[cache] records=3900000 chunks=4305753 buffer_tokens=676 elapsed=5862.8s recent=171.7s
+[cache] records=4000000 chunks=4413812 buffer_tokens=682 elapsed=6009.9s recent=147.1s
+[cache] records=4100000 chunks=4522560 buffer_tokens=647 elapsed=6158.3s recent=148.4s
+[cache] records=4200000 chunks=4632049 buffer_tokens=558 elapsed=6307.0s recent=148.7s
+[cache] records=4300000 chunks=4740354 buffer_tokens=841 elapsed=6454.7s recent=147.7s
+[cache] records=4400000 chunks=4849473 buffer_tokens=213 elapsed=6604.8s recent=150.1s
+[cache] records=4500000 chunks=4958322 buffer_tokens=735 elapsed=6754.8s recent=150.0s
+[cache] records=4600000 chunks=5067328 buffer_tokens=236 elapsed=6904.6s recent=149.8s
+[cache] records=4700000 chunks=5175728 buffer_tokens=876 elapsed=7053.9s recent=149.2s
+[cache] records=4800000 chunks=5283551 buffer_tokens=95 elapsed=7202.6s recent=148.7s
+[cache] records=4900000 chunks=5411131 buffer_tokens=453 elapsed=7376.1s recent=173.5s
+[cache] records=5000000 chunks=5519097 buffer_tokens=852 elapsed=7523.4s recent=147.3s
+[cache] records=5100000 chunks=5628354 buffer_tokens=500 elapsed=7672.1s recent=148.7s
+[cache] records=5200000 chunks=5737569 buffer_tokens=612 elapsed=7821.8s recent=149.7s
+[cache] records=5300000 chunks=5846808 buffer_tokens=279 elapsed=7971.0s recent=149.2s
+[cache] records=5400000 chunks=5954738 buffer_tokens=842 elapsed=8120.0s recent=149.0s
+[cache] records=5500000 chunks=6063404 buffer_tokens=780 elapsed=8269.6s recent=149.6s
+[cache] records=5600000 chunks=6172338 buffer_tokens=71 elapsed=8418.5s recent=148.9s
+[cache] records=5700000 chunks=6281171 buffer_tokens=125 elapsed=8568.2s recent=149.7s
+[cache] records=5800000 chunks=6407255 buffer_tokens=713 elapsed=8741.0s recent=172.8s
+[cache] records=5900000 chunks=6515836 buffer_tokens=484 elapsed=8889.8s recent=148.7s
+[cache] records=6000000 chunks=6624316 buffer_tokens=735 elapsed=9038.9s recent=149.1s
+[cache] records=6100000 chunks=6732297 buffer_tokens=387 elapsed=9186.4s recent=147.4s
+[cache] records=6200000 chunks=6840706 buffer_tokens=840 elapsed=9335.2s recent=148.9s
+[cache] records=6300000 chunks=6948960 buffer_tokens=730 elapsed=9483.9s recent=148.7s
+[cache] records=6400000 chunks=7057042 buffer_tokens=438 elapsed=9630.5s recent=146.6s
+[cache] records=6500000 chunks=7166013 buffer_tokens=783 elapsed=9778.9s recent=148.4s
+[cache] records=6600000 chunks=7273882 buffer_tokens=179 elapsed=9925.8s recent=146.9s
+[cache] records=6700000 chunks=7381269 buffer_tokens=200 elapsed=10072.4s recent=146.6s
+[cache] records=6800000 chunks=7507592 buffer_tokens=1000 elapsed=10244.1s recent=171.6s
+[cache] records=6900000 chunks=7616146 buffer_tokens=383 elapsed=10391.8s recent=147.8s
+[cache] records=7000000 chunks=7724177 buffer_tokens=569 elapsed=10539.5s recent=147.7s
+[cache] records=7100000 chunks=7832286 buffer_tokens=528 elapsed=10687.1s recent=147.5s
+[cache] records=7200000 chunks=7940257 buffer_tokens=491 elapsed=10833.1s recent=146.1s
+[cache] records=7300000 chunks=8048719 buffer_tokens=784 elapsed=10980.6s recent=147.5s
+[cache] records=7400000 chunks=8156886 buffer_tokens=224 elapsed=11127.1s recent=146.5s
+[cache] records=7500000 chunks=8264279 buffer_tokens=418 elapsed=11273.5s recent=146.4s
+[cache] records=7600000 chunks=8372134 buffer_tokens=795 elapsed=11420.1s recent=146.6s
+[cache] records=7700000 chunks=8481587 buffer_tokens=164 elapsed=11568.6s recent=148.4s
+[cache] records=7800000 chunks=8608641 buffer_tokens=245 elapsed=11742.0s recent=173.5s
+[cache] records=7900000 chunks=8716430 buffer_tokens=913 elapsed=11889.2s recent=147.2s
+{
+  "data_path": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext",
+  "text_column": "text",
+  "txt_record_mode": "auto",
+  "openwebtext_split": "train_minus_100k",
+  "detokenizer": null,
+  "max_len": 1024,
+  "payload_len": 1022,
+  "num_chunks": 8734897,
+  "num_records": 7913769,
+  "dtype": "int32",
+  "bos_id": 50256,
+  "eos_id": 50256,
+  "vocab_size": 50257,
+  "packing": "flm_stream_wrapped",
+  "dropped_remainder_tokens": 595,
+  "builder": "fast_encode_batch_ordered_v2",
+  "encode_batch_size": 16384,
+  "write_batch_chunks": 16384
+}
+[cache] done elapsed=11914.3s bytes=35778138112

LTA_openwebtext_dualt/logs/infer_owt_compact_v2048_latest_compare_dir_dual_steps128_c1024_t1p4_n8.log

@@ -0,0 +1,29 @@
+[infer] ckpt=runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/latest.pt
+[infer] rule=dirichlet_resample out=docs/lta_samples/metrics_20260519/owt_compact_v2048_latest_compare_dir_dual_steps128_c1024_t1p4_n8/dirichlet_resample.jsonl
+[ckpt] runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/latest.pt step=75000
+[decode-base] n=8 max_len=1024 steps=128 model_t=post
+[decode-time] schedule=linear s=[0.0,0.25] gumbel=(2.2,0.8) force_final=True t0=0.000000 t_mid=0.500000 t_end=1.000000 dt_mean=0.007812 dt_max=0.007812
+[decode] temp=1.40 final=state rule=dirichlet_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 1/8
+[decode] temp=1.40 final=state rule=dirichlet_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 2/8
+[decode] temp=1.40 final=state rule=dirichlet_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 3/8
+[decode] temp=1.40 final=state rule=dirichlet_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.40 final=state rule=dirichlet_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 5/8
+[decode] temp=1.40 final=state rule=dirichlet_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 6/8
+[decode] temp=1.40 final=state rule=dirichlet_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 7/8
+[decode] temp=1.40 final=state rule=dirichlet_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/latest.pt", "step": 75000, "decode": {"steps": 128, "model_t_mode": "post", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.0078125, 0.015625, 0.0234375, 0.03125, 0.0390625, 0.046875, 0.0546875, 0.0625, 0.0703125, 0.078125, 0.0859375, 0.09375, 0.1015625, 0.109375, 0.1171875, 0.125, 0.1328125, 0.140625, 0.1484375, 0.15625, 0.1640625, 0.171875, 0.1796875, 0.1875, 0.1953125, 0.203125, 0.2109375, 0.21875, 0.2265625, 0.234375, 0.2421875, 0.25, 0.2578125, 0.265625, 0.2734375, 0.28125, 0.2890625, 0.296875, 0.3046875, 0.3125, 0.3203125, 0.328125, 0.3359375, 0.34375, 0.3515625, 0.359375, 0.3671875, 0.375, 0.3828125, 0.390625, 0.3984375, 0.40625, 0.4140625, 0.421875, 0.4296875, 0.4375, 0.4453125, 0.453125, 0.4609375, 0.46875, 0.4765625, 0.484375, 0.4921875, 0.5, 0.5078125, 0.515625, 0.5234375, 0.53125, 0.5390625, 0.546875, 0.5546875, 0.5625, 0.5703125, 0.578125, 0.5859375, 0.59375, 0.6015625, 0.609375, 0.6171875, 0.625, 0.6328125, 0.640625, 0.6484375, 0.65625, 0.6640625, 0.671875, 0.6796875, 0.6875, 0.6953125, 0.703125, 0.7109375, 0.71875, 0.7265625, 0.734375, 0.7421875, 0.75, 0.7578125, 0.765625, 0.7734375, 0.78125, 0.7890625, 0.796875, 0.8046875, 0.8125, 0.8203125, 0.828125, 0.8359375, 0.84375, 0.8515625, 0.859375, 0.8671875, 0.875, 0.8828125, 0.890625, 0.8984375, 0.90625, 0.9140625, 0.921875, 0.9296875, 0.9375, 0.9453125, 0.953125, 0.9609375, 0.96875, 0.9765625, 0.984375, 0.9921875, 1.0], "decode_rule": "dirichlet_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 1024.0, "target_prob": 1.0, "endpoint_temp": 1.4, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 5.721501467126203, "nll_per_token": 1.7442312651989507, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 6.053917778986924, "nll_per_token": 1.800705629236558, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 0.8634121112371498, "unique_tokens": 30, "token_count": 8192, "distinct_1": 0.003662109375, "distinct_2": 0.02297165200391007, "top_token_mass": 0.3948974609375}}
+[done] docs/lta_samples/metrics_20260519/owt_compact_v2048_latest_compare_dir_dual_steps128_c1024_t1p4_n8/dirichlet_resample.jsonl
+[infer] rule=dual_line_resample out=docs/lta_samples/metrics_20260519/owt_compact_v2048_latest_compare_dir_dual_steps128_c1024_t1p4_n8/dual_line_resample.jsonl
+[ckpt] runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/latest.pt step=75000
+[decode-base] n=8 max_len=1024 steps=128 model_t=post
+[decode-time] schedule=linear s=[0.0,0.25] gumbel=(2.2,0.8) force_final=True t0=0.000000 t_mid=0.500000 t_end=1.000000 dt_mean=0.007812 dt_max=0.007812
+[decode] temp=1.40 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 1/8
+[decode] temp=1.40 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 2/8
+[decode] temp=1.40 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 3/8
+[decode] temp=1.40 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 4/8
+[decode] temp=1.40 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 5/8
+[decode] temp=1.40 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 6/8
+[decode] temp=1.40 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 7/8
+[decode] temp=1.40 final=state rule=dual_line_resample support=1 semantic=1 anchor=onehot cfg=0/1@0:uniform decode_freq_penalty=0/0/0-1^1 final_sample=argmax/1/k64/p0.95 freq_penalty=0/0/0 start_t=0 start_init=noise time_path=0.0000->1.0000 generated 8/8
+[summary] {"type": "summary", "checkpoint": "runs/lta_owt_compact_gpt2bpe_v2048_stream1024_fullycoupled_rmsnorm_nobias_adamw_wd0p1_logitnormal_m1p5_s0p8_hardce_mask1p0-1p0_fp32_ddit768x12_gbs512_8gpu_1m_20260517_141027/latest.pt", "step": 75000, "decode": {"steps": 128, "model_t_mode": "post", "decode_time_schedule": "linear", "decode_s_min_frac": 0.0, "decode_s_max_frac": 0.25, "decode_force_final_t": true, "decode_time_grid": [0.0, 0.0078125, 0.015625, 0.0234375, 0.03125, 0.0390625, 0.046875, 0.0546875, 0.0625, 0.0703125, 0.078125, 0.0859375, 0.09375, 0.1015625, 0.109375, 0.1171875, 0.125, 0.1328125, 0.140625, 0.1484375, 0.15625, 0.1640625, 0.171875, 0.1796875, 0.1875, 0.1953125, 0.203125, 0.2109375, 0.21875, 0.2265625, 0.234375, 0.2421875, 0.25, 0.2578125, 0.265625, 0.2734375, 0.28125, 0.2890625, 0.296875, 0.3046875, 0.3125, 0.3203125, 0.328125, 0.3359375, 0.34375, 0.3515625, 0.359375, 0.3671875, 0.375, 0.3828125, 0.390625, 0.3984375, 0.40625, 0.4140625, 0.421875, 0.4296875, 0.4375, 0.4453125, 0.453125, 0.4609375, 0.46875, 0.4765625, 0.484375, 0.4921875, 0.5, 0.5078125, 0.515625, 0.5234375, 0.53125, 0.5390625, 0.546875, 0.5546875, 0.5625, 0.5703125, 0.578125, 0.5859375, 0.59375, 0.6015625, 0.609375, 0.6171875, 0.625, 0.6328125, 0.640625, 0.6484375, 0.65625, 0.6640625, 0.671875, 0.6796875, 0.6875, 0.6953125, 0.703125, 0.7109375, 0.71875, 0.7265625, 0.734375, 0.7421875, 0.75, 0.7578125, 0.765625, 0.7734375, 0.78125, 0.7890625, 0.796875, 0.8046875, 0.8125, 0.8203125, 0.828125, 0.8359375, 0.84375, 0.8515625, 0.859375, 0.8671875, 0.875, 0.8828125, 0.890625, 0.8984375, 0.90625, 0.9140625, 0.921875, 0.9296875, 0.9375, 0.9453125, 0.953125, 0.9609375, 0.96875, 0.9765625, 0.984375, 0.9921875, 1.0], "decode_rule": "dual_line_resample", "support_power": 1.0, "semantic_power": 1.0, "anchor_mode": "onehot", "cfg_scale": 0.0, "cfg_power": 1.0, "cfg_start": 0.0, "cfg_prior": "uniform", "decode_freq_penalty_alpha": 0.0, "decode_freq_penalty_beta": 0.0, "decode_freq_penalty_floor": 0.0, "decode_freq_penalty_start": 0.0, "decode_freq_penalty_end": 1.0, "decode_freq_penalty_power": 1.0, "start_t": 0.0, "start_init": "noise", "noise_init": "dirichlet", "noise_sigma": -1.0, "dirichlet_concentration": 1.0, "concentration_min": 1.0, "concentration_max": 1024.0, "target_prob": 1.0, "endpoint_temp": 1.4, "final_from": "state", "final_sample_mode": "argmax", "final_sample_temp": 1.0, "final_top_k": 64, "final_top_p": 0.95, "final_freq_penalty_alpha": 0.0, "final_freq_penalty_beta": 0.0, "final_freq_penalty_floor": 0.0, "lock_bos": false, "n_samples": 8, "seed": 20260519}, "raw_genppl": {"ppl": 10.269459151870745, "nll_per_token": 2.329174359639486, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "stripped_genppl": {"ppl": 10.226794329481784, "nll_per_token": 2.3250111710791495, "tokens": 2040, "kept_samples": 8, "total_samples": 8, "empty_rate": 0.0, "skipped_samples": 0}, "diversity": {"sample_entropy": 0.8533172316541697, "unique_tokens": 60, "token_count": 8192, "distinct_1": 0.00732421875, "distinct_2": 0.028470185728250243, "top_token_mass": 0.4017333984375}}
+[done] docs/lta_samples/metrics_20260519/owt_compact_v2048_latest_compare_dir_dual_steps128_c1024_t1p4_n8/dual_line_resample.jsonl

LTA_openwebtext_dualt/logs/lm1b_classic_dirichlet_every1k_infer_watch/infer_lta_lm1b_classic_dirichlet_len512_gbs512_4gpu_20k_save1k_20260523_step_0004000_t1p45.log

@@ -0,0 +1,36 @@
+[watch-classic-1k] 2026-05-23_18:14:51 infer runs/lta_lm1b_classic_dirichlet_len512_gbs512_4gpu_20k_save1k_20260523/step_0004000.pt -> docs/lta_samples/metrics_20260523/lm1b_classic_dirichlet_len512_every1k_normal_steps_state_t1p45_c1024_n256/lta_lm1b_classic_dirichlet_len512_gbs512_4gpu_20k_save1k_20260523/step_0004000
+[ckpt] runs/lta_lm1b_classic_dirichlet_len512_gbs512_4gpu_20k_save1k_20260523/step_0004000.pt step=4000
+[decode] steps128_c1024_t1p45 generated 8/256
+[decode] steps128_c1024_t1p45 generated 16/256
+[decode] steps128_c1024_t1p45 generated 24/256
+[decode] steps128_c1024_t1p45 generated 32/256
+[decode] steps128_c1024_t1p45 generated 40/256
+[decode] steps128_c1024_t1p45 generated 48/256
+[decode] steps128_c1024_t1p45 generated 56/256
+[decode] steps128_c1024_t1p45 generated 64/256
+[decode] steps128_c1024_t1p45 generated 72/256
+[decode] steps128_c1024_t1p45 generated 80/256
+[decode] steps128_c1024_t1p45 generated 88/256
+[decode] steps128_c1024_t1p45 generated 96/256
+[decode] steps128_c1024_t1p45 generated 104/256
+[decode] steps128_c1024_t1p45 generated 112/256
+[decode] steps128_c1024_t1p45 generated 120/256
+[decode] steps128_c1024_t1p45 generated 128/256
+[decode] steps128_c1024_t1p45 generated 136/256
+[decode] steps128_c1024_t1p45 generated 144/256
+[decode] steps128_c1024_t1p45 generated 152/256
+[decode] steps128_c1024_t1p45 generated 160/256
+[decode] steps128_c1024_t1p45 generated 168/256
+[decode] steps128_c1024_t1p45 generated 176/256
+[decode] steps128_c1024_t1p45 generated 184/256
+[decode] steps128_c1024_t1p45 generated 192/256
+[decode] steps128_c1024_t1p45 generated 200/256
+[decode] steps128_c1024_t1p45 generated 208/256
+[decode] steps128_c1024_t1p45 generated 216/256
+[decode] steps128_c1024_t1p45 generated 224/256
+[decode] steps128_c1024_t1p45 generated 232/256
+[decode] steps128_c1024_t1p45 generated 240/256
+[decode] steps128_c1024_t1p45 generated 248/256
+[decode] steps128_c1024_t1p45 generated 256/256
+[summary] {"name": "steps128_c1024_t1p45", "step": 4000, "decode_steps": 128, "concentration_max": 1024.0, "raw_genppl": 11.650994762534163, "stripped_genppl": 11.735267921260704, "sample_entropy": 0.14561788086881036, "distinct_1": 0.0013427734375, "distinct_2": 0.00368456457925636, "top_token_mass": 0.9227676391601562, "raw_kept": 256, "stripped_kept": 256}
+[watch-classic-1k] 2026-05-23_18:20:56 done step_0004000

LTA_openwebtext_dualt/logs/lm1b_classic_dirichlet_every1k_infer_watch/infer_lta_lm1b_classic_dirichlet_len512_gbs512_4gpu_20k_save1k_20260523_step_0014000_t1p45.log

@@ -0,0 +1,36 @@
+[watch-classic-1k] 2026-05-23_21:55:07 infer runs/lta_lm1b_classic_dirichlet_len512_gbs512_4gpu_20k_save1k_20260523/step_0014000.pt -> docs/lta_samples/metrics_20260523/lm1b_classic_dirichlet_len512_every1k_normal_steps_state_t1p45_c1024_n256/lta_lm1b_classic_dirichlet_len512_gbs512_4gpu_20k_save1k_20260523/step_0014000
+[ckpt] runs/lta_lm1b_classic_dirichlet_len512_gbs512_4gpu_20k_save1k_20260523/step_0014000.pt step=14000
+[decode] steps128_c1024_t1p45 generated 8/256
+[decode] steps128_c1024_t1p45 generated 16/256
+[decode] steps128_c1024_t1p45 generated 24/256
+[decode] steps128_c1024_t1p45 generated 32/256
+[decode] steps128_c1024_t1p45 generated 40/256
+[decode] steps128_c1024_t1p45 generated 48/256
+[decode] steps128_c1024_t1p45 generated 56/256
+[decode] steps128_c1024_t1p45 generated 64/256
+[decode] steps128_c1024_t1p45 generated 72/256
+[decode] steps128_c1024_t1p45 generated 80/256
+[decode] steps128_c1024_t1p45 generated 88/256
+[decode] steps128_c1024_t1p45 generated 96/256
+[decode] steps128_c1024_t1p45 generated 104/256
+[decode] steps128_c1024_t1p45 generated 112/256
+[decode] steps128_c1024_t1p45 generated 120/256
+[decode] steps128_c1024_t1p45 generated 128/256
+[decode] steps128_c1024_t1p45 generated 136/256
+[decode] steps128_c1024_t1p45 generated 144/256
+[decode] steps128_c1024_t1p45 generated 152/256
+[decode] steps128_c1024_t1p45 generated 160/256
+[decode] steps128_c1024_t1p45 generated 168/256
+[decode] steps128_c1024_t1p45 generated 176/256
+[decode] steps128_c1024_t1p45 generated 184/256
+[decode] steps128_c1024_t1p45 generated 192/256
+[decode] steps128_c1024_t1p45 generated 200/256
+[decode] steps128_c1024_t1p45 generated 208/256
+[decode] steps128_c1024_t1p45 generated 216/256
+[decode] steps128_c1024_t1p45 generated 224/256
+[decode] steps128_c1024_t1p45 generated 232/256
+[decode] steps128_c1024_t1p45 generated 240/256
+[decode] steps128_c1024_t1p45 generated 248/256
+[decode] steps128_c1024_t1p45 generated 256/256
+[summary] {"name": "steps128_c1024_t1p45", "step": 14000, "decode_steps": 128, "concentration_max": 1024.0, "raw_genppl": 26.115239555828243, "stripped_genppl": 28.510617958544135, "sample_entropy": 4.184041060828689, "distinct_1": 0.02449798583984375, "distinct_2": 0.16947468199608612, "top_token_mass": 0.12488555908203125, "raw_kept": 256, "stripped_kept": 256}
+[watch-classic-1k] 2026-05-23_22:01:32 done step_0014000

LTA_openwebtext_dualt/logs/lta_lm1b_bert_absrope_time4_dirichlet_len128_C1_to_1024_mask1_sameT_gbs512_b32_4gpu_1m_save1k_20260526.log

@@ -0,0 +1,193 @@
+
+*****************************************
+Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+*****************************************
+NCCL version 2.25.1+cuda12.8
+{
+  "device": "cuda:0",
+  "rank": 0,
+  "world_size": 4,
+  "samples": "wrapped_stream",
+  "vocab_size": 30522,
+  "tokenizer_vocab_size": 30522,
+  "save_dir": "runs/lta_lm1b_bert_absrope_time4_dirichlet_len128_C1_to_1024_mask1_sameT_gbs512_b32_4gpu_1m_save1k_20260526",
+  "max_len": 128,
+  "effective_model_max_len": 128,
+  "batch_size": 32,
+  "grad_accum": 4,
+  "effective_batch_size": 512,
+  "global_batch_size": 512,
+  "lr_schedule": "constant_warmup",
+  "optimizer": "adamw",
+  "epochs": 0.0,
+  "steps_per_epoch": 0,
+  "total_steps": 1000000,
+  "warmup_steps": 2500,
+  "warmup_epochs": -1.0,
+  "min_lr": 6e-05,
+  "weight_decay": 0.0,
+  "output_weight_decay": -1.0,
+  "adamw_param_groups": "nanogpt",
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "adam_eps": 1e-08,
+  "muon_impl": "legacy",
+  "muon_momentum": 0.95,
+  "muon_ns_steps": 5,
+  "muon_update_scale": 1.0,
+  "muon_nesterov": false,
+  "muon_width_scale": false,
+  "muon_grouping": "",
+  "muon_param_count": 0,
+  "muon_adam_param_count": 0,
+  "muon_param_names": [],
+  "muon_adam_param_names": [],
+  "muon_effective_nesterov": false,
+  "muon_effective_width_scale": false,
+  "muon_effective_weight_decay": 0.0,
+  "muon_adam_fallback_nesterov": false,
+  "muon_adam_fallback_weight_decay": 0.0,
+  "ema_decay": 0.0,
+  "ema_start_step": 0,
+  "model_type": "ddit_elf",
+  "ddit_mlp_type": "gelu",
+  "elf_num_time_tokens": 4,
+  "elf_num_model_mode_tokens": 0,
+  "abs_pos_embed": true,
+  "qk_norm": true,
+  "output_bias": false,
+  "output_init_std": -1.0,
+  "norm_type": "rmsnorm",
+  "target_loss": "hard_ce",
+  "linear_soft_target_power": 1.0,
+  "linear_soft_target_min_conf": 0.0,
+  "linear_soft_target_max_conf": 1.0,
+  "t_sampling_mode": "uniform",
+  "t_sampling_power": 1.0,
+  "t_sampling_eps": 0.0001,
+  "t_sampling_logit_mean": -1.5,
+  "t_sampling_logit_std": 0.8,
+  "t_sampling_gumbel_loc": 2.2,
+  "t_sampling_gumbel_scale": 0.8,
+  "dual_t": true,
+  "corrupt_t_mode": "same",
+  "corrupt_min_t": 0.0,
+  "corrupt_max_t": 1.0,
+  "prefix_block_prob": 0.0,
+  "prefix_block_len": 128,
+  "block_ar_two_stream": false,
+  "block_ar_block_len": 128,
+  "mask_ratio_floor_schedule": "none",
+  "dirichlet_endpoint_mode": "categorical_dual_t",
+  "dirichlet_semantic_t_mode": "same",
+  "dirichlet_semantic_t_value": 0.0,
+  "dirichlet_semantic_t_curve": "linear",
+  "dirichlet_semantic_t_power": 1.0,
+  "dirichlet_support_t_curve": "linear",
+  "dirichlet_support_t_power": 1.0,
+  "endpoint_sequence_random_prob_alpha": 0.0,
+  "categorical_wrong_from_full_vocab": true,
+  "categorical_wrong_from_batch_valid_tokens": false,
+  "categorical_wrong_basin_token_ids": "",
+  "categorical_wrong_basin_prob": 0.0,
+  "categorical_wrong_unigram_prob": 0.0,
+  "categorical_wrong_uniform_prob": 0.0,
+  "categorical_wrong_prob_floor": 0.0,
+  "categorical_gold_prob_floor": 0.0,
+  "categorical_gold_prob_ceil": 1.0,
+  "categorical_wrong_corpus_unigram_path": "",
+  "categorical_wrong_corpus_unigram_alpha": 1.0,
+  "categorical_wrong_basin_shared_prob": 0.0,
+  "categorical_wrong_unigram_shared_prob": 0.0,
+  "mask_mixture_original_prob": 0.0,
+  "mask_mixture_lowk_prob": 0.0,
+  "mask_mixture_lowcorrupt_prob": 0.0,
+  "mask_mixture_block_prob": 0.0,
+  "mask_mixture_all_prob": 0.0,
+  "mask_mixture_lowk_clean_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_block_tokens": "64,128",
+  "simplex_bridge_sampler": "dirichlet",
+  "logistic_normal_sigma_min": 0.18,
+  "logistic_normal_sigma_max": 2.2,
+  "logistic_normal_tau_min": 0.65,
+  "logistic_normal_tau_max": 1.15,
+  "torch_compile": false,
+  "compile_mode": "max-autotune",
+  "state_format": "prob",
+  "meanflow_weight": 0.0,
+  "rollout_train_prob": 0.0,
+  "rollout_train_steps": 1,
+  "rollout_train_steps_min": -1,
+  "rollout_train_infer_steps": 64,
+  "rollout_train_time_mode": "fixed_steps",
+  "rollout_train_s_dist": "uniform",
+  "rollout_train_s_min_frac": 0.0,
+  "rollout_train_s_max_frac": 0.125,
+  "rollout_train_s_beta_alpha": 2.0,
+  "rollout_train_s_beta_beta": 6.0,
+  "rollout_train_temp": 1.0,
+  "rollout_train_max_gamma": 1.0,
+  "rollout_train_rule": "flowmap",
+  "rollout_train_corrupt_only": true,
+  "rollout_train_samplewise": false,
+  "rollout_train_compute_always": false,
+  "rollout_train_keep_grad": false,
+  "rollout_train_sync_t": false,
+  "rollout_train_state_mix_mode": "final",
+  "rollout_train_state_mix_alpha": 0.5,
+  "bridge_noise_init": "logistic_normal",
+  "noise_sigma": -1.0,
+  "allow_tf32": true,
+  "activation_checkpointing": false,
+  "activation_checkpoint_interval": 1,
+  "activation_checkpoint_scope": "block",
+  "ddp_static_graph": false,
+  "ddp_gradient_as_bucket_view": true,
+  "blocking_data_transfer": false,
+  "dataloader_prefetch_factor": 2,
+  "full_train_stats": false,
+  "tokenized_hf": false,
+  "tokenized_pad_token": "pad",
+  "elf_conditional_hf": false,
+  "record_pad_truncate": false,
+  "record_add_eos": false,
+  "record_add_special_tokens": false,
+  "record_pad_token": "pad",
+  "record_shuffle_buffer": 10000,
+  "wrap": true,
+  "wrap_mode": "stream",
+  "wrap_record_buffer_size": 200,
+  "owt_cached_chunks": false,
+  "owt_chunk_cache_dir": "",
+  "owt_chunk_cache_rebuild": false,
+  "owt_chunk_cache_write_batch": 4096,
+  "owt_exact_repeat_per_chunk": 0,
+  "online_chunk_shuffle": false,
+  "online_chunk_shuffle_buffer": 10000,
+  "openwebtext_split": "all",
+  "detokenizer": "auto",
+  "resolved_detokenizer": "lm1b",
+  "num_workers": 0,
+  "latest_every": 1000,
+  "resume_path": ""
+}
+step=100 micro_steps=400 elapsed=186.5s lr=1.212000e-05 loss=10.1123 loss_recon=10.1123 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0303 corrupt_frac=1.0000 acc_corrupt=0.0303 loss_corrupt=10.1123 wrong_frac=0.4988 init_acc_corrupt=0.4667 acc_corrupt_t_0p0_0p2=0.0293 corrupt_frac_t_0p0_0p2=0.2025 acc_corrupt_t_0p2_0p4=0.0302 corrupt_frac_t_0p2_0p4=0.1960 acc_corrupt_t_0p4_0p6=0.0309 corrupt_frac_t_0p4_0p6=0.1990 acc_corrupt_t_0p6_0p8=0.0314 corrupt_frac_t_0p6_0p8=0.1995 acc_corrupt_t_0p8_1p0=0.0300 corrupt_frac_t_0p8_1p0=0.2030 out_w_norm=0.8830 out_g_norm=1.5783 loss_all=9.6849 init_gold_top10=0.5627 init_gold_top100=0.5703
+step=200 micro_steps=800 elapsed=190.5s lr=2.412000e-05 loss=9.0364 loss_recon=9.0364 loss_meanflow=0.0000 mean_model_t=0.5016 mean_corrupt_t=0.5016 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0307 corrupt_frac=1.0000 acc_corrupt=0.0307 loss_corrupt=9.0364 wrong_frac=0.4983 init_acc_corrupt=0.4677 acc_corrupt_t_0p0_0p2=0.0307 corrupt_frac_t_0p0_0p2=0.2011 acc_corrupt_t_0p2_0p4=0.0315 corrupt_frac_t_0p2_0p4=0.1982 acc_corrupt_t_0p4_0p6=0.0308 corrupt_frac_t_0p4_0p6=0.1974 acc_corrupt_t_0p6_0p8=0.0302 corrupt_frac_t_0p6_0p8=0.1988 acc_corrupt_t_0p8_1p0=0.0304 corrupt_frac_t_0p8_1p0=0.2059 out_w_norm=5.6664 out_g_norm=1.7135 loss_all=8.3520 init_gold_top10=0.4731 init_gold_top100=0.4768
+step=300 micro_steps=1200 elapsed=191.3s lr=3.612000e-05 loss=7.7637 loss_recon=7.7637 loss_meanflow=0.0000 mean_model_t=0.4957 mean_corrupt_t=0.4957 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0421 corrupt_frac=1.0000 acc_corrupt=0.0421 loss_corrupt=7.7637 wrong_frac=0.5036 init_acc_corrupt=0.4610 acc_corrupt_t_0p0_0p2=0.0423 corrupt_frac_t_0p0_0p2=0.2036 acc_corrupt_t_0p2_0p4=0.0424 corrupt_frac_t_0p2_0p4=0.2020 acc_corrupt_t_0p4_0p6=0.0419 corrupt_frac_t_0p4_0p6=0.2033 acc_corrupt_t_0p6_0p8=0.0419 corrupt_frac_t_0p6_0p8=0.1990 acc_corrupt_t_0p8_1p0=0.0421 corrupt_frac_t_0p8_1p0=0.1941 out_w_norm=12.5051 out_g_norm=1.5277 loss_all=7.2514 init_gold_top10=0.5310 init_gold_top100=0.5371
+step=400 micro_steps=1600 elapsed=169.7s lr=4.812000e-05 loss=7.0507 loss_recon=7.0507 loss_meanflow=0.0000 mean_model_t=0.5004 mean_corrupt_t=0.5004 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0453 corrupt_frac=1.0000 acc_corrupt=0.0453 loss_corrupt=7.0507 wrong_frac=0.4999 init_acc_corrupt=0.4660 acc_corrupt_t_0p0_0p2=0.0449 corrupt_frac_t_0p0_0p2=0.2009 acc_corrupt_t_0p2_0p4=0.0457 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.0450 corrupt_frac_t_0p4_0p6=0.1975 acc_corrupt_t_0p6_0p8=0.0456 corrupt_frac_t_0p6_0p8=0.2042 acc_corrupt_t_0p8_1p0=0.0455 corrupt_frac_t_0p8_1p0=0.1983 out_w_norm=19.3630 out_g_norm=0.4905 loss_all=6.8370 init_gold_top10=0.5823 init_gold_top100=0.5859
+step=500 micro_steps=2000 elapsed=168.7s lr=6.012000e-05 loss=6.8048 loss_recon=6.8048 loss_meanflow=0.0000 mean_model_t=0.5017 mean_corrupt_t=0.5017 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0569 corrupt_frac=1.0000 acc_corrupt=0.0569 loss_corrupt=6.8048 wrong_frac=0.4982 init_acc_corrupt=0.4674 acc_corrupt_t_0p0_0p2=0.0563 corrupt_frac_t_0p0_0p2=0.1966 acc_corrupt_t_0p2_0p4=0.0573 corrupt_frac_t_0p2_0p4=0.1988 acc_corrupt_t_0p4_0p6=0.0564 corrupt_frac_t_0p4_0p6=0.2046 acc_corrupt_t_0p6_0p8=0.0570 corrupt_frac_t_0p6_0p8=0.1975 acc_corrupt_t_0p8_1p0=0.0576 corrupt_frac_t_0p8_1p0=0.2036 out_w_norm=24.2355 out_g_norm=0.2417 loss_all=6.7435 init_gold_top10=0.4373 init_gold_top100=0.4431
+step=600 micro_steps=2400 elapsed=168.4s lr=7.212000e-05 loss=6.4767 loss_recon=6.4767 loss_meanflow=0.0000 mean_model_t=0.4973 mean_corrupt_t=0.4973 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0951 corrupt_frac=1.0000 acc_corrupt=0.0951 loss_corrupt=6.4767 wrong_frac=0.5030 init_acc_corrupt=0.4624 acc_corrupt_t_0p0_0p2=0.0637 corrupt_frac_t_0p0_0p2=0.2057 acc_corrupt_t_0p2_0p4=0.0788 corrupt_frac_t_0p2_0p4=0.1970 acc_corrupt_t_0p4_0p6=0.0988 corrupt_frac_t_0p4_0p6=0.2034 acc_corrupt_t_0p6_0p8=0.1124 corrupt_frac_t_0p6_0p8=0.1965 acc_corrupt_t_0p8_1p0=0.1232 corrupt_frac_t_0p8_1p0=0.1980 out_w_norm=27.9859 out_g_norm=0.3179 loss_all=6.2120 init_gold_top10=0.5012 init_gold_top100=0.5088
+step=700 micro_steps=2800 elapsed=168.5s lr=8.412000e-05 loss=5.8641 loss_recon=5.8641 loss_meanflow=0.0000 mean_model_t=0.4936 mean_corrupt_t=0.4936 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.2140 corrupt_frac=1.0000 acc_corrupt=0.2140 loss_corrupt=5.8641 wrong_frac=0.5063 init_acc_corrupt=0.4583 acc_corrupt_t_0p0_0p2=0.0739 corrupt_frac_t_0p0_0p2=0.2048 acc_corrupt_t_0p2_0p4=0.1517 corrupt_frac_t_0p2_0p4=0.2040 acc_corrupt_t_0p4_0p6=0.2294 corrupt_frac_t_0p4_0p6=0.2013 acc_corrupt_t_0p6_0p8=0.2895 corrupt_frac_t_0p6_0p8=0.1971 acc_corrupt_t_0p8_1p0=0.3351 corrupt_frac_t_0p8_1p0=0.1947 out_w_norm=32.6670 out_g_norm=0.3211 loss_all=5.3731 init_gold_top10=0.5059 init_gold_top100=0.5112
+step=800 micro_steps=3200 elapsed=168.9s lr=9.612000e-05 loss=5.2292 loss_recon=5.2292 loss_meanflow=0.0000 mean_model_t=0.4981 mean_corrupt_t=0.4981 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3267 corrupt_frac=1.0000 acc_corrupt=0.3267 loss_corrupt=5.2292 wrong_frac=0.5016 init_acc_corrupt=0.4637 acc_corrupt_t_0p0_0p2=0.0815 corrupt_frac_t_0p0_0p2=0.2012 acc_corrupt_t_0p2_0p4=0.2131 corrupt_frac_t_0p2_0p4=0.2014 acc_corrupt_t_0p4_0p6=0.3479 corrupt_frac_t_0p4_0p6=0.1986 acc_corrupt_t_0p6_0p8=0.4521 corrupt_frac_t_0p6_0p8=0.2019 acc_corrupt_t_0p8_1p0=0.5435 corrupt_frac_t_0p8_1p0=0.1970 out_w_norm=37.6251 out_g_norm=0.2018 loss_all=4.9786 init_gold_top10=0.4893 init_gold_top100=0.4961
+step=900 micro_steps=3600 elapsed=168.9s lr=1.081200e-04 loss=4.7043 loss_recon=4.7043 loss_meanflow=0.0000 mean_model_t=0.5073 mean_corrupt_t=0.5073 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3935 corrupt_frac=1.0000 acc_corrupt=0.3935 loss_corrupt=4.7043 wrong_frac=0.4933 init_acc_corrupt=0.4729 acc_corrupt_t_0p0_0p2=0.0859 corrupt_frac_t_0p0_0p2=0.1908 acc_corrupt_t_0p2_0p4=0.2439 corrupt_frac_t_0p2_0p4=0.1955 acc_corrupt_t_0p4_0p6=0.4140 corrupt_frac_t_0p4_0p6=0.2045 acc_corrupt_t_0p6_0p8=0.5422 corrupt_frac_t_0p6_0p8=0.2084 acc_corrupt_t_0p8_1p0=0.6559 corrupt_frac_t_0p8_1p0=0.2014 out_w_norm=42.6753 out_g_norm=0.1519 loss_all=4.5171 init_gold_top10=0.5010 init_gold_top100=0.5071
+step=1000 micro_steps=4000 elapsed=169.5s lr=1.201200e-04 loss=4.4151 loss_recon=4.4151 loss_meanflow=0.0000 mean_model_t=0.4991 mean_corrupt_t=0.4991 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4339 corrupt_frac=1.0000 acc_corrupt=0.4339 loss_corrupt=4.4151 wrong_frac=0.5013 init_acc_corrupt=0.4646 acc_corrupt_t_0p0_0p2=0.0865 corrupt_frac_t_0p0_0p2=0.2031 acc_corrupt_t_0p2_0p4=0.2662 corrupt_frac_t_0p2_0p4=0.1991 acc_corrupt_t_0p4_0p6=0.4582 corrupt_frac_t_0p4_0p6=0.1998 acc_corrupt_t_0p6_0p8=0.6153 corrupt_frac_t_0p6_0p8=0.1950 acc_corrupt_t_0p8_1p0=0.7476 corrupt_frac_t_0p8_1p0=0.2034 out_w_norm=47.9426 out_g_norm=0.1355 loss_all=4.4417 init_gold_top10=0.4824 init_gold_top100=0.4880
+step=1100 micro_steps=4400 elapsed=210.8s lr=1.321200e-04 loss=4.1825 loss_recon=4.1825 loss_meanflow=0.0000 mean_model_t=0.4995 mean_corrupt_t=0.4995 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4663 corrupt_frac=1.0000 acc_corrupt=0.4663 loss_corrupt=4.1825 wrong_frac=0.5006 init_acc_corrupt=0.4649 acc_corrupt_t_0p0_0p2=0.0887 corrupt_frac_t_0p0_0p2=0.2027 acc_corrupt_t_0p2_0p4=0.2773 corrupt_frac_t_0p2_0p4=0.1986 acc_corrupt_t_0p4_0p6=0.4939 corrupt_frac_t_0p4_0p6=0.1970 acc_corrupt_t_0p6_0p8=0.6610 corrupt_frac_t_0p6_0p8=0.2014 acc_corrupt_t_0p8_1p0=0.8100 corrupt_frac_t_0p8_1p0=0.2013 out_w_norm=53.2951 out_g_norm=0.1301 loss_all=4.2400 init_gold_top10=0.4680 init_gold_top100=0.4744
+step=1200 micro_steps=4800 elapsed=217.5s lr=1.441200e-04 loss=4.0361 loss_recon=4.0361 loss_meanflow=0.0000 mean_model_t=0.4959 mean_corrupt_t=0.4959 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4813 corrupt_frac=1.0000 acc_corrupt=0.4813 loss_corrupt=4.0361 wrong_frac=0.5039 init_acc_corrupt=0.4606 acc_corrupt_t_0p0_0p2=0.0911 corrupt_frac_t_0p0_0p2=0.2047 acc_corrupt_t_0p2_0p4=0.2904 corrupt_frac_t_0p2_0p4=0.2081 acc_corrupt_t_0p4_0p6=0.5115 corrupt_frac_t_0p4_0p6=0.1931 acc_corrupt_t_0p6_0p8=0.6914 corrupt_frac_t_0p6_0p8=0.1963 acc_corrupt_t_0p8_1p0=0.8453 corrupt_frac_t_0p8_1p0=0.1993 out_w_norm=58.5677 out_g_norm=0.1257 loss_all=3.9159 init_gold_top10=0.5178 init_gold_top100=0.5210
+step=1300 micro_steps=5200 elapsed=213.6s lr=1.561200e-04 loss=3.8590 loss_recon=3.8590 loss_meanflow=0.0000 mean_model_t=0.5035 mean_corrupt_t=0.5035 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5001 corrupt_frac=1.0000 acc_corrupt=0.5001 loss_corrupt=3.8590 wrong_frac=0.4961 init_acc_corrupt=0.4694 acc_corrupt_t_0p0_0p2=0.0917 corrupt_frac_t_0p0_0p2=0.2007 acc_corrupt_t_0p2_0p4=0.2950 corrupt_frac_t_0p2_0p4=0.1952 acc_corrupt_t_0p4_0p6=0.5277 corrupt_frac_t_0p4_0p6=0.1989 acc_corrupt_t_0p6_0p8=0.7082 corrupt_frac_t_0p6_0p8=0.1996 acc_corrupt_t_0p8_1p0=0.8635 corrupt_frac_t_0p8_1p0=0.2066 out_w_norm=63.6326 out_g_norm=0.1224 loss_all=3.3433 init_gold_top10=0.5500 init_gold_top100=0.5549
+step=1400 micro_steps=5600 elapsed=170.8s lr=1.681200e-04 loss=3.8081 loss_recon=3.8081 loss_meanflow=0.0000 mean_model_t=0.4996 mean_corrupt_t=0.4996 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5029 corrupt_frac=1.0000 acc_corrupt=0.5029 loss_corrupt=3.8081 wrong_frac=0.5007 init_acc_corrupt=0.4649 acc_corrupt_t_0p0_0p2=0.0926 corrupt_frac_t_0p0_0p2=0.1982 acc_corrupt_t_0p2_0p4=0.3024 corrupt_frac_t_0p2_0p4=0.2064 acc_corrupt_t_0p4_0p6=0.5355 corrupt_frac_t_0p4_0p6=0.2013 acc_corrupt_t_0p6_0p8=0.7179 corrupt_frac_t_0p6_0p8=0.1927 acc_corrupt_t_0p8_1p0=0.8722 corrupt_frac_t_0p8_1p0=0.2020 out_w_norm=68.3892 out_g_norm=0.1220 loss_all=3.3125 init_gold_top10=0.5654 init_gold_top100=0.5728
+step=1500 micro_steps=6000 elapsed=172.2s lr=1.801200e-04 loss=3.7324 loss_recon=3.7324 loss_meanflow=0.0000 mean_model_t=0.5012 mean_corrupt_t=0.5012 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5097 corrupt_frac=1.0000 acc_corrupt=0.5097 loss_corrupt=3.7324 wrong_frac=0.4984 init_acc_corrupt=0.4672 acc_corrupt_t_0p0_0p2=0.0927 corrupt_frac_t_0p0_0p2=0.1995 acc_corrupt_t_0p2_0p4=0.3029 corrupt_frac_t_0p2_0p4=0.1982 acc_corrupt_t_0p4_0p6=0.5403 corrupt_frac_t_0p4_0p6=0.2019 acc_corrupt_t_0p6_0p8=0.7248 corrupt_frac_t_0p6_0p8=0.2023 acc_corrupt_t_0p8_1p0=0.8839 corrupt_frac_t_0p8_1p0=0.1987 out_w_norm=72.8833 out_g_norm=0.1208 loss_all=4.1249 init_gold_top10=0.4473 init_gold_top100=0.4573
+step=1600 micro_steps=6400 elapsed=172.8s lr=1.921200e-04 loss=3.6736 loss_recon=3.6736 loss_meanflow=0.0000 mean_model_t=0.5013 mean_corrupt_t=0.5013 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5143 corrupt_frac=1.0000 acc_corrupt=0.5143 loss_corrupt=3.6736 wrong_frac=0.4987 init_acc_corrupt=0.4670 acc_corrupt_t_0p0_0p2=0.0922 corrupt_frac_t_0p0_0p2=0.1989 acc_corrupt_t_0p2_0p4=0.3047 corrupt_frac_t_0p2_0p4=0.1967 acc_corrupt_t_0p4_0p6=0.5451 corrupt_frac_t_0p4_0p6=0.2019 acc_corrupt_t_0p6_0p8=0.7290 corrupt_frac_t_0p6_0p8=0.2000 acc_corrupt_t_0p8_1p0=0.8899 corrupt_frac_t_0p8_1p0=0.2025 out_w_norm=77.1516 out_g_norm=0.1206 loss_all=3.4935 init_gold_top10=0.5088 init_gold_top100=0.5166
+step=1700 micro_steps=6800 elapsed=171.9s lr=2.041200e-04 loss=3.5917 loss_recon=3.5917 loss_meanflow=0.0000 mean_model_t=0.5053 mean_corrupt_t=0.5053 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5223 corrupt_frac=1.0000 acc_corrupt=0.5223 loss_corrupt=3.5917 wrong_frac=0.4942 init_acc_corrupt=0.4717 acc_corrupt_t_0p0_0p2=0.0939 corrupt_frac_t_0p0_0p2=0.1970 acc_corrupt_t_0p2_0p4=0.3090 corrupt_frac_t_0p2_0p4=0.1931 acc_corrupt_t_0p4_0p6=0.5527 corrupt_frac_t_0p4_0p6=0.2037 acc_corrupt_t_0p6_0p8=0.7371 corrupt_frac_t_0p6_0p8=0.2016 acc_corrupt_t_0p8_1p0=0.8944 corrupt_frac_t_0p8_1p0=0.2050 out_w_norm=81.2332 out_g_norm=0.1188 loss_all=2.9977 init_gold_top10=0.5801 init_gold_top100=0.5813
+step=1800 micro_steps=7200 elapsed=171.1s lr=2.161200e-04 loss=3.6236 loss_recon=3.6236 loss_meanflow=0.0000 mean_model_t=0.4975 mean_corrupt_t=0.4975 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5167 corrupt_frac=1.0000 acc_corrupt=0.5167 loss_corrupt=3.6236 wrong_frac=0.5032 init_acc_corrupt=0.4621 acc_corrupt_t_0p0_0p2=0.0940 corrupt_frac_t_0p0_0p2=0.2045 acc_corrupt_t_0p2_0p4=0.3115 corrupt_frac_t_0p2_0p4=0.1988 acc_corrupt_t_0p4_0p6=0.5554 corrupt_frac_t_0p4_0p6=0.1986 acc_corrupt_t_0p6_0p8=0.7369 corrupt_frac_t_0p6_0p8=0.2023 acc_corrupt_t_0p8_1p0=0.8991 corrupt_frac_t_0p8_1p0=0.1967 out_w_norm=85.1363 out_g_norm=0.1236 loss_all=4.2999 init_gold_top10=0.3867 init_gold_top100=0.4004
+step=1900 micro_steps=7600 elapsed=169.2s lr=2.281200e-04 loss=3.5647 loss_recon=3.5647 loss_meanflow=0.0000 mean_model_t=0.4990 mean_corrupt_t=0.4990 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5224 corrupt_frac=1.0000 acc_corrupt=0.5224 loss_corrupt=3.5647 wrong_frac=0.5005 init_acc_corrupt=0.4652 acc_corrupt_t_0p0_0p2=0.0948 corrupt_frac_t_0p0_0p2=0.2051 acc_corrupt_t_0p2_0p4=0.3171 corrupt_frac_t_0p2_0p4=0.1954 acc_corrupt_t_0p4_0p6=0.5616 corrupt_frac_t_0p4_0p6=0.2013 acc_corrupt_t_0p6_0p8=0.7467 corrupt_frac_t_0p6_0p8=0.2020 acc_corrupt_t_0p8_1p0=0.9013 corrupt_frac_t_0p8_1p0=0.1967 out_w_norm=88.8849 out_g_norm=0.1189 loss_all=3.5332 init_gold_top10=0.4932 init_gold_top100=0.4985

LTA_openwebtext_dualt/logs/lta_lm1b_dirichlet_len1024_Cv_to_2v_gbs512_8gpu_20k_save1k_gumbelwatch_20260524_watcher.pid

@@ -0,0 +1 @@
+10234

LTA_openwebtext_dualt/logs/lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_4gpu_20k_save1k_20260525.nohup.log

@@ -0,0 +1,204 @@
+[launch] run=lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_4gpu_20k_save1k_20260525
+[launch] data_path=/e2e-data/evad-tech-vla/wanghan58/data/embedded-language-flows/openwebtext-t5 mode=tokenized_hf_pad_pad split=train_minus_100k text_column=text
+[launch] max_len=1024 gbs=512 per_gpu=32 total_steps=20000
+[launch] dirichlet C=32100->64200 wrong_floor=0.0
+[launch] abs_pos=0 force_special_corrupt=none special_endpoint_gold=none special_loss=nonex1.0
+[launch] watcher disabled
+
+*****************************************
+Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+*****************************************
+NCCL version 2.25.1+cuda12.8
+{
+  "device": "cuda:0",
+  "rank": 0,
+  "world_size": 4,
+  "samples": "tokenized_hf:9737184:pad=0",
+  "vocab_size": 32100,
+  "tokenizer_vocab_size": 32100,
+  "save_dir": "runs/lta_owt_t5elf_dirichlet_len1024_Cv_to_2v_mask1_gbs512_b32_4gpu_20k_save1k_20260525",
+  "max_len": 1024,
+  "effective_model_max_len": 1024,
+  "batch_size": 32,
+  "grad_accum": 4,
+  "effective_batch_size": 512,
+  "global_batch_size": 512,
+  "lr_schedule": "constant_warmup",
+  "optimizer": "adamw",
+  "epochs": 0.0,
+  "steps_per_epoch": 19018,
+  "total_steps": 20000,
+  "warmup_steps": 2500,
+  "warmup_epochs": -1.0,
+  "min_lr": 6e-05,
+  "weight_decay": 0.0,
+  "output_weight_decay": -1.0,
+  "adamw_param_groups": "nanogpt",
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "adam_eps": 1e-08,
+  "muon_impl": "legacy",
+  "muon_momentum": 0.95,
+  "muon_ns_steps": 5,
+  "muon_update_scale": 1.0,
+  "muon_nesterov": false,
+  "muon_width_scale": false,
+  "muon_grouping": "",
+  "muon_param_count": 0,
+  "muon_adam_param_count": 0,
+  "muon_param_names": [],
+  "muon_adam_param_names": [],
+  "muon_effective_nesterov": false,
+  "muon_effective_width_scale": false,
+  "muon_effective_weight_decay": 0.0,
+  "muon_adam_fallback_nesterov": false,
+  "muon_adam_fallback_weight_decay": 0.0,
+  "ema_decay": 0.0,
+  "ema_start_step": 0,
+  "model_type": "ddit",
+  "ddit_mlp_type": "gelu",
+  "block_anchor_every": 0,
+  "block_anchor_init_std": 0.02,
+  "bos_anchor_every": 0,
+  "bos_anchor_token_id": -1,
+  "bos_anchor_extra_len": 0,
+  "abs_pos_embed": false,
+  "abs_pos_init_std": 0.02,
+  "elf_num_time_tokens": 4,
+  "elf_num_model_mode_tokens": 0,
+  "qk_norm": true,
+  "output_bias": false,
+  "output_init_std": -1.0,
+  "norm_type": "rmsnorm",
+  "target_loss": "hard_ce",
+  "linear_soft_target_power": 1.0,
+  "linear_soft_target_min_conf": 0.0,
+  "linear_soft_target_max_conf": 1.0,
+  "t_sampling_mode": "uniform",
+  "t_sampling_power": 1.0,
+  "t_sampling_eps": 0.0001,
+  "t_sampling_logit_mean": -1.5,
+  "t_sampling_logit_std": 0.8,
+  "t_sampling_gumbel_loc": 2.2,
+  "t_sampling_gumbel_scale": 0.8,
+  "dual_t": true,
+  "corrupt_t_mode": "independent",
+  "corrupt_min_t": 0.0,
+  "corrupt_max_t": 1.0,
+  "prefix_block_prob": 0.0,
+  "prefix_block_len": 128,
+  "block_ar_two_stream": false,
+  "block_ar_block_len": 128,
+  "mask_ratio_floor_schedule": "none",
+  "dirichlet_endpoint_mode": "categorical_dual_t",
+  "dirichlet_semantic_t_mode": "same",
+  "dirichlet_semantic_t_value": 0.0,
+  "dirichlet_semantic_t_curve": "linear",
+  "dirichlet_semantic_t_power": 1.0,
+  "dirichlet_support_t_curve": "linear",
+  "dirichlet_support_t_power": 1.0,
+  "endpoint_sequence_random_prob_alpha": 0.0,
+  "categorical_wrong_from_full_vocab": true,
+  "categorical_wrong_from_batch_valid_tokens": false,
+  "categorical_wrong_basin_token_ids": "",
+  "categorical_wrong_basin_prob": 0.0,
+  "categorical_wrong_unigram_prob": 0.0,
+  "categorical_wrong_uniform_prob": 0.0,
+  "categorical_wrong_prob_floor": 0.0,
+  "categorical_gold_prob_floor": 0.0,
+  "categorical_gold_prob_ceil": 1.0,
+  "categorical_wrong_corpus_unigram_path": "",
+  "categorical_wrong_corpus_unigram_alpha": 1.0,
+  "categorical_wrong_basin_shared_prob": 0.0,
+  "categorical_wrong_unigram_shared_prob": 0.0,
+  "mask_mixture_original_prob": 0.0,
+  "mask_mixture_lowk_prob": 0.0,
+  "mask_mixture_lowcorrupt_prob": 0.0,
+  "mask_mixture_block_prob": 0.0,
+  "mask_mixture_all_prob": 0.0,
+  "mask_mixture_lowk_clean_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_block_tokens": "64,128",
+  "simplex_bridge_sampler": "dirichlet",
+  "logistic_normal_sigma_min": 0.18,
+  "logistic_normal_sigma_max": 2.2,
+  "logistic_normal_tau_min": 0.65,
+  "logistic_normal_tau_max": 1.15,
+  "torch_compile": false,
+  "compile_mode": "max-autotune",
+  "state_format": "prob",
+  "meanflow_weight": 0.0,
+  "rollout_train_prob": 0.0,
+  "rollout_train_steps": 1,
+  "rollout_train_steps_min": -1,
+  "rollout_train_infer_steps": 64,
+  "rollout_train_time_mode": "fixed_steps",
+  "rollout_train_s_dist": "uniform",
+  "rollout_train_s_min_frac": 0.0,
+  "rollout_train_s_max_frac": 0.125,
+  "rollout_train_s_beta_alpha": 2.0,
+  "rollout_train_s_beta_beta": 6.0,
+  "rollout_train_temp": 1.0,
+  "rollout_train_max_gamma": 1.0,
+  "rollout_train_rule": "flowmap",
+  "rollout_train_corrupt_only": true,
+  "rollout_train_samplewise": false,
+  "rollout_train_compute_always": false,
+  "rollout_train_keep_grad": false,
+  "rollout_train_sync_t": false,
+  "rollout_train_state_mix_mode": "final",
+  "rollout_train_state_mix_alpha": 0.5,
+  "bridge_noise_init": "logistic_normal",
+  "noise_sigma": -1.0,
+  "allow_tf32": true,
+  "activation_checkpointing": false,
+  "activation_checkpoint_interval": 1,
+  "activation_checkpoint_scope": "block",
+  "ddp_static_graph": false,
+  "ddp_gradient_as_bucket_view": true,
+  "blocking_data_transfer": false,
+  "dataloader_prefetch_factor": 2,
+  "full_train_stats": false,
+  "tokenized_hf": true,
+  "tokenized_pad_token": "pad",
+  "elf_conditional_hf": false,
+  "record_pad_truncate": false,
+  "record_add_eos": false,
+  "record_add_special_tokens": false,
+  "record_pad_token": "pad",
+  "record_shuffle_buffer": 10000,
+  "wrap": false,
+  "wrap_mode": "stream",
+  "wrap_record_buffer_size": 200,
+  "owt_cached_chunks": false,
+  "owt_chunk_cache_dir": "",
+  "owt_chunk_cache_rebuild": false,
+  "owt_chunk_cache_write_batch": 4096,
+  "owt_exact_repeat_per_chunk": 0,
+  "online_chunk_shuffle": false,
+  "online_chunk_shuffle_buffer": 10000,
+  "openwebtext_split": "all",
+  "detokenizer": "auto",
+  "resolved_detokenizer": null,
+  "num_workers": 8,
+  "latest_every": 1000,
+  "resume_path": ""
+}
+step=100 epoch=1/2 epoch_step=100/19018 micro_steps=400 elapsed=200.7s lr=1.212000e-05 loss=10.1985 loss_recon=10.1985 loss_meanflow=0.0000 mean_model_t=0.5004 mean_corrupt_t=0.4966 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3703 corrupt_frac=1.0000 acc_corrupt=0.3703 loss_corrupt=10.1985 wrong_frac=0.5036 init_acc_corrupt=0.4964 acc_corrupt_t_0p0_0p2=0.0746 corrupt_frac_t_0p0_0p2=0.2026 acc_corrupt_t_0p2_0p4=0.1967 corrupt_frac_t_0p2_0p4=0.2058 acc_corrupt_t_0p4_0p6=0.3447 corrupt_frac_t_0p4_0p6=0.1937 acc_corrupt_t_0p6_0p8=0.5246 corrupt_frac_t_0p6_0p8=0.2019 acc_corrupt_t_0p8_1p0=0.7248 corrupt_frac_t_0p8_1p0=0.1960 out_w_norm=1.0610 out_g_norm=1.1618 loss_all=9.7534 init_gold_top10=0.4680 init_gold_top100=0.4698
+step=200 epoch=1/2 epoch_step=200/19018 micro_steps=800 elapsed=199.4s lr=2.412000e-05 loss=8.9228 loss_recon=8.9228 loss_meanflow=0.0000 mean_model_t=0.4969 mean_corrupt_t=0.5032 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0558 corrupt_frac=1.0000 acc_corrupt=0.0558 loss_corrupt=8.9228 wrong_frac=0.4974 init_acc_corrupt=0.5026 acc_corrupt_t_0p0_0p2=0.0439 corrupt_frac_t_0p0_0p2=0.1932 acc_corrupt_t_0p2_0p4=0.0445 corrupt_frac_t_0p2_0p4=0.2028 acc_corrupt_t_0p4_0p6=0.0465 corrupt_frac_t_0p4_0p6=0.2034 acc_corrupt_t_0p6_0p8=0.0586 corrupt_frac_t_0p6_0p8=0.1996 acc_corrupt_t_0p8_1p0=0.0852 corrupt_frac_t_0p8_1p0=0.2009 out_w_norm=7.1043 out_g_norm=1.6873 loss_all=8.0208 init_gold_top10=0.4885 init_gold_top100=0.4902
+step=300 epoch=1/2 epoch_step=300/19018 micro_steps=1200 elapsed=226.1s lr=3.612000e-05 loss=7.0822 loss_recon=7.0822 loss_meanflow=0.0000 mean_model_t=0.4995 mean_corrupt_t=0.4971 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1080 corrupt_frac=1.0000 acc_corrupt=0.1080 loss_corrupt=7.0822 wrong_frac=0.5029 init_acc_corrupt=0.4971 acc_corrupt_t_0p0_0p2=0.0511 corrupt_frac_t_0p0_0p2=0.2049 acc_corrupt_t_0p2_0p4=0.0777 corrupt_frac_t_0p2_0p4=0.1954 acc_corrupt_t_0p4_0p6=0.1070 corrupt_frac_t_0p4_0p6=0.2034 acc_corrupt_t_0p6_0p8=0.1400 corrupt_frac_t_0p6_0p8=0.2009 acc_corrupt_t_0p8_1p0=0.1660 corrupt_frac_t_0p8_1p0=0.1954 out_w_norm=13.5033 out_g_norm=1.2325 loss_all=5.8280 init_gold_top10=0.5261 init_gold_top100=0.5270
+step=400 epoch=1/2 epoch_step=400/19018 micro_steps=1600 elapsed=199.5s lr=4.812000e-05 loss=4.4781 loss_recon=4.4781 loss_meanflow=0.0000 mean_model_t=0.4998 mean_corrupt_t=0.4972 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.4401 corrupt_frac=1.0000 acc_corrupt=0.4401 loss_corrupt=4.4781 wrong_frac=0.5029 init_acc_corrupt=0.4971 acc_corrupt_t_0p0_0p2=0.1077 corrupt_frac_t_0p0_0p2=0.2051 acc_corrupt_t_0p2_0p4=0.2716 corrupt_frac_t_0p2_0p4=0.2005 acc_corrupt_t_0p4_0p6=0.4383 corrupt_frac_t_0p4_0p6=0.2012 acc_corrupt_t_0p6_0p8=0.6137 corrupt_frac_t_0p6_0p8=0.1928 acc_corrupt_t_0p8_1p0=0.7803 corrupt_frac_t_0p8_1p0=0.2018 out_w_norm=19.2735 out_g_norm=0.4549 loss_all=3.8788 init_gold_top10=0.5142 init_gold_top100=0.5154
+step=500 epoch=1/2 epoch_step=500/19018 micro_steps=2000 elapsed=199.5s lr=6.012000e-05 loss=3.9170 loss_recon=3.9170 loss_meanflow=0.0000 mean_model_t=0.4974 mean_corrupt_t=0.5002 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5027 corrupt_frac=1.0000 acc_corrupt=0.5027 loss_corrupt=3.9170 wrong_frac=0.5004 init_acc_corrupt=0.4996 acc_corrupt_t_0p0_0p2=0.1147 corrupt_frac_t_0p0_0p2=0.2013 acc_corrupt_t_0p2_0p4=0.3073 corrupt_frac_t_0p2_0p4=0.1991 acc_corrupt_t_0p4_0p6=0.5015 corrupt_frac_t_0p4_0p6=0.1994 acc_corrupt_t_0p6_0p8=0.6964 corrupt_frac_t_0p6_0p8=0.1978 acc_corrupt_t_0p8_1p0=0.8928 corrupt_frac_t_0p8_1p0=0.2025 out_w_norm=22.6140 out_g_norm=0.3801 loss_all=3.6717 init_gold_top10=0.5317 init_gold_top100=0.5332
+step=600 epoch=1/2 epoch_step=600/19018 micro_steps=2400 elapsed=199.5s lr=7.212000e-05 loss=3.8422 loss_recon=3.8422 loss_meanflow=0.0000 mean_model_t=0.4983 mean_corrupt_t=0.5039 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5099 corrupt_frac=1.0000 acc_corrupt=0.5099 loss_corrupt=3.8422 wrong_frac=0.4967 init_acc_corrupt=0.5033 acc_corrupt_t_0p0_0p2=0.1191 corrupt_frac_t_0p0_0p2=0.1962 acc_corrupt_t_0p2_0p4=0.3077 corrupt_frac_t_0p2_0p4=0.1950 acc_corrupt_t_0p4_0p6=0.5053 corrupt_frac_t_0p4_0p6=0.2069 acc_corrupt_t_0p6_0p8=0.7012 corrupt_frac_t_0p6_0p8=0.2041 acc_corrupt_t_0p8_1p0=0.9011 corrupt_frac_t_0p8_1p0=0.1993 out_w_norm=23.8739 out_g_norm=0.3429 loss_all=4.1737 init_gold_top10=0.4535 init_gold_top100=0.4548
+step=700 epoch=1/2 epoch_step=700/19018 micro_steps=2800 elapsed=199.5s lr=8.412000e-05 loss=3.7273 loss_recon=3.7273 loss_meanflow=0.0000 mean_model_t=0.4997 mean_corrupt_t=0.5042 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5165 corrupt_frac=1.0000 acc_corrupt=0.5165 loss_corrupt=3.7273 wrong_frac=0.4958 init_acc_corrupt=0.5042 acc_corrupt_t_0p0_0p2=0.1227 corrupt_frac_t_0p0_0p2=0.1949 acc_corrupt_t_0p2_0p4=0.3148 corrupt_frac_t_0p2_0p4=0.1977 acc_corrupt_t_0p4_0p6=0.5121 corrupt_frac_t_0p4_0p6=0.2004 acc_corrupt_t_0p6_0p8=0.7082 corrupt_frac_t_0p6_0p8=0.2050 acc_corrupt_t_0p8_1p0=0.9019 corrupt_frac_t_0p8_1p0=0.2030 out_w_norm=24.9630 out_g_norm=0.3548 loss_all=4.1789 init_gold_top10=0.4437 init_gold_top100=0.4450
+step=800 epoch=1/2 epoch_step=800/19018 micro_steps=3200 elapsed=200.5s lr=9.612000e-05 loss=3.6741 loss_recon=3.6741 loss_meanflow=0.0000 mean_model_t=0.5031 mean_corrupt_t=0.4978 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5160 corrupt_frac=1.0000 acc_corrupt=0.5160 loss_corrupt=3.6741 wrong_frac=0.5020 init_acc_corrupt=0.4980 acc_corrupt_t_0p0_0p2=0.1243 corrupt_frac_t_0p0_0p2=0.2044 acc_corrupt_t_0p2_0p4=0.3250 corrupt_frac_t_0p2_0p4=0.1984 acc_corrupt_t_0p4_0p6=0.5212 corrupt_frac_t_0p4_0p6=0.1988 acc_corrupt_t_0p6_0p8=0.7155 corrupt_frac_t_0p6_0p8=0.1989 acc_corrupt_t_0p8_1p0=0.9035 corrupt_frac_t_0p8_1p0=0.1999 out_w_norm=26.0897 out_g_norm=0.3698 loss_all=3.3709 init_gold_top10=0.5284 init_gold_top100=0.5297
+step=900 epoch=1/2 epoch_step=900/19018 micro_steps=3600 elapsed=200.5s lr=1.081200e-04 loss=3.5740 loss_recon=3.5740 loss_meanflow=0.0000 mean_model_t=0.4997 mean_corrupt_t=0.5014 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5228 corrupt_frac=1.0000 acc_corrupt=0.5228 loss_corrupt=3.5740 wrong_frac=0.4989 init_acc_corrupt=0.5012 acc_corrupt_t_0p0_0p2=0.1279 corrupt_frac_t_0p0_0p2=0.1955 acc_corrupt_t_0p2_0p4=0.3258 corrupt_frac_t_0p2_0p4=0.2058 acc_corrupt_t_0p4_0p6=0.5272 corrupt_frac_t_0p4_0p6=0.1979 acc_corrupt_t_0p6_0p8=0.7187 corrupt_frac_t_0p6_0p8=0.1978 acc_corrupt_t_0p8_1p0=0.9052 corrupt_frac_t_0p8_1p0=0.2040 out_w_norm=27.1010 out_g_norm=0.3675 loss_all=3.9080 init_gold_top10=0.4464 init_gold_top100=0.4482
+step=1000 epoch=1/2 epoch_step=1000/19018 micro_steps=4000 elapsed=210.6s lr=1.201200e-04 loss=3.5327 loss_recon=3.5327 loss_meanflow=0.0000 mean_model_t=0.5026 mean_corrupt_t=0.4997 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5239 corrupt_frac=1.0000 acc_corrupt=0.5239 loss_corrupt=3.5327 wrong_frac=0.5001 init_acc_corrupt=0.4999 acc_corrupt_t_0p0_0p2=0.1297 corrupt_frac_t_0p0_0p2=0.2010 acc_corrupt_t_0p2_0p4=0.3304 corrupt_frac_t_0p2_0p4=0.2006 acc_corrupt_t_0p4_0p6=0.5322 corrupt_frac_t_0p4_0p6=0.1958 acc_corrupt_t_0p6_0p8=0.7208 corrupt_frac_t_0p6_0p8=0.2033 acc_corrupt_t_0p8_1p0=0.9072 corrupt_frac_t_0p8_1p0=0.1993 out_w_norm=28.0792 out_g_norm=0.3662 loss_all=3.1546 init_gold_top10=0.5438 init_gold_top100=0.5452
+step=1100 epoch=1/2 epoch_step=1100/19018 micro_steps=4400 elapsed=225.3s lr=1.321200e-04 loss=3.4852 loss_recon=3.4852 loss_meanflow=0.0000 mean_model_t=0.5014 mean_corrupt_t=0.5007 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5269 corrupt_frac=1.0000 acc_corrupt=0.5269 loss_corrupt=3.4852 wrong_frac=0.4993 init_acc_corrupt=0.5008 acc_corrupt_t_0p0_0p2=0.1325 corrupt_frac_t_0p0_0p2=0.1999 acc_corrupt_t_0p2_0p4=0.3354 corrupt_frac_t_0p2_0p4=0.1979 acc_corrupt_t_0p4_0p6=0.5330 corrupt_frac_t_0p4_0p6=0.2044 acc_corrupt_t_0p6_0p8=0.7262 corrupt_frac_t_0p6_0p8=0.1983 acc_corrupt_t_0p8_1p0=0.9077 corrupt_frac_t_0p8_1p0=0.1995 out_w_norm=29.1077 out_g_norm=0.3601 loss_all=3.1611 init_gold_top10=0.5427 init_gold_top100=0.5437
+step=1200 epoch=1/2 epoch_step=1200/19018 micro_steps=4800 elapsed=199.5s lr=1.441200e-04 loss=3.4507 loss_recon=3.4507 loss_meanflow=0.0000 mean_model_t=0.5030 mean_corrupt_t=0.5005 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5286 corrupt_frac=1.0000 acc_corrupt=0.5286 loss_corrupt=3.4507 wrong_frac=0.4995 init_acc_corrupt=0.5005 acc_corrupt_t_0p0_0p2=0.1307 corrupt_frac_t_0p0_0p2=0.1980 acc_corrupt_t_0p2_0p4=0.3380 corrupt_frac_t_0p2_0p4=0.2011 acc_corrupt_t_0p4_0p6=0.5372 corrupt_frac_t_0p4_0p6=0.2018 acc_corrupt_t_0p6_0p8=0.7286 corrupt_frac_t_0p6_0p8=0.2042 acc_corrupt_t_0p8_1p0=0.9088 corrupt_frac_t_0p8_1p0=0.1959 out_w_norm=30.2570 out_g_norm=0.3400 loss_all=3.7855 init_gold_top10=0.4513 init_gold_top100=0.4527
+step=1300 epoch=1/2 epoch_step=1300/19018 micro_steps=5200 elapsed=199.5s lr=1.561200e-04 loss=3.4260 loss_recon=3.4260 loss_meanflow=0.0000 mean_model_t=0.5031 mean_corrupt_t=0.5000 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5299 corrupt_frac=1.0000 acc_corrupt=0.5299 loss_corrupt=3.4260 wrong_frac=0.4997 init_acc_corrupt=0.5003 acc_corrupt_t_0p0_0p2=0.1330 corrupt_frac_t_0p0_0p2=0.1987 acc_corrupt_t_0p2_0p4=0.3374 corrupt_frac_t_0p2_0p4=0.1973 acc_corrupt_t_0p4_0p6=0.5402 corrupt_frac_t_0p4_0p6=0.2074 acc_corrupt_t_0p6_0p8=0.7292 corrupt_frac_t_0p6_0p8=0.2066 acc_corrupt_t_0p8_1p0=0.9107 corrupt_frac_t_0p8_1p0=0.1925 out_w_norm=31.5035 out_g_norm=0.3191 loss_all=2.8906 init_gold_top10=0.5694 init_gold_top100=0.5705
+step=1400 epoch=1/2 epoch_step=1400/19018 micro_steps=5600 elapsed=199.5s lr=1.681200e-04 loss=3.4040 loss_recon=3.4040 loss_meanflow=0.0000 mean_model_t=0.4973 mean_corrupt_t=0.5001 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5313 corrupt_frac=1.0000 acc_corrupt=0.5313 loss_corrupt=3.4040 wrong_frac=0.4995 init_acc_corrupt=0.5005 acc_corrupt_t_0p0_0p2=0.1332 corrupt_frac_t_0p0_0p2=0.1993 acc_corrupt_t_0p2_0p4=0.3377 corrupt_frac_t_0p2_0p4=0.1990 acc_corrupt_t_0p4_0p6=0.5388 corrupt_frac_t_0p4_0p6=0.2007 acc_corrupt_t_0p6_0p8=0.7315 corrupt_frac_t_0p6_0p8=0.2052 acc_corrupt_t_0p8_1p0=0.9111 corrupt_frac_t_0p8_1p0=0.1983 out_w_norm=32.9606 out_g_norm=0.2942 loss_all=2.8138 init_gold_top10=0.5705 init_gold_top100=0.5716
+step=1500 epoch=1/2 epoch_step=1500/19018 micro_steps=6000 elapsed=199.6s lr=1.801200e-04 loss=3.3915 loss_recon=3.3915 loss_meanflow=0.0000 mean_model_t=0.5006 mean_corrupt_t=0.4996 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5313 corrupt_frac=1.0000 acc_corrupt=0.5313 loss_corrupt=3.3915 wrong_frac=0.5006 init_acc_corrupt=0.4994 acc_corrupt_t_0p0_0p2=0.1336 corrupt_frac_t_0p0_0p2=0.1960 acc_corrupt_t_0p2_0p4=0.3407 corrupt_frac_t_0p2_0p4=0.2036 acc_corrupt_t_0p4_0p6=0.5395 corrupt_frac_t_0p4_0p6=0.2020 acc_corrupt_t_0p6_0p8=0.7314 corrupt_frac_t_0p6_0p8=0.2017 acc_corrupt_t_0p8_1p0=0.9111 corrupt_frac_t_0p8_1p0=0.1967 out_w_norm=34.6646 out_g_norm=0.2818 loss_all=3.3737 init_gold_top10=0.5000 init_gold_top100=0.5015
+step=1600 epoch=1/2 epoch_step=1600/19018 micro_steps=6400 elapsed=199.5s lr=1.921200e-04 loss=3.3626 loss_recon=3.3626 loss_meanflow=0.0000 mean_model_t=0.4998 mean_corrupt_t=0.5003 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5335 corrupt_frac=1.0000 acc_corrupt=0.5335 loss_corrupt=3.3626 wrong_frac=0.4994 init_acc_corrupt=0.5006 acc_corrupt_t_0p0_0p2=0.1365 corrupt_frac_t_0p0_0p2=0.2023 acc_corrupt_t_0p2_0p4=0.3397 corrupt_frac_t_0p2_0p4=0.1965 acc_corrupt_t_0p4_0p6=0.5408 corrupt_frac_t_0p4_0p6=0.1999 acc_corrupt_t_0p6_0p8=0.7328 corrupt_frac_t_0p6_0p8=0.1957 acc_corrupt_t_0p8_1p0=0.9115 corrupt_frac_t_0p8_1p0=0.2065 out_w_norm=36.5363 out_g_norm=0.2637 loss_all=3.5901 init_gold_top10=0.4688 init_gold_top100=0.4702
+step=1700 epoch=1/2 epoch_step=1700/19018 micro_steps=6800 elapsed=199.6s lr=2.041200e-04 loss=3.3443 loss_recon=3.3443 loss_meanflow=0.0000 mean_model_t=0.4981 mean_corrupt_t=0.4997 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.5344 corrupt_frac=1.0000 acc_corrupt=0.5344 loss_corrupt=3.3443 wrong_frac=0.4994 init_acc_corrupt=0.5006 acc_corrupt_t_0p0_0p2=0.1348 corrupt_frac_t_0p0_0p2=0.2017 acc_corrupt_t_0p2_0p4=0.3426 corrupt_frac_t_0p2_0p4=0.1997 acc_corrupt_t_0p4_0p6=0.5449 corrupt_frac_t_0p4_0p6=0.1972 acc_corrupt_t_0p6_0p8=0.7340 corrupt_frac_t_0p6_0p8=0.1989 acc_corrupt_t_0p8_1p0=0.9131 corrupt_frac_t_0p8_1p0=0.2034 out_w_norm=38.5982 out_g_norm=0.2560 loss_all=3.2181 init_gold_top10=0.5173 init_gold_top100=0.5193
+Terminated

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/packaging/_structures.py

@@ -0,0 +1,33 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+"""Backward-compatibility shim for unpickling Version objects serialized before
+packaging 26.1.
+
+Old pickles reference ``packaging._structures.InfinityType`` and
+``packaging._structures.NegativeInfinityType``.  This module provides minimal
+stand-in classes so that ``pickle.loads()`` can resolve those references.
+The deserialized objects are not used for comparisons — ``Version.__setstate__``
+discards the stale ``_key`` cache and recomputes it from the core version fields.
+"""
+
+from __future__ import annotations
+
+
+class InfinityType:
+    """Stand-in for the removed ``InfinityType`` used in old comparison keys."""
+
+    def __repr__(self) -> str:
+        return "Infinity"
+
+
+class NegativeInfinityType:
+    """Stand-in for the removed ``NegativeInfinityType`` used in old comparison keys."""
+
+    def __repr__(self) -> str:
+        return "-Infinity"
+
+
+Infinity = InfinityType()
+NegativeInfinity = NegativeInfinityType()

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/packaging/requirements.py

@@ -0,0 +1,129 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+from __future__ import annotations
+
+from typing import Iterator
+
+from ._parser import parse_requirement as _parse_requirement
+from ._tokenizer import ParserSyntaxError
+from .markers import Marker, _normalize_extra_values
+from .specifiers import SpecifierSet
+from .utils import canonicalize_name
+
+__all__ = [
+    "InvalidRequirement",
+    "Requirement",
+]
+
+
+def __dir__() -> list[str]:
+    return __all__
+
+
+class InvalidRequirement(ValueError):
+    """
+    An invalid requirement was found, users should refer to PEP 508.
+    """
+
+
+class Requirement:
+    """Parse a requirement.
+
+    Parse a given requirement string into its parts, such as name, specifier,
+    URL, and extras. Raises InvalidRequirement on a badly-formed requirement
+    string.
+
+    Instances are safe to serialize with :mod:`pickle`. They use a stable
+    format so the same pickle can be loaded in future packaging releases.
+
+    .. versionchanged:: 26.2
+
+        Added a stable pickle format. Pickles created with packaging 26.2+ can
+        be unpickled with future releases.  Backward compatibility with pickles
+        from packaging < 26.2 is supported but may be removed in a future
+        release.
+    """
+
+    # TODO: Can we test whether something is contained within a requirement?
+    #       If so how do we do that? Do we need to test against the _name_ of
+    #       the thing as well as the version? What about the markers?
+    # TODO: Can we normalize the name and extra name?
+
+    def __init__(self, requirement_string: str) -> None:
+        try:
+            parsed = _parse_requirement(requirement_string)
+        except ParserSyntaxError as e:
+            raise InvalidRequirement(str(e)) from e
+
+        self.name: str = parsed.name
+        self.url: str | None = parsed.url or None
+        self.extras: set[str] = set(parsed.extras or [])
+        self.specifier: SpecifierSet = SpecifierSet(parsed.specifier)
+        self.marker: Marker | None = None
+        if parsed.marker is not None:
+            self.marker = Marker.__new__(Marker)
+            self.marker._markers = _normalize_extra_values(parsed.marker)
+
+    def _iter_parts(self, name: str) -> Iterator[str]:
+        yield name
+
+        if self.extras:
+            formatted_extras = ",".join(sorted(self.extras))
+            yield f"[{formatted_extras}]"
+
+        if self.specifier:
+            yield str(self.specifier)
+
+        if self.url:
+            yield f" @ {self.url}"
+            if self.marker:
+                yield " "
+
+        if self.marker:
+            yield f"; {self.marker}"
+
+    def __getstate__(self) -> str:
+        # Return the requirement string for compactness and stability.
+        # Re-parsed on load to reconstruct all fields.
+        return str(self)
+
+    def __setstate__(self, state: object) -> None:
+        if isinstance(state, str):
+            # New format (26.2+): just the requirement string.
+            try:
+                tmp = Requirement(state)
+            except InvalidRequirement as exc:
+                raise TypeError(f"Cannot restore Requirement from {state!r}") from exc
+            self.name = tmp.name
+            self.url = tmp.url
+            self.extras = tmp.extras
+            self.specifier = tmp.specifier
+            self.marker = tmp.marker
+            return
+        if isinstance(state, dict):
+            # Old format (packaging <= 26.1, no __slots__): plain __dict__.
+            self.__dict__.update(state)
+            return
+        raise TypeError(f"Cannot restore Requirement from {state!r}")
+
+    def __str__(self) -> str:
+        return "".join(self._iter_parts(self.name))
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__}({str(self)!r})>"
+
+    def __hash__(self) -> int:
+        return hash(tuple(self._iter_parts(canonicalize_name(self.name))))
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Requirement):
+            return NotImplemented
+
+        return (
+            canonicalize_name(self.name) == canonicalize_name(other.name)
+            and self.extras == other.extras
+            and self.specifier == other.specifier
+            and self.url == other.url
+            and self.marker == other.marker
+        )

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/packaging/tags.py

@@ -0,0 +1,932 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import annotations
+
+import logging
+import operator
+import platform
+import re
+import struct
+import subprocess
+import sys
+import sysconfig
+from importlib.machinery import EXTENSION_SUFFIXES
+from typing import (
+    TYPE_CHECKING,
+    Iterable,
+    Iterator,
+    Sequence,
+    Tuple,
+    TypeVar,
+    cast,
+)
+
+from . import _manylinux, _musllinux
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable
+    from typing import AbstractSet
+
+
+__all__ = [
+    "INTERPRETER_SHORT_NAMES",
+    "AppleVersion",
+    "PythonVersion",
+    "Tag",
+    "UnsortedTagsError",
+    "android_platforms",
+    "compatible_tags",
+    "cpython_tags",
+    "create_compatible_tags_selector",
+    "generic_tags",
+    "interpreter_name",
+    "interpreter_version",
+    "ios_platforms",
+    "mac_platforms",
+    "parse_tag",
+    "platform_tags",
+    "sys_tags",
+]
+
+
+def __dir__() -> list[str]:
+    return __all__
+
+
+logger = logging.getLogger(__name__)
+
+PythonVersion = Sequence[int]
+AppleVersion = Tuple[int, int]
+_T = TypeVar("_T")
+
+INTERPRETER_SHORT_NAMES: dict[str, str] = {
+    "python": "py",  # Generic.
+    "cpython": "cp",
+    "pypy": "pp",
+    "ironpython": "ip",
+    "jython": "jy",
+}
+
+
+# This function can be unit tested without reloading the module
+# (Unlike _32_BIT_INTERPRETER)
+def _compute_32_bit_interpreter() -> bool:
+    return struct.calcsize("P") == 4
+
+
+_32_BIT_INTERPRETER = _compute_32_bit_interpreter()
+
+
+class UnsortedTagsError(ValueError):
+    """
+    Raised when a tag component is not in sorted order per PEP 425.
+    """
+
+
+class Tag:
+    """
+    A representation of the tag triple for a wheel.
+
+    Instances are considered immutable and thus are hashable. Equality checking
+    is also supported.
+
+    Instances are safe to serialize with :mod:`pickle`. They use a stable
+    format so the same pickle can be loaded in future packaging releases.
+
+    .. versionchanged:: 26.2
+
+        Added a stable pickle format. Pickles created with packaging 26.2+ can
+        be unpickled with future releases.  Backward compatibility with pickles
+        from packaging < 26.2 is supported but may be removed in a future
+        release.
+    """
+
+    __slots__ = ["_abi", "_hash", "_interpreter", "_platform"]
+
+    def __init__(self, interpreter: str, abi: str, platform: str) -> None:
+        """
+        :param str interpreter: The interpreter name, e.g. ``"py"``
+                                (see :attr:`INTERPRETER_SHORT_NAMES` for mapping
+                                well-known interpreter names to their short names).
+        :param str abi: The ABI that a wheel supports, e.g. ``"cp37m"``.
+        :param str platform: The OS/platform the wheel supports,
+                            e.g. ``"win_amd64"``.
+        """
+        self._interpreter = interpreter.lower()
+        self._abi = abi.lower()
+        self._platform = platform.lower()
+        # The __hash__ of every single element in a Set[Tag] will be evaluated each time
+        # that a set calls its `.disjoint()` method, which may be called hundreds of
+        # times when scanning a page of links for packages with tags matching that
+        # Set[Tag]. Pre-computing the value here produces significant speedups for
+        # downstream consumers.
+        self._hash = hash((self._interpreter, self._abi, self._platform))
+
+    @property
+    def interpreter(self) -> str:
+        """
+        The interpreter name, e.g. ``"py"`` (see
+        :attr:`INTERPRETER_SHORT_NAMES` for mapping well-known interpreter
+        names to their short names).
+        """
+        return self._interpreter
+
+    @property
+    def abi(self) -> str:
+        """
+        The supported ABI.
+        """
+        return self._abi
+
+    @property
+    def platform(self) -> str:
+        """
+        The OS/platform.
+        """
+        return self._platform
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Tag):
+            return NotImplemented
+
+        return (
+            (self._hash == other._hash)  # Short-circuit ASAP for perf reasons.
+            and (self._platform == other._platform)
+            and (self._abi == other._abi)
+            and (self._interpreter == other._interpreter)
+        )
+
+    def __hash__(self) -> int:
+        return self._hash
+
+    def __str__(self) -> str:
+        return f"{self._interpreter}-{self._abi}-{self._platform}"
+
+    def __repr__(self) -> str:
+        return f"<{self} @ {id(self)}>"
+
+    def __getstate__(self) -> tuple[str, str, str]:
+        # Return state as a 3-item tuple: (interpreter, abi, platform).
+        # Cache member _hash is excluded and will be recomputed.
+        return (self._interpreter, self._abi, self._platform)
+
+    def __setstate__(self, state: object) -> None:
+        if isinstance(state, tuple):
+            if len(state) == 3 and all(isinstance(s, str) for s in state):
+                # New format (26.2+): (interpreter, abi, platform)
+                self._interpreter, self._abi, self._platform = state
+                self._hash = hash((self._interpreter, self._abi, self._platform))
+                return
+            if len(state) == 2 and isinstance(state[1], dict):
+                # Old format (packaging <= 26.1, __slots__): (None, {slot: value}).
+                _, slots = state
+                try:
+                    interpreter = slots["_interpreter"]
+                    abi = slots["_abi"]
+                    platform = slots["_platform"]
+                except KeyError:
+                    raise TypeError(f"Cannot restore Tag from {state!r}") from None
+                if not all(
+                    isinstance(value, str) for value in (interpreter, abi, platform)
+                ):
+                    raise TypeError(f"Cannot restore Tag from {state!r}")
+                self._interpreter = interpreter.lower()
+                self._abi = abi.lower()
+                self._platform = platform.lower()
+                self._hash = hash((self._interpreter, self._abi, self._platform))
+                return
+        raise TypeError(f"Cannot restore Tag from {state!r}")
+
+
+def parse_tag(tag: str, *, validate_order: bool = False) -> frozenset[Tag]:
+    """
+    Parses the provided tag (e.g. `py3-none-any`) into a frozenset of
+    :class:`Tag` instances.
+
+    Returning a set is required due to the possibility that the tag is a
+    `compressed tag set`_, e.g. ``"py2.py3-none-any"`` which supports both
+    Python 2 and Python 3.
+
+    If **validate_order** is true, compressed tag set components are checked
+    to be in sorted order as required by PEP 425.
+
+    :param str tag: The tag to parse, e.g. ``"py3-none-any"``.
+    :param bool validate_order: Check whether compressed tag set components
+        are in sorted order.
+    :raises UnsortedTagsError: If **validate_order** is true and any compressed tag
+        set component is not in sorted order.
+
+    .. versionadded:: 26.1
+       The *validate_order* parameter.
+    """
+    tags = set()
+    interpreters, abis, platforms = tag.split("-")
+    if validate_order:
+        for component in (interpreters, abis, platforms):
+            parts = component.split(".")
+            if parts != sorted(parts):
+                raise UnsortedTagsError(
+                    f"Tag component {component!r} is not in sorted order per PEP 425"
+                )
+    for interpreter in interpreters.split("."):
+        for abi in abis.split("."):
+            for platform_ in platforms.split("."):
+                tags.add(Tag(interpreter, abi, platform_))
+    return frozenset(tags)
+
+
+def _get_config_var(name: str, warn: bool = False) -> int | str | None:
+    value: int | str | None = sysconfig.get_config_var(name)
+    if value is None and warn:
+        logger.debug(
+            "Config variable '%s' is unset, Python ABI tag may be incorrect", name
+        )
+    return value
+
+
+def _normalize_string(string: str) -> str:
+    return string.replace(".", "_").replace("-", "_").replace(" ", "_")
+
+
+def _is_threaded_cpython(abis: list[str]) -> bool:
+    """
+    Determine if the ABI corresponds to a threaded (`--disable-gil`) build.
+
+    The threaded builds are indicated by a "t" in the abiflags.
+    """
+    if len(abis) == 0:
+        return False
+    # expect e.g., cp313
+    m = re.match(r"cp\d+(.*)", abis[0])
+    if not m:
+        return False
+    abiflags = m.group(1)
+    return "t" in abiflags
+
+
+def _abi3_applies(python_version: PythonVersion, threading: bool) -> bool:
+    """
+    Determine if the Python version supports abi3.
+
+    PEP 384 was first implemented in Python 3.2. The free-threaded
+    builds do not support abi3.
+    """
+    return len(python_version) > 1 and tuple(python_version) >= (3, 2) and not threading
+
+
+def _abi3t_applies(python_version: PythonVersion, threading: bool) -> bool:
+    """
+    Determine if the Python version supports abi3t.
+
+    PEP 803 was first implemented in Python 3.15 but, per PEP 803, this
+    returns tags going back to Python 3.2 to mirror the abi3
+    implementation and leave open the possibility of abi3t wheels
+    supporting older Python versions.
+
+    """
+    return len(python_version) > 1 and tuple(python_version) >= (3, 2) and threading
+
+
+def _cpython_abis(py_version: PythonVersion, warn: bool = False) -> list[str]:
+    py_version = tuple(py_version)  # To allow for version comparison.
+    abis = []
+    version = _version_nodot(py_version[:2])
+    threading = debug = pymalloc = ucs4 = ""
+    with_debug = _get_config_var("Py_DEBUG", warn)
+    has_refcount = hasattr(sys, "gettotalrefcount")
+    # Windows doesn't set Py_DEBUG, so checking for support of debug-compiled
+    # extension modules is the best option.
+    # https://github.com/pypa/pip/issues/3383#issuecomment-173267692
+    has_ext = "_d.pyd" in EXTENSION_SUFFIXES
+    if with_debug or (with_debug is None and (has_refcount or has_ext)):
+        debug = "d"
+    if py_version >= (3, 13) and _get_config_var("Py_GIL_DISABLED", warn):
+        threading = "t"
+    if py_version < (3, 8):
+        with_pymalloc = _get_config_var("WITH_PYMALLOC", warn)
+        if with_pymalloc or with_pymalloc is None:
+            pymalloc = "m"
+        if py_version < (3, 3):
+            unicode_size = _get_config_var("Py_UNICODE_SIZE", warn)
+            if unicode_size == 4 or (
+                unicode_size is None and sys.maxunicode == 0x10FFFF
+            ):
+                ucs4 = "u"
+    elif debug:
+        # Debug builds can also load "normal" extension modules.
+        # We can also assume no UCS-4 or pymalloc requirement.
+        abis.append(f"cp{version}{threading}")
+    abis.insert(0, f"cp{version}{threading}{debug}{pymalloc}{ucs4}")
+    return abis
+
+
+def cpython_tags(
+    python_version: PythonVersion | None = None,
+    abis: Iterable[str] | None = None,
+    platforms: Iterable[str] | None = None,
+    *,
+    warn: bool = False,
+) -> Iterator[Tag]:
+    """
+    Yields the tags for the CPython interpreter.
+
+    The specific tags generated are:
+
+    - ``cp<python_version>-<abi>-<platform>``
+    - ``cp<python_version>-<stable_abi>-<platform>``
+    - ``cp<python_version>-none-<platform>``
+    - ``cp<older version>-<stable_abi>-<platform>`` where "older version" is all older
+      minor versions down to Python 3.2 (when ``abi3`` was introduced)
+
+    If ``python_version`` only provides a major-only version then only
+    user-provided ABIs via ``abis`` and the ``none`` ABI will be used.
+
+    The ``stable_abi`` will be either ``abi3`` or ``abi3t`` if `abi` is a
+    GIL-enabled ABI like `"cp315"` or a free-threaded ABI like `"cp315t"`,
+    respectively.
+
+    :param Sequence python_version: A one- or two-item sequence representing the
+                                 targeted Python version. Defaults to
+                                 ``sys.version_info[:2]``.
+    :param Iterable abis: Iterable of compatible ABIs. Defaults to the ABIs
+                          compatible with the current system.
+    :param Iterable platforms: Iterable of compatible platforms. Defaults to the
+                               platforms compatible with the current system.
+    :param bool warn: Whether warnings should be logged. Defaults to ``False``.
+    """
+    if not python_version:
+        python_version = sys.version_info[:2]
+
+    interpreter = f"cp{_version_nodot(python_version[:2])}"
+
+    if abis is None:
+        abis = _cpython_abis(python_version, warn) if len(python_version) > 1 else []
+    abis = list(abis)
+    # 'abi3' and 'none' are explicitly handled later.
+    for explicit_abi in ("abi3", "none"):
+        try:
+            abis.remove(explicit_abi)
+        except ValueError:  # noqa: PERF203
+            pass
+
+    platforms = list(platforms or platform_tags())
+    for abi in abis:
+        for platform_ in platforms:
+            yield Tag(interpreter, abi, platform_)
+
+    threading = _is_threaded_cpython(abis)
+    use_abi3 = _abi3_applies(python_version, threading)
+    use_abi3t = _abi3t_applies(python_version, threading)
+
+    if use_abi3:
+        yield from (Tag(interpreter, "abi3", platform_) for platform_ in platforms)
+    if use_abi3t:
+        yield from (Tag(interpreter, "abi3t", platform_) for platform_ in platforms)
+
+    yield from (Tag(interpreter, "none", platform_) for platform_ in platforms)
+
+    if use_abi3 or use_abi3t:
+        for minor_version in range(python_version[1] - 1, 1, -1):
+            for platform_ in platforms:
+                version = _version_nodot((python_version[0], minor_version))
+                interpreter = f"cp{version}"
+                if use_abi3:
+                    yield Tag(interpreter, "abi3", platform_)
+                if use_abi3t:
+                    # Support for abi3t was introduced in Python 3.15, but in
+                    # principle abi3t wheels are possible for older limited API
+                    # versions, so allow things like ("cp37", "abi3t", "platform")
+                    yield Tag(interpreter, "abi3t", platform_)
+
+
+def _generic_abi() -> list[str]:
+    """
+    Return the ABI tag based on EXT_SUFFIX.
+    """
+    # The following are examples of `EXT_SUFFIX`.
+    # We want to keep the parts which are related to the ABI and remove the
+    # parts which are related to the platform:
+    # - linux:   '.cpython-310-x86_64-linux-gnu.so' => cp310
+    # - mac:     '.cpython-310-darwin.so'           => cp310
+    # - win:     '.cp310-win_amd64.pyd'             => cp310
+    # - win:     '.pyd'                             => cp37 (uses _cpython_abis())
+    # - pypy:    '.pypy38-pp73-x86_64-linux-gnu.so' => pypy38_pp73
+    # - graalpy: '.graalpy-38-native-x86_64-darwin.dylib'
+    #                                               => graalpy_38_native
+
+    ext_suffix = _get_config_var("EXT_SUFFIX", warn=True)
+    if not isinstance(ext_suffix, str) or ext_suffix[0] != ".":
+        raise SystemError("invalid sysconfig.get_config_var('EXT_SUFFIX')")
+    parts = ext_suffix.split(".")
+    if len(parts) < 3:
+        # CPython3.7 and earlier uses ".pyd" on Windows.
+        return _cpython_abis(sys.version_info[:2])
+    soabi = parts[1]
+    if soabi.startswith("cpython"):
+        # non-windows
+        abi = "cp" + soabi.split("-")[1]
+    elif soabi.startswith("cp"):
+        # windows
+        abi = soabi.split("-")[0]
+    elif soabi.startswith("pypy"):
+        abi = "-".join(soabi.split("-")[:2])
+    elif soabi.startswith("graalpy"):
+        abi = "-".join(soabi.split("-")[:3])
+    elif soabi:
+        # pyston, ironpython, others?
+        abi = soabi
+    else:
+        return []
+    return [_normalize_string(abi)]
+
+
+def generic_tags(
+    interpreter: str | None = None,
+    abis: Iterable[str] | None = None,
+    platforms: Iterable[str] | None = None,
+    *,
+    warn: bool = False,
+) -> Iterator[Tag]:
+    """
+    Yields the tags for an interpreter which requires no specialization.
+
+    This function should be used if one of the other interpreter-specific
+    functions provided by this module is not appropriate (i.e. not calculating
+    tags for a CPython interpreter).
+
+    The specific tags generated are:
+
+    - ``<interpreter>-<abi>-<platform>``
+
+    The ``"none"`` ABI will be added if it was not explicitly provided.
+
+    :param str interpreter: The name of the interpreter. Defaults to being
+                            calculated.
+    :param Iterable abis: Iterable of compatible ABIs. Defaults to the ABIs
+                          compatible with the current system.
+    :param Iterable platforms: Iterable of compatible platforms. Defaults to the
+                               platforms compatible with the current system.
+    :param bool warn: Whether warnings should be logged. Defaults to ``False``.
+    """
+    if not interpreter:
+        interp_name = interpreter_name()
+        interp_version = interpreter_version(warn=warn)
+        interpreter = f"{interp_name}{interp_version}"
+    abis = _generic_abi() if abis is None else list(abis)
+    platforms = list(platforms or platform_tags())
+    if "none" not in abis:
+        abis.append("none")
+    for abi in abis:
+        for platform_ in platforms:
+            yield Tag(interpreter, abi, platform_)
+
+
+def _py_interpreter_range(py_version: PythonVersion) -> Iterator[str]:
+    """
+    Yields Python versions in descending order.
+
+    After the latest version, the major-only version will be yielded, and then
+    all previous versions of that major version.
+    """
+    if len(py_version) > 1:
+        yield f"py{_version_nodot(py_version[:2])}"
+    yield f"py{py_version[0]}"
+    if len(py_version) > 1:
+        for minor in range(py_version[1] - 1, -1, -1):
+            yield f"py{_version_nodot((py_version[0], minor))}"
+
+
+def compatible_tags(
+    python_version: PythonVersion | None = None,
+    interpreter: str | None = None,
+    platforms: Iterable[str] | None = None,
+) -> Iterator[Tag]:
+    """
+    Yields the tags for an interpreter compatible with the Python version
+    specified by ``python_version``.
+
+    The specific tags generated are:
+
+    - ``py*-none-<platform>``
+    - ``<interpreter>-none-any`` if ``interpreter`` is provided
+    - ``py*-none-any``
+
+    :param Sequence python_version: A one- or two-item sequence representing the
+                                 compatible version of Python. Defaults to
+                                 ``sys.version_info[:2]``.
+    :param str interpreter: The name of the interpreter (if known), e.g.
+                            ``"cp38"``. Defaults to the current interpreter.
+    :param Iterable platforms: Iterable of compatible platforms. Defaults to the
+                               platforms compatible with the current system.
+    """
+    if not python_version:
+        python_version = sys.version_info[:2]
+    platforms = list(platforms or platform_tags())
+    for version in _py_interpreter_range(python_version):
+        for platform_ in platforms:
+            yield Tag(version, "none", platform_)
+    if interpreter:
+        yield Tag(interpreter, "none", "any")
+    for version in _py_interpreter_range(python_version):
+        yield Tag(version, "none", "any")
+
+
+def _mac_arch(arch: str, is_32bit: bool = _32_BIT_INTERPRETER) -> str:
+    if not is_32bit:
+        return arch
+
+    if arch.startswith("ppc"):
+        return "ppc"
+
+    return "i386"
+
+
+def _mac_binary_formats(version: AppleVersion, cpu_arch: str) -> list[str]:
+    formats = [cpu_arch]
+    if cpu_arch == "x86_64":
+        if version < (10, 4):
+            return []
+        formats.extend(["intel", "fat64", "fat32"])
+
+    elif cpu_arch == "i386":
+        if version < (10, 4):
+            return []
+        formats.extend(["intel", "fat32", "fat"])
+
+    elif cpu_arch == "ppc64":
+        # TODO: Need to care about 32-bit PPC for ppc64 through 10.2?
+        if version > (10, 5) or version < (10, 4):
+            return []
+        formats.append("fat64")
+
+    elif cpu_arch == "ppc":
+        if version > (10, 6):
+            return []
+        formats.extend(["fat32", "fat"])
+
+    if cpu_arch in {"arm64", "x86_64"}:
+        formats.append("universal2")
+
+    if cpu_arch in {"x86_64", "i386", "ppc64", "ppc", "intel"}:
+        formats.append("universal")
+
+    return formats
+
+
+def mac_platforms(
+    version: AppleVersion | None = None, arch: str | None = None
+) -> Iterator[str]:
+    """
+    Yields the :attr:`~Tag.platform` tags for macOS.
+
+    The `version` parameter is a two-item tuple specifying the macOS version to
+    generate platform tags for. The `arch` parameter is the CPU architecture to
+    generate platform tags for. Both parameters default to the appropriate value
+    for the current system.
+
+    :param tuple version: A two-item tuple representing the version of macOS.
+                          Defaults to the current system's version.
+    :param str arch: The CPU architecture. Defaults to the architecture of the
+                     current system, e.g. ``"x86_64"``.
+
+    .. note::
+        Equivalent support for the other major platforms is purposefully not
+        provided:
+
+        - On Windows, platform compatibility is statically specified
+        - On Linux, code must be run on the system itself to determine
+          compatibility
+    """
+    version_str, _, cpu_arch = platform.mac_ver()
+    if version is None:
+        version = cast("AppleVersion", tuple(map(int, version_str.split(".")[:2])))
+        if version == (10, 16):
+            # When built against an older macOS SDK, Python will report macOS 10.16
+            # instead of the real version.
+            version_str = subprocess.run(
+                [
+                    sys.executable,
+                    "-sS",
+                    "-c",
+                    "import platform; print(platform.mac_ver()[0])",
+                ],
+                check=True,
+                env={"SYSTEM_VERSION_COMPAT": "0"},
+                stdout=subprocess.PIPE,
+                text=True,
+            ).stdout
+            version = cast("AppleVersion", tuple(map(int, version_str.split(".")[:2])))
+
+    if arch is None:
+        arch = _mac_arch(cpu_arch)
+
+    if (10, 0) <= version < (11, 0):
+        # Prior to Mac OS 11, each yearly release of Mac OS bumped the
+        # "minor" version number.  The major version was always 10.
+        major_version = 10
+        for minor_version in range(version[1], -1, -1):
+            compat_version = major_version, minor_version
+            binary_formats = _mac_binary_formats(compat_version, arch)
+            for binary_format in binary_formats:
+                yield f"macosx_{major_version}_{minor_version}_{binary_format}"
+
+    if version >= (11, 0):
+        # Starting with Mac OS 11, each yearly release bumps the major version
+        # number.   The minor versions are now the midyear updates.
+        minor_version = 0
+        for major_version in range(version[0], 10, -1):
+            compat_version = major_version, minor_version
+            binary_formats = _mac_binary_formats(compat_version, arch)
+            for binary_format in binary_formats:
+                yield f"macosx_{major_version}_{minor_version}_{binary_format}"
+
+    if version >= (11, 0):
+        # Mac OS 11 on x86_64 is compatible with binaries from previous releases.
+        # Arm64 support was introduced in 11.0, so no Arm binaries from previous
+        # releases exist.
+        #
+        # However, the "universal2" binary format can have a
+        # macOS version earlier than 11.0 when the x86_64 part of the binary supports
+        # that version of macOS.
+        major_version = 10
+        if arch == "x86_64":
+            for minor_version in range(16, 3, -1):
+                compat_version = major_version, minor_version
+                binary_formats = _mac_binary_formats(compat_version, arch)
+                for binary_format in binary_formats:
+                    yield f"macosx_{major_version}_{minor_version}_{binary_format}"
+        else:
+            for minor_version in range(16, 3, -1):
+                compat_version = major_version, minor_version
+                binary_format = "universal2"
+                yield f"macosx_{major_version}_{minor_version}_{binary_format}"
+
+
+def ios_platforms(
+    version: AppleVersion | None = None, multiarch: str | None = None
+) -> Iterator[str]:
+    """
+
+    Yields the :attr:`~Tag.platform` tags for iOS.
+
+    :param tuple version: A two-item tuple representing the version of iOS.
+                          Defaults to the current system's version.
+    :param str multiarch: The CPU architecture+ABI to be used. This should be in
+                          the format by ``sys.implementation._multiarch`` (e.g.,
+                          ``arm64_iphoneos`` or ``x86_64_iphonesimulator``).
+                          Defaults to the current system's multiarch value.
+
+    .. note::
+        Behavior of this method is undefined if invoked on non-iOS platforms
+        without providing explicit version and multiarch arguments.
+    """
+    if version is None:
+        # if iOS is the current platform, ios_ver *must* be defined. However,
+        # it won't exist for CPython versions before 3.13, which causes a mypy
+        # error.
+        _, release, _, _ = platform.ios_ver()  # type: ignore[attr-defined, unused-ignore]
+        version = cast("AppleVersion", tuple(map(int, release.split(".")[:2])))
+
+    if multiarch is None:
+        multiarch = sys.implementation._multiarch
+    multiarch = multiarch.replace("-", "_")
+
+    ios_platform_template = "ios_{major}_{minor}_{multiarch}"
+
+    # Consider any iOS major.minor version from the version requested, down to
+    # 12.0. 12.0 is the first iOS version that is known to have enough features
+    # to support CPython. Consider every possible minor release up to X.9. There
+    # highest the minor has ever gone is 8 (14.8 and 15.8) but having some extra
+    # candidates that won't ever match doesn't really hurt, and it saves us from
+    # having to keep an explicit list of known iOS versions in the code. Return
+    # the results descending order of version number.
+
+    # If the requested major version is less than 12, there won't be any matches.
+    if version[0] < 12:
+        return
+
+    # Consider the actual X.Y version that was requested.
+    yield ios_platform_template.format(
+        major=version[0], minor=version[1], multiarch=multiarch
+    )
+
+    # Consider every minor version from X.0 to the minor version prior to the
+    # version requested by the platform.
+    for minor in range(version[1] - 1, -1, -1):
+        yield ios_platform_template.format(
+            major=version[0], minor=minor, multiarch=multiarch
+        )
+
+    for major in range(version[0] - 1, 11, -1):
+        for minor in range(9, -1, -1):
+            yield ios_platform_template.format(
+                major=major, minor=minor, multiarch=multiarch
+            )
+
+
+def android_platforms(
+    api_level: int | None = None, abi: str | None = None
+) -> Iterator[str]:
+    """
+    Yields the :attr:`~Tag.platform` tags for Android. If this function is invoked on
+    non-Android platforms, the ``api_level`` and ``abi`` arguments are required.
+
+    :param int api_level: The maximum `API level
+        <https://developer.android.com/tools/releases/platforms>`__ to return. Defaults
+        to the current system's version, as returned by ``platform.android_ver``.
+    :param str abi: The `Android ABI <https://developer.android.com/ndk/guides/abis>`__,
+        e.g. ``arm64_v8a``. Defaults to the current system's ABI , as returned by
+        ``sysconfig.get_platform``. Hyphens and periods will be replaced with
+        underscores.
+    """
+    if platform.system() != "Android" and (api_level is None or abi is None):
+        raise TypeError(
+            "on non-Android platforms, the api_level and abi arguments are required"
+        )
+
+    if api_level is None:
+        # Python 3.13 was the first version to return platform.system() == "Android",
+        # and also the first version to define platform.android_ver().
+        api_level = platform.android_ver().api_level  # type: ignore[attr-defined]
+
+    if abi is None:
+        abi = sysconfig.get_platform().split("-")[-1]
+    abi = _normalize_string(abi)
+
+    # 16 is the minimum API level known to have enough features to support CPython
+    # without major patching. Yield every API level from the maximum down to the
+    # minimum, inclusive.
+    min_api_level = 16
+    for ver in range(api_level, min_api_level - 1, -1):
+        yield f"android_{ver}_{abi}"
+
+
+def _linux_platforms(is_32bit: bool = _32_BIT_INTERPRETER) -> Iterator[str]:
+    linux = _normalize_string(sysconfig.get_platform())
+    if not linux.startswith("linux_"):
+        # we should never be here, just yield the sysconfig one and return
+        yield linux
+        return
+    if is_32bit:
+        if linux == "linux_x86_64":
+            linux = "linux_i686"
+        elif linux == "linux_aarch64":
+            linux = "linux_armv8l"
+    _, arch = linux.split("_", 1)
+    archs = {"armv8l": ["armv8l", "armv7l"]}.get(arch, [arch])
+    yield from _manylinux.platform_tags(archs)
+    yield from _musllinux.platform_tags(archs)
+    for arch in archs:
+        yield f"linux_{arch}"
+
+
+def _emscripten_platforms() -> Iterator[str]:
+    pyemscripten_platform_version = sysconfig.get_config_var(
+        "PYEMSCRIPTEN_PLATFORM_VERSION"
+    )
+    if pyemscripten_platform_version:
+        yield f"pyemscripten_{pyemscripten_platform_version}_wasm32"
+    yield from _generic_platforms()
+
+
+def _generic_platforms() -> Iterator[str]:
+    yield _normalize_string(sysconfig.get_platform())
+
+
+def platform_tags() -> Iterator[str]:
+    """
+    Yields the :attr:`~Tag.platform` tags for the running interpreter.
+    """
+    if platform.system() == "Darwin":
+        return mac_platforms()
+    elif platform.system() == "iOS":
+        return ios_platforms()
+    elif platform.system() == "Android":
+        return android_platforms()
+    elif platform.system() == "Linux":
+        return _linux_platforms()
+    elif platform.system() == "Emscripten":
+        return _emscripten_platforms()
+    else:
+        return _generic_platforms()
+
+
+def interpreter_name() -> str:
+    """
+    Returns the name of the running interpreter.
+
+    Some implementations have a reserved, two-letter abbreviation which will
+    be returned when appropriate.
+
+    This typically acts as the prefix to the :attr:`~Tag.interpreter` tag.
+    """
+    name = sys.implementation.name
+    return INTERPRETER_SHORT_NAMES.get(name) or name
+
+
+def interpreter_version(*, warn: bool = False) -> str:
+    """
+    Returns the running interpreter's version.
+
+    This typically acts as the suffix to the :attr:`~Tag.interpreter` tag.
+
+    :param bool warn: Whether warnings should be logged. Defaults to ``False``.
+    """
+    version = _get_config_var("py_version_nodot", warn=warn)
+    return str(version) if version else _version_nodot(sys.version_info[:2])
+
+
+def _version_nodot(version: PythonVersion) -> str:
+    return "".join(map(str, version))
+
+
+def sys_tags(*, warn: bool = False) -> Iterator[Tag]:
+    """
+    Yields the sequence of tag triples that the running interpreter supports.
+
+    The iterable is ordered so that the best-matching tag is first in the
+    sequence. The exact preferential order to tags is interpreter-specific, but
+    in general the tag importance is in the order of:
+
+    1. Interpreter
+    2. Platform
+    3. ABI
+
+    This order is due to the fact that an ABI is inherently tied to the
+    platform, but platform-specific code is not necessarily tied to the ABI. The
+    interpreter is the most important tag as it dictates basic support for any
+    wheel.
+
+    The function returns an iterable in order to allow for the possible
+    short-circuiting of tag generation if the entire sequence is not necessary
+    and tag calculation happens to be expensive.
+
+    :param bool warn: Whether warnings should be logged. Defaults to ``False``.
+
+    .. versionchanged:: 21.3
+        Added the `pp3-none-any` tag (:issue:`311`).
+    .. versionchanged:: 27.0
+        Added the `abi3t` tag (:issue:`1099`).
+    """
+
+    interp_name = interpreter_name()
+    if interp_name == "cp":
+        yield from cpython_tags(warn=warn)
+    else:
+        yield from generic_tags()
+
+    if interp_name == "pp":
+        interp = "pp3"
+    elif interp_name == "cp":
+        interp = "cp" + interpreter_version(warn=warn)
+    else:
+        interp = None
+    yield from compatible_tags(interpreter=interp)
+
+
+def create_compatible_tags_selector(
+    tags: Iterable[Tag],
+) -> Callable[[Iterable[tuple[_T, AbstractSet[Tag]]]], Iterator[_T]]:
+    """Create a callable to select things compatible with supported tags.
+
+    This function accepts an ordered sequence of tags, with the preferred
+    tags first.
+
+    The returned callable accepts an iterable of tuples (thing, set[Tag]),
+    and returns an iterator of things, with the things with the best
+    matching tags first.
+
+    Example to select compatible wheel filenames:
+
+    >>> from packaging import tags
+    >>> from packaging.utils import parse_wheel_filename
+    >>> selector = tags.create_compatible_tags_selector(tags.sys_tags())
+    >>> filenames = ["foo-1.0-py3-none-any.whl", "foo-1.0-py2-none-any.whl"]
+    >>> list(selector([
+    ...     (filename, parse_wheel_filename(filename)[-1]) for filename in filenames
+    ... ]))
+    ['foo-1.0-py3-none-any.whl']
+
+    .. versionadded:: 26.1
+    """
+    tag_ranks: dict[Tag, int] = {}
+    for rank, tag in enumerate(tags):
+        tag_ranks.setdefault(tag, rank)  # ignore duplicate tags, keep first
+    supported_tags = tag_ranks.keys()
+
+    def selector(
+        tagged_things: Iterable[tuple[_T, AbstractSet[Tag]]],
+    ) -> Iterator[_T]:
+        ranked_things: list[tuple[_T, int]] = []
+        for thing, thing_tags in tagged_things:
+            supported_thing_tags = thing_tags & supported_tags
+            if supported_thing_tags:
+                thing_rank = min(tag_ranks[t] for t in supported_thing_tags)
+                ranked_things.append((thing, thing_rank))
+        return iter(
+            thing for thing, _ in sorted(ranked_things, key=operator.itemgetter(1))
+        )
+
+    return selector

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/shellingham/__init__.py

@@ -0,0 +1,23 @@
+import importlib
+import os
+
+from ._core import ShellDetectionFailure
+
+__version__ = "1.5.4"
+
+
+def detect_shell(pid=None, max_depth=10):
+    name = os.name
+    try:
+        impl = importlib.import_module(".{}".format(name), __name__)
+    except ImportError:
+        message = "Shell detection not implemented for {0!r}".format(name)
+        raise RuntimeError(message)
+    try:
+        get_shell = impl.get_shell
+    except AttributeError:
+        raise RuntimeError("get_shell not implemented for {0!r}".format(name))
+    shell = get_shell(pid, max_depth=max_depth)
+    if shell:
+        return shell
+    raise ShellDetectionFailure()

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/shellingham/_core.py

@@ -0,0 +1,11 @@
+SHELL_NAMES = (
+    {"sh", "bash", "dash", "ash"}  # Bourne.
+    | {"csh", "tcsh"}  # C.
+    | {"ksh", "zsh", "fish"}  # Common alternatives.
+    | {"cmd", "powershell", "pwsh"}  # Microsoft.
+    | {"elvish", "xonsh", "nu"}  # More exotic.
+)
+
+
+class ShellDetectionFailure(EnvironmentError):
+    pass

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/aria/configuration_aria.py

@@ -0,0 +1,149 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/aria/modular_aria.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_aria.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2024 The Rhymes-AI Teams Authors and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...modeling_rope_utils import RopeParameters
+from ...utils import auto_docstring
+from ...utils.type_validators import interval
+from ..auto import CONFIG_MAPPING, AutoConfig
+
+
+@auto_docstring(checkpoint="rhymes-ai/Aria")
+@strict
+class AriaTextConfig(PreTrainedConfig):
+    r"""
+    moe_num_experts (`int`, *optional*, defaults to 8):
+        The number of experts in the MoE layer.
+    moe_topk (`int`, *optional*, defaults to 2):
+        The number of top experts to route to for each token.
+    moe_num_shared_experts (`int`, *optional*, defaults to 2):
+        The number of shared experts.
+    """
+
+    model_type = "aria_text"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.shared_experts.gate_proj": "colwise",
+        "layers.*.mlp.shared_experts.up_proj": "colwise",
+        "layers.*.mlp.shared_experts.down_proj": "rowwise",
+    }
+    base_model_pp_plan = {
+        "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+        "norm": (["hidden_states"], ["hidden_states"]),
+    }
+
+    vocab_size: int = 32000
+    hidden_size: int = 4096
+
+    intermediate_size: int = 4096
+    num_hidden_layers: int = 32
+    num_attention_heads: int = 32
+    num_key_value_heads: int | None = None
+    hidden_act: str = "silu"
+    max_position_embeddings: int = 2048
+    initializer_range: float = interval(min=0.0, max=1.0)(default=0.02)
+    rms_norm_eps: float = 1e-6
+    use_cache: bool = True
+    pad_token_id: int | None = 2
+    bos_token_id: int | None = 1
+    eos_token_id: int | list[int] | None = 2
+    pretraining_tp: int | None = 1
+    tie_word_embeddings: bool = False
+    rope_parameters: RopeParameters | dict | None = None
+    attention_bias: bool = False
+    attention_dropout: int | float | None = 0.0
+    mlp_bias: bool = False
+    head_dim: int | None = None
+    base_config_key = "text_config"
+    moe_num_experts: int = 8
+    moe_topk: int = 2
+    moe_num_shared_experts: int = 2
+
+    def __post_init__(self, **kwargs):
+        if self.head_dim is None:
+            self.head_dim = self.hidden_size // self.num_attention_heads
+        if self.num_key_value_heads is None:
+            self.num_key_value_heads = self.num_attention_heads
+
+        super().__post_init__(**kwargs)
+
+    def validate_architecture(self):
+        """Part of `@strict`-powered validation. Validates the architecture of the config."""
+        if self.hidden_size % self.num_attention_heads != 0:
+            raise ValueError(
+                f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention "
+                f"heads ({self.num_attention_heads})."
+            )
+
+
+@auto_docstring(checkpoint="rhymes-ai/Aria")
+@strict
+class AriaConfig(PreTrainedConfig):
+    r"""
+    projector_patch_to_query_dict (`dict`, *optional*):
+        Mapping of patch sizes to query dimensions.
+    """
+
+    model_type = "aria"
+    attribute_map = {
+        "image_token_id": "image_token_index",
+    }
+    sub_configs = {"text_config": AriaTextConfig, "vision_config": AutoConfig}
+
+    vision_config: dict | PreTrainedConfig | None = None
+    text_config: dict | AriaTextConfig | None = None
+    vision_feature_layer: int | list[int] = -1
+    projector_patch_to_query_dict: dict | None = None
+    image_token_index: int = 9
+    initializer_range: float = 0.02
+    tie_word_embeddings: bool = False
+
+    def __post_init__(self, **kwargs):
+        # Convert the keys and values of projector_patch_to_query_dict to integers
+        # This ensures consistency even if they were provided as strings
+        if self.projector_patch_to_query_dict is None:
+            self.projector_patch_to_query_dict = {
+                1225: 128,
+                4900: 256,
+            }
+        self.projector_patch_to_query_dict = {int(k): int(v) for k, v in self.projector_patch_to_query_dict.items()}
+        self.max_value_projector_patch_to_query_dict = max(self.projector_patch_to_query_dict.values())
+
+        if isinstance(self.vision_config, dict):
+            self.vision_config["model_type"] = "idefics3_vision"
+            self.vision_config = CONFIG_MAPPING[self.vision_config["model_type"]](**self.vision_config)
+        elif self.vision_config is None:
+            self.vision_config = CONFIG_MAPPING["idefics3_vision"]()
+
+        if isinstance(self.text_config, dict) and "model_type" in self.text_config:
+            self.text_config = AriaTextConfig(**self.text_config)
+        elif self.text_config is None:
+            self.text_config = AriaTextConfig()
+
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["AriaConfig", "AriaTextConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ovis2/__init__.py

@@ -0,0 +1,31 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_ovis2 import *
+    from .image_processing_ovis2 import *
+    from .image_processing_pil_ovis2 import *
+    from .modeling_ovis2 import *
+    from .processing_ovis2 import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ovis2/configuration_ovis2.py

@@ -0,0 +1,102 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+from ..qwen2.configuration_qwen2 import Qwen2Config
+
+
+@auto_docstring(checkpoint="thisisiron/Ovis2-1B-hf")
+@strict
+class Ovis2VisionConfig(PreTrainedConfig):
+    r"""
+    hidden_stride (`int`, *optional*, defaults to 1):
+        The stride of the hidden layer in the Vision Transformer.
+    num_visual_indicator_tokens (`int`, *optional*, defaults to 5):
+        Number of visual indicator tokens.
+    tokenize_function (`str`, *optional*, defaults to `"softmax"`):
+        The function used to tokenize the visual indicator tokens.
+    """
+
+    base_config_key = "vision_config"
+
+    hidden_size: int = 1024
+    intermediate_size: int = 2816
+    num_hidden_layers: int = 24
+    num_attention_heads: int = 8
+    num_channels: int = 3
+    image_size: int | list[int] | tuple[int, int] = 224
+    patch_size: int | list[int] | tuple[int, int] = 14
+    rms_norm_eps: float = 1e-5
+    attention_dropout: float | int = 0.0
+    qkv_bias: bool = False
+    mlp_bias: bool = False
+    hidden_act: str = "silu"
+    vocab_size: int = 16384
+    hidden_stride: int = 1
+    num_visual_indicator_tokens: int = 5
+    initializer_range: float = 0.02
+    tokenize_function: str = "softmax"
+
+
+@auto_docstring(checkpoint="thisisiron/Ovis2-1B-hf")
+@strict
+class Ovis2Config(PreTrainedConfig):
+    r"""
+    visual_indicator_token_ids (`List[int]`, *optional*, defaults to `[151666, 151667, 151668, 151669, 151670]`):
+        The visual indicator token ids to encode the image prompt.
+
+    ```python
+    >>> from transformers import Ovis2ForConditionalGeneration, Ovis2Config
+
+    >>> # Initializing a Ovis2 style configuration
+    >>> configuration = Ovis2Config()
+
+    >>> # Initializing a model from the Ovis2-2B style configuration
+    >>> model = Ovis2ForConditionalGeneration(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```
+    """
+
+    model_type = "ovis2"
+    sub_configs = {"text_config": Qwen2Config, "vision_config": Ovis2VisionConfig}
+
+    vision_config: dict | PreTrainedConfig | None = None
+    text_config: dict | PreTrainedConfig | None = None
+    image_token_id: int = 151665
+    visual_indicator_token_ids: list[int] | tuple[int, ...] = (151666, 151667, 151668, 151669, 151670)
+    vocab_size: int = 151643
+    hidden_size: int = 1536
+    tie_word_embeddings: bool = True
+
+    def __post_init__(self, **kwargs):
+        if isinstance(self.vision_config, dict):
+            self.vision_config = Ovis2VisionConfig(**self.vision_config)
+        if self.vision_config is None:
+            self.vision_config = Ovis2VisionConfig(num_visual_indicator_tokens=len(self.visual_indicator_token_ids))
+
+        if isinstance(self.text_config, dict):
+            self.text_config = Qwen2Config(**self.text_config)
+        elif self.text_config is None:
+            self.text_config = Qwen2Config()
+
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["Ovis2VisionConfig", "Ovis2Config"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ovis2/image_processing_ovis2.py

@@ -0,0 +1,327 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image processor class for OVIS2."""
+
+from functools import lru_cache
+
+import torch
+from torchvision.transforms.v2 import functional as tvF
+
+from ...image_processing_backends import TorchvisionBackend
+from ...image_processing_utils import BatchFeature
+from ...image_transforms import group_images_by_shape, reorder_images
+from ...image_utils import (
+    OPENAI_CLIP_MEAN,
+    OPENAI_CLIP_STD,
+    ImageInput,
+    PILImageResampling,
+    SizeDict,
+)
+from ...processing_utils import ImagesKwargs, Unpack
+from ...utils import TensorType, auto_docstring
+
+
+# Helper functions for patch/tile calculations
+@lru_cache(maxsize=10)
+def get_all_supported_aspect_ratios(min_image_tiles: int, max_image_tiles: int) -> list[tuple[int, int]]:
+    """Computes all allowed aspect ratios for a given minimum and maximum number of input tiles."""
+    aspect_ratios = []
+    for width in range(1, max_image_tiles + 1):
+        for height in range(1, max_image_tiles + 1):
+            if width * height <= max_image_tiles and width * height >= min_image_tiles:
+                aspect_ratios.append((width, height))
+    return sorted(aspect_ratios, key=lambda x: x[0] * x[1])
+
+
+@lru_cache(maxsize=100)
+def get_optimal_tiled_canvas(
+    original_image_size: tuple[int, int],
+    target_tile_size: tuple[int, int],
+    min_image_tiles: int,
+    max_image_tiles: int,
+) -> tuple[int, int]:
+    """Find the canvas with the closest aspect ratio to the original image aspect ratio."""
+    possible_tile_arrangements = get_all_supported_aspect_ratios(min_image_tiles, max_image_tiles)
+    original_height, original_width = original_image_size
+    target_tile_height, target_tile_width = target_tile_size
+    aspect_ratio = original_width / original_height
+    area = original_width * original_height
+
+    best_ratio_diff = float("inf")
+    best_grid = (1, 1)
+    for grid in possible_tile_arrangements:
+        grid_aspect_ratio = grid[0] / grid[1]
+        ratio_diff = abs(aspect_ratio - grid_aspect_ratio)
+        if ratio_diff < best_ratio_diff:
+            best_ratio_diff = ratio_diff
+            best_grid = grid
+        elif ratio_diff == best_ratio_diff:
+            if area > 0.5 * target_tile_height * target_tile_width * grid[0] * grid[1]:
+                best_grid = grid
+    return best_grid
+
+
+def compute_patch_covering_area(left: int, upper: int, right: int, lower: int, side: int) -> float:
+    w = right - left
+    h = lower - upper
+    w, h = max(w, h), min(w, h)
+    if w > side:
+        h = h / w * side
+        w = side
+    return w * h
+
+
+def split_image_into_grid(h: int, w: int, grid: tuple[int, int]) -> list[tuple[int, int, int, int]]:
+    row_height = h // grid[0]
+    col_width = w // grid[1]
+    return [
+        (
+            col * col_width,
+            row * row_height,
+            w if col == grid[1] - 1 else (col + 1) * col_width,
+            h if row == grid[0] - 1 else (row + 1) * row_height,
+        )
+        for row in range(grid[0])
+        for col in range(grid[1])
+    ]
+
+
+@lru_cache(maxsize=100)
+def get_min_tile_covering_grid(
+    image_size: tuple[int, int],
+    target_patch_size: int,
+    max_image_tiles: int,
+    covering_threshold: float = 0.9,
+) -> tuple[int, int]:
+    image_height, image_width = image_size
+    image_area = image_width * image_height
+    candidate_tile_grids = get_all_supported_aspect_ratios(1, max_image_tiles)
+    evaluated_grids = []
+    sufficient_covering_grids = []
+
+    for tile_grid in candidate_tile_grids:
+        tile_regions = split_image_into_grid(image_height, image_width, tile_grid)
+        tile_covering_ratio = (
+            sum(compute_patch_covering_area(*region, target_patch_size) for region in tile_regions) / image_area
+        )
+        evaluated_grids.append((tile_grid, tile_covering_ratio))
+        if tile_covering_ratio > covering_threshold:
+            sufficient_covering_grids.append((tile_grid, tile_covering_ratio))
+
+    if sufficient_covering_grids:
+        return min(sufficient_covering_grids, key=lambda x: (x[0][0] * x[0][1], -x[1]))[0]
+    return min(evaluated_grids, key=lambda x: (-x[1], x[0][0] * x[0][1]))[0]
+
+
+class Ovis2ImageProcessorKwargs(ImagesKwargs, total=False):
+    """
+    crop_to_patches (`bool`, *optional*, defaults to `False`):
+        Whether to crop the image to patches. Can be overridden by the `crop_to_patches` parameter in the
+        `preprocess` method.
+    min_patches (`int`, *optional*, defaults to 1):
+        The minimum number of patches to be extracted from the image. Only has an effect if `crop_to_patches` is
+        set to `True`. Can be overridden by the `min_patches` parameter in the `preprocess` method.
+    max_patches (`int`, *optional*, defaults to 12):
+        The maximum number of patches to be extracted from the image. Only has an effect if `crop_to_patches` is
+        set to `True`. Can be overridden by the `max_patches` parameter in the `preprocess` method.
+    use_covering_area_grid (`bool`, *optional*, defaults to `True`):
+        Whether to use the covering area grid to determine the number of patches. Only has an effect if
+        `crop_to_patches` is set to `True`. Can be overridden by the `use_covering_area_grid` parameter in the
+        `preprocess` method.
+    """
+
+    crop_to_patches: bool
+    min_patches: int
+    max_patches: int
+    use_covering_area_grid: bool
+
+
+@auto_docstring
+class Ovis2ImageProcessor(TorchvisionBackend):
+    resample = PILImageResampling.BICUBIC
+    image_mean = OPENAI_CLIP_MEAN
+    image_std = OPENAI_CLIP_STD
+    size = {"height": 384, "width": 384}
+    default_to_square = True
+    do_resize = True
+    do_rescale = True
+    do_normalize = True
+    do_convert_rgb = True
+    crop_to_patches = False
+    min_patches = 1
+    max_patches = 12
+    use_covering_area_grid = True
+    valid_kwargs = Ovis2ImageProcessorKwargs
+
+    def __init__(self, **kwargs: Unpack[Ovis2ImageProcessorKwargs]):
+        super().__init__(**kwargs)
+
+    @auto_docstring
+    def preprocess(self, images: ImageInput, **kwargs: Unpack[Ovis2ImageProcessorKwargs]) -> BatchFeature:
+        return super().preprocess(images, **kwargs)
+
+    def crop_image_to_patches(
+        self,
+        images: "torch.Tensor",
+        min_patches: int,
+        max_patches: int,
+        use_covering_area_grid: bool = True,
+        covering_threshold: float = 0.9,
+        patch_size: SizeDict | None = None,
+        resample: "PILImageResampling | tvF.InterpolationMode | int | None" = None,
+    ):
+        """
+        Crop the images to patches and return a list of cropped images.
+        The number of patches and their grid arrangement are determined by the original image size,
+        the target patch size and the minimum and maximum number of patches.
+        The aspect ratio of the patches grid is chosen to be the closest to the original image aspect ratio.
+
+        Args:
+            images (`torch.Tensor`):
+                The images to be cropped.
+            min_patches (`int`):
+                The minimum number of patches to be extracted from the image.
+            max_patches (`int`):
+                The maximum number of patches to be extracted from the image.
+            use_covering_area_grid (`bool`, *optional*, defaults to `True`):
+                Whether to use the original OVIS2 approach: compute the minimal number of tiles that cover at least 90%
+                of the image area. If `False`, the closest aspect ratio to the target is used.
+            covering_threshold (`float`, *optional*, defaults to `0.9`):
+                The threshold for the covering area. Only has an effect if `use_covering_area_grid` is set to `True`.
+            patch_size (`SizeDict`, *optional*):
+                The size of the output patches.
+            resample (`PILImageResampling | tvF.InterpolationMode | int | None`, *optional*):
+                Resampling filter to use if resizing the image.
+
+        Returns:
+            Tuple[`torch.Tensor`, `list`]: A tuple containing the processed images tensor and the grid information.
+        """
+        num_image = images.shape[0]
+        patch_size_height, patch_size_width = patch_size.height, patch_size.width
+        original_height, original_width = images.shape[-2:]
+
+        if use_covering_area_grid:
+            # Use the original OVIS2 approach: compute the minimal number of tiles that cover at least 90% of the image area
+            num_columns, num_rows = get_min_tile_covering_grid(
+                (original_height, original_width),
+                target_patch_size=patch_size_height,  # square patch size
+                max_image_tiles=max_patches,
+                covering_threshold=covering_threshold,
+            )
+        else:
+            # find the closest aspect ratio to the target
+            num_columns, num_rows = get_optimal_tiled_canvas(
+                (original_height, original_width), (patch_size_height, patch_size_width), min_patches, max_patches
+            )
+
+        # calculate the target width and height
+        target_width = patch_size_width * num_columns
+        target_height = patch_size_height * num_rows
+        num_blocks = num_columns * num_rows
+
+        # resize the image so that each patch is of patch_size
+        resized_image = self.resize(images, SizeDict(height=target_height, width=target_width), resample=resample)
+        # split the image into patches
+        processed_images = []
+        for i in range(num_blocks):
+            column = i % num_columns
+            row = i // num_columns
+            box = (
+                column * patch_size_width,
+                row * patch_size_height,
+                (column + 1) * patch_size_width,
+                (row + 1) * patch_size_height,
+            )
+            # split the image
+            patch_image = resized_image[..., box[1] : box[3], box[0] : box[2]]
+            processed_images.append(patch_image)
+
+        if len(processed_images) != 1:
+            thumbnail_img = self.resize(images, patch_size, resample=resample)
+            processed_images.insert(0, thumbnail_img)
+
+        processed_images = torch.stack(processed_images, dim=0).transpose(0, 1).contiguous()
+        grid = [[num_rows, num_columns] for _ in range(num_image)]
+
+        return processed_images, grid
+
+    def _preprocess(
+        self,
+        images: list["torch.Tensor"],
+        do_resize: bool,
+        size: SizeDict,
+        resample: "PILImageResampling | tvF.InterpolationMode | int | None",
+        do_center_crop: bool,
+        crop_size: SizeDict,
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: float | list[float] | None,
+        image_std: float | list[float] | None,
+        disable_grouping: bool | None,
+        return_tensors: str | TensorType | None,
+        crop_to_patches: bool = False,
+        min_patches: int = 1,
+        max_patches: int = 12,
+        use_covering_area_grid: bool = True,
+        **kwargs,
+    ) -> BatchFeature:
+        if crop_to_patches and max_patches > 1:
+            grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)
+            processed_images_grouped = {}
+            grids = {}
+            for shape, stacked_images in grouped_images.items():
+                stacked_images, grid = self.crop_image_to_patches(
+                    stacked_images,
+                    min_patches,
+                    max_patches,
+                    patch_size=size,
+                    use_covering_area_grid=use_covering_area_grid,
+                    resample=resample,
+                )
+                processed_images_grouped[shape] = stacked_images
+                grids[shape] = grid
+            images = reorder_images(processed_images_grouped, grouped_images_index)
+            images = [image for images_list in images for image in images_list]
+            grids = reorder_images(grids, grouped_images_index)
+        else:
+            grids = [[1, 1] for _ in range(len(images))]
+
+        # Group images by size for batched resizing
+        grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)
+        resized_images_grouped = {}
+        for shape, stacked_images in grouped_images.items():
+            if do_resize:
+                stacked_images = self.resize(image=stacked_images, size=size, resample=resample)
+            resized_images_grouped[shape] = stacked_images
+        resized_images = reorder_images(resized_images_grouped, grouped_images_index)
+
+        # Group images by size for further processing
+        # Needed in case do_resize is False, or resize returns images with different sizes
+        grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping)
+        processed_images_grouped = {}
+        for shape, stacked_images in grouped_images.items():
+            if do_center_crop:
+                stacked_images = self.center_crop(stacked_images, crop_size)
+            # Fused rescale and normalize
+            stacked_images = self.rescale_and_normalize(
+                stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std
+            )
+            processed_images_grouped[shape] = stacked_images
+
+        processed_images = reorder_images(processed_images_grouped, grouped_images_index)
+        return BatchFeature(data={"pixel_values": processed_images, "grids": grids}, tensor_type=return_tensors)
+
+
+__all__ = ["Ovis2ImageProcessor"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ovis2/image_processing_pil_ovis2.py

@@ -0,0 +1,297 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PIL Image processor class for OVIS2."""
+
+from functools import lru_cache
+
+import numpy as np
+
+from ...image_processing_backends import PilBackend
+from ...image_processing_utils import BatchFeature
+from ...image_transforms import to_channel_dimension_format
+from ...image_utils import (
+    OPENAI_CLIP_MEAN,
+    OPENAI_CLIP_STD,
+    ChannelDimension,
+    ImageInput,
+    PILImageResampling,
+    SizeDict,
+    infer_channel_dimension_format,
+)
+from ...processing_utils import ImagesKwargs, Unpack
+from ...utils import TensorType, auto_docstring
+
+
+# Adapted from transformers.models.ovis2.image_processing_ovis2.Ovis2ImageProcessorKwargs
+class Ovis2ImageProcessorKwargs(ImagesKwargs, total=False):
+    """
+    crop_to_patches (`bool`, *optional*, defaults to `False`):
+        Whether to crop the image to patches. Can be overridden by the `crop_to_patches` parameter in the
+        `preprocess` method.
+    min_patches (`int`, *optional*, defaults to 1):
+        The minimum number of patches to be extracted from the image. Only has an effect if `crop_to_patches` is
+        set to `True`. Can be overridden by the `min_patches` parameter in the `preprocess` method.
+    max_patches (`int`, *optional*, defaults to 12):
+        The maximum number of patches to be extracted from the image. Only has an effect if `crop_to_patches` is
+        set to `True`. Can be overridden by the `max_patches` parameter in the `preprocess` method.
+    use_covering_area_grid (`bool`, *optional*, defaults to `True`):
+        Whether to use the covering area grid to determine the number of patches. Only has an effect if
+        `crop_to_patches` is set to `True`. Can be overridden by the `use_covering_area_grid` parameter in the
+        `preprocess` method.
+    """
+
+    crop_to_patches: bool
+    min_patches: int
+    max_patches: int
+    use_covering_area_grid: bool
+
+
+# Adapted from transformers.models.ovis2.image_processing_ovis2.get_all_supported_aspect_ratios
+@lru_cache(maxsize=10)
+def get_all_supported_aspect_ratios(min_image_tiles: int, max_image_tiles: int) -> list[tuple[int, int]]:
+    """Computes all allowed aspect ratios for a given minimum and maximum number of input tiles."""
+    aspect_ratios = []
+    for width in range(1, max_image_tiles + 1):
+        for height in range(1, max_image_tiles + 1):
+            if width * height <= max_image_tiles and width * height >= min_image_tiles:
+                aspect_ratios.append((width, height))
+    return sorted(aspect_ratios, key=lambda x: x[0] * x[1])
+
+
+# Adapted from transformers.models.ovis2.image_processing_ovis2.compute_patch_covering_area
+def compute_patch_covering_area(left: int, upper: int, right: int, lower: int, side: int) -> float:
+    w = right - left
+    h = lower - upper
+    w, h = max(w, h), min(w, h)
+    if w > side:
+        h = h / w * side
+        w = side
+    return w * h
+
+
+# Adapted from transformers.models.ovis2.image_processing_ovis2.split_image_into_grid
+def split_image_into_grid(h: int, w: int, grid: tuple[int, int]) -> list[tuple[int, int, int, int]]:
+    row_height = h // grid[0]
+    col_width = w // grid[1]
+    return [
+        (
+            col * col_width,
+            row * row_height,
+            w if col == grid[1] - 1 else (col + 1) * col_width,
+            h if row == grid[0] - 1 else (row + 1) * row_height,
+        )
+        for row in range(grid[0])
+        for col in range(grid[1])
+    ]
+
+
+# Adapted from transformers.models.ovis2.image_processing_ovis2.get_min_tile_covering_grid
+@lru_cache(maxsize=100)
+def get_min_tile_covering_grid(
+    image_size: tuple[int, int],
+    target_patch_size: int,
+    max_image_tiles: int,
+    covering_threshold: float = 0.9,
+) -> tuple[int, int]:
+    image_height, image_width = image_size
+    image_area = image_width * image_height
+    candidate_tile_grids = get_all_supported_aspect_ratios(1, max_image_tiles)
+    evaluated_grids = []
+    sufficient_covering_grids = []
+
+    for tile_grid in candidate_tile_grids:
+        tile_regions = split_image_into_grid(image_height, image_width, tile_grid)
+        tile_covering_ratio = (
+            sum(compute_patch_covering_area(*region, target_patch_size) for region in tile_regions) / image_area
+        )
+        evaluated_grids.append((tile_grid, tile_covering_ratio))
+        if tile_covering_ratio > covering_threshold:
+            sufficient_covering_grids.append((tile_grid, tile_covering_ratio))
+
+    if sufficient_covering_grids:
+        return min(sufficient_covering_grids, key=lambda x: (x[0][0] * x[0][1], -x[1]))[0]
+    return min(evaluated_grids, key=lambda x: (-x[1], x[0][0] * x[0][1]))[0]
+
+
+# Adapted from transformers.models.ovis2.image_processing_ovis2.get_optimal_tiled_canvas
+@lru_cache(maxsize=100)
+def get_optimal_tiled_canvas(
+    original_image_size: tuple[int, int],
+    target_tile_size: tuple[int, int],
+    min_image_tiles: int,
+    max_image_tiles: int,
+) -> tuple[int, int]:
+    """Find the canvas with the closest aspect ratio to the original image aspect ratio."""
+    possible_tile_arrangements = get_all_supported_aspect_ratios(min_image_tiles, max_image_tiles)
+    original_height, original_width = original_image_size
+    target_tile_height, target_tile_width = target_tile_size
+    aspect_ratio = original_width / original_height
+    area = original_width * original_height
+
+    best_ratio_diff = float("inf")
+    best_grid = (1, 1)
+    for grid in possible_tile_arrangements:
+        grid_aspect_ratio = grid[0] / grid[1]
+        ratio_diff = abs(aspect_ratio - grid_aspect_ratio)
+        if ratio_diff < best_ratio_diff:
+            best_ratio_diff = ratio_diff
+            best_grid = grid
+        elif ratio_diff == best_ratio_diff:
+            if area > 0.5 * target_tile_height * target_tile_width * grid[0] * grid[1]:
+                best_grid = grid
+    return best_grid
+
+
+@auto_docstring
+class Ovis2ImageProcessorPil(PilBackend):
+    resample = PILImageResampling.BICUBIC
+    image_mean = OPENAI_CLIP_MEAN
+    image_std = OPENAI_CLIP_STD
+    size = {"height": 384, "width": 384}
+    default_to_square = True
+    do_resize = True
+    do_rescale = True
+    do_normalize = True
+    do_convert_rgb = True
+    crop_to_patches = False
+    min_patches = 1
+    max_patches = 12
+    use_covering_area_grid = True
+    valid_kwargs = Ovis2ImageProcessorKwargs
+
+    def __init__(self, **kwargs: Unpack[Ovis2ImageProcessorKwargs]):
+        super().__init__(**kwargs)
+
+    @auto_docstring
+    def preprocess(self, images: ImageInput, **kwargs: Unpack[Ovis2ImageProcessorKwargs]) -> BatchFeature:
+        return super().preprocess(images, **kwargs)
+
+    def crop_image_to_patches(
+        self,
+        image: np.ndarray,
+        min_patches: int,
+        max_patches: int,
+        use_covering_area_grid: bool = True,
+        covering_threshold: float = 0.9,
+        patch_size: SizeDict | None = None,
+        resample: "PILImageResampling | None" = None,
+    ):
+        """
+        Crop the image to patches and return a list of cropped images.
+        Mirrors TorchvisionBackend.crop_image_to_patches.
+        """
+        # Normalize to CHW when called directly (e.g. from tests); _preprocess already receives CHW
+        input_data_format = infer_channel_dimension_format(image)
+        image = to_channel_dimension_format(image, ChannelDimension.FIRST, input_data_format)
+
+        patch_size_height, patch_size_width = patch_size.height, patch_size.width
+        original_height, original_width = image.shape[-2:]
+
+        if use_covering_area_grid:
+            # Use the original OVIS2 approach: compute the minimal number of tiles that cover at least 90% of the image area
+            num_columns, num_rows = get_min_tile_covering_grid(
+                (original_height, original_width),
+                target_patch_size=patch_size_height,  # square patch size
+                max_image_tiles=max_patches,
+                covering_threshold=covering_threshold,
+            )
+        else:
+            # find the closest aspect ratio to the target
+            num_columns, num_rows = get_optimal_tiled_canvas(
+                (original_height, original_width), (patch_size_height, patch_size_width), min_patches, max_patches
+            )
+
+        # calculate the target width and height
+        target_width = patch_size_width * num_columns
+        target_height = patch_size_height * num_rows
+        num_blocks = num_columns * num_rows
+
+        # resize the image so that each patch is of patch_size
+        resized_image = self.resize(image, SizeDict(height=target_height, width=target_width), resample=resample)
+        # split the image into patches
+        processed_images = []
+        for i in range(num_blocks):
+            column = i % num_columns
+            row = i // num_columns
+            box = (
+                column * patch_size_width,
+                row * patch_size_height,
+                (column + 1) * patch_size_width,
+                (row + 1) * patch_size_height,
+            )
+            patch_image = resized_image[:, box[1] : box[3], box[0] : box[2]]
+            processed_images.append(patch_image)
+
+        if len(processed_images) != 1:
+            thumbnail_img = self.resize(image, patch_size, resample=resample)
+            processed_images.insert(0, thumbnail_img)
+
+        return processed_images, [num_rows, num_columns]
+
+    def _preprocess(
+        self,
+        images: list[np.ndarray],
+        do_resize: bool,
+        size: SizeDict,
+        resample: "PILImageResampling | None",
+        do_center_crop: bool,
+        crop_size: SizeDict,
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: float | list[float] | None,
+        image_std: float | list[float] | None,
+        return_tensors: str | TensorType | None,
+        crop_to_patches: bool = False,
+        min_patches: int = 1,
+        max_patches: int = 12,
+        use_covering_area_grid: bool = True,
+        **kwargs,
+    ) -> BatchFeature:
+        if crop_to_patches and max_patches > 1:
+            # Crop to patches first
+            processed_images = []
+            grids = []
+            for image in images:
+                patches, grid = self.crop_image_to_patches(
+                    image,
+                    min_patches,
+                    max_patches,
+                    patch_size=size,
+                    use_covering_area_grid=use_covering_area_grid,
+                    resample=resample,
+                )
+                processed_images.extend(patches)
+                grids.append(grid)
+            images = processed_images
+        else:
+            grids = [[1, 1] for _ in range(len(images))]
+
+        # Process all images (including patches if any) through the standard pipeline
+        processed_images = []
+        for image in images:
+            if do_resize:
+                image = self.resize(image, size=size, resample=resample)
+            if do_center_crop:
+                image = self.center_crop(image, crop_size)
+            if do_rescale:
+                image = self.rescale(image, rescale_factor)
+            if do_normalize:
+                image = self.normalize(image, image_mean, image_std)
+            processed_images.append(image)
+
+        return BatchFeature(data={"pixel_values": processed_images, "grids": grids}, tensor_type=return_tensors)
+
+
+__all__ = ["Ovis2ImageProcessorPil"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ovis2/modular_ovis2.py

@@ -0,0 +1,448 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from dataclasses import dataclass
+
+import torch
+from torch import nn
+
+from ... import initialization as init
+from ...cache_utils import Cache
+from ...generation import GenerationMixin
+from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
+from ...modeling_utils import PreTrainedModel
+from ...processing_utils import Unpack
+from ...utils import TransformersKwargs, auto_docstring, can_return_tuple
+from ...utils.generic import merge_with_config_defaults
+from ...utils.output_capturing import capture_outputs
+from ..aimv2.modeling_aimv2 import Aimv2Attention, Aimv2EncoderLayer
+from ..auto import AutoModel
+from ..llama.modeling_llama import LlamaMLP, LlamaRMSNorm
+from ..llava.modeling_llava import LlavaForConditionalGeneration, LlavaModel
+from ..llava_next.modeling_llava_next import LlavaNextCausalLMOutputWithPast, LlavaNextModelOutputWithPast
+from ..siglip.modeling_siglip import SiglipEncoder, SiglipVisionEmbeddings
+from .configuration_ovis2 import Ovis2Config, Ovis2VisionConfig
+
+
+def hard_softmax(logits: torch.Tensor, dim: int):
+    y_soft = logits.softmax(dim)
+    # Straight through.
+    index = y_soft.max(dim, keepdim=True)[1]
+    y_hard = torch.zeros_like(logits, memory_format=torch.legacy_contiguous_format).scatter_(dim, index, 1.0)
+    ret = y_hard - y_soft.detach() + y_soft
+
+    return ret
+
+
+@auto_docstring
+@dataclass
+class BaseModelOutputWithVisualIndicatorFeatures(BaseModelOutputWithPooling):
+    r"""
+    visual_indicator_features (`torch.FloatTensor` of shape `(batch_size, visual_indicator_size)`):
+        Visual indicator features extracted from the model, which can be used for auxiliary tasks or further processing.
+    """
+
+    visual_indicator_features: torch.FloatTensor | None = None
+
+
+class Ovis2ModelOutputWithPast(LlavaNextModelOutputWithPast):
+    pass
+
+
+class Ovis2CausalLMOutputWithPast(LlavaNextCausalLMOutputWithPast):
+    pass
+
+
+class Ovis2RMSNorm(LlamaRMSNorm):
+    pass
+
+
+class Ovis2VisionMLP(LlamaMLP):
+    pass
+
+
+class Ovis2VisionEmbeddings(SiglipVisionEmbeddings):
+    def __init__(self, config: Ovis2VisionConfig):
+        super().__init__(config)
+        self.rms_norm = Ovis2RMSNorm(config.hidden_size, config.rms_norm_eps)
+
+    def interpolate_pos_encoding(self):
+        raise NotImplementedError("Not needed for Ovis2")
+
+    def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
+        target_dtype = self.patch_embedding.weight.dtype
+        patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype))
+        embeddings = patch_embeds.flatten(2).transpose(1, 2)
+        embeddings = self.rms_norm(embeddings)
+
+        embeddings = embeddings + self.position_embedding(self.position_ids)
+
+        return embeddings
+
+
+class Ovis2VisionAttention(Aimv2Attention):
+    pass
+
+
+class Ovis2VisionEncoderLayer(Aimv2EncoderLayer):
+    def __init__(self, config: Ovis2VisionConfig):
+        super().__init__()
+        self.attention = Ovis2VisionAttention(config)
+
+
+class Ovis2VisionEncoder(SiglipEncoder):
+    def __init__(self, config: Ovis2VisionConfig):
+        super().__init__(config)
+        self.layers = nn.ModuleList([Ovis2VisionEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        inputs_embeds,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutput:
+        hidden_states = inputs_embeds
+        for encoder_layer in self.layers:
+            hidden_states = encoder_layer(hidden_states, attention_mask, **kwargs)
+
+        return BaseModelOutput(last_hidden_state=hidden_states)
+
+
+class Ovis2VisionTransformer(nn.Module):
+    def __init__(self, config: Ovis2VisionConfig):
+        super().__init__()
+        self.config = config
+        self.embeddings = Ovis2VisionEmbeddings(config)
+        self.encoder = Ovis2VisionEncoder(config)
+        self.rms_norm = Ovis2RMSNorm(config.hidden_size, config.rms_norm_eps)
+        self.gradient_checkpointing = False
+
+    @can_return_tuple
+    def forward(
+        self,
+        pixel_values,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs,
+    ):
+        hidden_states = self.embeddings(pixel_values)
+
+        encoder_outputs: BaseModelOutput = self.encoder(
+            inputs_embeds=hidden_states,
+            attention_mask=attention_mask,
+            **kwargs,
+        )
+
+        last_hidden_state = encoder_outputs.last_hidden_state
+        last_hidden_state = self.rms_norm(last_hidden_state)
+
+        return BaseModelOutput(last_hidden_state=last_hidden_state)
+
+
+class Ovis2VisualEmbeddingTable(nn.Embedding):
+    def forward(self, visual_tokens: torch.Tensor) -> torch.Tensor:
+        if visual_tokens.dtype in [torch.int8, torch.int16, torch.int32, torch.int64, torch.long]:
+            return super().forward(visual_tokens)
+        return torch.matmul(visual_tokens, self.weight)
+
+
+class Ovis2PreTrainedModel(PreTrainedModel):
+    config: Ovis2Config
+    base_model_prefix = "model"
+    input_modalities = ("image", "text")
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Ovis2VisionAttention"]
+    _skip_keys_device_placement = ["past_key_values"]
+    _supports_cache_class = True
+    _supports_flash_attn = True
+    _supports_flex_attn = True
+    _supports_sdpa = True
+
+    _can_compile_fullgraph = True
+    _supports_attention_backend = True
+
+    def _init_weights(self, module):
+        super()._init_weights(module)
+        if isinstance(module, Ovis2VisionEmbeddings):
+            init.copy_(module.position_ids, torch.arange(module.position_ids.shape[-1]).expand((1, -1)))
+
+
+class Ovis2VisionModel(Ovis2PreTrainedModel):
+    config: Ovis2VisionConfig
+    _can_record_outputs = {
+        "hidden_states": Ovis2VisionEncoderLayer,
+        "attentions": Ovis2VisionAttention,
+    }
+
+    def __init__(self, config: Ovis2VisionConfig):
+        super().__init__(config)
+        self.config = config
+        self.transformer = Ovis2VisionTransformer(config)
+        self.num_visual_indicator_tokens = config.num_visual_indicator_tokens
+        self.vocab_size = config.vocab_size
+        self.head_linear = nn.Linear(
+            config.hidden_size * config.hidden_stride * config.hidden_stride,
+            self.vocab_size - self.num_visual_indicator_tokens,
+            bias=False,
+        )
+        self.head_norm = nn.LayerNorm(self.vocab_size - self.num_visual_indicator_tokens)
+
+        self.post_init()
+
+    @merge_with_config_defaults
+    @capture_outputs
+    def forward(
+        self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]
+    ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures:
+        outputs = self.transformer(pixel_values, **kwargs)
+        last_hidden_state = outputs[0]
+        if self.config.hidden_stride > 1:
+            num_images, seq_len, hidden_dim = last_hidden_state.shape
+            hidden_stride = self.config.hidden_stride
+
+            sqrt_l = int(math.sqrt(seq_len))
+            if sqrt_l * sqrt_l != seq_len:
+                raise ValueError("Token sequence length must be a perfect square")
+
+            pad_size = (hidden_stride - (sqrt_l % hidden_stride)) % hidden_stride
+            last_hidden_state = nn.functional.pad(last_hidden_state, (0, 0, 0, pad_size, 0, pad_size), "constant", 0)
+            sqrt_l += pad_size
+
+            last_hidden_state = last_hidden_state.reshape(
+                num_images, sqrt_l // hidden_stride, hidden_stride, sqrt_l // hidden_stride, hidden_stride, hidden_dim
+            )
+            last_hidden_state = last_hidden_state.permute(0, 1, 3, 2, 4, 5)
+            last_hidden_state = last_hidden_state.reshape(
+                num_images, -1, hidden_stride * hidden_stride * hidden_dim
+            )  # (n, (sqrt_l//hs)^2, hs^2*d)
+
+        logits = self.head_linear(last_hidden_state)
+        logits = self.head_norm(logits)
+
+        if self.config.tokenize_function == "gumbel_argmax":
+            prob_token = nn.functional.gumbel_softmax(logits, dim=-1, hard=True)
+        elif self.config.tokenize_function == "st_argmax":
+            prob_token = hard_softmax(logits, dim=-1)
+        elif self.config.tokenize_function == "softmax":
+            prob_token = nn.functional.softmax(logits, dim=-1)
+
+        return BaseModelOutputWithVisualIndicatorFeatures(
+            last_hidden_state=last_hidden_state,
+            pooler_output=prob_token,
+        )
+
+
+class Ovis2Model(LlavaModel):
+    def __init__(self, config: Ovis2Config):
+        super().__init__(config)
+        self.vision_tower = Ovis2VisionModel(config.vision_config)
+        self.visual_embeddings_table = Ovis2VisualEmbeddingTable(config.vision_config.vocab_size, config.hidden_size)
+
+        self.visual_vocab_size = config.vision_config.vocab_size
+        self.vocab_size = config.vocab_size
+        self.visual_indicator_token_ids = config.visual_indicator_token_ids
+        self.language_model = AutoModel.from_config(config.text_config)
+        del self.multi_modal_projector
+
+    @can_return_tuple
+    @auto_docstring(
+        custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection."
+    )
+    def get_image_features(
+        self,
+        pixel_values: torch.FloatTensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures:
+        image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs)
+        image_features = image_outputs.pooler_output
+        batch_size, img_seq_len, _ = image_features.shape
+        padding_tensor = torch.zeros(
+            (batch_size, img_seq_len, self.vision_tower.num_visual_indicator_tokens),
+            dtype=image_features.dtype,
+            device=image_features.device,
+            requires_grad=False,
+            layout=image_features.layout,
+        )
+        image_features = torch.cat([image_features, padding_tensor], dim=2)
+        image_features = self.visual_embeddings_table(image_features)
+
+        visual_indicator = torch.arange(
+            self.visual_vocab_size - self.vision_tower.num_visual_indicator_tokens,
+            self.visual_vocab_size,
+            dtype=torch.long,
+        ).to(image_features.device)
+        image_outputs.pooler_output = image_features
+        image_outputs.visual_indicator_features = self.visual_embeddings_table(visual_indicator)
+
+        return image_outputs
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        pixel_values: torch.FloatTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        use_cache: bool | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs,
+    ) -> tuple | Ovis2ModelOutputWithPast:
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        if pixel_values is not None:
+            image_outputs = self.get_image_features(pixel_values=pixel_values, return_dict=True)
+            image_features = image_outputs.pooler_output
+            visual_indicator_features = image_outputs.visual_indicator_features
+
+            special_image_mask = self.get_placeholder_mask(
+                input_ids,
+                inputs_embeds=inputs_embeds,
+                image_features=image_features,
+            )
+            inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features)
+
+            for i, visual_indicator_id in enumerate(self.visual_indicator_token_ids):
+                if input_ids is None:
+                    mask = inputs_embeds == self.get_input_embeddings()(
+                        torch.tensor(visual_indicator_id, dtype=torch.long, device=inputs_embeds.device)
+                    )
+                    mask = mask.all(-1)
+                else:
+                    mask = (input_ids == visual_indicator_id).to(inputs_embeds.device)
+
+                if mask.any():
+                    inputs_embeds[mask] = (
+                        visual_indicator_features[i]
+                        .expand_as(inputs_embeds[mask])
+                        .to(inputs_embeds.device, inputs_embeds.dtype)
+                    )
+
+        outputs = self.language_model(
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            logits_to_keep=logits_to_keep,
+            **kwargs,
+        )
+
+        return Ovis2ModelOutputWithPast(
+            last_hidden_state=outputs.last_hidden_state,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            image_hidden_states=image_features if pixel_values is not None else None,
+        )
+
+
+@auto_docstring
+class Ovis2ForConditionalGeneration(LlavaForConditionalGeneration, GenerationMixin):
+    def __init__(self, config: Ovis2Config):
+        super().__init__(config)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+    @auto_docstring
+    def get_image_features(
+        self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]
+    ) -> tuple | BaseModelOutputWithVisualIndicatorFeatures:
+        return self.model.get_image_features(pixel_values=pixel_values, **kwargs)
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        pixel_values: torch.FloatTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        use_cache: bool | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs,
+    ) -> tuple | Ovis2CausalLMOutputWithPast:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        Example:
+
+        ```python
+        >>> from PIL import Image
+        >>> import httpx
+        >>> from io import BytesIO
+        >>> from transformers import AutoProcessor, Ovis2ForConditionalGeneration
+
+        >>> model = Ovis2ForConditionalGeneration.from_pretrained("thisisiron/Ovis2-2B-hf")
+        >>> processor = AutoProcessor.from_pretrained("thisisiron/Ovis2-2B-hf")
+
+        >>> prompt = "<|im_start|>user\n<image>\nDescribe the image.<|im_end|>\n<|im_start|>assistant\n"
+        >>> url = "http://images.cocodataset.org/val2014/COCO_val2014_000000537955.jpg"
+        >>> with httpx.stream("GET", url) as response:
+        ...     image = Image.open(BytesIO(response.read()))
+
+        >>> inputs = processor(images=image, text=prompt, return_tensors="pt")
+
+        >>> # Generate
+        >>> generate_ids = model.generate(**inputs, max_new_tokens=15)
+        >>> processor.batch_decode(generate_ids, skip_special_tokens=True)[0]
+        "user\n\nDescribe the image.\nassistant\nThe image features a brown dog standing on a wooden floor, looking up with"
+        ```"""
+        outputs = self.model(
+            input_ids=input_ids,
+            pixel_values=pixel_values,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            **kwargs,
+        )
+
+        hidden_states = outputs[0]
+        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(
+                logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **kwargs
+            )
+
+        return Ovis2CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            image_hidden_states=outputs.image_hidden_states,
+        )
+
+
+__all__ = ["Ovis2PreTrainedModel", "Ovis2Model", "Ovis2ForConditionalGeneration"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ovis2/processing_ovis2.py

@@ -0,0 +1,149 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from ...feature_extraction_utils import BatchFeature
+from ...image_utils import ImageInput
+from ...processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
+from ...tokenization_utils_base import PreTokenizedInput, TextInput
+from ...utils import auto_docstring, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class Ovis2ProcessorKwargs(ProcessingKwargs, total=False):
+    _defaults = {
+        "text_kwargs": {
+            "padding": False,
+        },
+        "image_kwargs": {},
+    }
+
+
+@auto_docstring
+class Ovis2Processor(ProcessorMixin):
+    def __init__(
+        self,
+        image_processor=None,
+        tokenizer=None,
+        chat_template=None,
+        image_token="<image>",
+        image_seq_length=256,
+        **kwargs,
+    ):
+        r"""
+        image_token (`str`, *optional*, defaults to `"<image>"`):
+            Special token used to denote image location.
+        image_seq_length (`int`, *optional*, defaults to 256):
+            The number of image tokens to be used for each image in the input.
+        """
+        self.image_seq_length = image_seq_length
+        self.image_token = tokenizer.image_token if hasattr(tokenizer, "image_token") else image_token
+        self.image_token_id = (
+            tokenizer.image_token_id
+            if getattr(tokenizer, "image_token_id", None)
+            else tokenizer.convert_tokens_to_ids(self.image_token)
+        )
+        super().__init__(image_processor, tokenizer, chat_template=chat_template, **kwargs)
+
+    @auto_docstring
+    def __call__(
+        self,
+        images: ImageInput | None = None,
+        text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] = None,
+        **kwargs: Unpack[Ovis2ProcessorKwargs],
+    ) -> BatchFeature:
+        r"""
+        Returns:
+            [`BatchFeature`]: A [`BatchFeature`] with the following fields:
+
+            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
+            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
+              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
+              `None`).
+            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
+            - **image_sizes** -- Size of each image that will be used to unpad an image. Returned when `images` is not `None`.
+        """
+
+        output_kwargs = self._merge_kwargs(
+            Ovis2ProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+
+        if isinstance(text, str):
+            text = [text]
+        elif not isinstance(text, list) and not isinstance(text[0], str):
+            raise TypeError("Invalid input text. Please provide a string, or a list of strings")
+
+        image_inputs = {}
+
+        if images is not None:
+            image_inputs = self.image_processor(images, **output_kwargs["images_kwargs"])
+            image_grids = image_inputs.pop("grids").tolist()
+            text = self._expand_image_tokens(text, image_grids)
+
+        text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
+        return BatchFeature(data={**text_inputs, **image_inputs})
+
+    def _expand_image_tokens(
+        self,
+        text: list[TextInput],
+        grids: list[list[int]],
+    ):
+        processed_text = []
+        grid_index = 0
+        for sample in text:
+            while "<image>" in sample:
+                grid = grids[grid_index]
+                row, col = grid[0], grid[1]
+                placeholder = f"<IMG_START>{'<IMG_ATOM>' * self.image_seq_length}<IMG_GRID>"
+                if row * col > 1:
+                    for r in range(row):
+                        for c in range(col):
+                            placeholder += f"{'<IMG_ATOM>' * self.image_seq_length}"
+                            if c < col - 1:
+                                placeholder += "<IMG_COL>"
+                        if r < row - 1:
+                            placeholder += "<IMG_ROW>"
+                placeholder += "<IMG_END>"
+
+                sample = sample.replace("<image>", placeholder, 1)
+                grid_index += 1
+            processed_text.append(sample)
+        return processed_text
+
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+        the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, **kwargs)
+
+    @property
+    def model_input_names(self):
+        tokenizer_input_names = self.tokenizer.model_input_names
+        image_processor_input_names = self.image_processor.model_input_names
+        return list(tokenizer_input_names) + list(image_processor_input_names)
+
+
+__all__ = ["Ovis2Processor"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/timesformer/__init__.py

@@ -0,0 +1,27 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_timesformer import *
+    from .modeling_timesformer import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)