Add files using upload-large-folder tool

LTA_openwebtext_dualt/logs/build_owt_gpt2_len1024_train_minus_100k_cache_fast.log

@@ -0,0 +1,82 @@
+[fast-cache] start data=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext split=train_minus_100k max_len=1024 cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train_minus_100k
+[fast-cache] batch_records=8192 write_batch=8192 detokenizer=None
+[fast-cache] records=100000 chunks=110531 tmp=0.41GB rate=561.6 rec/s dt=178.0s
+[fast-cache] records=200000 chunks=221984 tmp=0.14GB rate=573.0 rec/s dt=171.0s
+[fast-cache] records=300000 chunks=333067 tmp=0.47GB rate=577.8 rec/s dt=170.2s
+[fast-cache] records=400000 chunks=444201 tmp=0.80GB rate=580.0 rec/s dt=170.4s
+[fast-cache] records=500000 chunks=554143 tmp=1.14GB rate=574.2 rec/s dt=181.2s
+[fast-cache] records=600000 chunks=664175 tmp=1.16GB rate=562.6 rec/s dt=195.7s
+[fast-cache] records=700000 chunks=775113 tmp=1.16GB rate=566.4 rec/s dt=169.4s
+[fast-cache] records=800000 chunks=884668 tmp=0.28GB rate=570.0 rec/s dt=167.8s
+[fast-cache] records=900000 chunks=994931 tmp=0.62GB rate=572.0 rec/s dt=169.9s
+[fast-cache] records=1000000 chunks=1105352 tmp=0.97GB rate=569.7 rec/s dt=181.8s
+[fast-cache] records=1100000 chunks=1215581 tmp=1.17GB rate=571.6 rec/s dt=169.3s
+[fast-cache] records=1200000 chunks=1326186 tmp=0.25GB rate=573.2 rec/s dt=169.0s
+[fast-cache] records=1300000 chunks=1436243 tmp=0.58GB rate=575.1 rec/s dt=167.0s
+[fast-cache] records=1400000 chunks=1546223 tmp=0.89GB rate=577.7 rec/s dt=162.6s
+[fast-cache] records=1500000 chunks=1657123 tmp=1.17GB rate=575.6 rec/s dt=182.6s
+[fast-cache] records=1600000 chunks=1767742 tmp=0.19GB rate=576.7 rec/s dt=168.6s
+[fast-cache] records=1700000 chunks=1878959 tmp=0.52GB rate=577.6 rec/s dt=168.9s
+[fast-cache] records=1800000 chunks=1988825 tmp=0.84GB rate=578.6 rec/s dt=167.8s
+[fast-cache] records=1900000 chunks=2098555 tmp=1.16GB rate=579.4 rec/s dt=168.3s
+[fast-cache] records=2000000 chunks=2208500 tmp=1.16GB rate=578.1 rec/s dt=180.0s
+[fast-cache] records=2100000 chunks=2319549 tmp=1.16GB rate=578.8 rec/s dt=169.1s
+[fast-cache] records=2200000 chunks=2430443 tmp=0.28GB rate=579.5 rec/s dt=168.2s
+[fast-cache] records=2300000 chunks=2541342 tmp=0.61GB rate=580.0 rec/s dt=169.1s
+[fast-cache] records=2400000 chunks=2652688 tmp=0.95GB rate=580.3 rec/s dt=170.0s
+[fast-cache] records=2500000 chunks=2762560 tmp=1.17GB rate=579.1 rec/s dt=181.1s
+[fast-cache] records=2600000 chunks=2873567 tmp=0.25GB rate=579.6 rec/s dt=169.0s
+[fast-cache] records=2700000 chunks=2983083 tmp=0.58GB rate=580.2 rec/s dt=167.6s
+[fast-cache] records=2800000 chunks=3093194 tmp=0.91GB rate=580.8 rec/s dt=167.5s
+[fast-cache] records=2900000 chunks=3203699 tmp=1.17GB rate=579.7 rec/s dt=181.7s
+[fast-cache] records=3000000 chunks=3313755 tmp=0.20GB rate=580.2 rec/s dt=167.9s
+[fast-cache] records=3100000 chunks=3423161 tmp=0.53GB rate=580.9 rec/s dt=166.3s
+[fast-cache] records=3200000 chunks=3533464 tmp=0.86GB rate=581.2 rec/s dt=168.6s
+[fast-cache] records=3300000 chunks=3644326 tmp=1.17GB rate=581.6 rec/s dt=168.9s
+[fast-cache] records=3400000 chunks=3754158 tmp=1.17GB rate=580.6 rec/s dt=181.3s
+[fast-cache] records=3500000 chunks=3864926 tmp=1.17GB rate=581.0 rec/s dt=168.4s
+[fast-cache] records=3600000 chunks=3975286 tmp=0.09GB rate=581.5 rec/s dt=167.1s
+[fast-cache] records=3700000 chunks=4085755 tmp=0.42GB rate=581.9 rec/s dt=167.0s
+[fast-cache] records=3800000 chunks=4195623 tmp=0.75GB rate=582.3 rec/s dt=167.8s
+[fast-cache] records=3900000 chunks=4306405 tmp=1.11GB rate=581.4 rec/s dt=181.6s
+[fast-cache] records=4000000 chunks=4416413 tmp=1.17GB rate=581.8 rec/s dt=167.8s
+[fast-cache] records=4100000 chunks=4527089 tmp=1.17GB rate=582.2 rec/s dt=167.0s
+[fast-cache] records=4200000 chunks=4638228 tmp=1.17GB rate=582.5 rec/s dt=167.5s
+[fast-cache] records=4300000 chunks=4748237 tmp=1.17GB rate=582.9 rec/s dt=166.5s
+[fast-cache] records=4400000 chunks=4859560 tmp=1.17GB rate=582.1 rec/s dt=182.8s
+[fast-cache] records=4500000 chunks=4970270 tmp=1.17GB rate=582.4 rec/s dt=167.7s
+[fast-cache] records=4600000 chunks=5080968 tmp=1.17GB rate=582.6 rec/s dt=168.6s
+[fast-cache] records=4700000 chunks=5191426 tmp=1.17GB rate=582.8 rec/s dt=168.1s
+[fast-cache] records=4800000 chunks=5301202 tmp=1.17GB rate=583.2 rec/s dt=167.0s
+[fast-cache] records=4900000 chunks=5412403 tmp=1.17GB rate=582.4 rec/s dt=182.1s
+[fast-cache] records=5000000 chunks=5522299 tmp=1.17GB rate=582.6 rec/s dt=168.6s
+[fast-cache] records=5100000 chunks=5633519 tmp=1.17GB rate=582.8 rec/s dt=168.6s
+[fast-cache] records=5200000 chunks=5744407 tmp=1.17GB rate=583.1 rec/s dt=168.1s
+[fast-cache] records=5300000 chunks=5855652 tmp=1.17GB rate=583.4 rec/s dt=166.4s
+[fast-cache] records=5400000 chunks=5965114 tmp=1.17GB rate=583.0 rec/s dt=177.4s
+[fast-cache] records=5500000 chunks=6076031 tmp=1.17GB rate=583.3 rec/s dt=166.2s
+[fast-cache] records=5600000 chunks=6186495 tmp=1.17GB rate=583.7 rec/s dt=166.3s
+[fast-cache] records=5700000 chunks=6297525 tmp=1.17GB rate=583.9 rec/s dt=167.2s
+[fast-cache] records=5800000 chunks=6407324 tmp=1.17GB rate=583.4 rec/s dt=179.1s
+[fast-cache] records=5900000 chunks=6518060 tmp=1.17GB rate=583.8 rec/s dt=165.2s
+[fast-cache] records=6000000 chunks=6628110 tmp=1.17GB rate=584.1 rec/s dt=166.2s
+[fast-cache] records=6100000 chunks=6738088 tmp=1.17GB rate=584.4 rec/s dt=165.8s
+[fast-cache] records=6200000 chunks=6848287 tmp=1.17GB rate=584.7 rec/s dt=165.8s
+[fast-cache] records=6300000 chunks=6958445 tmp=1.17GB rate=584.2 rec/s dt=180.2s
+[fast-cache] records=6400000 chunks=7068420 tmp=1.17GB rate=584.5 rec/s dt=165.8s
+[fast-cache] records=6500000 chunks=7179301 tmp=1.17GB rate=584.6 rec/s dt=168.4s
+[fast-cache] records=6600000 chunks=7288969 tmp=1.17GB rate=584.9 rec/s dt=165.7s
+[fast-cache] records=6700000 chunks=7398234 tmp=1.17GB rate=585.2 rec/s dt=165.7s
+[fast-cache] records=6800000 chunks=7508294 tmp=1.17GB rate=584.8 rec/s dt=179.0s
+[fast-cache] records=6900000 chunks=7618742 tmp=1.17GB rate=584.9 rec/s dt=167.6s
+[fast-cache] records=7000000 chunks=7728620 tmp=1.17GB rate=585.1 rec/s dt=166.9s
+[fast-cache] records=7100000 chunks=7838485 tmp=1.17GB rate=585.4 rec/s dt=165.6s
+[fast-cache] records=7200000 chunks=7948428 tmp=1.17GB rate=585.6 rec/s dt=167.0s
+[fast-cache] records=7300000 chunks=8058799 tmp=1.17GB rate=585.2 rec/s dt=179.4s
+[fast-cache] records=7400000 chunks=8168253 tmp=1.17GB rate=585.4 rec/s dt=166.5s
+[fast-cache] records=7500000 chunks=8278051 tmp=1.17GB rate=585.6 rec/s dt=166.3s
+[fast-cache] records=7600000 chunks=8388185 tmp=1.17GB rate=585.7 rec/s dt=167.5s
+[fast-cache] records=7700000 chunks=8499029 tmp=1.17GB rate=585.8 rec/s dt=168.7s
+[fast-cache] records=7800000 chunks=8609940 tmp=1.17GB rate=585.4 rec/s dt=181.1s
+[fast-cache] records=7900000 chunks=8719806 tmp=1.17GB rate=585.6 rec/s dt=166.0s
+[fast-cache] done records=7913769 chunks=8734897 elapsed=13506.9s size=1.17GB

LTA_openwebtext_dualt/logs/cmp_owt100k_dirichlet_wrongfix_ddit_6x384_len256_gbs256_steps10000_parallel.log

@@ -0,0 +1,216 @@
+{
+  "device": "cuda",
+  "samples": "wrapped_streaming",
+  "vocab_size": 50257,
+  "save_dir": "runs/cmp_owt100k_dirichlet_wrongfix_ddit_6x384_len256_gbs256_steps10000_parallel",
+  "batch_size": 16,
+  "grad_accum": 16,
+  "effective_batch_size": 256,
+  "global_batch_size": 256,
+  "lr_schedule": "constant_warmup",
+  "warmup_steps": 500,
+  "model_type": "ddit",
+  "state_format": "prob",
+  "wrap": true,
+  "num_workers": 0
+}
+step=50 micro_steps=800 elapsed=134.1s lr=3.060000e-05 loss_all=10.7668 acc_all=0.4461 loss_corrupt=10.7770 acc_corrupt=0.1881 corrupt_frac=0.5511 loss=10.7770 mean_t=0.5018 wrong_frac=0.6995 init_acc_corrupt=0.2278 init_gold_top10=0.2755 init_gold_top100=0.3117
+step=100 micro_steps=1600 elapsed=105.5s lr=6.060000e-05 loss_all=9.9349 acc_all=0.0696 loss_corrupt=9.9394 acc_corrupt=0.0454 corrupt_frac=0.5497 loss=9.9394 mean_t=0.5014 wrong_frac=0.7002 init_acc_corrupt=0.2272 init_gold_top10=0.2754 init_gold_top100=0.3112
+step=150 micro_steps=2400 elapsed=96.7s lr=9.060000e-05 loss_all=8.0411 acc_all=0.0373 loss_corrupt=8.0409 acc_corrupt=0.0371 corrupt_frac=0.5496 loss=8.0409 mean_t=0.4970 wrong_frac=0.7000 init_acc_corrupt=0.2264 init_gold_top10=0.2745 init_gold_top100=0.3117
+step=200 micro_steps=3200 elapsed=105.6s lr=1.206000e-04 loss_all=7.5220 acc_all=0.0378 loss_corrupt=7.5876 acc_corrupt=0.0377 corrupt_frac=0.5551 loss=7.5876 mean_t=0.5036 wrong_frac=0.6998 init_acc_corrupt=0.2278 init_gold_top10=0.2758 init_gold_top100=0.3123
+step=250 micro_steps=4000 elapsed=105.5s lr=1.506000e-04 loss_all=7.3699 acc_all=0.0383 loss_corrupt=7.5160 acc_corrupt=0.0378 corrupt_frac=0.5472 loss=7.5160 mean_t=0.4968 wrong_frac=0.6995 init_acc_corrupt=0.2265 init_gold_top10=0.2752 init_gold_top100=0.3124
+step=300 micro_steps=4800 elapsed=105.3s lr=1.806000e-04 loss_all=7.2481 acc_all=0.0548 loss_corrupt=7.4411 acc_corrupt=0.0474 corrupt_frac=0.5513 loss=7.4411 mean_t=0.4996 wrong_frac=0.6999 init_acc_corrupt=0.2264 init_gold_top10=0.2741 init_gold_top100=0.3108
+step=350 micro_steps=5600 elapsed=105.3s lr=2.106000e-04 loss_all=6.5422 acc_all=0.1604 loss_corrupt=6.9923 acc_corrupt=0.1073 corrupt_frac=0.5538 loss=6.9923 mean_t=0.4988 wrong_frac=0.7000 init_acc_corrupt=0.2265 init_gold_top10=0.2748 init_gold_top100=0.3114
+step=400 micro_steps=6400 elapsed=105.5s lr=2.406000e-04 loss_all=5.7315 acc_all=0.2843 loss_corrupt=6.5090 acc_corrupt=0.1724 corrupt_frac=0.5480 loss=6.5090 mean_t=0.5017 wrong_frac=0.6997 init_acc_corrupt=0.2280 init_gold_top10=0.2758 init_gold_top100=0.3119
+step=450 micro_steps=7200 elapsed=105.5s lr=2.706000e-04 loss_all=5.0510 acc_all=0.3733 loss_corrupt=6.1205 acc_corrupt=0.2183 corrupt_frac=0.5456 loss=6.1205 mean_t=0.4970 wrong_frac=0.6998 init_acc_corrupt=0.2260 init_gold_top10=0.2740 init_gold_top100=0.3110
+step=500 micro_steps=8000 elapsed=105.3s lr=3.000000e-04 loss_all=4.4190 acc_all=0.4552 loss_corrupt=5.7779 acc_corrupt=0.2573 corrupt_frac=0.5551 loss=5.7779 mean_t=0.4987 wrong_frac=0.7003 init_acc_corrupt=0.2256 init_gold_top10=0.2747 init_gold_top100=0.3121
+step=550 micro_steps=8800 elapsed=105.2s lr=3.000000e-04 loss_all=4.0293 acc_all=0.4992 loss_corrupt=5.5551 acc_corrupt=0.2792 corrupt_frac=0.5499 loss=5.5551 mean_t=0.4975 wrong_frac=0.7006 init_acc_corrupt=0.2256 init_gold_top10=0.2736 init_gold_top100=0.3103
+step=600 micro_steps=9600 elapsed=105.2s lr=3.000000e-04 loss_all=3.7658 acc_all=0.5238 loss_corrupt=5.3635 acc_corrupt=0.2938 corrupt_frac=0.5479 loss=5.3635 mean_t=0.5009 wrong_frac=0.6998 init_acc_corrupt=0.2274 init_gold_top10=0.2757 init_gold_top100=0.3116
+step=650 micro_steps=10400 elapsed=105.5s lr=3.000000e-04 loss_all=3.5808 acc_all=0.5396 loss_corrupt=5.2251 acc_corrupt=0.3033 corrupt_frac=0.5490 loss=5.2251 mean_t=0.5004 wrong_frac=0.6996 init_acc_corrupt=0.2287 init_gold_top10=0.2759 init_gold_top100=0.3117
+step=700 micro_steps=11200 elapsed=104.4s lr=3.000000e-04 loss_all=3.4302 acc_all=0.5522 loss_corrupt=5.1098 acc_corrupt=0.3107 corrupt_frac=0.5498 loss=5.1098 mean_t=0.5028 wrong_frac=0.6998 init_acc_corrupt=0.2264 init_gold_top10=0.2756 init_gold_top100=0.3124
+step=750 micro_steps=12000 elapsed=98.3s lr=3.000000e-04 loss_all=3.3181 acc_all=0.5611 loss_corrupt=5.0145 acc_corrupt=0.3168 corrupt_frac=0.5486 loss=5.0145 mean_t=0.4957 wrong_frac=0.6994 init_acc_corrupt=0.2256 init_gold_top10=0.2746 init_gold_top100=0.3126
+step=800 micro_steps=12800 elapsed=105.4s lr=3.000000e-04 loss_all=3.2271 acc_all=0.5678 loss_corrupt=4.9277 acc_corrupt=0.3232 corrupt_frac=0.5518 loss=4.9277 mean_t=0.5007 wrong_frac=0.6994 init_acc_corrupt=0.2272 init_gold_top10=0.2748 init_gold_top100=0.3117
+step=850 micro_steps=13600 elapsed=105.6s lr=3.000000e-04 loss_all=3.1430 acc_all=0.5748 loss_corrupt=4.8527 acc_corrupt=0.3284 corrupt_frac=0.5521 loss=4.8527 mean_t=0.4987 wrong_frac=0.6998 init_acc_corrupt=0.2261 init_gold_top10=0.2747 init_gold_top100=0.3119
+step=900 micro_steps=14400 elapsed=106.1s lr=3.000000e-04 loss_all=3.0449 acc_all=0.5835 loss_corrupt=4.7593 acc_corrupt=0.3351 corrupt_frac=0.5495 loss=4.7593 mean_t=0.5035 wrong_frac=0.7004 init_acc_corrupt=0.2282 init_gold_top10=0.2758 init_gold_top100=0.3116
+step=950 micro_steps=15200 elapsed=105.5s lr=3.000000e-04 loss_all=2.9813 acc_all=0.5878 loss_corrupt=4.7023 acc_corrupt=0.3375 corrupt_frac=0.5498 loss=4.7023 mean_t=0.4980 wrong_frac=0.6995 init_acc_corrupt=0.2257 init_gold_top10=0.2749 init_gold_top100=0.3120
+step=1000 micro_steps=16000 elapsed=105.3s lr=3.000000e-04 loss_all=2.9095 acc_all=0.5938 loss_corrupt=4.6192 acc_corrupt=0.3444 corrupt_frac=0.5513 loss=4.6192 mean_t=0.5036 wrong_frac=0.6999 init_acc_corrupt=0.2281 init_gold_top10=0.2754 init_gold_top100=0.3108
+step=1050 micro_steps=16800 elapsed=121.6s lr=3.000000e-04 loss_all=2.8622 acc_all=0.5968 loss_corrupt=4.5656 acc_corrupt=0.3470 corrupt_frac=0.5527 loss=4.5656 mean_t=0.4998 wrong_frac=0.7003 init_acc_corrupt=0.2263 init_gold_top10=0.2743 init_gold_top100=0.3113
+step=1100 micro_steps=17600 elapsed=105.6s lr=3.000000e-04 loss_all=2.8087 acc_all=0.6021 loss_corrupt=4.5162 acc_corrupt=0.3513 corrupt_frac=0.5507 loss=4.5162 mean_t=0.4990 wrong_frac=0.7001 init_acc_corrupt=0.2272 init_gold_top10=0.2756 init_gold_top100=0.3118
+step=1150 micro_steps=18400 elapsed=105.6s lr=3.000000e-04 loss_all=2.7695 acc_all=0.6050 loss_corrupt=4.4690 acc_corrupt=0.3540 corrupt_frac=0.5516 loss=4.4690 mean_t=0.5016 wrong_frac=0.6994 init_acc_corrupt=0.2277 init_gold_top10=0.2753 init_gold_top100=0.3121
+step=1200 micro_steps=19200 elapsed=105.5s lr=3.000000e-04 loss_all=2.6982 acc_all=0.6121 loss_corrupt=4.4015 acc_corrupt=0.3594 corrupt_frac=0.5478 loss=4.4015 mean_t=0.5038 wrong_frac=0.7001 init_acc_corrupt=0.2284 init_gold_top10=0.2756 init_gold_top100=0.3115
+step=1250 micro_steps=20000 elapsed=105.7s lr=3.000000e-04 loss_all=2.6762 acc_all=0.6137 loss_corrupt=4.3900 acc_corrupt=0.3595 corrupt_frac=0.5484 loss=4.3900 mean_t=0.4944 wrong_frac=0.7001 init_acc_corrupt=0.2255 init_gold_top10=0.2750 init_gold_top100=0.3123
+step=1300 micro_steps=20800 elapsed=63.9s lr=3.000000e-04 loss_all=2.6480 acc_all=0.6150 loss_corrupt=4.3416 acc_corrupt=0.3625 corrupt_frac=0.5514 loss=4.3416 mean_t=0.4941 wrong_frac=0.6998 init_acc_corrupt=0.2252 init_gold_top10=0.2739 init_gold_top100=0.3114
+step=1350 micro_steps=21600 elapsed=54.5s lr=3.000000e-04 loss_all=2.6112 acc_all=0.6185 loss_corrupt=4.2952 acc_corrupt=0.3662 corrupt_frac=0.5515 loss=4.2952 mean_t=0.4995 wrong_frac=0.7003 init_acc_corrupt=0.2258 init_gold_top10=0.2736 init_gold_top100=0.3105
+step=1400 micro_steps=22400 elapsed=54.4s lr=3.000000e-04 loss_all=2.5706 acc_all=0.6222 loss_corrupt=4.2506 acc_corrupt=0.3693 corrupt_frac=0.5501 loss=4.2506 mean_t=0.4997 wrong_frac=0.7008 init_acc_corrupt=0.2263 init_gold_top10=0.2741 init_gold_top100=0.3111
+step=1450 micro_steps=23200 elapsed=54.6s lr=3.000000e-04 loss_all=2.5276 acc_all=0.6266 loss_corrupt=4.1992 acc_corrupt=0.3740 corrupt_frac=0.5494 loss=4.1992 mean_t=0.5023 wrong_frac=0.6996 init_acc_corrupt=0.2276 init_gold_top10=0.2756 init_gold_top100=0.3122
+step=1500 micro_steps=24000 elapsed=54.5s lr=3.000000e-04 loss_all=2.5249 acc_all=0.6263 loss_corrupt=4.1886 acc_corrupt=0.3746 corrupt_frac=0.5514 loss=4.1886 mean_t=0.5000 wrong_frac=0.6998 init_acc_corrupt=0.2260 init_gold_top10=0.2746 init_gold_top100=0.3121
+step=1550 micro_steps=24800 elapsed=54.4s lr=3.000000e-04 loss_all=2.4652 acc_all=0.6327 loss_corrupt=4.1304 acc_corrupt=0.3791 corrupt_frac=0.5470 loss=4.1304 mean_t=0.5033 wrong_frac=0.7003 init_acc_corrupt=0.2276 init_gold_top10=0.2749 init_gold_top100=0.3108
+step=1600 micro_steps=25600 elapsed=54.5s lr=3.000000e-04 loss_all=2.4779 acc_all=0.6298 loss_corrupt=4.1169 acc_corrupt=0.3796 corrupt_frac=0.5528 loss=4.1169 mean_t=0.4998 wrong_frac=0.6993 init_acc_corrupt=0.2268 init_gold_top10=0.2753 init_gold_top100=0.3118
+step=1650 micro_steps=26400 elapsed=54.3s lr=3.000000e-04 loss_all=2.4167 acc_all=0.6368 loss_corrupt=4.0626 acc_corrupt=0.3843 corrupt_frac=0.5483 loss=4.0626 mean_t=0.4984 wrong_frac=0.6996 init_acc_corrupt=0.2269 init_gold_top10=0.2749 init_gold_top100=0.3113
+step=1700 micro_steps=27200 elapsed=54.7s lr=3.000000e-04 loss_all=2.4076 acc_all=0.6371 loss_corrupt=4.0415 acc_corrupt=0.3859 corrupt_frac=0.5506 loss=4.0415 mean_t=0.5023 wrong_frac=0.6999 init_acc_corrupt=0.2283 init_gold_top10=0.2748 init_gold_top100=0.3112
+step=1750 micro_steps=28000 elapsed=54.7s lr=3.000000e-04 loss_all=2.3943 acc_all=0.6380 loss_corrupt=4.0269 acc_corrupt=0.3861 corrupt_frac=0.5506 loss=4.0269 mean_t=0.4983 wrong_frac=0.7001 init_acc_corrupt=0.2255 init_gold_top10=0.2742 init_gold_top100=0.3113
+step=1800 micro_steps=28800 elapsed=55.0s lr=3.000000e-04 loss_all=2.3673 acc_all=0.6404 loss_corrupt=3.9861 acc_corrupt=0.3894 corrupt_frac=0.5508 loss=3.9861 mean_t=0.4983 wrong_frac=0.7003 init_acc_corrupt=0.2263 init_gold_top10=0.2738 init_gold_top100=0.3112
+step=1850 micro_steps=29600 elapsed=54.5s lr=3.000000e-04 loss_all=2.3550 acc_all=0.6412 loss_corrupt=3.9643 acc_corrupt=0.3911 corrupt_frac=0.5527 loss=3.9643 mean_t=0.5022 wrong_frac=0.7001 init_acc_corrupt=0.2268 init_gold_top10=0.2749 init_gold_top100=0.3113
+step=1900 micro_steps=30400 elapsed=54.4s lr=3.000000e-04 loss_all=2.3276 acc_all=0.6436 loss_corrupt=3.9319 acc_corrupt=0.3931 corrupt_frac=0.5513 loss=3.9319 mean_t=0.5002 wrong_frac=0.7001 init_acc_corrupt=0.2263 init_gold_top10=0.2747 init_gold_top100=0.3116
+step=1950 micro_steps=31200 elapsed=54.3s lr=3.000000e-04 loss_all=2.2978 acc_all=0.6467 loss_corrupt=3.8819 acc_corrupt=0.3981 corrupt_frac=0.5524 loss=3.8819 mean_t=0.5036 wrong_frac=0.6995 init_acc_corrupt=0.2284 init_gold_top10=0.2759 init_gold_top100=0.3117
+step=2000 micro_steps=32000 elapsed=54.8s lr=3.000000e-04 loss_all=2.3150 acc_all=0.6440 loss_corrupt=3.8932 acc_corrupt=0.3965 corrupt_frac=0.5565 loss=3.8932 mean_t=0.4972 wrong_frac=0.7002 init_acc_corrupt=0.2263 init_gold_top10=0.2746 init_gold_top100=0.3110
+step=2050 micro_steps=32800 elapsed=57.2s lr=3.000000e-04 loss_all=2.2709 acc_all=0.6500 loss_corrupt=3.8538 acc_corrupt=0.4010 corrupt_frac=0.5517 loss=3.8538 mean_t=0.5019 wrong_frac=0.7000 init_acc_corrupt=0.2272 init_gold_top10=0.2757 init_gold_top100=0.3123
+step=2100 micro_steps=33600 elapsed=67.2s lr=3.000000e-04 loss_all=2.2463 acc_all=0.6523 loss_corrupt=3.8198 acc_corrupt=0.4039 corrupt_frac=0.5510 loss=3.8198 mean_t=0.5022 wrong_frac=0.7001 init_acc_corrupt=0.2274 init_gold_top10=0.2751 init_gold_top100=0.3116
+step=2150 micro_steps=34400 elapsed=54.1s lr=3.000000e-04 loss_all=2.2359 acc_all=0.6528 loss_corrupt=3.8069 acc_corrupt=0.4038 corrupt_frac=0.5512 loss=3.8069 mean_t=0.4990 wrong_frac=0.7001 init_acc_corrupt=0.2273 init_gold_top10=0.2751 init_gold_top100=0.3114
+step=2200 micro_steps=35200 elapsed=53.8s lr=3.000000e-04 loss_all=2.2087 acc_all=0.6565 loss_corrupt=3.7870 acc_corrupt=0.4057 corrupt_frac=0.5482 loss=3.7870 mean_t=0.5005 wrong_frac=0.7003 init_acc_corrupt=0.2268 init_gold_top10=0.2742 init_gold_top100=0.3106
+step=2250 micro_steps=36000 elapsed=54.1s lr=3.000000e-04 loss_all=2.2056 acc_all=0.6558 loss_corrupt=3.7660 acc_corrupt=0.4072 corrupt_frac=0.5515 loss=3.7660 mean_t=0.4998 wrong_frac=0.7002 init_acc_corrupt=0.2257 init_gold_top10=0.2747 init_gold_top100=0.3118
+step=2300 micro_steps=36800 elapsed=53.8s lr=3.000000e-04 loss_all=2.1810 acc_all=0.6587 loss_corrupt=3.7337 acc_corrupt=0.4103 corrupt_frac=0.5509 loss=3.7337 mean_t=0.5017 wrong_frac=0.6998 init_acc_corrupt=0.2272 init_gold_top10=0.2747 init_gold_top100=0.3112
+step=2350 micro_steps=37600 elapsed=54.2s lr=3.000000e-04 loss_all=2.1618 acc_all=0.6606 loss_corrupt=3.7090 acc_corrupt=0.4121 corrupt_frac=0.5500 loss=3.7090 mean_t=0.4987 wrong_frac=0.6994 init_acc_corrupt=0.2256 init_gold_top10=0.2752 init_gold_top100=0.3126
+step=2400 micro_steps=38400 elapsed=53.8s lr=3.000000e-04 loss_all=2.1462 acc_all=0.6630 loss_corrupt=3.6985 acc_corrupt=0.4137 corrupt_frac=0.5483 loss=3.6985 mean_t=0.4981 wrong_frac=0.7001 init_acc_corrupt=0.2267 init_gold_top10=0.2750 init_gold_top100=0.3114
+step=2450 micro_steps=39200 elapsed=53.8s lr=3.000000e-04 loss_all=2.1166 acc_all=0.6657 loss_corrupt=3.6661 acc_corrupt=0.4157 corrupt_frac=0.5460 loss=3.6661 mean_t=0.5003 wrong_frac=0.7000 init_acc_corrupt=0.2263 init_gold_top10=0.2749 init_gold_top100=0.3124
+step=2500 micro_steps=40000 elapsed=53.8s lr=3.000000e-04 loss_all=2.1221 acc_all=0.6653 loss_corrupt=3.6571 acc_corrupt=0.4176 corrupt_frac=0.5496 loss=3.6571 mean_t=0.5030 wrong_frac=0.6999 init_acc_corrupt=0.2276 init_gold_top10=0.2756 init_gold_top100=0.3119
+step=2550 micro_steps=40800 elapsed=53.8s lr=3.000000e-04 loss_all=2.1060 acc_all=0.6663 loss_corrupt=3.6349 acc_corrupt=0.4185 corrupt_frac=0.5495 loss=3.6349 mean_t=0.4972 wrong_frac=0.7001 init_acc_corrupt=0.2267 init_gold_top10=0.2748 init_gold_top100=0.3110
+step=2600 micro_steps=41600 elapsed=53.8s lr=3.000000e-04 loss_all=2.0740 acc_all=0.6704 loss_corrupt=3.6028 acc_corrupt=0.4217 corrupt_frac=0.5468 loss=3.6028 mean_t=0.4999 wrong_frac=0.7003 init_acc_corrupt=0.2268 init_gold_top10=0.2752 init_gold_top100=0.3115
+step=2650 micro_steps=42400 elapsed=53.8s lr=3.000000e-04 loss_all=2.0886 acc_all=0.6676 loss_corrupt=3.5986 acc_corrupt=0.4217 corrupt_frac=0.5527 loss=3.5986 mean_t=0.5050 wrong_frac=0.6998 init_acc_corrupt=0.2282 init_gold_top10=0.2758 init_gold_top100=0.3117
+step=2700 micro_steps=43200 elapsed=53.9s lr=3.000000e-04 loss_all=2.0636 acc_all=0.6711 loss_corrupt=3.5799 acc_corrupt=0.4237 corrupt_frac=0.5489 loss=3.5799 mean_t=0.5010 wrong_frac=0.6998 init_acc_corrupt=0.2267 init_gold_top10=0.2748 init_gold_top100=0.3117
+step=2750 micro_steps=44000 elapsed=54.2s lr=3.000000e-04 loss_all=2.0399 acc_all=0.6734 loss_corrupt=3.5460 acc_corrupt=0.4266 corrupt_frac=0.5476 loss=3.5460 mean_t=0.5011 wrong_frac=0.7003 init_acc_corrupt=0.2266 init_gold_top10=0.2743 init_gold_top100=0.3115
+step=2800 micro_steps=44800 elapsed=53.7s lr=3.000000e-04 loss_all=2.0597 acc_all=0.6706 loss_corrupt=3.5543 acc_corrupt=0.4261 corrupt_frac=0.5530 loss=3.5543 mean_t=0.5017 wrong_frac=0.6999 init_acc_corrupt=0.2282 init_gold_top10=0.2748 init_gold_top100=0.3106
+step=2850 micro_steps=45600 elapsed=57.3s lr=3.000000e-04 loss_all=2.0211 acc_all=0.6754 loss_corrupt=3.5283 acc_corrupt=0.4278 corrupt_frac=0.5470 loss=3.5283 mean_t=0.5007 wrong_frac=0.6999 init_acc_corrupt=0.2270 init_gold_top10=0.2754 init_gold_top100=0.3119
+step=2900 micro_steps=46400 elapsed=53.5s lr=3.000000e-04 loss_all=2.0404 acc_all=0.6722 loss_corrupt=3.5336 acc_corrupt=0.4267 corrupt_frac=0.5522 loss=3.5336 mean_t=0.4984 wrong_frac=0.7005 init_acc_corrupt=0.2262 init_gold_top10=0.2740 init_gold_top100=0.3105
+step=2950 micro_steps=47200 elapsed=53.5s lr=3.000000e-04 loss_all=2.0138 acc_all=0.6758 loss_corrupt=3.5003 acc_corrupt=0.4308 corrupt_frac=0.5505 loss=3.5003 mean_t=0.4977 wrong_frac=0.7002 init_acc_corrupt=0.2267 init_gold_top10=0.2743 init_gold_top100=0.3106
+step=3000 micro_steps=48000 elapsed=55.7s lr=3.000000e-04 loss_all=1.9832 acc_all=0.6794 loss_corrupt=3.4830 acc_corrupt=0.4313 corrupt_frac=0.5447 loss=3.4830 mean_t=0.4954 wrong_frac=0.7000 init_acc_corrupt=0.2251 init_gold_top10=0.2740 init_gold_top100=0.3119
+step=3050 micro_steps=48800 elapsed=70.1s lr=3.000000e-04 loss_all=1.9915 acc_all=0.6781 loss_corrupt=3.4697 acc_corrupt=0.4334 corrupt_frac=0.5496 loss=3.4697 mean_t=0.4977 wrong_frac=0.6995 init_acc_corrupt=0.2261 init_gold_top10=0.2753 init_gold_top100=0.3126
+step=3100 micro_steps=49600 elapsed=53.8s lr=3.000000e-04 loss_all=2.0096 acc_all=0.6751 loss_corrupt=3.4741 acc_corrupt=0.4327 corrupt_frac=0.5551 loss=3.4741 mean_t=0.4970 wrong_frac=0.6998 init_acc_corrupt=0.2262 init_gold_top10=0.2745 init_gold_top100=0.3117
+step=3150 micro_steps=50400 elapsed=53.9s lr=3.000000e-04 loss_all=1.9817 acc_all=0.6792 loss_corrupt=3.4595 acc_corrupt=0.4344 corrupt_frac=0.5497 loss=3.4595 mean_t=0.4988 wrong_frac=0.6994 init_acc_corrupt=0.2255 init_gold_top10=0.2752 init_gold_top100=0.3127
+step=3200 micro_steps=51200 elapsed=53.8s lr=3.000000e-04 loss_all=1.9751 acc_all=0.6791 loss_corrupt=3.4341 acc_corrupt=0.4362 corrupt_frac=0.5515 loss=3.4341 mean_t=0.4982 wrong_frac=0.6994 init_acc_corrupt=0.2269 init_gold_top10=0.2755 init_gold_top100=0.3118
+step=3250 micro_steps=52000 elapsed=53.8s lr=3.000000e-04 loss_all=1.9645 acc_all=0.6805 loss_corrupt=3.4289 acc_corrupt=0.4364 corrupt_frac=0.5500 loss=3.4289 mean_t=0.4992 wrong_frac=0.7001 init_acc_corrupt=0.2263 init_gold_top10=0.2741 init_gold_top100=0.3115
+step=3300 micro_steps=52800 elapsed=53.9s lr=3.000000e-04 loss_all=1.9566 acc_all=0.6811 loss_corrupt=3.4048 acc_corrupt=0.4393 corrupt_frac=0.5521 loss=3.4048 mean_t=0.5028 wrong_frac=0.6997 init_acc_corrupt=0.2272 init_gold_top10=0.2758 init_gold_top100=0.3123
+step=3350 micro_steps=53600 elapsed=53.9s lr=3.000000e-04 loss_all=1.9507 acc_all=0.6817 loss_corrupt=3.4046 acc_corrupt=0.4389 corrupt_frac=0.5513 loss=3.4046 mean_t=0.5000 wrong_frac=0.7002 init_acc_corrupt=0.2253 init_gold_top10=0.2744 init_gold_top100=0.3116
+step=3400 micro_steps=54400 elapsed=53.8s lr=3.000000e-04 loss_all=1.9368 acc_all=0.6832 loss_corrupt=3.3804 acc_corrupt=0.4411 corrupt_frac=0.5502 loss=3.3804 mean_t=0.5012 wrong_frac=0.7006 init_acc_corrupt=0.2259 init_gold_top10=0.2742 init_gold_top100=0.3116
+step=3450 micro_steps=55200 elapsed=53.9s lr=3.000000e-04 loss_all=1.9260 acc_all=0.6844 loss_corrupt=3.3673 acc_corrupt=0.4426 corrupt_frac=0.5503 loss=3.3673 mean_t=0.5011 wrong_frac=0.6997 init_acc_corrupt=0.2273 init_gold_top10=0.2761 init_gold_top100=0.3121
+step=3500 micro_steps=56000 elapsed=53.6s lr=3.000000e-04 loss_all=1.9521 acc_all=0.6803 loss_corrupt=3.3866 acc_corrupt=0.4396 corrupt_frac=0.5555 loss=3.3866 mean_t=0.4953 wrong_frac=0.7003 init_acc_corrupt=0.2246 init_gold_top10=0.2731 init_gold_top100=0.3108
+step=3550 micro_steps=56800 elapsed=54.2s lr=3.000000e-04 loss_all=1.8994 acc_all=0.6876 loss_corrupt=3.3391 acc_corrupt=0.4449 corrupt_frac=0.5479 loss=3.3391 mean_t=0.4982 wrong_frac=0.6995 init_acc_corrupt=0.2274 init_gold_top10=0.2751 init_gold_top100=0.3118
+step=3600 micro_steps=57600 elapsed=53.9s lr=3.000000e-04 loss_all=1.9192 acc_all=0.6846 loss_corrupt=3.3538 acc_corrupt=0.4433 corrupt_frac=0.5523 loss=3.3538 mean_t=0.4988 wrong_frac=0.7000 init_acc_corrupt=0.2251 init_gold_top10=0.2736 init_gold_top100=0.3110
+step=3650 micro_steps=58400 elapsed=54.4s lr=3.000000e-04 loss_all=1.8857 acc_all=0.6891 loss_corrupt=3.3195 acc_corrupt=0.4471 corrupt_frac=0.5471 loss=3.3195 mean_t=0.5037 wrong_frac=0.6999 init_acc_corrupt=0.2270 init_gold_top10=0.2743 init_gold_top100=0.3108
+step=3700 micro_steps=59200 elapsed=54.2s lr=3.000000e-04 loss_all=1.8729 acc_all=0.6907 loss_corrupt=3.2987 acc_corrupt=0.4496 corrupt_frac=0.5478 loss=3.2987 mean_t=0.5001 wrong_frac=0.6998 init_acc_corrupt=0.2279 init_gold_top10=0.2759 init_gold_top100=0.3118
+step=3750 micro_steps=60000 elapsed=54.2s lr=3.000000e-04 loss_all=1.8678 acc_all=0.6911 loss_corrupt=3.2994 acc_corrupt=0.4488 corrupt_frac=0.5458 loss=3.2994 mean_t=0.5039 wrong_frac=0.7000 init_acc_corrupt=0.2272 init_gold_top10=0.2745 init_gold_top100=0.3114
+step=3800 micro_steps=60800 elapsed=54.5s lr=3.000000e-04 loss_all=1.8839 acc_all=0.6886 loss_corrupt=3.3073 acc_corrupt=0.4479 corrupt_frac=0.5501 loss=3.3073 mean_t=0.4974 wrong_frac=0.6992 init_acc_corrupt=0.2266 init_gold_top10=0.2751 init_gold_top100=0.3123
+step=3850 micro_steps=61600 elapsed=54.5s lr=3.000000e-04 loss_all=1.8565 acc_all=0.6916 loss_corrupt=3.2639 acc_corrupt=0.4520 corrupt_frac=0.5485 loss=3.2639 mean_t=0.4986 wrong_frac=0.6997 init_acc_corrupt=0.2271 init_gold_top10=0.2761 init_gold_top100=0.3118
+step=3900 micro_steps=62400 elapsed=54.4s lr=3.000000e-04 loss_all=1.8642 acc_all=0.6900 loss_corrupt=3.2655 acc_corrupt=0.4516 corrupt_frac=0.5516 loss=3.2655 mean_t=0.5017 wrong_frac=0.7005 init_acc_corrupt=0.2272 init_gold_top10=0.2753 init_gold_top100=0.3109
+step=3950 micro_steps=63200 elapsed=57.9s lr=3.000000e-04 loss_all=1.8377 acc_all=0.6936 loss_corrupt=3.2412 acc_corrupt=0.4543 corrupt_frac=0.5480 loss=3.2412 mean_t=0.5028 wrong_frac=0.7003 init_acc_corrupt=0.2284 init_gold_top10=0.2759 init_gold_top100=0.3115
+step=4000 micro_steps=64000 elapsed=65.7s lr=3.000000e-04 loss_all=1.8519 acc_all=0.6915 loss_corrupt=3.2490 acc_corrupt=0.4534 corrupt_frac=0.5512 loss=3.2490 mean_t=0.5008 wrong_frac=0.7001 init_acc_corrupt=0.2263 init_gold_top10=0.2748 init_gold_top100=0.3113
+step=4050 micro_steps=64800 elapsed=61.8s lr=3.000000e-04 loss_all=1.8528 acc_all=0.6912 loss_corrupt=3.2440 acc_corrupt=0.4541 corrupt_frac=0.5523 loss=3.2440 mean_t=0.5007 wrong_frac=0.7005 init_acc_corrupt=0.2271 init_gold_top10=0.2746 init_gold_top100=0.3109
+step=4100 micro_steps=65600 elapsed=62.9s lr=3.000000e-04 loss_all=1.8451 acc_all=0.6925 loss_corrupt=3.2463 acc_corrupt=0.4536 corrupt_frac=0.5495 loss=3.2463 mean_t=0.4956 wrong_frac=0.6997 init_acc_corrupt=0.2261 init_gold_top10=0.2743 init_gold_top100=0.3115
+step=4150 micro_steps=66400 elapsed=53.9s lr=3.000000e-04 loss_all=1.8382 acc_all=0.6930 loss_corrupt=3.2346 acc_corrupt=0.4545 corrupt_frac=0.5497 loss=3.2346 mean_t=0.5005 wrong_frac=0.7003 init_acc_corrupt=0.2263 init_gold_top10=0.2740 init_gold_top100=0.3109
+step=4200 micro_steps=67200 elapsed=53.9s lr=3.000000e-04 loss_all=1.8483 acc_all=0.6908 loss_corrupt=3.2330 acc_corrupt=0.4542 corrupt_frac=0.5536 loss=3.2330 mean_t=0.5009 wrong_frac=0.7000 init_acc_corrupt=0.2256 init_gold_top10=0.2743 init_gold_top100=0.3114
+step=4250 micro_steps=68000 elapsed=53.9s lr=3.000000e-04 loss_all=1.8289 acc_all=0.6935 loss_corrupt=3.2070 acc_corrupt=0.4571 corrupt_frac=0.5518 loss=3.2070 mean_t=0.5005 wrong_frac=0.7005 init_acc_corrupt=0.2268 init_gold_top10=0.2744 init_gold_top100=0.3110
+step=4300 micro_steps=68800 elapsed=53.9s lr=3.000000e-04 loss_all=1.8142 acc_all=0.6954 loss_corrupt=3.1901 acc_corrupt=0.4591 corrupt_frac=0.5502 loss=3.1901 mean_t=0.4979 wrong_frac=0.6999 init_acc_corrupt=0.2255 init_gold_top10=0.2749 init_gold_top100=0.3121
+step=4350 micro_steps=69600 elapsed=54.2s lr=3.000000e-04 loss_all=1.8094 acc_all=0.6958 loss_corrupt=3.1786 acc_corrupt=0.4603 corrupt_frac=0.5513 loss=3.1786 mean_t=0.5018 wrong_frac=0.6998 init_acc_corrupt=0.2281 init_gold_top10=0.2751 init_gold_top100=0.3115
+step=4400 micro_steps=70400 elapsed=54.4s lr=3.000000e-04 loss_all=1.7855 acc_all=0.6997 loss_corrupt=3.1678 acc_corrupt=0.4620 corrupt_frac=0.5460 loss=3.1678 mean_t=0.5025 wrong_frac=0.6992 init_acc_corrupt=0.2282 init_gold_top10=0.2757 init_gold_top100=0.3118
+step=4450 micro_steps=71200 elapsed=54.4s lr=3.000000e-04 loss_all=1.7829 acc_all=0.6987 loss_corrupt=3.1487 acc_corrupt=0.4628 corrupt_frac=0.5488 loss=3.1487 mean_t=0.5023 wrong_frac=0.6997 init_acc_corrupt=0.2278 init_gold_top10=0.2753 init_gold_top100=0.3111
+step=4500 micro_steps=72000 elapsed=54.4s lr=3.000000e-04 loss_all=1.7906 acc_all=0.6977 loss_corrupt=3.1509 acc_corrupt=0.4628 corrupt_frac=0.5503 loss=3.1509 mean_t=0.5012 wrong_frac=0.6996 init_acc_corrupt=0.2282 init_gold_top10=0.2761 init_gold_top100=0.3119
+step=4550 micro_steps=72800 elapsed=54.4s lr=3.000000e-04 loss_all=1.7611 acc_all=0.7018 loss_corrupt=3.1259 acc_corrupt=0.4655 corrupt_frac=0.5459 loss=3.1259 mean_t=0.4989 wrong_frac=0.6992 init_acc_corrupt=0.2276 init_gold_top10=0.2757 init_gold_top100=0.3125
+step=4600 micro_steps=73600 elapsed=54.4s lr=3.000000e-04 loss_all=1.7884 acc_all=0.6981 loss_corrupt=3.1565 acc_corrupt=0.4620 corrupt_frac=0.5497 loss=3.1565 mean_t=0.5012 wrong_frac=0.7007 init_acc_corrupt=0.2260 init_gold_top10=0.2743 init_gold_top100=0.3111
+step=4650 micro_steps=74400 elapsed=54.3s lr=3.000000e-04 loss_all=1.7657 acc_all=0.7005 loss_corrupt=3.1149 acc_corrupt=0.4664 corrupt_frac=0.5495 loss=3.1149 mean_t=0.5007 wrong_frac=0.6998 init_acc_corrupt=0.2277 init_gold_top10=0.2762 init_gold_top100=0.3126
+step=4700 micro_steps=75200 elapsed=53.9s lr=3.000000e-04 loss_all=1.7764 acc_all=0.6994 loss_corrupt=3.1288 acc_corrupt=0.4655 corrupt_frac=0.5510 loss=3.1288 mean_t=0.4987 wrong_frac=0.7004 init_acc_corrupt=0.2256 init_gold_top10=0.2743 init_gold_top100=0.3115
+step=4750 micro_steps=76000 elapsed=54.1s lr=3.000000e-04 loss_all=1.7684 acc_all=0.7000 loss_corrupt=3.1188 acc_corrupt=0.4658 corrupt_frac=0.5501 loss=3.1188 mean_t=0.5012 wrong_frac=0.6995 init_acc_corrupt=0.2264 init_gold_top10=0.2756 init_gold_top100=0.3126
+step=4800 micro_steps=76800 elapsed=54.1s lr=3.000000e-04 loss_all=1.7626 acc_all=0.7011 loss_corrupt=3.1295 acc_corrupt=0.4643 corrupt_frac=0.5465 loss=3.1295 mean_t=0.4968 wrong_frac=0.7002 init_acc_corrupt=0.2246 init_gold_top10=0.2740 init_gold_top100=0.3119
+step=4850 micro_steps=77600 elapsed=53.8s lr=3.000000e-04 loss_all=1.7504 acc_all=0.7021 loss_corrupt=3.0908 acc_corrupt=0.4691 corrupt_frac=0.5499 loss=3.0908 mean_t=0.5004 wrong_frac=0.6997 init_acc_corrupt=0.2283 init_gold_top10=0.2758 init_gold_top100=0.3115
+step=4900 micro_steps=78400 elapsed=54.2s lr=3.000000e-04 loss_all=1.7656 acc_all=0.6998 loss_corrupt=3.0992 acc_corrupt=0.4681 corrupt_frac=0.5529 loss=3.0992 mean_t=0.4993 wrong_frac=0.6997 init_acc_corrupt=0.2279 init_gold_top10=0.2756 init_gold_top100=0.3115
+step=4950 micro_steps=79200 elapsed=54.5s lr=3.000000e-04 loss_all=1.7401 acc_all=0.7032 loss_corrupt=3.0801 acc_corrupt=0.4697 corrupt_frac=0.5489 loss=3.0801 mean_t=0.4974 wrong_frac=0.7005 init_acc_corrupt=0.2259 init_gold_top10=0.2743 init_gold_top100=0.3111
+step=5000 micro_steps=80000 elapsed=54.5s lr=3.000000e-04 loss_all=1.7395 acc_all=0.7034 loss_corrupt=3.0845 acc_corrupt=0.4689 corrupt_frac=0.5473 loss=3.0845 mean_t=0.4997 wrong_frac=0.6999 init_acc_corrupt=0.2266 init_gold_top10=0.2745 init_gold_top100=0.3114
+step=5050 micro_steps=80800 elapsed=70.1s lr=3.000000e-04 loss_all=1.7487 acc_all=0.7022 loss_corrupt=3.0914 acc_corrupt=0.4686 corrupt_frac=0.5490 loss=3.0914 mean_t=0.4976 wrong_frac=0.7000 init_acc_corrupt=0.2248 init_gold_top10=0.2730 init_gold_top100=0.3116
+step=5100 micro_steps=81600 elapsed=53.9s lr=3.000000e-04 loss_all=1.7399 acc_all=0.7030 loss_corrupt=3.0755 acc_corrupt=0.4701 corrupt_frac=0.5493 loss=3.0755 mean_t=0.4979 wrong_frac=0.6996 init_acc_corrupt=0.2258 init_gold_top10=0.2744 init_gold_top100=0.3118
+step=5150 micro_steps=82400 elapsed=54.3s lr=3.000000e-04 loss_all=1.7418 acc_all=0.7025 loss_corrupt=3.0698 acc_corrupt=0.4705 corrupt_frac=0.5510 loss=3.0698 mean_t=0.4982 wrong_frac=0.7001 init_acc_corrupt=0.2259 init_gold_top10=0.2742 init_gold_top100=0.3114
+step=5200 micro_steps=83200 elapsed=53.8s lr=3.000000e-04 loss_all=1.7329 acc_all=0.7038 loss_corrupt=3.0611 acc_corrupt=0.4719 corrupt_frac=0.5503 loss=3.0611 mean_t=0.5007 wrong_frac=0.6995 init_acc_corrupt=0.2272 init_gold_top10=0.2750 init_gold_top100=0.3116
+step=5250 micro_steps=84000 elapsed=53.8s lr=3.000000e-04 loss_all=1.7249 acc_all=0.7047 loss_corrupt=3.0519 acc_corrupt=0.4726 corrupt_frac=0.5490 loss=3.0519 mean_t=0.4980 wrong_frac=0.6990 init_acc_corrupt=0.2260 init_gold_top10=0.2750 init_gold_top100=0.3127
+step=5300 micro_steps=84800 elapsed=53.7s lr=3.000000e-04 loss_all=1.7279 acc_all=0.7044 loss_corrupt=3.0439 acc_corrupt=0.4744 corrupt_frac=0.5515 loss=3.0439 mean_t=0.4979 wrong_frac=0.6996 init_acc_corrupt=0.2272 init_gold_top10=0.2750 init_gold_top100=0.3120
+step=5350 micro_steps=85600 elapsed=53.5s lr=3.000000e-04 loss_all=1.7095 acc_all=0.7066 loss_corrupt=3.0302 acc_corrupt=0.4751 corrupt_frac=0.5481 loss=3.0302 mean_t=0.4970 wrong_frac=0.7003 init_acc_corrupt=0.2249 init_gold_top10=0.2742 init_gold_top100=0.3121
+step=5400 micro_steps=86400 elapsed=53.6s lr=3.000000e-04 loss_all=1.7046 acc_all=0.7076 loss_corrupt=3.0326 acc_corrupt=0.4749 corrupt_frac=0.5461 loss=3.0326 mean_t=0.5021 wrong_frac=0.7000 init_acc_corrupt=0.2268 init_gold_top10=0.2745 init_gold_top100=0.3113
+step=5450 micro_steps=87200 elapsed=53.5s lr=3.000000e-04 loss_all=1.7316 acc_all=0.7026 loss_corrupt=3.0441 acc_corrupt=0.4727 corrupt_frac=0.5538 loss=3.0441 mean_t=0.4986 wrong_frac=0.6995 init_acc_corrupt=0.2260 init_gold_top10=0.2751 init_gold_top100=0.3121
+step=5500 micro_steps=88000 elapsed=53.5s lr=3.000000e-04 loss_all=1.7123 acc_all=0.7055 loss_corrupt=3.0166 acc_corrupt=0.4764 corrupt_frac=0.5520 loss=3.0166 mean_t=0.5031 wrong_frac=0.6993 init_acc_corrupt=0.2286 init_gold_top10=0.2762 init_gold_top100=0.3119
+step=5550 micro_steps=88800 elapsed=53.5s lr=3.000000e-04 loss_all=1.7026 acc_all=0.7071 loss_corrupt=3.0160 acc_corrupt=0.4761 corrupt_frac=0.5485 loss=3.0160 mean_t=0.4945 wrong_frac=0.6998 init_acc_corrupt=0.2258 init_gold_top10=0.2748 init_gold_top100=0.3116
+step=5600 micro_steps=89600 elapsed=53.5s lr=3.000000e-04 loss_all=1.7061 acc_all=0.7065 loss_corrupt=2.9973 acc_corrupt=0.4792 corrupt_frac=0.5524 loss=2.9973 mean_t=0.5031 wrong_frac=0.6997 init_acc_corrupt=0.2276 init_gold_top10=0.2752 init_gold_top100=0.3113
+step=5650 micro_steps=90400 elapsed=53.5s lr=3.000000e-04 loss_all=1.7087 acc_all=0.7053 loss_corrupt=3.0063 acc_corrupt=0.4766 corrupt_frac=0.5530 loss=3.0063 mean_t=0.4984 wrong_frac=0.7008 init_acc_corrupt=0.2272 init_gold_top10=0.2750 init_gold_top100=0.3110
+step=5700 micro_steps=91200 elapsed=53.5s lr=3.000000e-04 loss_all=1.6979 acc_all=0.7077 loss_corrupt=3.0012 acc_corrupt=0.4784 corrupt_frac=0.5497 loss=3.0012 mean_t=0.5021 wrong_frac=0.6994 init_acc_corrupt=0.2275 init_gold_top10=0.2751 init_gold_top100=0.3114
+step=5750 micro_steps=92000 elapsed=53.5s lr=3.000000e-04 loss_all=1.7046 acc_all=0.7058 loss_corrupt=3.0027 acc_corrupt=0.4770 corrupt_frac=0.5527 loss=3.0027 mean_t=0.4952 wrong_frac=0.7003 init_acc_corrupt=0.2254 init_gold_top10=0.2743 init_gold_top100=0.3112
+step=5800 micro_steps=92800 elapsed=53.5s lr=3.000000e-04 loss_all=1.7074 acc_all=0.7055 loss_corrupt=3.0033 acc_corrupt=0.4772 corrupt_frac=0.5529 loss=3.0033 mean_t=0.4989 wrong_frac=0.7001 init_acc_corrupt=0.2261 init_gold_top10=0.2750 init_gold_top100=0.3118
+step=5850 micro_steps=93600 elapsed=53.4s lr=3.000000e-04 loss_all=1.6698 acc_all=0.7110 loss_corrupt=2.9766 acc_corrupt=0.4802 corrupt_frac=0.5459 loss=2.9766 mean_t=0.4977 wrong_frac=0.7004 init_acc_corrupt=0.2259 init_gold_top10=0.2740 init_gold_top100=0.3107
+step=5900 micro_steps=94400 elapsed=53.5s lr=3.000000e-04 loss_all=1.6801 acc_all=0.7087 loss_corrupt=2.9627 acc_corrupt=0.4816 corrupt_frac=0.5521 loss=2.9627 mean_t=0.5024 wrong_frac=0.7003 init_acc_corrupt=0.2276 init_gold_top10=0.2754 init_gold_top100=0.3114
+step=5950 micro_steps=95200 elapsed=54.0s lr=3.000000e-04 loss_all=1.6587 acc_all=0.7125 loss_corrupt=2.9507 acc_corrupt=0.4838 corrupt_frac=0.5467 loss=2.9507 mean_t=0.4980 wrong_frac=0.6998 init_acc_corrupt=0.2278 init_gold_top10=0.2749 init_gold_top100=0.3112
+step=6000 micro_steps=96000 elapsed=54.5s lr=3.000000e-04 loss_all=1.6782 acc_all=0.7091 loss_corrupt=2.9655 acc_corrupt=0.4818 corrupt_frac=0.5512 loss=2.9655 mean_t=0.5049 wrong_frac=0.7002 init_acc_corrupt=0.2273 init_gold_top10=0.2752 init_gold_top100=0.3117
+step=6050 micro_steps=96800 elapsed=60.4s lr=3.000000e-04 loss_all=1.6813 acc_all=0.7090 loss_corrupt=2.9698 acc_corrupt=0.4815 corrupt_frac=0.5511 loss=2.9698 mean_t=0.4988 wrong_frac=0.6999 init_acc_corrupt=0.2261 init_gold_top10=0.2751 init_gold_top100=0.3121
+step=6100 micro_steps=97600 elapsed=63.9s lr=3.000000e-04 loss_all=1.6839 acc_all=0.7081 loss_corrupt=2.9694 acc_corrupt=0.4807 corrupt_frac=0.5526 loss=2.9694 mean_t=0.5024 wrong_frac=0.7001 init_acc_corrupt=0.2261 init_gold_top10=0.2743 init_gold_top100=0.3113
+step=6150 micro_steps=98400 elapsed=53.9s lr=3.000000e-04 loss_all=1.6589 acc_all=0.7113 loss_corrupt=2.9422 acc_corrupt=0.4835 corrupt_frac=0.5488 loss=2.9422 mean_t=0.5039 wrong_frac=0.7003 init_acc_corrupt=0.2271 init_gold_top10=0.2750 init_gold_top100=0.3109
+step=6200 micro_steps=99200 elapsed=54.0s lr=3.000000e-04 loss_all=1.6651 acc_all=0.7103 loss_corrupt=2.9457 acc_corrupt=0.4831 corrupt_frac=0.5509 loss=2.9457 mean_t=0.4997 wrong_frac=0.6996 init_acc_corrupt=0.2271 init_gold_top10=0.2755 init_gold_top100=0.3122
+step=6250 micro_steps=100000 elapsed=53.8s lr=3.000000e-04 loss_all=1.6482 acc_all=0.7129 loss_corrupt=2.9270 acc_corrupt=0.4854 corrupt_frac=0.5481 loss=2.9270 mean_t=0.4985 wrong_frac=0.7001 init_acc_corrupt=0.2265 init_gold_top10=0.2747 init_gold_top100=0.3120
+step=6300 micro_steps=100800 elapsed=53.7s lr=3.000000e-04 loss_all=1.6623 acc_all=0.7105 loss_corrupt=2.9344 acc_corrupt=0.4846 corrupt_frac=0.5522 loss=2.9344 mean_t=0.5011 wrong_frac=0.7002 init_acc_corrupt=0.2262 init_gold_top10=0.2747 init_gold_top100=0.3120
+step=6350 micro_steps=101600 elapsed=58.7s lr=3.000000e-04 loss_all=1.6481 acc_all=0.7130 loss_corrupt=2.9234 acc_corrupt=0.4864 corrupt_frac=0.5486 loss=2.9234 mean_t=0.4986 wrong_frac=0.6998 init_acc_corrupt=0.2269 init_gold_top10=0.2752 init_gold_top100=0.3118
+step=6400 micro_steps=102400 elapsed=53.8s lr=3.000000e-04 loss_all=1.6661 acc_all=0.7095 loss_corrupt=2.9368 acc_corrupt=0.4836 corrupt_frac=0.5531 loss=2.9368 mean_t=0.4985 wrong_frac=0.7000 init_acc_corrupt=0.2263 init_gold_top10=0.2747 init_gold_top100=0.3115
+step=6450 micro_steps=103200 elapsed=53.8s lr=3.000000e-04 loss_all=1.6521 acc_all=0.7120 loss_corrupt=2.9257 acc_corrupt=0.4856 corrupt_frac=0.5507 loss=2.9257 mean_t=0.5023 wrong_frac=0.6995 init_acc_corrupt=0.2271 init_gold_top10=0.2753 init_gold_top100=0.3115
+step=6500 micro_steps=104000 elapsed=53.8s lr=3.000000e-04 loss_all=1.6655 acc_all=0.7096 loss_corrupt=2.9261 acc_corrupt=0.4854 corrupt_frac=0.5548 loss=2.9261 mean_t=0.5002 wrong_frac=0.7001 init_acc_corrupt=0.2272 init_gold_top10=0.2749 init_gold_top100=0.3111
+step=6550 micro_steps=104800 elapsed=53.8s lr=3.000000e-04 loss_all=1.6448 acc_all=0.7131 loss_corrupt=2.9205 acc_corrupt=0.4859 corrupt_frac=0.5482 loss=2.9205 mean_t=0.4961 wrong_frac=0.7007 init_acc_corrupt=0.2246 init_gold_top10=0.2731 init_gold_top100=0.3105
+step=6600 micro_steps=105600 elapsed=53.7s lr=3.000000e-04 loss_all=1.6428 acc_all=0.7129 loss_corrupt=2.8973 acc_corrupt=0.4894 corrupt_frac=0.5528 loss=2.8973 mean_t=0.5060 wrong_frac=0.6997 init_acc_corrupt=0.2300 init_gold_top10=0.2759 init_gold_top100=0.3113
+step=6650 micro_steps=106400 elapsed=71.1s lr=3.000000e-04 loss_all=1.6636 acc_all=0.7096 loss_corrupt=2.9137 acc_corrupt=0.4869 corrupt_frac=0.5568 loss=2.9137 mean_t=0.5012 wrong_frac=0.7001 init_acc_corrupt=0.2266 init_gold_top10=0.2745 init_gold_top100=0.3117
+step=6700 micro_steps=107200 elapsed=54.1s lr=3.000000e-04 loss_all=1.6563 acc_all=0.7109 loss_corrupt=2.9097 acc_corrupt=0.4876 corrupt_frac=0.5542 loss=2.9097 mean_t=0.5031 wrong_frac=0.6999 init_acc_corrupt=0.2280 init_gold_top10=0.2755 init_gold_top100=0.3116
+step=6750 micro_steps=108000 elapsed=54.1s lr=3.000000e-04 loss_all=1.6580 acc_all=0.7102 loss_corrupt=2.9181 acc_corrupt=0.4855 corrupt_frac=0.5538 loss=2.9181 mean_t=0.4957 wrong_frac=0.7000 init_acc_corrupt=0.2248 init_gold_top10=0.2739 init_gold_top100=0.3116
+step=6800 micro_steps=108800 elapsed=53.8s lr=3.000000e-04 loss_all=1.6242 acc_all=0.7153 loss_corrupt=2.8787 acc_corrupt=0.4910 corrupt_frac=0.5502 loss=2.8787 mean_t=0.5008 wrong_frac=0.6998 init_acc_corrupt=0.2286 init_gold_top10=0.2765 init_gold_top100=0.3116
+step=6850 micro_steps=109600 elapsed=53.8s lr=3.000000e-04 loss_all=1.6186 acc_all=0.7163 loss_corrupt=2.8721 acc_corrupt=0.4923 corrupt_frac=0.5494 loss=2.8721 mean_t=0.5029 wrong_frac=0.6996 init_acc_corrupt=0.2286 init_gold_top10=0.2765 init_gold_top100=0.3123
+step=6900 micro_steps=110400 elapsed=53.8s lr=3.000000e-04 loss_all=1.6265 acc_all=0.7149 loss_corrupt=2.8772 acc_corrupt=0.4912 corrupt_frac=0.5512 loss=2.8772 mean_t=0.5019 wrong_frac=0.7000 init_acc_corrupt=0.2272 init_gold_top10=0.2751 init_gold_top100=0.3115
+step=6950 micro_steps=111200 elapsed=53.8s lr=3.000000e-04 loss_all=1.6363 acc_all=0.7130 loss_corrupt=2.8829 acc_corrupt=0.4901 corrupt_frac=0.5534 loss=2.8829 mean_t=0.4989 wrong_frac=0.7001 init_acc_corrupt=0.2265 init_gold_top10=0.2746 init_gold_top100=0.3118
+step=7000 micro_steps=112000 elapsed=53.9s lr=3.000000e-04 loss_all=1.6347 acc_all=0.7135 loss_corrupt=2.8817 acc_corrupt=0.4906 corrupt_frac=0.5529 loss=2.8817 mean_t=0.4988 wrong_frac=0.7001 init_acc_corrupt=0.2263 init_gold_top10=0.2747 init_gold_top100=0.3115
+step=7050 micro_steps=112800 elapsed=55.9s lr=3.000000e-04 loss_all=1.6213 acc_all=0.7149 loss_corrupt=2.8574 acc_corrupt=0.4930 corrupt_frac=0.5526 loss=2.8574 mean_t=0.5051 wrong_frac=0.6998 init_acc_corrupt=0.2287 init_gold_top10=0.2756 init_gold_top100=0.3117
+step=7100 micro_steps=113600 elapsed=68.2s lr=3.000000e-04 loss_all=1.6264 acc_all=0.7143 loss_corrupt=2.8783 acc_corrupt=0.4900 corrupt_frac=0.5509 loss=2.8783 mean_t=0.4977 wrong_frac=0.7003 init_acc_corrupt=0.2259 init_gold_top10=0.2740 init_gold_top100=0.3107
+step=7150 micro_steps=114400 elapsed=54.0s lr=3.000000e-04 loss_all=1.6208 acc_all=0.7151 loss_corrupt=2.8648 acc_corrupt=0.4922 corrupt_frac=0.5521 loss=2.8648 mean_t=0.5037 wrong_frac=0.7004 init_acc_corrupt=0.2277 init_gold_top10=0.2749 init_gold_top100=0.3107
+step=7200 micro_steps=115200 elapsed=55.7s lr=3.000000e-04 loss_all=1.6170 acc_all=0.7157 loss_corrupt=2.8623 acc_corrupt=0.4923 corrupt_frac=0.5507 loss=2.8623 mean_t=0.4990 wrong_frac=0.6999 init_acc_corrupt=0.2270 init_gold_top10=0.2756 init_gold_top100=0.3123
+step=7250 micro_steps=116000 elapsed=54.1s lr=3.000000e-04 loss_all=1.5985 acc_all=0.7183 loss_corrupt=2.8470 acc_corrupt=0.4940 corrupt_frac=0.5474 loss=2.8470 mean_t=0.4971 wrong_frac=0.7000 init_acc_corrupt=0.2247 init_gold_top10=0.2739 init_gold_top100=0.3113
+step=7300 micro_steps=116800 elapsed=54.6s lr=3.000000e-04 loss_all=1.5972 acc_all=0.7180 loss_corrupt=2.8285 acc_corrupt=0.4961 corrupt_frac=0.5502 loss=2.8285 mean_t=0.4990 wrong_frac=0.7003 init_acc_corrupt=0.2263 init_gold_top10=0.2748 init_gold_top100=0.3117
+step=7350 micro_steps=117600 elapsed=54.3s lr=3.000000e-04 loss_all=1.6147 acc_all=0.7159 loss_corrupt=2.8480 acc_corrupt=0.4944 corrupt_frac=0.5520 loss=2.8480 mean_t=0.5004 wrong_frac=0.6999 init_acc_corrupt=0.2277 init_gold_top10=0.2751 init_gold_top100=0.3109
+step=7400 micro_steps=118400 elapsed=53.6s lr=3.000000e-04 loss_all=1.6097 acc_all=0.7162 loss_corrupt=2.8525 acc_corrupt=0.4929 corrupt_frac=0.5503 loss=2.8525 mean_t=0.4972 wrong_frac=0.7000 init_acc_corrupt=0.2247 init_gold_top10=0.2744 init_gold_top100=0.3124
+step=7450 micro_steps=119200 elapsed=54.3s lr=3.000000e-04 loss_all=1.6152 acc_all=0.7154 loss_corrupt=2.8520 acc_corrupt=0.4932 corrupt_frac=0.5526 loss=2.8520 mean_t=0.5013 wrong_frac=0.7005 init_acc_corrupt=0.2260 init_gold_top10=0.2737 init_gold_top100=0.3106
+step=7500 micro_steps=120000 elapsed=54.5s lr=3.000000e-04 loss_all=1.5871 acc_all=0.7193 loss_corrupt=2.8222 acc_corrupt=0.4965 corrupt_frac=0.5479 loss=2.8222 mean_t=0.4972 wrong_frac=0.7003 init_acc_corrupt=0.2263 init_gold_top10=0.2747 init_gold_top100=0.3112
+step=7550 micro_steps=120800 elapsed=54.5s lr=3.000000e-04 loss_all=1.5896 acc_all=0.7194 loss_corrupt=2.8216 acc_corrupt=0.4974 corrupt_frac=0.5491 loss=2.8216 mean_t=0.5020 wrong_frac=0.6999 init_acc_corrupt=0.2270 init_gold_top10=0.2747 init_gold_top100=0.3115
+step=7600 micro_steps=121600 elapsed=54.5s lr=3.000000e-04 loss_all=1.6009 acc_all=0.7171 loss_corrupt=2.8272 acc_corrupt=0.4963 corrupt_frac=0.5530 loss=2.8272 mean_t=0.5013 wrong_frac=0.7011 init_acc_corrupt=0.2266 init_gold_top10=0.2736 init_gold_top100=0.3102
+step=7650 micro_steps=122400 elapsed=53.6s lr=3.000000e-04 loss_all=1.5859 acc_all=0.7196 loss_corrupt=2.8208 acc_corrupt=0.4971 corrupt_frac=0.5486 loss=2.8208 mean_t=0.4974 wrong_frac=0.6997 init_acc_corrupt=0.2262 init_gold_top10=0.2745 init_gold_top100=0.3119
+step=7700 micro_steps=123200 elapsed=53.5s lr=3.000000e-04 loss_all=1.5855 acc_all=0.7190 loss_corrupt=2.8073 acc_corrupt=0.4984 corrupt_frac=0.5510 loss=2.8073 mean_t=0.4999 wrong_frac=0.7001 init_acc_corrupt=0.2271 init_gold_top10=0.2752 init_gold_top100=0.3118
+step=7750 micro_steps=124000 elapsed=53.5s lr=3.000000e-04 loss_all=1.5885 acc_all=0.7186 loss_corrupt=2.8101 acc_corrupt=0.4980 corrupt_frac=0.5518 loss=2.8101 mean_t=0.5055 wrong_frac=0.7001 init_acc_corrupt=0.2280 init_gold_top10=0.2755 init_gold_top100=0.3113
+step=7800 micro_steps=124800 elapsed=53.5s lr=3.000000e-04 loss_all=1.5982 acc_all=0.7173 loss_corrupt=2.8251 acc_corrupt=0.4960 corrupt_frac=0.5520 loss=2.8251 mean_t=0.4963 wrong_frac=0.6996 init_acc_corrupt=0.2258 init_gold_top10=0.2749 init_gold_top100=0.3121
+step=7850 micro_steps=125600 elapsed=53.5s lr=3.000000e-04 loss_all=1.5945 acc_all=0.7182 loss_corrupt=2.8315 acc_corrupt=0.4951 corrupt_frac=0.5489 loss=2.8315 mean_t=0.4977 wrong_frac=0.7002 init_acc_corrupt=0.2250 init_gold_top10=0.2736 init_gold_top100=0.3115
+step=7900 micro_steps=126400 elapsed=53.5s lr=3.000000e-04 loss_all=1.5946 acc_all=0.7177 loss_corrupt=2.8085 acc_corrupt=0.4984 corrupt_frac=0.5533 loss=2.8085 mean_t=0.4990 wrong_frac=0.6998 init_acc_corrupt=0.2267 init_gold_top10=0.2750 init_gold_top100=0.3116
+step=7950 micro_steps=127200 elapsed=53.5s lr=3.000000e-04 loss_all=1.5667 acc_all=0.7213 loss_corrupt=2.7845 acc_corrupt=0.5006 corrupt_frac=0.5490 loss=2.7845 mean_t=0.5020 wrong_frac=0.7000 init_acc_corrupt=0.2258 init_gold_top10=0.2751 init_gold_top100=0.3127
+step=8000 micro_steps=128000 elapsed=53.6s lr=3.000000e-04 loss_all=1.5797 acc_all=0.7200 loss_corrupt=2.8032 acc_corrupt=0.4988 corrupt_frac=0.5498 loss=2.8032 mean_t=0.5014 wrong_frac=0.7000 init_acc_corrupt=0.2271 init_gold_top10=0.2749 init_gold_top100=0.3112
+step=8050 micro_steps=128800 elapsed=69.6s lr=3.000000e-04 loss_all=1.5695 acc_all=0.7212 loss_corrupt=2.7909 acc_corrupt=0.5000 corrupt_frac=0.5480 loss=2.7909 mean_t=0.5017 wrong_frac=0.7003 init_acc_corrupt=0.2269 init_gold_top10=0.2747 init_gold_top100=0.3112
+step=8100 micro_steps=129600 elapsed=54.6s lr=3.000000e-04 loss_all=1.5886 acc_all=0.7182 loss_corrupt=2.8009 acc_corrupt=0.4989 corrupt_frac=0.5534 loss=2.8009 mean_t=0.5013 wrong_frac=0.6998 init_acc_corrupt=0.2270 init_gold_top10=0.2755 init_gold_top100=0.3117
+step=8150 micro_steps=130400 elapsed=54.2s lr=3.000000e-04 loss_all=1.5626 acc_all=0.7220 loss_corrupt=2.7885 acc_corrupt=0.4998 corrupt_frac=0.5470 loss=2.7885 mean_t=0.4998 wrong_frac=0.7001 init_acc_corrupt=0.2252 init_gold_top10=0.2739 init_gold_top100=0.3116
+step=8200 micro_steps=131200 elapsed=53.9s lr=3.000000e-04 loss_all=1.5581 acc_all=0.7228 loss_corrupt=2.7806 acc_corrupt=0.5014 corrupt_frac=0.5474 loss=2.7806 mean_t=0.4994 wrong_frac=0.7002 init_acc_corrupt=0.2261 init_gold_top10=0.2745 init_gold_top100=0.3115
+step=8250 micro_steps=132000 elapsed=53.8s lr=3.000000e-04 loss_all=1.5548 acc_all=0.7229 loss_corrupt=2.7627 acc_corrupt=0.5034 corrupt_frac=0.5492 loss=2.7627 mean_t=0.5021 wrong_frac=0.6996 init_acc_corrupt=0.2287 init_gold_top10=0.2761 init_gold_top100=0.3120
+step=8300 micro_steps=132800 elapsed=53.9s lr=3.000000e-04 loss_all=1.5628 acc_all=0.7224 loss_corrupt=2.7747 acc_corrupt=0.5031 corrupt_frac=0.5497 loss=2.7747 mean_t=0.5025 wrong_frac=0.7000 init_acc_corrupt=0.2285 init_gold_top10=0.2764 init_gold_top100=0.3121
+step=8350 micro_steps=133600 elapsed=53.9s lr=3.000000e-04 loss_all=1.5546 acc_all=0.7225 loss_corrupt=2.7594 acc_corrupt=0.5033 corrupt_frac=0.5495 loss=2.7594 mean_t=0.4976 wrong_frac=0.7003 init_acc_corrupt=0.2266 init_gold_top10=0.2755 init_gold_top100=0.3119
+step=8400 micro_steps=134400 elapsed=53.9s lr=3.000000e-04 loss_all=1.5662 acc_all=0.7210 loss_corrupt=2.7759 acc_corrupt=0.5013 corrupt_frac=0.5505 loss=2.7759 mean_t=0.4989 wrong_frac=0.7003 init_acc_corrupt=0.2260 init_gold_top10=0.2747 init_gold_top100=0.3114
+step=8450 micro_steps=135200 elapsed=53.7s lr=3.000000e-04 loss_all=1.5540 acc_all=0.7232 loss_corrupt=2.7651 acc_corrupt=0.5034 corrupt_frac=0.5488 loss=2.7651 mean_t=0.5000 wrong_frac=0.6998 init_acc_corrupt=0.2261 init_gold_top10=0.2752 init_gold_top100=0.3121
+step=8500 micro_steps=136000 elapsed=53.5s lr=3.000000e-04 loss_all=1.5661 acc_all=0.7212 loss_corrupt=2.7805 acc_corrupt=0.5011 corrupt_frac=0.5503 loss=2.7805 mean_t=0.5000 wrong_frac=0.7000 init_acc_corrupt=0.2259 init_gold_top10=0.2747 init_gold_top100=0.3122
+step=8550 micro_steps=136800 elapsed=53.5s lr=3.000000e-04 loss_all=1.5491 acc_all=0.7239 loss_corrupt=2.7537 acc_corrupt=0.5048 corrupt_frac=0.5483 loss=2.7537 mean_t=0.5015 wrong_frac=0.7002 init_acc_corrupt=0.2272 init_gold_top10=0.2751 init_gold_top100=0.3111
+step=8600 micro_steps=137600 elapsed=53.5s lr=3.000000e-04 loss_all=1.5532 acc_all=0.7226 loss_corrupt=2.7517 acc_corrupt=0.5043 corrupt_frac=0.5505 loss=2.7517 mean_t=0.5003 wrong_frac=0.7000 init_acc_corrupt=0.2270 init_gold_top10=0.2757 init_gold_top100=0.3128
+step=8650 micro_steps=138400 elapsed=53.5s lr=3.000000e-04 loss_all=1.5554 acc_all=0.7227 loss_corrupt=2.7551 acc_corrupt=0.5047 corrupt_frac=0.5510 loss=2.7551 mean_t=0.4999 wrong_frac=0.6997 init_acc_corrupt=0.2282 init_gold_top10=0.2764 init_gold_top100=0.3122
+step=8700 micro_steps=139200 elapsed=53.4s lr=3.000000e-04 loss_all=1.5556 acc_all=0.7220 loss_corrupt=2.7557 acc_corrupt=0.5036 corrupt_frac=0.5512 loss=2.7557 mean_t=0.4948 wrong_frac=0.6998 init_acc_corrupt=0.2260 init_gold_top10=0.2751 init_gold_top100=0.3121
+step=8750 micro_steps=140000 elapsed=53.4s lr=3.000000e-04 loss_all=1.5611 acc_all=0.7218 loss_corrupt=2.7653 acc_corrupt=0.5033 corrupt_frac=0.5515 loss=2.7653 mean_t=0.5023 wrong_frac=0.7002 init_acc_corrupt=0.2258 init_gold_top10=0.2746 init_gold_top100=0.3117
+step=8800 micro_steps=140800 elapsed=53.4s lr=3.000000e-04 loss_all=1.5451 acc_all=0.7241 loss_corrupt=2.7491 acc_corrupt=0.5049 corrupt_frac=0.5483 loss=2.7491 mean_t=0.5019 wrong_frac=0.7003 init_acc_corrupt=0.2262 init_gold_top10=0.2747 init_gold_top100=0.3111
+step=8850 micro_steps=141600 elapsed=53.5s lr=3.000000e-04 loss_all=1.5463 acc_all=0.7240 loss_corrupt=2.7411 acc_corrupt=0.5066 corrupt_frac=0.5506 loss=2.7411 mean_t=0.5031 wrong_frac=0.6995 init_acc_corrupt=0.2277 init_gold_top10=0.2759 init_gold_top100=0.3122
+step=8900 micro_steps=142400 elapsed=53.4s lr=3.000000e-04 loss_all=1.5342 acc_all=0.7255 loss_corrupt=2.7303 acc_corrupt=0.5077 corrupt_frac=0.5497 loss=2.7303 mean_t=0.5005 wrong_frac=0.6997 init_acc_corrupt=0.2282 init_gold_top10=0.2762 init_gold_top100=0.3123
+step=8950 micro_steps=143200 elapsed=53.7s lr=3.000000e-04 loss_all=1.5731 acc_all=0.7199 loss_corrupt=2.7806 acc_corrupt=0.5007 corrupt_frac=0.5522 loss=2.7806 mean_t=0.4974 wrong_frac=0.7002 init_acc_corrupt=0.2243 init_gold_top10=0.2733 init_gold_top100=0.3113
+step=9000 micro_steps=144000 elapsed=54.0s lr=3.000000e-04 loss_all=1.5237 acc_all=0.7263 loss_corrupt=2.7206 acc_corrupt=0.5074 corrupt_frac=0.5472 loss=2.7206 mean_t=0.5005 wrong_frac=0.7003 init_acc_corrupt=0.2271 init_gold_top10=0.2752 init_gold_top100=0.3115
+step=9050 micro_steps=144800 elapsed=70.0s lr=3.000000e-04 loss_all=1.5387 acc_all=0.7244 loss_corrupt=2.7312 acc_corrupt=0.5068 corrupt_frac=0.5503 loss=2.7312 mean_t=0.5007 wrong_frac=0.7003 init_acc_corrupt=0.2275 init_gold_top10=0.2750 init_gold_top100=0.3111
+step=9100 micro_steps=145600 elapsed=54.3s lr=3.000000e-04 loss_all=1.5374 acc_all=0.7242 loss_corrupt=2.7244 acc_corrupt=0.5074 corrupt_frac=0.5514 loss=2.7244 mean_t=0.5026 wrong_frac=0.7001 init_acc_corrupt=0.2276 init_gold_top10=0.2752 init_gold_top100=0.3106
+step=9150 micro_steps=146400 elapsed=54.4s lr=3.000000e-04 loss_all=1.5342 acc_all=0.7246 loss_corrupt=2.7279 acc_corrupt=0.5062 corrupt_frac=0.5486 loss=2.7279 mean_t=0.4939 wrong_frac=0.7001 init_acc_corrupt=0.2251 init_gold_top10=0.2739 init_gold_top100=0.3117
+step=9200 micro_steps=147200 elapsed=53.7s lr=3.000000e-04 loss_all=1.5326 acc_all=0.7253 loss_corrupt=2.7300 acc_corrupt=0.5066 corrupt_frac=0.5484 loss=2.7300 mean_t=0.4999 wrong_frac=0.7002 init_acc_corrupt=0.2260 init_gold_top10=0.2743 init_gold_top100=0.3112
+step=9250 micro_steps=148000 elapsed=53.4s lr=3.000000e-04 loss_all=1.5441 acc_all=0.7232 loss_corrupt=2.7237 acc_corrupt=0.5078 corrupt_frac=0.5536 loss=2.7237 mean_t=0.4990 wrong_frac=0.6995 init_acc_corrupt=0.2272 init_gold_top10=0.2754 init_gold_top100=0.3126
+step=9300 micro_steps=148800 elapsed=53.6s lr=3.000000e-04 loss_all=1.5334 acc_all=0.7250 loss_corrupt=2.7293 acc_corrupt=0.5065 corrupt_frac=0.5490 loss=2.7293 mean_t=0.5005 wrong_frac=0.7008 init_acc_corrupt=0.2265 init_gold_top10=0.2735 init_gold_top100=0.3104
+step=9350 micro_steps=149600 elapsed=54.5s lr=3.000000e-04 loss_all=1.5100 acc_all=0.7279 loss_corrupt=2.6909 acc_corrupt=0.5109 corrupt_frac=0.5476 loss=2.6909 mean_t=0.5012 wrong_frac=0.7005 init_acc_corrupt=0.2272 init_gold_top10=0.2747 init_gold_top100=0.3106
+step=9400 micro_steps=150400 elapsed=54.5s lr=3.000000e-04 loss_all=1.5446 acc_all=0.7230 loss_corrupt=2.7298 acc_corrupt=0.5065 corrupt_frac=0.5528 loss=2.7298 mean_t=0.4989 wrong_frac=0.7005 init_acc_corrupt=0.2255 init_gold_top10=0.2741 init_gold_top100=0.3108
+step=9450 micro_steps=151200 elapsed=53.7s lr=3.000000e-04 loss_all=1.5156 acc_all=0.7275 loss_corrupt=2.7049 acc_corrupt=0.5097 corrupt_frac=0.5472 loss=2.7049 mean_t=0.5006 wrong_frac=0.6994 init_acc_corrupt=0.2278 init_gold_top10=0.2757 init_gold_top100=0.3116
+step=9500 micro_steps=152000 elapsed=53.7s lr=3.000000e-04 loss_all=1.5247 acc_all=0.7256 loss_corrupt=2.6931 acc_corrupt=0.5114 corrupt_frac=0.5527 loss=2.6931 mean_t=0.5014 wrong_frac=0.7001 init_acc_corrupt=0.2279 init_gold_top10=0.2753 init_gold_top100=0.3115
+step=9550 micro_steps=152800 elapsed=53.9s lr=3.000000e-04 loss_all=1.4995 acc_all=0.7299 loss_corrupt=2.6835 acc_corrupt=0.5128 corrupt_frac=0.5453 loss=2.6835 mean_t=0.5042 wrong_frac=0.7001 init_acc_corrupt=0.2278 init_gold_top10=0.2756 init_gold_top100=0.3118
+step=9600 micro_steps=153600 elapsed=54.2s lr=3.000000e-04 loss_all=1.5401 acc_all=0.7232 loss_corrupt=2.7116 acc_corrupt=0.5087 corrupt_frac=0.5548 loss=2.7116 mean_t=0.5016 wrong_frac=0.6999 init_acc_corrupt=0.2278 init_gold_top10=0.2751 init_gold_top100=0.3111
+step=9650 micro_steps=154400 elapsed=54.1s lr=3.000000e-04 loss_all=1.5310 acc_all=0.7248 loss_corrupt=2.7159 acc_corrupt=0.5079 corrupt_frac=0.5504 loss=2.7159 mean_t=0.4985 wrong_frac=0.7007 init_acc_corrupt=0.2253 init_gold_top10=0.2740 init_gold_top100=0.3109
+step=9700 micro_steps=155200 elapsed=53.8s lr=3.000000e-04 loss_all=1.5152 acc_all=0.7266 loss_corrupt=2.6887 acc_corrupt=0.5110 corrupt_frac=0.5502 loss=2.6887 mean_t=0.4996 wrong_frac=0.6997 init_acc_corrupt=0.2260 init_gold_top10=0.2751 init_gold_top100=0.3119
+step=9750 micro_steps=156000 elapsed=53.8s lr=3.000000e-04 loss_all=1.5177 acc_all=0.7267 loss_corrupt=2.7043 acc_corrupt=0.5089 corrupt_frac=0.5477 loss=2.7043 mean_t=0.4976 wrong_frac=0.7008 init_acc_corrupt=0.2242 init_gold_top10=0.2734 init_gold_top100=0.3109
+step=9800 micro_steps=156800 elapsed=53.8s lr=3.000000e-04 loss_all=1.5043 acc_all=0.7284 loss_corrupt=2.6745 acc_corrupt=0.5134 corrupt_frac=0.5494 loss=2.6745 mean_t=0.4978 wrong_frac=0.6995 init_acc_corrupt=0.2269 init_gold_top10=0.2759 init_gold_top100=0.3123
+step=9850 micro_steps=157600 elapsed=53.8s lr=3.000000e-04 loss_all=1.4900 acc_all=0.7308 loss_corrupt=2.6656 acc_corrupt=0.5144 corrupt_frac=0.5460 loss=2.6656 mean_t=0.5012 wrong_frac=0.6996 init_acc_corrupt=0.2272 init_gold_top10=0.2754 init_gold_top100=0.3119
+step=9900 micro_steps=158400 elapsed=53.8s lr=3.000000e-04 loss_all=1.5044 acc_all=0.7286 loss_corrupt=2.6804 acc_corrupt=0.5128 corrupt_frac=0.5487 loss=2.6804 mean_t=0.4978 wrong_frac=0.7000 init_acc_corrupt=0.2252 init_gold_top10=0.2749 init_gold_top100=0.3127
+step=9950 micro_steps=159200 elapsed=53.9s lr=3.000000e-04 loss_all=1.5173 acc_all=0.7262 loss_corrupt=2.6824 acc_corrupt=0.5119 corrupt_frac=0.5521 loss=2.6824 mean_t=0.4993 wrong_frac=0.7003 init_acc_corrupt=0.2269 init_gold_top10=0.2749 init_gold_top100=0.3114
+step=10000 micro_steps=160000 elapsed=53.8s lr=3.000000e-04 loss_all=1.5098 acc_all=0.7275 loss_corrupt=2.6827 acc_corrupt=0.5120 corrupt_frac=0.5495 loss=2.6827 mean_t=0.5013 wrong_frac=0.6996 init_acc_corrupt=0.2274 init_gold_top10=0.2751 init_gold_top100=0.3119

LTA_openwebtext_dualt/logs/genppl_lm1b_latest_dirichlet_sweep.log

@@ -0,0 +1,312 @@
+=== DIRICHLET INIT C=1 K=128 S=32 ===
+{
+  "mask_ratio_0.10": {
+    "corrupt_tokens": 4,
+    "endpoint_loss": 1.5438232421875,
+    "endpoint_acc": 0.5,
+    "final_acc": 0.5,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did alexei : dominate the field ofversion golf and rake in endorsements. imagery",
+    "endpoint": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did most : dominate the field of professional golf and rake in endorsements.",
+    "final": "and while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did most : dominate the field of professional golf and rake in endorsements."
+  },
+  "mask_ratio_0.20": {
+    "corrupt_tokens": 17,
+    "endpoint_loss": 2.2356011867523193,
+    "endpoint_acc": 0.47058823704719543,
+    "final_acc": 0.47058823704719543,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "while athletes in harris professions dealt with dopingistic and other hatch, bundesliga continued whenever do what he did best galway dominate the 1400 violin professional golf [unused416] rake iniards.",
+    "endpoint": "while athletes in other sport dealt with dopings and other problems, he continued to do what he did best to dominate the world of professional golf and and in sponsorships.",
+    "final": "and while athletes in the sport dealt with dopings and other problems, he continued to do what he did best to dominate the world of professional golf and rake in sprinters."
+  },
+  "mask_ratio_0.50": {
+    "corrupt_tokens": 37,
+    "endpoint_loss": 5.348362922668457,
+    "endpoint_acc": 0.18918919563293457,
+    "final_acc": 0.1621621549129486,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "##ented while mendes loans danielsrook武 with doping scandals and other carbonate, ye continued abolition kilometres rohan he [unused949] best atv dominate salsa recipe of commands juarez carriers rake ruling endorsements. taleager",
+    "endpoint": "while mr. has was charged with doping scandals and other scandals, he continued to argue that he could best to dominate the list of the ' ' s s endorsements.",
+    "final": "and while mr. daniels was charged with doping scandals and other scandals, he continued to argue that he did best to dominate the list of the city ' s top endorsements."
+  },
+  "mask_ratio_1.00": {
+    "corrupt_tokens": 74,
+    "endpoint_loss": 6.789273738861084,
+    "endpoint_acc": 0.13513512909412384,
+    "final_acc": 0.04054053872823715,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "[unused150]pass plunged battista supplemented wyominggating economies remarkablyylus heights [unused531] flood [unused587] ² acknowledges drop arsenicballsrtingien bolsheviksese balance viceroy syllable stefan stump 1895 unconscious bismarck rating absolute software roar testingbrates",
+    "endpoint": "",
+    "final": "wednesday."
+  },
+  "pure_noise": [
+    "to a federal investigation."
+  ],
+  "gen_ppl": 15.357786529758505,
+  "gen_nll_per_token": 2.7316226111997253,
+  "gen_tokens": 6191,
+  "gen_scored_samples": 32,
+  "gen_skipped_samples": 0,
+  "gen_empty_rate": 0.0,
+  "gen_kept_samples": 32,
+  "gen_total_samples": 32,
+  "gen_ppl_model": "/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard",
+  "gen_ppl_output": "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/runs/lta_lm1b_duo_aligned_dirichlet_dualt_onehot_hardce_b128_4xh20_1m_nw0_compile_ro/genppl_lm1b_latest_dirichletC1_k128_s32_flm.jsonl",
+  "gen_ppl_flm_formula": true,
+  "gen_ppl_full_decode": true,
+  "decode_solver": "flowmap",
+  "noise_init": "dirichlet",
+  "noise_sigma": -1.0,
+  "dirichlet_init_concentration": 1.0,
+  "gen_ppl_min_chars": 0,
+  "gen_ppl_normalize_whitespace": false,
+  "gen_ppl_drop_remainder": false,
+  "gen_sample_entropy": 3.53966880233913,
+  "gen_unique_tokens": 874,
+  "gen_token_count": 4096,
+  "gen_distinct_1": 0.21337890625,
+  "gen_distinct_2": 0.5059055118110236,
+  "gen_top_token_mass": 0.108154296875,
+  "gen_samples_preview": [
+    "[CLS], he said. [SEP] [SEP] [CLS] the commission refused to take a preliminary hearing in october and issued a grudge. [SEP] [SEP] [CLS] the company ' s stock fell, or 4. 6 cents, or 4. 5 cents, or 2. 58, to $ 4. 95. [SEP] [SEP] [CLS] in contrast, rose $ 2. 50 to $ $ 20 a share. [SEP] [SEP] [CLS] the company, however, and alzheimer ' s, respectively, others, are the backbone of alzheimer ' s. [SEP] [SEP] [CLS] early investigators were investigating heath ledger ' s sudden death in the middle of pregnancy. [SEP] [SEP] [CLS] the second was a ' [SEP]",
+    "[CLS] attorneys, the former director of the state ' s investigative department. [SEP] [SEP] [CLS] a member of the uaw ' s a good lawyer. [SEP] [SEP] [CLS] demand, for example, assumptions, product, and otherties, projections and financial statements. subject to the forward - looking statements. [SEP] [SEP] [CLS] in the case, it ' s a virus, a virus, a new virus, and a ability to contact a doctor - - is likely to be brought. [SEP] [SEP] [CLS] \" in fact, it ' s not a good idea, \" he added, laughing. [SEP] [SEP] [CLS] the verdict is a desire to be entirely due to [SEP]",
+    "[CLS] the country ' s recovery. [SEP] [SEP] [CLS] the new budget is scheduled to be released on a jan. [SEP] [SEP] [CLS] \" the family of the people, politicians, the parties, \" he said. [SEP] [SEP] [CLS] that ' s edmonton, connecticut, connecticut, new jersey, new jersey, connecticut, $ 10. 99. [SEP] [SEP] [CLS] the new project is expected to be delayed for a two - hour period in the eastern part of the city, the company said. [SEP] [SEP] [CLS] the starter, waile, a former former draft receiver, was in 2007, giving a boost to clemens, contributing to a possible 10 - year [SEP]",
+    "[CLS] the nortel. [SEP] [SEP] [CLS] \" it is a great disease, the disease, and the impact, of course, the melting, \" he said. [SEP] [SEP] [CLS] the woman ' s body was found in the netherlands, located in connecticut, new york, connecticut, and denmark, in philadelphia, norway, and elsewhere on the university ' s corporate campus in the chicago area. [SEP] [SEP] [CLS] 18 to 17, he said. [SEP] [SEP] [CLS] i ' m going to have his head on that. [SEP] [SEP] [CLS] i am in the wild... [SEP] [SEP] [CLS] \" the economy, the economy, seems stronger in the award [SEP]",
+    "[CLS], and opposition activists in protest on friday. [SEP] [SEP] [CLS] that ' s not going to be a delegate to florida.... obama. [SEP] [SEP] [CLS] \" such policy is needed in the future, and the measures, the policies and policies, \" he said. [SEP] [SEP] [CLS] this is a victory for the future, \" he said. [SEP] [SEP] [CLS] it is not a tragedy story. [SEP] [SEP] [CLS] and the answer is,, and harder, it ' s better. [SEP] [SEP] [CLS] in general, it ' s not easy to predict. [SEP] [SEP] [CLS] but that ' s just 0. [SEP] [SEP] [CLS] in fact, [SEP]"
+  ]
+}
+=== DIRICHLET INIT C=4 K=128 S=32 ===
+{
+  "mask_ratio_0.10": {
+    "corrupt_tokens": 4,
+    "endpoint_loss": 3.001953125,
+    "endpoint_acc": 0.25,
+    "final_acc": 0.25,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "while athletes in instrument professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the confrontation of professional golf and rake in endorsements.",
+    "endpoint": "while athletes in the professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the world of professional golf and rake in endorsements.",
+    "final": "and while athletes in the sport dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the confrontation of professional golf and rake in endorsements."
+  },
+  "mask_ratio_0.20": {
+    "corrupt_tokens": 18,
+    "endpoint_loss": 2.8520872592926025,
+    "endpoint_acc": 0.3888888955116272,
+    "final_acc": 0.3333333432674408,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "upward athletes proves different professions dealt else doping scandals portugal other controversies, woods continued to do what he did eighty : dominate rover facebook of orchestrated golf and restriction in endorsements.",
+    "endpoint": "as athletes from different professions dealt with doping scandals and other controversies, woods continued to do what he did most : dominate the sponsorship of american golf and compete in endorsements.",
+    "final": "as athletes from different professions dealt with doping scandals and other controversies, woods continued to do what he did most : dominate the business of world golf and compete in endorsements."
+  },
+  "mask_ratio_0.50": {
+    "corrupt_tokens": 42,
+    "endpoint_loss": 5.020946502685547,
+    "endpoint_acc": 0.2380952388048172,
+    "final_acc": 0.1666666716337204,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "##ʃ athletes abbey different“ dealt withzekᆨrrediled controversies, programmeriring tanks speculated what he did best : dominate ' surname norway professionalsett caste helmgraph blend defend.",
+    "endpoint": "athletes from different disciplines dealt with the the of and controversies, and the to of what he did best : he ' s, professionalism and regraph f defend.",
+    "final": ""
+  },
+  "mask_ratio_1.00": {
+    "corrupt_tokens": 74,
+    "endpoint_loss": 7.154612064361572,
+    "endpoint_acc": 0.09459459781646729,
+    "final_acc": 0.027027027681469917,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "frs rejected bangkok lehigh presidential plaid him och arabic transforms engineer virus jerusalem drank cosmos festivals city fenderulum eleanorᆸ dowagerrb selena vendor jo stresses refrain barker firth secrecy designedwalker daytimeque designation mars",
+    "endpoint": "",
+    "final": "he rejected the request to succeed him."
+  },
+  "pure_noise": [
+    "europeans."
+  ],
+  "gen_ppl": 16.622931512600037,
+  "gen_nll_per_token": 2.8107831585063363,
+  "gen_tokens": 5997,
+  "gen_scored_samples": 32,
+  "gen_skipped_samples": 0,
+  "gen_empty_rate": 0.0,
+  "gen_kept_samples": 32,
+  "gen_total_samples": 32,
+  "gen_ppl_model": "/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard",
+  "gen_ppl_output": "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/runs/lta_lm1b_duo_aligned_dirichlet_dualt_onehot_hardce_b128_4xh20_1m_nw0_compile_ro/genppl_lm1b_latest_dirichletC4_k128_s32_flm.jsonl",
+  "gen_ppl_flm_formula": true,
+  "gen_ppl_full_decode": true,
+  "decode_solver": "flowmap",
+  "noise_init": "dirichlet",
+  "noise_sigma": -1.0,
+  "dirichlet_init_concentration": 4.0,
+  "gen_ppl_min_chars": 0,
+  "gen_ppl_normalize_whitespace": false,
+  "gen_ppl_drop_remainder": false,
+  "gen_sample_entropy": 3.5170808405450784,
+  "gen_unique_tokens": 859,
+  "gen_token_count": 4096,
+  "gen_distinct_1": 0.209716796875,
+  "gen_distinct_2": 0.49606299212598426,
+  "gen_top_token_mass": 0.13720703125,
+  "gen_samples_preview": [
+    "[CLS], stability, security and political stability, \" simm alfeevi, the lead foreign minister of the mdc, said. [SEP] [SEP] [CLS] the effect of the onset of malaria, for example, in this year, is to be felt in the u. s., mexico, canada, mexico and the netherlands. [SEP] [SEP] [CLS] in indonesia, pakistan, pakistan, pakistan, pakistan, iran, saudi arabia, nigeria, and pakistan, it ' s holy. [SEP] [SEP] [CLS] the man, meanwhile, is believed to be included in the legislation, and is not likely to appear. [SEP] [SEP] [CLS] the company ' s biggest [SEP]",
+    "[CLS]. [SEP] [SEP] [CLS] h. gen. saeed al zardari, the former minister of pakistan, and the minister, are linked to the former prime minister, the u. s. trade commissioner, and the head of the country ' s parliament. [SEP] [SEP] [CLS] the man is in the us, west germany, denmark and ireland, and is believed to be dead. [SEP] [SEP] [CLS] \" it ' s very tough, \" he said. [SEP] [SEP] [CLS] the company reached the company ' s finals in china, germany, and the netherlands in the united states. [SEP] [SEP] [CLS] the obama administration ' s comments on the possibility of [SEP]",
+    "[CLS] [SEP] [SEP] [CLS] romney comes in a state of massachusetts, and retires in a. n. [SEP] [SEP] [CLS] the government is also seeking to bring stability to its properties in the dollar, which is a measure of the u. s. dollar. [SEP] [SEP] [CLS] it is a kind of a good story. [SEP] [SEP] [CLS] the idea of the bill could be the approval of the u. s. u. s. telecommunications commissioner, including a. lloyd a. bruno, the u. s. secretary of state for the oversight of the union ' s audits, and the chairman of the legislation. [SEP] [SEP] [CLS] there has been [SEP]",
+    "[CLS] the olympics, and the importance of the u. s., the olympics, the ioc and the beijing olympics. [SEP] [SEP] [CLS] the us, japan, south korea, japan, japan, japan, japan and the interests of its eu partners, china, japan, japan, and japan, japan, south korea, india, south korea, north korea, japan, japan, japan, japan, japan, japan, and japan, japan, china, the company ' s subsidiaries, and russia, and the united republic of georgia,, according to reports. [SEP] [SEP] [CLS] on a battlefield, the u. s. government announced a [SEP]",
+    "[CLS] impact on our country, \" mr. mccain said. [SEP] [SEP] [CLS] he ' ll be counselor to the campaigns on the campaign, in florida and virginia. [SEP] [SEP] [CLS] the 26 - year - old was diagnosed in the current phase of the family ' s death, he said. [SEP] [SEP] [CLS] the republic of connecticut, new zealand, norway, norway, and norway, norway and norway. [SEP] [SEP] [CLS] the men, 36, of connecticut, arizona, connecticut, and massachusetts, are working on a four - year contract. [SEP] [SEP] [CLS] the account of twenty - two, microsoft ' s online advertising threatens to be $ 1, [SEP]"
+  ]
+}
+=== DIRICHLET INIT C=16 K=128 S=32 ===
+{
+  "mask_ratio_0.10": {
+    "corrupt_tokens": 6,
+    "endpoint_loss": 2.2595112323760986,
+    "endpoint_acc": 0.5,
+    "final_acc": 0.5,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "while athletes in different professions dealt with doping noticeably and other controversies, woods continued to do what he didlates : dominate the field of professional golf and rake in banes.",
+    "endpoint": "while athletes in different professions dealt with dopings and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in golds.",
+    "final": "and while athletes in different professions dealt with dopings and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in golds."
+  },
+  "mask_ratio_0.20": {
+    "corrupt_tokens": 19,
+    "endpoint_loss": 3.2713687419891357,
+    "endpoint_acc": 0.31578946113586426,
+    "final_acc": 0.2631579041481018,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "fabrics while athletes in different participating dealt with haute scandals and other controversiesک woods continued to do 1743 he consequences best : dominate the ever of professional golf and rake 1707 mlbs.",
+    "endpoint": ", while athletes in different countries dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the world of professional golf and rake in golfs.",
+    "final": ", while athletes in different sports dealt with the scandals and other controversies, woods continued to do what he did best : dominate the world of professional golf and rake in golfs."
+  },
+  "mask_ratio_0.50": {
+    "corrupt_tokens": 40,
+    "endpoint_loss": 4.8522138595581055,
+    "endpoint_acc": 0.20000000298023224,
+    "final_acc": 0.22499999403953552,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "post athletes dellised greeted dealt with clothed } and distributed controversies thereby eireann continued 2500 do what he did suggestioncrats dominate the fieldllin 弘 gustav and rake lambert revoked 1650 ineffective benjamin",
+    "endpoint": "the",
+    "final": "."
+  },
+  "mask_ratio_1.00": {
+    "corrupt_tokens": 74,
+    "endpoint_loss": 6.878285884857178,
+    "endpoint_acc": 0.12162162363529205,
+    "final_acc": 0.013513513840734959,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "##hop various chesterfield stray transactionvus convoys yates saharawala paralympics horticultural capcom bergen risking hesitantly crafts remind potsdam though biggest 車 approximate pediatricrer fitzgerald confederate il deposed luciferffie囗 earlsم clubhouse isle [unused245]",
+    "endpoint": "",
+    "final": "indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia"
+  },
+  "pure_noise": [
+    "indonesia, indonesia, indonesia, and costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, sri rica, costa rica, indonesia, indonesia, indonesia, indonesia, sri rica, costa rica, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia."
+  ],
+  "gen_ppl": 2.1056201858297925,
+  "gen_nll_per_token": 0.7446100488381823,
+  "gen_tokens": 7768,
+  "gen_scored_samples": 32,
+  "gen_skipped_samples": 0,
+  "gen_empty_rate": 0.0,
+  "gen_kept_samples": 32,
+  "gen_total_samples": 32,
+  "gen_ppl_model": "/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard",
+  "gen_ppl_output": "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/runs/lta_lm1b_duo_aligned_dirichlet_dualt_onehot_hardce_b128_4xh20_1m_nw0_compile_ro/genppl_lm1b_latest_dirichletC16_k128_s32_flm.jsonl",
+  "gen_ppl_flm_formula": true,
+  "gen_ppl_full_decode": true,
+  "decode_solver": "flowmap",
+  "noise_init": "dirichlet",
+  "noise_sigma": -1.0,
+  "dirichlet_init_concentration": 16.0,
+  "gen_ppl_min_chars": 0,
+  "gen_ppl_normalize_whitespace": false,
+  "gen_ppl_drop_remainder": false,
+  "gen_sample_entropy": 1.4798635938254432,
+  "gen_unique_tokens": 72,
+  "gen_token_count": 4096,
+  "gen_distinct_1": 0.017578125,
+  "gen_distinct_2": 0.03740157480314961,
+  "gen_top_token_mass": 0.424560546875,
+  "gen_samples_preview": [
+    "[CLS] indonesia, indonesia, indonesia, indonesia, indonesia, indonesia ( south korea ), costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, sri lanka, indonesia, costa rica, indonesia, sri lanka, indonesia, sri lanka, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, sri lanka, sri lanka, sri lanka, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, sri lanka, indonesia, sri lanka, indonesia, indonesia, indonesia, sri lanka, indonesia, indonesia, [SEP]",
+    "indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, and costa rica, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, costa rica, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, costa rica, indonesia, costa rica, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia,",
+    "[CLS] rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, china, indonesia, indonesia, costa rica, costa rica, indonesia, indonesia, indonesia, costa rica, costa rica, indonesia, indonesia, and indonesia, indonesia, indonesia, equatorial indonesia, indonesia, indonesia, costa rica, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia [SEP]",
+    "[CLS] indonesia, costa rica, indonesia, costa rica, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, costa rica, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, [SEP]",
+    "[CLS] rica, indonesia, costa rica, indonesia, indonesia, taiwan, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, costa rica, indonesia, indonesia, indonesia, indonesia and the women ' s republic, and indonesia. [SEP] [SEP] [CLS] indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, costa rica, indonesia, costa rica, indonesia, indonesia, indonesia, costa rica, costa rica, costa rica, indonesia [SEP]"
+  ]
+}
+=== DIRICHLET INIT C=64 K=128 S=32 ===
+{
+  "mask_ratio_0.10": {
+    "corrupt_tokens": 7,
+    "endpoint_loss": 2.3433337211608887,
+    "endpoint_acc": 0.7142857313156128,
+    "final_acc": 0.7142857313156128,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "while athletes in different professions dealt with commit scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake ineens.",
+    "endpoint": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in golfs.",
+    "final": "and while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in golfs."
+  },
+  "mask_ratio_0.20": {
+    "corrupt_tokens": 11,
+    "endpoint_loss": 2.6198785305023193,
+    "endpoint_acc": 0.4545454680919647,
+    "final_acc": 0.5454545617103577,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "while athletes in different professions dealt montane doping scandals ticking other controversies, woods continued to do what he did addiction : mistakenly the field of crying [unused987] and rake in endorsements.",
+    "endpoint": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : in the field of thes and rake in endorsements.",
+    "final": "and while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : divide the field of positives and rake in endorsements."
+  },
+  "mask_ratio_0.50": {
+    "corrupt_tokens": 38,
+    "endpoint_loss": 5.730685710906982,
+    "endpoint_acc": 0.18421052396297455,
+    "final_acc": 0.21052631735801697,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "accompany masonry athletes layla different professions incorrectly with doping scandals theaters lincolnshire controversies, woods museums to appearing oval he didger 1658 her thenailrization accessories golfvating rake in endorsementsович",
+    "endpoint": "and and athletes in different professions, with doping scandals and the controversies, woods sought to the that he did not and for the widely - of golf and rake in endorsements.",
+    "final": "criticized by athletes in different professions, with doping scandals and other controversies, woods sought to acknowledge that he did not agree on the much - criticized golf course or in endorsements."
+  },
+  "mask_ratio_1.00": {
+    "corrupt_tokens": 74,
+    "endpoint_loss": 7.769108772277832,
+    "endpoint_acc": 0.027027027681469917,
+    "final_acc": 0.013513513840734959,
+    "target": "while athletes in different professions dealt with doping scandals and other controversies, woods continued to do what he did best : dominate the field of professional golf and rake in endorsements.",
+    "init": "barbados monkeys dellalham eco mock defiant jena preston marker hurrying blossoms [unused810]lio expectation transient linemmeresa [unused27] amnesia 下 [unused388] ャ 32nd | woodland mix pendleton 125 paste sts screwing addicted ministerial bbjing",
+    "endpoint": "and and and and and and and and and and and and and and and and and, and and and and and and and and and and and and and and and and and and and",
+    "final": ", indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, costa rica, indonesia, indonesia, costa rica, indonesia, indonesia, indonesia, sri rica, indonesia, indonesia,"
+  },
+  "pure_noise": [
+    "indonesia, indonesia, indonesia, indonesia, indonesia, and indonesia, indonesia, and indonesia, and indonesia, indonesia, indonesia, indonesia, indonesia, and sri rica, indonesia, indonesia, indonesia, sri lanka, indonesia, and indonesia, indonesia, sri lanka, indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, sri lanka, and indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, sri lanka, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, i"
+  ],
+  "gen_ppl": 1.7528559673432922,
+  "gen_nll_per_token": 0.5612464390399833,
+  "gen_tokens": 7772,
+  "gen_scored_samples": 32,
+  "gen_skipped_samples": 0,
+  "gen_empty_rate": 0.0,
+  "gen_kept_samples": 32,
+  "gen_total_samples": 32,
+  "gen_ppl_model": "/e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard",
+  "gen_ppl_output": "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/runs/lta_lm1b_duo_aligned_dirichlet_dualt_onehot_hardce_b128_4xh20_1m_nw0_compile_ro/genppl_lm1b_latest_dirichletC64_k128_s32_flm.jsonl",
+  "gen_ppl_flm_formula": true,
+  "gen_ppl_full_decode": true,
+  "decode_solver": "flowmap",
+  "noise_init": "dirichlet",
+  "noise_sigma": -1.0,
+  "dirichlet_init_concentration": 64.0,
+  "gen_ppl_min_chars": 0,
+  "gen_ppl_normalize_whitespace": false,
+  "gen_ppl_drop_remainder": false,
+  "gen_sample_entropy": 1.2600845033217614,
+  "gen_unique_tokens": 13,
+  "gen_token_count": 4096,
+  "gen_distinct_1": 0.003173828125,
+  "gen_distinct_2": 0.006151574803149607,
+  "gen_top_token_mass": 0.44287109375,
+  "gen_samples_preview": [
+    "[CLS], indonesia, indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, sri lanka, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, south korea, indonesia, sri lanka, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, sri lanka, indonesia, indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia, sri lanka, indonesia, and indonesia, sri lanka, indonesia, and indonesia, and indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, sri lanka, indonesia, indonesia, indonesia, [SEP]",
+    "[CLS], indonesia, indonesia, indonesia, indonesia, indonesia, sri lanka, south rica, and indonesia, indonesia, indonesia, sri lanka, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia, indonesia, indonesia, sri rica, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, equatorial indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, and indonesia, and indonesia, [SEP]",
+    ", indonesia, indonesia, sri lanka, indonesia, and indonesia, indonesia, and indonesia, and indonesia, indonesia, indonesia, indonesia, sri lanka, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, sri lanka, sri lanka, costa rica, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia, sri lanka, sri lanka, indonesia, indonesia, indonesia, costa rica, indonesia, indonesia, sri lanka, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia,",
+    "indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, sri lanka, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia, indonesia, sri lanka, sri lanka, indonesia, sri lanka, indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, and indonesia, indonesia, and indonesia, indonesia, and indonesia, indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia, indonesia,",
+    "[CLS] indonesia, sri lanka, indonesia, indonesia, indonesia, and indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, and indonesia, and indonesia, indonesia, and indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, and indonesia, sri lanka, indonesia, and indonesia, indonesia, sri lanka, sri lanka, sri lanka, indonesia, sri lanka, and indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, sri lanka, indonesia, indonesia, indonesia, indonesia, indonesia, indonesia, [SEP]"
+  ]
+}

LTA_openwebtext_dualt/logs/lm1b_dirichlet_len1024_Cv_to_2v_gumbel_sde_watch/infer_lta_lm1b_dirichlet_len1024_Cv_to_2v_gbs512_4gpu_20k_save1k_gumbelwatch_20260524_step_0007000.log

@@ -0,0 +1,132 @@
+[watch-gumbel] 2026-05-24_07:10:26 infer runs/lta_lm1b_dirichlet_len1024_Cv_to_2v_gbs512_4gpu_20k_save1k_gumbelwatch_20260524/step_0007000.pt -> docs/lta_samples/metrics_20260524/lm1b_dirichlet_len1024_Cv_to_2v_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_lm1b_dirichlet_len1024_Cv_to_2v_gbs512_4gpu_20k_save1k_gumbelwatch_20260524/step_0007000
+[load] runs/lta_lm1b_dirichlet_len1024_Cv_to_2v_gbs512_4gpu_20k_save1k_gumbelwatch_20260524/step_0007000.pt
+[ckpt] step=7000
+[sde] generated 2/128
+[sde] generated 4/128
+[sde] generated 6/128
+[sde] generated 8/128
+[sde] generated 10/128
+[sde] generated 12/128
+[sde] generated 14/128
+[sde] generated 16/128
+[sde] generated 18/128
+[sde] generated 20/128
+[sde] generated 22/128
+[sde] generated 24/128
+[sde] generated 26/128
+[sde] generated 28/128
+[sde] generated 30/128
+[sde] generated 32/128
+[sde] generated 34/128
+[sde] generated 36/128
+[sde] generated 38/128
+[sde] generated 40/128
+[sde] generated 42/128
+[sde] generated 44/128
+[sde] generated 46/128
+[sde] generated 48/128
+[sde] generated 50/128
+[sde] generated 52/128
+[sde] generated 54/128
+[sde] generated 56/128
+[sde] generated 58/128
+[sde] generated 60/128
+[sde] generated 62/128
+[sde] generated 64/128
+[sde] generated 66/128
+[sde] generated 68/128
+[sde] generated 70/128
+[sde] generated 72/128
+[sde] generated 74/128
+[sde] generated 76/128
+[sde] generated 78/128
+[sde] generated 80/128
+[sde] generated 82/128
+[sde] generated 84/128
+[sde] generated 86/128
+[sde] generated 88/128
+[sde] generated 90/128
+[sde] generated 92/128
+[sde] generated 94/128
+[sde] generated 96/128
+[sde] generated 98/128
+[sde] generated 100/128
+[sde] generated 102/128
+[sde] generated 104/128
+[sde] generated 106/128
+[sde] generated 108/128
+[sde] generated 110/128
+[sde] generated 112/128
+[sde] generated 114/128
+[sde] generated 116/128
+[sde] generated 118/128
+[sde] generated 120/128
+[sde] generated 122/128
+[sde] generated 124/128
+[sde] generated 126/128
+[sde] generated 128/128
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_dirichlet_len1024_Cv_to_2v_gbs512_4gpu_20k_save1k_gumbelwatch_20260524/step_0007000.pt",
+  "step": 7000,
+  "decode": {
+    "decode_rule": "dirichlet_resample_sde",
+    "steps": 128,
+    "model_t_mode": "support_t",
+    "mean_mode": "endpoint_only",
+    "anchor_gamma": 1.0,
+    "endpoint_floor": 0.0,
+    "concentration_min": 30522.0,
+    "concentration_max": 61044.0,
+    "endpoint_temp": 1.45,
+    "endpoint_projection": "gumbel_softmax",
+    "endpoint_top_k": 0,
+    "endpoint_top_p": 0.95,
+    "gumbel_tau_start": 1.0,
+    "gumbel_tau_end": 0.2,
+    "ban_special_tokens": false,
+    "banned_endpoint_ids": [],
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "dirichlet",
+    "noise_sigma": -1.0,
+    "noise_dirichlet_concentration": 30522.0,
+    "sde_resample": "dirichlet",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 128,
+    "seed": 20260524
+  },
+  "raw_genppl": {
+    "ppl": 28.963689020402576,
+    "nll_per_token": 3.366042942706146,
+    "tokens": 130939,
+    "kept_samples": 128,
+    "total_samples": 128,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 30.354979241180427,
+    "nll_per_token": 3.4129605647139516,
+    "tokens": 130924,
+    "kept_samples": 128,
+    "total_samples": 128,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.7047035738409604,
+    "unique_tokens": 2735,
+    "token_count": 131072,
+    "distinct_1": 0.02086639404296875,
+    "distinct_2": 0.1436950146627566,
+    "top_token_mass": 0.15491485595703125
+  }
+}
+[done] docs/lta_samples/metrics_20260524/lm1b_dirichlet_len1024_Cv_to_2v_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_lm1b_dirichlet_len1024_Cv_to_2v_gbs512_4gpu_20k_save1k_gumbelwatch_20260524/step_0007000/sde_steps128_samples128_scored.jsonl
+[watch-gumbel] 2026-05-24_07:21:53 done step_0007000

LTA_openwebtext_dualt/logs/rollin_focused_4gpu/20260517_1733focused.log

@@ -0,0 +1,823 @@
+[rollin-focused] start stamp=20260517_1733focused len=256 vocab=969 out=docs/lta_samples/metrics_20260517/rollin_focused_len256_bs512_ode128_20260517_1733focused
+[rollin-focused] round=1 Sun May 17 17:48:07 UTC 2026
+[rollin-focused] train config=rollin_p50_s4_i32 from=0 to=500 rollout=0.50/s4/i32/temp1.45
+[rollin-focused] eval config=rollin_p50_s4_i32 step=500
+[eval-decode-acc] train8_rollin_focused_len256_rollin_p50_s4_i32_20260517_1733focused step=500 soft=none
+[decode] max_len=256 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_rollin_focused_len256_rollin_p50_s4_i32_20260517_1733focused::none": {
+      "run": "train8_rollin_focused_len256_rollin_p50_s4_i32_20260517_1733focused",
+      "checkpoint": "runs/train8_rollin_focused_len256_rollin_p50_s4_i32_20260517_1733focused/step_0000500.pt",
+      "ckpt_step": 500,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.04901123046875,
+      "token_acc_min": 0.02734375,
+      "token_acc_max": 0.09375,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        2,
+        5,
+        3,
+        3,
+        3,
+        3,
+        2,
+        2,
+        2,
+        6,
+        3,
+        0,
+        0,
+        3,
+        2,
+        2,
+        3,
+        3,
+        2,
+        6,
+        7,
+        7,
+        3,
+        2,
+        2,
+        0,
+        2,
+        0,
+        3,
+        0,
+        3,
+        3,
+        7,
+        3,
+        7,
+        0,
+        7,
+        3,
+        0,
+        0,
+        3,
+        6,
+        1,
+        7,
+        1,
+        3,
+        7,
+        2,
+        3,
+        3,
+        2,
+        3,
+        2,
+        1,
+        7,
+        1,
+        3,
+        3,
+        0,
+        1,
+        3,
+        7,
+        0,
+        0
+      ],
+      "best_token_acc": [
+        0.0390625,
+        0.046875,
+        0.0390625,
+        0.07421875,
+        0.0546875,
+        0.05859375,
+        0.0390625,
+        0.046875,
+        0.02734375,
+        0.02734375,
+        0.0546875,
+        0.05859375,
+        0.03515625,
+        0.0625,
+        0.05078125,
+        0.0625,
+        0.04296875,
+        0.0625,
+        0.0390625,
+        0.0390625,
+        0.09375,
+        0.046875,
+        0.04296875,
+        0.0546875,
+        0.05078125,
+        0.0546875,
+        0.03125,
+        0.03515625,
+        0.046875,
+        0.04296875,
+        0.0546875,
+        0.04296875,
+        0.07421875,
+        0.04296875,
+        0.06640625,
+        0.03515625,
+        0.05078125,
+        0.04296875,
+        0.0625,
+        0.05078125,
+        0.0546875,
+        0.05078125,
+        0.0546875,
+        0.05859375,
+        0.05078125,
+        0.046875,
+        0.05859375,
+        0.03125,
+        0.046875,
+        0.0390625,
+        0.046875,
+        0.05078125,
+        0.02734375,
+        0.0546875,
+        0.0546875,
+        0.0390625,
+        0.0546875,
+        0.04296875,
+        0.05078125,
+        0.04296875,
+        0.0546875,
+        0.046875,
+        0.0546875,
+        0.0390625
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=rollin_p50_s4_i32 ckpt_step=500 views=256000 token_acc=0.0490 exact=0/64 exact_refs=0 hits=[]
+[rollin-focused] train config=rollin_p75_s4_i32 from=0 to=500 rollout=0.75/s4/i32/temp1.45
+[rollin-focused] eval config=rollin_p75_s4_i32 step=500
+[eval-decode-acc] train8_rollin_focused_len256_rollin_p75_s4_i32_20260517_1733focused step=500 soft=none
+[decode] max_len=256 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_rollin_focused_len256_rollin_p75_s4_i32_20260517_1733focused::none": {
+      "run": "train8_rollin_focused_len256_rollin_p75_s4_i32_20260517_1733focused",
+      "checkpoint": "runs/train8_rollin_focused_len256_rollin_p75_s4_i32_20260517_1733focused/step_0000500.pt",
+      "ckpt_step": 500,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.04876708984375,
+      "token_acc_min": 0.0234375,
+      "token_acc_max": 0.08203125,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        5,
+        5,
+        0,
+        3,
+        3,
+        3,
+        3,
+        7,
+        0,
+        3,
+        0,
+        0,
+        0,
+        0,
+        7,
+        2,
+        3,
+        3,
+        7,
+        6,
+        7,
+        7,
+        7,
+        2,
+        2,
+        0,
+        1,
+        0,
+        7,
+        7,
+        3,
+        5,
+        7,
+        3,
+        7,
+        7,
+        7,
+        4,
+        4,
+        0,
+        3,
+        6,
+        7,
+        7,
+        7,
+        3,
+        7,
+        2,
+        5,
+        3,
+        3,
+        1,
+        3,
+        7,
+        7,
+        3,
+        5,
+        4,
+        0,
+        0,
+        3,
+        2,
+        0,
+        7
+      ],
+      "best_token_acc": [
+        0.05078125,
+        0.046875,
+        0.05078125,
+        0.07421875,
+        0.0390625,
+        0.046875,
+        0.046875,
+        0.0390625,
+        0.04296875,
+        0.03125,
+        0.0546875,
+        0.06640625,
+        0.03125,
+        0.0390625,
+        0.0546875,
+        0.05859375,
+        0.05078125,
+        0.0546875,
+        0.03515625,
+        0.0546875,
+        0.08203125,
+        0.046875,
+        0.046875,
+        0.0625,
+        0.04296875,
+        0.046875,
+        0.03515625,
+        0.0234375,
+        0.03515625,
+        0.0625,
+        0.0390625,
+        0.04296875,
+        0.08203125,
+        0.046875,
+        0.05859375,
+        0.04296875,
+        0.04296875,
+        0.046875,
+        0.0390625,
+        0.0390625,
+        0.0546875,
+        0.03515625,
+        0.0546875,
+        0.05859375,
+        0.05859375,
+        0.046875,
+        0.0625,
+        0.0390625,
+        0.05859375,
+        0.05859375,
+        0.05078125,
+        0.05859375,
+        0.02734375,
+        0.04296875,
+        0.0390625,
+        0.05859375,
+        0.0390625,
+        0.05078125,
+        0.0546875,
+        0.046875,
+        0.0625,
+        0.02734375,
+        0.046875,
+        0.0546875
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=rollin_p75_s4_i32 ckpt_step=500 views=256000 token_acc=0.0488 exact=0/64 exact_refs=0 hits=[]
+[rollin-focused] train config=rollin_p100_s4_i32 from=0 to=500 rollout=1.00/s4/i32/temp1.45
+[rollin-focused] eval config=rollin_p100_s4_i32 step=500
+[eval-decode-acc] train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused step=500 soft=none
+[decode] max_len=256 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused::none": {
+      "run": "train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused",
+      "checkpoint": "runs/train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused/step_0000500.pt",
+      "ckpt_step": 500,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.0469970703125,
+      "token_acc_min": 0.02734375,
+      "token_acc_max": 0.078125,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        5,
+        5,
+        0,
+        3,
+        0,
+        3,
+        0,
+        7,
+        3,
+        0,
+        3,
+        5,
+        0,
+        3,
+        2,
+        2,
+        3,
+        3,
+        7,
+        3,
+        7,
+        7,
+        2,
+        2,
+        2,
+        3,
+        7,
+        0,
+        2,
+        3,
+        3,
+        3,
+        7,
+        2,
+        7,
+        3,
+        7,
+        3,
+        0,
+        3,
+        3,
+        5,
+        2,
+        0,
+        5,
+        2,
+        7,
+        2,
+        3,
+        3,
+        3,
+        3,
+        7,
+        3,
+        7,
+        3,
+        3,
+        3,
+        2,
+        3,
+        3,
+        7,
+        0,
+        0
+      ],
+      "best_token_acc": [
+        0.046875,
+        0.0546875,
+        0.02734375,
+        0.04296875,
+        0.05859375,
+        0.05078125,
+        0.03125,
+        0.046875,
+        0.05859375,
+        0.03125,
+        0.05078125,
+        0.03125,
+        0.046875,
+        0.0625,
+        0.0625,
+        0.046875,
+        0.0390625,
+        0.0390625,
+        0.046875,
+        0.04296875,
+        0.078125,
+        0.02734375,
+        0.03125,
+        0.03125,
+        0.078125,
+        0.0703125,
+        0.02734375,
+        0.0390625,
+        0.0546875,
+        0.03125,
+        0.03515625,
+        0.04296875,
+        0.078125,
+        0.05078125,
+        0.05078125,
+        0.03515625,
+        0.04296875,
+        0.03515625,
+        0.03515625,
+        0.0703125,
+        0.07421875,
+        0.06640625,
+        0.03515625,
+        0.046875,
+        0.0390625,
+        0.03125,
+        0.0703125,
+        0.0625,
+        0.0546875,
+        0.0625,
+        0.0390625,
+        0.0546875,
+        0.03515625,
+        0.04296875,
+        0.04296875,
+        0.0546875,
+        0.03515625,
+        0.03125,
+        0.02734375,
+        0.046875,
+        0.03125,
+        0.06640625,
+        0.0546875,
+        0.0390625
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=rollin_p100_s4_i32 ckpt_step=500 views=256000 token_acc=0.0470 exact=0/64 exact_refs=0 hits=[]
+[rollin-focused] train config=rollin_p50_s8_i64 from=0 to=500 rollout=0.50/s8/i64/temp1.45
+[rollin-focused] eval config=rollin_p50_s8_i64 step=500
+[eval-decode-acc] train8_rollin_focused_len256_rollin_p50_s8_i64_20260517_1733focused step=500 soft=none
+[decode] max_len=256 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_rollin_focused_len256_rollin_p50_s8_i64_20260517_1733focused::none": {
+      "run": "train8_rollin_focused_len256_rollin_p50_s8_i64_20260517_1733focused",
+      "checkpoint": "runs/train8_rollin_focused_len256_rollin_p50_s8_i64_20260517_1733focused/step_0000500.pt",
+      "ckpt_step": 500,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.04052734375,
+      "token_acc_min": 0.015625,
+      "token_acc_max": 0.078125,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        3,
+        5,
+        0,
+        3,
+        0,
+        7,
+        3,
+        1,
+        3,
+        6,
+        3,
+        0,
+        0,
+        3,
+        2,
+        2,
+        3,
+        3,
+        2,
+        6,
+        7,
+        2,
+        3,
+        0,
+        2,
+        3,
+        1,
+        5,
+        3,
+        0,
+        3,
+        3,
+        7,
+        0,
+        7,
+        7,
+        1,
+        3,
+        4,
+        3,
+        3,
+        6,
+        1,
+        7,
+        1,
+        2,
+        7,
+        3,
+        5,
+        3,
+        3,
+        1,
+        1,
+        1,
+        7,
+        3,
+        5,
+        3,
+        0,
+        2,
+        7,
+        7,
+        0,
+        0
+      ],
+      "best_token_acc": [
+        0.02734375,
+        0.046875,
+        0.0625,
+        0.05078125,
+        0.03125,
+        0.0390625,
+        0.03515625,
+        0.015625,
+        0.03515625,
+        0.02734375,
+        0.05078125,
+        0.0390625,
+        0.03515625,
+        0.04296875,
+        0.02734375,
+        0.04296875,
+        0.0390625,
+        0.04296875,
+        0.03125,
+        0.0234375,
+        0.078125,
+        0.0234375,
+        0.03125,
+        0.02734375,
+        0.01953125,
+        0.0546875,
+        0.02734375,
+        0.02734375,
+        0.02734375,
+        0.03515625,
+        0.05859375,
+        0.03125,
+        0.04296875,
+        0.03125,
+        0.06640625,
+        0.04296875,
+        0.03515625,
+        0.0625,
+        0.04296875,
+        0.05078125,
+        0.05078125,
+        0.03125,
+        0.046875,
+        0.046875,
+        0.0390625,
+        0.04296875,
+        0.0390625,
+        0.02734375,
+        0.04296875,
+        0.05859375,
+        0.05859375,
+        0.05078125,
+        0.03515625,
+        0.04296875,
+        0.046875,
+        0.05078125,
+        0.05078125,
+        0.03515625,
+        0.046875,
+        0.04296875,
+        0.046875,
+        0.03125,
+        0.046875,
+        0.01953125
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=rollin_p50_s8_i64 ckpt_step=500 views=256000 token_acc=0.0405 exact=0/64 exact_refs=0 hits=[]
+[rollin-focused] train config=rollin_p75_s8_i64 from=0 to=500 rollout=0.75/s8/i64/temp1.45
+[rollin-focused] eval config=rollin_p75_s8_i64 step=500
+[eval-decode-acc] train8_rollin_focused_len256_rollin_p75_s8_i64_20260517_1733focused step=500 soft=none
+[decode] max_len=256 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_rollin_focused_len256_rollin_p75_s8_i64_20260517_1733focused::none": {
+      "run": "train8_rollin_focused_len256_rollin_p75_s8_i64_20260517_1733focused",
+      "checkpoint": "runs/train8_rollin_focused_len256_rollin_p75_s8_i64_20260517_1733focused/step_0000500.pt",
+      "ckpt_step": 500,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.05255126953125,
+      "token_acc_min": 0.01953125,
+      "token_acc_max": 0.08984375,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        5,
+        5,
+        0,
+        3,
+        0,
+        2,
+        0,
+        7,
+        0,
+        3,
+        0,
+        0,
+        0,
+        0,
+        5,
+        2,
+        0,
+        0,
+        5,
+        7,
+        7,
+        7,
+        7,
+        2,
+        2,
+        0,
+        5,
+        0,
+        1,
+        0,
+        3,
+        0,
+        7,
+        0,
+        7,
+        5,
+        7,
+        0,
+        0,
+        3,
+        3,
+        2,
+        4,
+        0,
+        0,
+        3,
+        7,
+        5,
+        5,
+        7,
+        3,
+        7,
+        0,
+        7,
+        7,
+        3,
+        5,
+        4,
+        0,
+        0,
+        7,
+        7,
+        0,
+        0
+      ],
+      "best_token_acc": [
+        0.06640625,
+        0.046875,
+        0.05859375,
+        0.0703125,
+        0.046875,
+        0.046875,
+        0.04296875,
+        0.05078125,
+        0.04296875,
+        0.03515625,
+        0.05078125,
+        0.0546875,
+        0.02734375,
+        0.04296875,
+        0.05078125,
+        0.06640625,
+        0.046875,
+        0.046875,
+        0.0390625,
+        0.05078125,
+        0.08984375,
+        0.0625,
+        0.05078125,
+        0.0390625,
+        0.05078125,
+        0.0703125,
+        0.04296875,
+        0.046875,
+        0.01953125,
+        0.05859375,
+        0.06640625,
+        0.04296875,
+        0.0703125,
+        0.0625,
+        0.04296875,
+        0.03515625,
+        0.0625,
+        0.05078125,
+        0.05078125,
+        0.0546875,
+        0.0859375,
+        0.03515625,
+        0.03125,
+        0.07421875,
+        0.05859375,
+        0.05078125,
+        0.08984375,
+        0.05078125,
+        0.05859375,
+        0.05078125,
+        0.0390625,
+        0.04296875,
+        0.03515625,
+        0.05859375,
+        0.04296875,
+        0.0546875,
+        0.05078125,
+        0.05078125,
+        0.05859375,
+        0.05078125,
+        0.05078125,
+        0.04296875,
+        0.05859375,
+        0.0859375
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=rollin_p75_s8_i64 ckpt_step=500 views=256000 token_acc=0.0526 exact=0/64 exact_refs=0 hits=[]
+[rollin-focused] train config=rollin_p50_s4_i32_temp1p0 from=0 to=500 rollout=0.50/s4/i32/temp1.0

LTA_openwebtext_dualt/logs/rollin_focused_4gpu/current.nohup

@@ -0,0 +1,2095 @@
+[rollin-focused] start stamp=20260517_1733focused len=256 vocab=969 out=docs/lta_samples/metrics_20260517/rollin_focused_len256_bs512_ode128_20260517_1733focused
+[rollin-focused] round=1 Sun May 17 17:48:07 UTC 2026
+[rollin-focused] train config=rollin_p50_s4_i32 from=0 to=500 rollout=0.50/s4/i32/temp1.45
+[launch] gpt2 cached OWT soft-endpoint m/n pilot
+[launch] run_name=train8_rollin_focused_len256_rollin_p50_s4_i32_20260517_1733focused
+[launch] save_dir=runs/train8_rollin_focused_len256_rollin_p50_s4_i32_20260517_1733focused
+[launch] n=256 m=0 clean_state_mode=onehot
+[launch] mask_mixture lowk=0.0 all=1.0
+[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=969
+[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1
+[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0
+[launch] mask_ratio=1.0->1.0
+[launch] mask_ratio_floor_schedule=none
+[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet
+[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids=
+[launch] rollout_train prob=0.50 steps=4 infer_steps=32 temp=1.45 corrupt_only=1 samplewise=1
+[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit exact_repeat_per_chunk=64
+NCCL version 2.25.1+cuda12.8
+{
+  "device": "cuda:0",
+  "rank": 0,
+  "world_size": 4,
+  "samples": "owt_cached_chunks:8",
+  "vocab_size": 969,
+  "tokenizer_vocab_size": 50257,
+  "save_dir": "runs/train8_rollin_focused_len256_rollin_p50_s4_i32_20260517_1733focused",
+  "batch_size": 128,
+  "grad_accum": 1,
+  "effective_batch_size": 512,
+  "global_batch_size": 512,
+  "lr_schedule": "constant_warmup",
+  "optimizer": "muon",
+  "epochs": 0.0,
+  "steps_per_epoch": 1,
+  "total_steps": 500,
+  "warmup_steps": 10,
+  "warmup_epochs": -1.0,
+  "min_lr": 0.0,
+  "weight_decay": 0.1,
+  "output_weight_decay": -1.0,
+  "adamw_param_groups": "nanogpt",
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_eps": 1e-08,
+  "muon_impl": "legacy",
+  "muon_momentum": 0.95,
+  "muon_ns_steps": 5,
+  "muon_update_scale": 1.0,
+  "muon_nesterov": false,
+  "muon_width_scale": false,
+  "muon_grouping": "legacy_dim_ge_2",
+  "muon_param_count": 1965440,
+  "muon_adam_param_count": 8192,
+  "muon_param_names": [
+    "vocab_embed.embedding",
+    "sigma_map.net.0.weight",
+    "sigma_map.net.2.weight",
+    "blocks.0.attn_qkv.weight",
+    "blocks.0.attn_out.weight",
+    "blocks.0.mlp.0.weight",
+    "blocks.0.mlp.2.weight",
+    "blocks.0.adaLN_modulation.weight",
+    "blocks.1.attn_qkv.weight",
+    "blocks.1.attn_out.weight",
+    "blocks.1.mlp.0.weight",
+    "blocks.1.mlp.2.weight",
+    "blocks.1.adaLN_modulation.weight",
+    "blocks.2.attn_qkv.weight",
+    "blocks.2.attn_out.weight",
+    "blocks.2.mlp.0.weight",
+    "blocks.2.mlp.2.weight",
+    "blocks.2.adaLN_modulation.weight",
+    "output_layer.linear.weight",
+    "output_layer.adaLN_modulation.weight"
+  ],
+  "muon_adam_param_names": [
+    "sigma_map.net.0.bias",
+    "sigma_map.net.2.bias",
+    "blocks.0.norm1.weight",
+    "blocks.0.norm2.weight",
+    "blocks.0.mlp.0.bias",
+    "blocks.0.mlp.2.bias",
+    "blocks.0.adaLN_modulation.bias",
+    "blocks.1.norm1.weight",
+    "blocks.1.norm2.weight",
+    "blocks.1.mlp.0.bias",
+    "blocks.1.mlp.2.bias",
+    "blocks.1.adaLN_modulation.bias",
+    "blocks.2.norm1.weight",
+    "blocks.2.norm2.weight",
+    "blocks.2.mlp.0.bias",
+    "blocks.2.mlp.2.bias",
+    "blocks.2.adaLN_modulation.bias",
+    "output_layer.norm_final.weight",
+    "output_layer.adaLN_modulation.bias"
+  ],
+  "muon_effective_nesterov": false,
+  "muon_effective_width_scale": false,
+  "muon_effective_weight_decay": 0.1,
+  "muon_adam_fallback_nesterov": false,
+  "muon_adam_fallback_weight_decay": 0.1,
+  "ema_decay": 0.9999,
+  "ema_start_step": 0,
+  "model_type": "ddit",
+  "ddit_mlp_type": "gelu",
+  "elf_num_time_tokens": 4,
+  "elf_num_model_mode_tokens": 0,
+  "qk_norm": true,
+  "output_bias": false,
+  "output_init_std": -1.0,
+  "norm_type": "rmsnorm",
+  "target_loss": "hard_ce",
+  "linear_soft_target_power": 1.0,
+  "linear_soft_target_min_conf": 0.0,
+  "linear_soft_target_max_conf": 1.0,
+  "t_sampling_mode": "logit_normal",
+  "t_sampling_power": 1.0,
+  "t_sampling_eps": 0.0001,
+  "t_sampling_logit_mean": -1.5,
+  "t_sampling_logit_std": 0.8,
+  "dual_t": true,
+  "corrupt_t_mode": "same",
+  "corrupt_min_t": 0.0,
+  "corrupt_max_t": 1.0,
+  "prefix_block_prob": 0.0,
+  "prefix_block_len": 128,
+  "mask_ratio_floor_schedule": "none",
+  "dirichlet_endpoint_mode": "categorical_dual_t",
+  "dirichlet_semantic_t_mode": "same",
+  "dirichlet_semantic_t_value": 0.0,
+  "dirichlet_semantic_t_curve": "linear",
+  "dirichlet_semantic_t_power": 1.0,
+  "endpoint_sequence_random_prob_alpha": 0.0,
+  "categorical_wrong_from_full_vocab": true,
+  "categorical_wrong_from_batch_valid_tokens": false,
+  "categorical_wrong_basin_token_ids": "",
+  "categorical_wrong_basin_prob": 0.0,
+  "categorical_wrong_unigram_prob": 0.0,
+  "categorical_wrong_uniform_prob": 0.0,
+  "categorical_wrong_prob_floor": 0.0,
+  "categorical_wrong_corpus_unigram_path": "",
+  "categorical_wrong_corpus_unigram_alpha": 1.0,
+  "categorical_wrong_basin_shared_prob": 0.0,
+  "categorical_wrong_unigram_shared_prob": 0.0,
+  "mask_mixture_original_prob": 0.0,
+  "mask_mixture_lowk_prob": 0.0,
+  "mask_mixture_lowcorrupt_prob": 0.0,
+  "mask_mixture_block_prob": 0.0,
+  "mask_mixture_all_prob": 1.0,
+  "mask_mixture_lowk_clean_tokens": "0",
+  "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_block_tokens": "64,128",
+  "simplex_bridge_sampler": "dirichlet",
+  "logistic_normal_sigma_min": 0.1,
+  "logistic_normal_sigma_max": 1.0,
+  "logistic_normal_tau_min": 1.0,
+  "logistic_normal_tau_max": 1.0,
+  "torch_compile": false,
+  "compile_mode": "max-autotune",
+  "state_format": "prob",
+  "meanflow_weight": 0.0,
+  "rollout_train_prob": 0.5,
+  "rollout_train_steps": 4,
+  "rollout_train_infer_steps": 32,
+  "rollout_train_temp": 1.45,
+  "rollout_train_max_gamma": 1.0,
+  "rollout_train_corrupt_only": true,
+  "rollout_train_samplewise": true,
+  "rollout_train_compute_always": false,
+  "bridge_noise_init": "logistic_normal",
+  "noise_sigma": -1.0,
+  "allow_tf32": true,
+  "activation_checkpointing": false,
+  "activation_checkpoint_interval": 1,
+  "activation_checkpoint_scope": "block",
+  "ddp_static_graph": false,
+  "ddp_gradient_as_bucket_view": true,
+  "blocking_data_transfer": false,
+  "dataloader_prefetch_factor": 4,
+  "full_train_stats": false,
+  "tokenized_hf": false,
+  "tokenized_pad_token": "pad",
+  "elf_conditional_hf": false,
+  "record_pad_truncate": false,
+  "record_add_eos": false,
+  "record_add_special_tokens": false,
+  "record_pad_token": "pad",
+  "record_shuffle_buffer": 10000,
+  "wrap": true,
+  "wrap_mode": "stream",
+  "wrap_record_buffer_size": 200,
+  "owt_cached_chunks": true,
+  "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit",
+  "owt_chunk_cache_rebuild": false,
+  "owt_chunk_cache_write_batch": 4096,
+  "owt_exact_repeat_per_chunk": 64,
+  "online_chunk_shuffle": false,
+  "online_chunk_shuffle_buffer": 10000,
+  "openwebtext_split": "train_minus_100k",
+  "detokenizer": "auto",
+  "resolved_detokenizer": null,
+  "num_workers": 0,
+  "latest_every": 500,
+  "resume_path": ""
+}
+step=100 epoch=100/500 epoch_step=1/1 micro_steps=100 elapsed=8.1s lr=2.000000e-03 loss=6.7066 loss_recon=6.7066 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0996 corrupt_frac=1.0000 acc_corrupt=0.0996 loss_corrupt=6.7066 wrong_frac=0.7915 init_acc_corrupt=0.1159 acc_corrupt_t_0p0_0p2=0.0486 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.1327 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.2813 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.4044 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=1.0999 out_g_norm=1.0063 loss_all=6.4485 init_gold_top10=0.2091 init_gold_top100=0.4880 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1132 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.1071 logit_acc_rollout_kept=0.1002
+step=200 epoch=200/500 epoch_step=1/1 micro_steps=200 elapsed=7.3s lr=2.000000e-03 loss=6.0967 loss_recon=6.0967 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1105 corrupt_frac=1.0000 acc_corrupt=0.1105 loss_corrupt=6.0967 wrong_frac=0.7892 init_acc_corrupt=0.1187 acc_corrupt_t_0p0_0p2=0.0550 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.1484 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.2931 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.4273 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=3.3153 out_g_norm=1.4045 acc_corrupt_t_0p8_1p0=0.4753 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.8271 init_gold_top10=0.2011 init_gold_top100=0.5196 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.0963 init_acc_rollout_kept=0.1255 logit_acc_rollout_applied=0.1037 logit_acc_rollout_kept=0.1171
+step=300 epoch=300/500 epoch_step=1/1 micro_steps=300 elapsed=7.3s lr=2.000000e-03 loss=5.5738 loss_recon=5.5738 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1199 corrupt_frac=1.0000 acc_corrupt=0.1199 loss_corrupt=5.5738 wrong_frac=0.7935 init_acc_corrupt=0.1146 acc_corrupt_t_0p0_0p2=0.0584 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.1673 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.3229 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.4766 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=5.1952 out_g_norm=0.7234 acc_corrupt_t_0p8_1p0=0.6445 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.3223 init_gold_top10=0.2068 init_gold_top100=0.5396 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1296 init_acc_rollout_kept=0.1153 logit_acc_rollout_applied=0.1337 logit_acc_rollout_kept=0.1255
+step=400 epoch=400/500 epoch_step=1/1 micro_steps=400 elapsed=7.4s lr=2.000000e-03 loss=5.0170 loss_recon=5.0170 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1455 corrupt_frac=1.0000 acc_corrupt=0.1455 loss_corrupt=5.0170 wrong_frac=0.7917 init_acc_corrupt=0.1171 acc_corrupt_t_0p0_0p2=0.0635 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.1998 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.4382 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=6.8540 out_g_norm=0.4136 acc_corrupt_t_0p6_0p8=0.6488 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.7832 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.8079 init_gold_top10=0.1987 init_gold_top100=0.5691 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.0888 init_acc_rollout_kept=0.1080 logit_acc_rollout_applied=0.1315 logit_acc_rollout_kept=0.1581
+step=500 epoch=500/500 epoch_step=1/1 micro_steps=500 elapsed=7.4s lr=2.000000e-03 loss=4.2786 loss_recon=4.2786 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1817 corrupt_frac=1.0000 acc_corrupt=0.1817 loss_corrupt=4.2786 wrong_frac=0.7928 init_acc_corrupt=0.1164 acc_corrupt_t_0p0_0p2=0.0737 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.2701 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.5240 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.6918 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.8594 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=8.3727 out_g_norm=0.4580 loss_all=3.8022 init_gold_top10=0.2265 init_gold_top100=0.6773 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1161 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.2017 logit_acc_rollout_kept=0.2186
+[rollin-focused] eval config=rollin_p50_s4_i32 step=500
+[eval-decode-acc] train8_rollin_focused_len256_rollin_p50_s4_i32_20260517_1733focused step=500 soft=none
+[decode] max_len=256 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_rollin_focused_len256_rollin_p50_s4_i32_20260517_1733focused::none": {
+      "run": "train8_rollin_focused_len256_rollin_p50_s4_i32_20260517_1733focused",
+      "checkpoint": "runs/train8_rollin_focused_len256_rollin_p50_s4_i32_20260517_1733focused/step_0000500.pt",
+      "ckpt_step": 500,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.04901123046875,
+      "token_acc_min": 0.02734375,
+      "token_acc_max": 0.09375,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        2,
+        5,
+        3,
+        3,
+        3,
+        3,
+        2,
+        2,
+        2,
+        6,
+        3,
+        0,
+        0,
+        3,
+        2,
+        2,
+        3,
+        3,
+        2,
+        6,
+        7,
+        7,
+        3,
+        2,
+        2,
+        0,
+        2,
+        0,
+        3,
+        0,
+        3,
+        3,
+        7,
+        3,
+        7,
+        0,
+        7,
+        3,
+        0,
+        0,
+        3,
+        6,
+        1,
+        7,
+        1,
+        3,
+        7,
+        2,
+        3,
+        3,
+        2,
+        3,
+        2,
+        1,
+        7,
+        1,
+        3,
+        3,
+        0,
+        1,
+        3,
+        7,
+        0,
+        0
+      ],
+      "best_token_acc": [
+        0.0390625,
+        0.046875,
+        0.0390625,
+        0.07421875,
+        0.0546875,
+        0.05859375,
+        0.0390625,
+        0.046875,
+        0.02734375,
+        0.02734375,
+        0.0546875,
+        0.05859375,
+        0.03515625,
+        0.0625,
+        0.05078125,
+        0.0625,
+        0.04296875,
+        0.0625,
+        0.0390625,
+        0.0390625,
+        0.09375,
+        0.046875,
+        0.04296875,
+        0.0546875,
+        0.05078125,
+        0.0546875,
+        0.03125,
+        0.03515625,
+        0.046875,
+        0.04296875,
+        0.0546875,
+        0.04296875,
+        0.07421875,
+        0.04296875,
+        0.06640625,
+        0.03515625,
+        0.05078125,
+        0.04296875,
+        0.0625,
+        0.05078125,
+        0.0546875,
+        0.05078125,
+        0.0546875,
+        0.05859375,
+        0.05078125,
+        0.046875,
+        0.05859375,
+        0.03125,
+        0.046875,
+        0.0390625,
+        0.046875,
+        0.05078125,
+        0.02734375,
+        0.0546875,
+        0.0546875,
+        0.0390625,
+        0.0546875,
+        0.04296875,
+        0.05078125,
+        0.04296875,
+        0.0546875,
+        0.046875,
+        0.0546875,
+        0.0390625
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=rollin_p50_s4_i32 ckpt_step=500 views=256000 token_acc=0.0490 exact=0/64 exact_refs=0 hits=[]
+[rollin-focused] train config=rollin_p75_s4_i32 from=0 to=500 rollout=0.75/s4/i32/temp1.45
+[launch] gpt2 cached OWT soft-endpoint m/n pilot
+[launch] run_name=train8_rollin_focused_len256_rollin_p75_s4_i32_20260517_1733focused
+[launch] save_dir=runs/train8_rollin_focused_len256_rollin_p75_s4_i32_20260517_1733focused
+[launch] n=256 m=0 clean_state_mode=onehot
+[launch] mask_mixture lowk=0.0 all=1.0
+[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=969
+[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1
+[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0
+[launch] mask_ratio=1.0->1.0
+[launch] mask_ratio_floor_schedule=none
+[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet
+[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids=
+[launch] rollout_train prob=0.75 steps=4 infer_steps=32 temp=1.45 corrupt_only=1 samplewise=1
+[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit exact_repeat_per_chunk=64
+NCCL version 2.25.1+cuda12.8
+{
+  "device": "cuda:0",
+  "rank": 0,
+  "world_size": 4,
+  "samples": "owt_cached_chunks:8",
+  "vocab_size": 969,
+  "tokenizer_vocab_size": 50257,
+  "save_dir": "runs/train8_rollin_focused_len256_rollin_p75_s4_i32_20260517_1733focused",
+  "batch_size": 128,
+  "grad_accum": 1,
+  "effective_batch_size": 512,
+  "global_batch_size": 512,
+  "lr_schedule": "constant_warmup",
+  "optimizer": "muon",
+  "epochs": 0.0,
+  "steps_per_epoch": 1,
+  "total_steps": 500,
+  "warmup_steps": 10,
+  "warmup_epochs": -1.0,
+  "min_lr": 0.0,
+  "weight_decay": 0.1,
+  "output_weight_decay": -1.0,
+  "adamw_param_groups": "nanogpt",
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_eps": 1e-08,
+  "muon_impl": "legacy",
+  "muon_momentum": 0.95,
+  "muon_ns_steps": 5,
+  "muon_update_scale": 1.0,
+  "muon_nesterov": false,
+  "muon_width_scale": false,
+  "muon_grouping": "legacy_dim_ge_2",
+  "muon_param_count": 1965440,
+  "muon_adam_param_count": 8192,
+  "muon_param_names": [
+    "vocab_embed.embedding",
+    "sigma_map.net.0.weight",
+    "sigma_map.net.2.weight",
+    "blocks.0.attn_qkv.weight",
+    "blocks.0.attn_out.weight",
+    "blocks.0.mlp.0.weight",
+    "blocks.0.mlp.2.weight",
+    "blocks.0.adaLN_modulation.weight",
+    "blocks.1.attn_qkv.weight",
+    "blocks.1.attn_out.weight",
+    "blocks.1.mlp.0.weight",
+    "blocks.1.mlp.2.weight",
+    "blocks.1.adaLN_modulation.weight",
+    "blocks.2.attn_qkv.weight",
+    "blocks.2.attn_out.weight",
+    "blocks.2.mlp.0.weight",
+    "blocks.2.mlp.2.weight",
+    "blocks.2.adaLN_modulation.weight",
+    "output_layer.linear.weight",
+    "output_layer.adaLN_modulation.weight"
+  ],
+  "muon_adam_param_names": [
+    "sigma_map.net.0.bias",
+    "sigma_map.net.2.bias",
+    "blocks.0.norm1.weight",
+    "blocks.0.norm2.weight",
+    "blocks.0.mlp.0.bias",
+    "blocks.0.mlp.2.bias",
+    "blocks.0.adaLN_modulation.bias",
+    "blocks.1.norm1.weight",
+    "blocks.1.norm2.weight",
+    "blocks.1.mlp.0.bias",
+    "blocks.1.mlp.2.bias",
+    "blocks.1.adaLN_modulation.bias",
+    "blocks.2.norm1.weight",
+    "blocks.2.norm2.weight",
+    "blocks.2.mlp.0.bias",
+    "blocks.2.mlp.2.bias",
+    "blocks.2.adaLN_modulation.bias",
+    "output_layer.norm_final.weight",
+    "output_layer.adaLN_modulation.bias"
+  ],
+  "muon_effective_nesterov": false,
+  "muon_effective_width_scale": false,
+  "muon_effective_weight_decay": 0.1,
+  "muon_adam_fallback_nesterov": false,
+  "muon_adam_fallback_weight_decay": 0.1,
+  "ema_decay": 0.9999,
+  "ema_start_step": 0,
+  "model_type": "ddit",
+  "ddit_mlp_type": "gelu",
+  "elf_num_time_tokens": 4,
+  "elf_num_model_mode_tokens": 0,
+  "qk_norm": true,
+  "output_bias": false,
+  "output_init_std": -1.0,
+  "norm_type": "rmsnorm",
+  "target_loss": "hard_ce",
+  "linear_soft_target_power": 1.0,
+  "linear_soft_target_min_conf": 0.0,
+  "linear_soft_target_max_conf": 1.0,
+  "t_sampling_mode": "logit_normal",
+  "t_sampling_power": 1.0,
+  "t_sampling_eps": 0.0001,
+  "t_sampling_logit_mean": -1.5,
+  "t_sampling_logit_std": 0.8,
+  "dual_t": true,
+  "corrupt_t_mode": "same",
+  "corrupt_min_t": 0.0,
+  "corrupt_max_t": 1.0,
+  "prefix_block_prob": 0.0,
+  "prefix_block_len": 128,
+  "mask_ratio_floor_schedule": "none",
+  "dirichlet_endpoint_mode": "categorical_dual_t",
+  "dirichlet_semantic_t_mode": "same",
+  "dirichlet_semantic_t_value": 0.0,
+  "dirichlet_semantic_t_curve": "linear",
+  "dirichlet_semantic_t_power": 1.0,
+  "endpoint_sequence_random_prob_alpha": 0.0,
+  "categorical_wrong_from_full_vocab": true,
+  "categorical_wrong_from_batch_valid_tokens": false,
+  "categorical_wrong_basin_token_ids": "",
+  "categorical_wrong_basin_prob": 0.0,
+  "categorical_wrong_unigram_prob": 0.0,
+  "categorical_wrong_uniform_prob": 0.0,
+  "categorical_wrong_prob_floor": 0.0,
+  "categorical_wrong_corpus_unigram_path": "",
+  "categorical_wrong_corpus_unigram_alpha": 1.0,
+  "categorical_wrong_basin_shared_prob": 0.0,
+  "categorical_wrong_unigram_shared_prob": 0.0,
+  "mask_mixture_original_prob": 0.0,
+  "mask_mixture_lowk_prob": 0.0,
+  "mask_mixture_lowcorrupt_prob": 0.0,
+  "mask_mixture_block_prob": 0.0,
+  "mask_mixture_all_prob": 1.0,
+  "mask_mixture_lowk_clean_tokens": "0",
+  "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_block_tokens": "64,128",
+  "simplex_bridge_sampler": "dirichlet",
+  "logistic_normal_sigma_min": 0.1,
+  "logistic_normal_sigma_max": 1.0,
+  "logistic_normal_tau_min": 1.0,
+  "logistic_normal_tau_max": 1.0,
+  "torch_compile": false,
+  "compile_mode": "max-autotune",
+  "state_format": "prob",
+  "meanflow_weight": 0.0,
+  "rollout_train_prob": 0.75,
+  "rollout_train_steps": 4,
+  "rollout_train_infer_steps": 32,
+  "rollout_train_temp": 1.45,
+  "rollout_train_max_gamma": 1.0,
+  "rollout_train_corrupt_only": true,
+  "rollout_train_samplewise": true,
+  "rollout_train_compute_always": false,
+  "bridge_noise_init": "logistic_normal",
+  "noise_sigma": -1.0,
+  "allow_tf32": true,
+  "activation_checkpointing": false,
+  "activation_checkpoint_interval": 1,
+  "activation_checkpoint_scope": "block",
+  "ddp_static_graph": false,
+  "ddp_gradient_as_bucket_view": true,
+  "blocking_data_transfer": false,
+  "dataloader_prefetch_factor": 4,
+  "full_train_stats": false,
+  "tokenized_hf": false,
+  "tokenized_pad_token": "pad",
+  "elf_conditional_hf": false,
+  "record_pad_truncate": false,
+  "record_add_eos": false,
+  "record_add_special_tokens": false,
+  "record_pad_token": "pad",
+  "record_shuffle_buffer": 10000,
+  "wrap": true,
+  "wrap_mode": "stream",
+  "wrap_record_buffer_size": 200,
+  "owt_cached_chunks": true,
+  "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit",
+  "owt_chunk_cache_rebuild": false,
+  "owt_chunk_cache_write_batch": 4096,
+  "owt_exact_repeat_per_chunk": 64,
+  "online_chunk_shuffle": false,
+  "online_chunk_shuffle_buffer": 10000,
+  "openwebtext_split": "train_minus_100k",
+  "detokenizer": "auto",
+  "resolved_detokenizer": null,
+  "num_workers": 0,
+  "latest_every": 500,
+  "resume_path": ""
+}
+step=100 epoch=100/500 epoch_step=1/1 micro_steps=100 elapsed=7.9s lr=2.000000e-03 loss=6.7062 loss_recon=6.7062 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7551 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0995 corrupt_frac=1.0000 acc_corrupt=0.0995 loss_corrupt=6.7062 wrong_frac=0.7915 init_acc_corrupt=0.1159 acc_corrupt_t_0p0_0p2=0.0487 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.1328 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.2793 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.3955 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=1.0988 out_g_norm=1.0088 loss_all=6.4472 init_gold_top10=0.2094 init_gold_top100=0.5158 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.1260 init_acc_rollout_kept=0.0945 logit_acc_rollout_applied=0.1110 logit_acc_rollout_kept=0.0877
+step=200 epoch=200/500 epoch_step=1/1 micro_steps=200 elapsed=7.2s lr=2.000000e-03 loss=6.0952 loss_recon=6.0952 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7441 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1107 corrupt_frac=1.0000 acc_corrupt=0.1107 loss_corrupt=6.0952 wrong_frac=0.7892 init_acc_corrupt=0.1189 acc_corrupt_t_0p0_0p2=0.0550 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.1490 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.2934 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.4247 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=3.3189 out_g_norm=1.4035 acc_corrupt_t_0p8_1p0=0.4753 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.8276 init_gold_top10=0.2034 init_gold_top100=0.5537 rollout_applied_pos_frac=0.7422 init_acc_rollout_applied=0.0933 init_acc_rollout_kept=0.1610 logit_acc_rollout_applied=0.1023 logit_acc_rollout_kept=0.1327
+step=300 epoch=300/500 epoch_step=1/1 micro_steps=300 elapsed=7.2s lr=2.000000e-03 loss=5.5678 loss_recon=5.5678 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7523 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1201 corrupt_frac=1.0000 acc_corrupt=0.1201 loss_corrupt=5.5678 wrong_frac=0.7935 init_acc_corrupt=0.1150 acc_corrupt_t_0p0_0p2=0.0586 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.1676 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.3226 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.4741 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=5.1933 out_g_norm=0.7206 acc_corrupt_t_0p8_1p0=0.6419 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.3093 init_gold_top10=0.2147 init_gold_top100=0.5972 rollout_applied_pos_frac=0.7266 init_acc_rollout_applied=0.1141 init_acc_rollout_kept=0.1450 logit_acc_rollout_applied=0.1243 logit_acc_rollout_kept=0.1458
+step=400 epoch=400/500 epoch_step=1/1 micro_steps=400 elapsed=7.2s lr=2.000000e-03 loss=4.9989 loss_recon=4.9989 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1464 corrupt_frac=1.0000 acc_corrupt=0.1464 loss_corrupt=4.9989 wrong_frac=0.7917 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.0638 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.2015 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.4397 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=6.8482 out_g_norm=0.4141 acc_corrupt_t_0p6_0p8=0.6478 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.7051 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.7662 init_gold_top10=0.2048 init_gold_top100=0.6762 rollout_applied_pos_frac=0.7188 init_acc_rollout_applied=0.0821 init_acc_rollout_kept=0.1494 logit_acc_rollout_applied=0.1291 logit_acc_rollout_kept=0.2026
+step=500 epoch=500/500 epoch_step=1/1 micro_steps=500 elapsed=7.2s lr=2.000000e-03 loss=4.2177 loss_recon=4.2177 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7552 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1858 corrupt_frac=1.0000 acc_corrupt=0.1858 loss_corrupt=4.2177 wrong_frac=0.7928 init_acc_corrupt=0.1175 acc_corrupt_t_0p0_0p2=0.0755 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.2769 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.5320 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.6984 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.8555 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=8.3516 out_g_norm=0.4557 loss_all=3.7268 init_gold_top10=0.2427 init_gold_top100=0.7842 rollout_applied_pos_frac=0.7344 init_acc_rollout_applied=0.1315 init_acc_rollout_kept=0.1149 logit_acc_rollout_applied=0.2242 logit_acc_rollout_kept=0.2061
+[rollin-focused] eval config=rollin_p75_s4_i32 step=500
+[eval-decode-acc] train8_rollin_focused_len256_rollin_p75_s4_i32_20260517_1733focused step=500 soft=none
+[decode] max_len=256 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_rollin_focused_len256_rollin_p75_s4_i32_20260517_1733focused::none": {
+      "run": "train8_rollin_focused_len256_rollin_p75_s4_i32_20260517_1733focused",
+      "checkpoint": "runs/train8_rollin_focused_len256_rollin_p75_s4_i32_20260517_1733focused/step_0000500.pt",
+      "ckpt_step": 500,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.04876708984375,
+      "token_acc_min": 0.0234375,
+      "token_acc_max": 0.08203125,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        5,
+        5,
+        0,
+        3,
+        3,
+        3,
+        3,
+        7,
+        0,
+        3,
+        0,
+        0,
+        0,
+        0,
+        7,
+        2,
+        3,
+        3,
+        7,
+        6,
+        7,
+        7,
+        7,
+        2,
+        2,
+        0,
+        1,
+        0,
+        7,
+        7,
+        3,
+        5,
+        7,
+        3,
+        7,
+        7,
+        7,
+        4,
+        4,
+        0,
+        3,
+        6,
+        7,
+        7,
+        7,
+        3,
+        7,
+        2,
+        5,
+        3,
+        3,
+        1,
+        3,
+        7,
+        7,
+        3,
+        5,
+        4,
+        0,
+        0,
+        3,
+        2,
+        0,
+        7
+      ],
+      "best_token_acc": [
+        0.05078125,
+        0.046875,
+        0.05078125,
+        0.07421875,
+        0.0390625,
+        0.046875,
+        0.046875,
+        0.0390625,
+        0.04296875,
+        0.03125,
+        0.0546875,
+        0.06640625,
+        0.03125,
+        0.0390625,
+        0.0546875,
+        0.05859375,
+        0.05078125,
+        0.0546875,
+        0.03515625,
+        0.0546875,
+        0.08203125,
+        0.046875,
+        0.046875,
+        0.0625,
+        0.04296875,
+        0.046875,
+        0.03515625,
+        0.0234375,
+        0.03515625,
+        0.0625,
+        0.0390625,
+        0.04296875,
+        0.08203125,
+        0.046875,
+        0.05859375,
+        0.04296875,
+        0.04296875,
+        0.046875,
+        0.0390625,
+        0.0390625,
+        0.0546875,
+        0.03515625,
+        0.0546875,
+        0.05859375,
+        0.05859375,
+        0.046875,
+        0.0625,
+        0.0390625,
+        0.05859375,
+        0.05859375,
+        0.05078125,
+        0.05859375,
+        0.02734375,
+        0.04296875,
+        0.0390625,
+        0.05859375,
+        0.0390625,
+        0.05078125,
+        0.0546875,
+        0.046875,
+        0.0625,
+        0.02734375,
+        0.046875,
+        0.0546875
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=rollin_p75_s4_i32 ckpt_step=500 views=256000 token_acc=0.0488 exact=0/64 exact_refs=0 hits=[]
+[rollin-focused] train config=rollin_p100_s4_i32 from=0 to=500 rollout=1.00/s4/i32/temp1.45
+[launch] gpt2 cached OWT soft-endpoint m/n pilot
+[launch] run_name=train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused
+[launch] save_dir=runs/train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused
+[launch] n=256 m=0 clean_state_mode=onehot
+[launch] mask_mixture lowk=0.0 all=1.0
+[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=969
+[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1
+[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0
+[launch] mask_ratio=1.0->1.0
+[launch] mask_ratio_floor_schedule=none
+[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet
+[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids=
+[launch] rollout_train prob=1.00 steps=4 infer_steps=32 temp=1.45 corrupt_only=1 samplewise=1
+[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit exact_repeat_per_chunk=64
+NCCL version 2.25.1+cuda12.8
+{
+  "device": "cuda:0",
+  "rank": 0,
+  "world_size": 4,
+  "samples": "owt_cached_chunks:8",
+  "vocab_size": 969,
+  "tokenizer_vocab_size": 50257,
+  "save_dir": "runs/train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused",
+  "batch_size": 128,
+  "grad_accum": 1,
+  "effective_batch_size": 512,
+  "global_batch_size": 512,
+  "lr_schedule": "constant_warmup",
+  "optimizer": "muon",
+  "epochs": 0.0,
+  "steps_per_epoch": 1,
+  "total_steps": 500,
+  "warmup_steps": 10,
+  "warmup_epochs": -1.0,
+  "min_lr": 0.0,
+  "weight_decay": 0.1,
+  "output_weight_decay": -1.0,
+  "adamw_param_groups": "nanogpt",
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_eps": 1e-08,
+  "muon_impl": "legacy",
+  "muon_momentum": 0.95,
+  "muon_ns_steps": 5,
+  "muon_update_scale": 1.0,
+  "muon_nesterov": false,
+  "muon_width_scale": false,
+  "muon_grouping": "legacy_dim_ge_2",
+  "muon_param_count": 1965440,
+  "muon_adam_param_count": 8192,
+  "muon_param_names": [
+    "vocab_embed.embedding",
+    "sigma_map.net.0.weight",
+    "sigma_map.net.2.weight",
+    "blocks.0.attn_qkv.weight",
+    "blocks.0.attn_out.weight",
+    "blocks.0.mlp.0.weight",
+    "blocks.0.mlp.2.weight",
+    "blocks.0.adaLN_modulation.weight",
+    "blocks.1.attn_qkv.weight",
+    "blocks.1.attn_out.weight",
+    "blocks.1.mlp.0.weight",
+    "blocks.1.mlp.2.weight",
+    "blocks.1.adaLN_modulation.weight",
+    "blocks.2.attn_qkv.weight",
+    "blocks.2.attn_out.weight",
+    "blocks.2.mlp.0.weight",
+    "blocks.2.mlp.2.weight",
+    "blocks.2.adaLN_modulation.weight",
+    "output_layer.linear.weight",
+    "output_layer.adaLN_modulation.weight"
+  ],
+  "muon_adam_param_names": [
+    "sigma_map.net.0.bias",
+    "sigma_map.net.2.bias",
+    "blocks.0.norm1.weight",
+    "blocks.0.norm2.weight",
+    "blocks.0.mlp.0.bias",
+    "blocks.0.mlp.2.bias",
+    "blocks.0.adaLN_modulation.bias",
+    "blocks.1.norm1.weight",
+    "blocks.1.norm2.weight",
+    "blocks.1.mlp.0.bias",
+    "blocks.1.mlp.2.bias",
+    "blocks.1.adaLN_modulation.bias",
+    "blocks.2.norm1.weight",
+    "blocks.2.norm2.weight",
+    "blocks.2.mlp.0.bias",
+    "blocks.2.mlp.2.bias",
+    "blocks.2.adaLN_modulation.bias",
+    "output_layer.norm_final.weight",
+    "output_layer.adaLN_modulation.bias"
+  ],
+  "muon_effective_nesterov": false,
+  "muon_effective_width_scale": false,
+  "muon_effective_weight_decay": 0.1,
+  "muon_adam_fallback_nesterov": false,
+  "muon_adam_fallback_weight_decay": 0.1,
+  "ema_decay": 0.9999,
+  "ema_start_step": 0,
+  "model_type": "ddit",
+  "ddit_mlp_type": "gelu",
+  "elf_num_time_tokens": 4,
+  "elf_num_model_mode_tokens": 0,
+  "qk_norm": true,
+  "output_bias": false,
+  "output_init_std": -1.0,
+  "norm_type": "rmsnorm",
+  "target_loss": "hard_ce",
+  "linear_soft_target_power": 1.0,
+  "linear_soft_target_min_conf": 0.0,
+  "linear_soft_target_max_conf": 1.0,
+  "t_sampling_mode": "logit_normal",
+  "t_sampling_power": 1.0,
+  "t_sampling_eps": 0.0001,
+  "t_sampling_logit_mean": -1.5,
+  "t_sampling_logit_std": 0.8,
+  "dual_t": true,
+  "corrupt_t_mode": "same",
+  "corrupt_min_t": 0.0,
+  "corrupt_max_t": 1.0,
+  "prefix_block_prob": 0.0,
+  "prefix_block_len": 128,
+  "mask_ratio_floor_schedule": "none",
+  "dirichlet_endpoint_mode": "categorical_dual_t",
+  "dirichlet_semantic_t_mode": "same",
+  "dirichlet_semantic_t_value": 0.0,
+  "dirichlet_semantic_t_curve": "linear",
+  "dirichlet_semantic_t_power": 1.0,
+  "endpoint_sequence_random_prob_alpha": 0.0,
+  "categorical_wrong_from_full_vocab": true,
+  "categorical_wrong_from_batch_valid_tokens": false,
+  "categorical_wrong_basin_token_ids": "",
+  "categorical_wrong_basin_prob": 0.0,
+  "categorical_wrong_unigram_prob": 0.0,
+  "categorical_wrong_uniform_prob": 0.0,
+  "categorical_wrong_prob_floor": 0.0,
+  "categorical_wrong_corpus_unigram_path": "",
+  "categorical_wrong_corpus_unigram_alpha": 1.0,
+  "categorical_wrong_basin_shared_prob": 0.0,
+  "categorical_wrong_unigram_shared_prob": 0.0,
+  "mask_mixture_original_prob": 0.0,
+  "mask_mixture_lowk_prob": 0.0,
+  "mask_mixture_lowcorrupt_prob": 0.0,
+  "mask_mixture_block_prob": 0.0,
+  "mask_mixture_all_prob": 1.0,
+  "mask_mixture_lowk_clean_tokens": "0",
+  "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_block_tokens": "64,128",
+  "simplex_bridge_sampler": "dirichlet",
+  "logistic_normal_sigma_min": 0.1,
+  "logistic_normal_sigma_max": 1.0,
+  "logistic_normal_tau_min": 1.0,
+  "logistic_normal_tau_max": 1.0,
+  "torch_compile": false,
+  "compile_mode": "max-autotune",
+  "state_format": "prob",
+  "meanflow_weight": 0.0,
+  "rollout_train_prob": 1.0,
+  "rollout_train_steps": 4,
+  "rollout_train_infer_steps": 32,
+  "rollout_train_temp": 1.45,
+  "rollout_train_max_gamma": 1.0,
+  "rollout_train_corrupt_only": true,
+  "rollout_train_samplewise": true,
+  "rollout_train_compute_always": false,
+  "bridge_noise_init": "logistic_normal",
+  "noise_sigma": -1.0,
+  "allow_tf32": true,
+  "activation_checkpointing": false,
+  "activation_checkpoint_interval": 1,
+  "activation_checkpoint_scope": "block",
+  "ddp_static_graph": false,
+  "ddp_gradient_as_bucket_view": true,
+  "blocking_data_transfer": false,
+  "dataloader_prefetch_factor": 4,
+  "full_train_stats": false,
+  "tokenized_hf": false,
+  "tokenized_pad_token": "pad",
+  "elf_conditional_hf": false,
+  "record_pad_truncate": false,
+  "record_add_eos": false,
+  "record_add_special_tokens": false,
+  "record_pad_token": "pad",
+  "record_shuffle_buffer": 10000,
+  "wrap": true,
+  "wrap_mode": "stream",
+  "wrap_record_buffer_size": 200,
+  "owt_cached_chunks": true,
+  "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit",
+  "owt_chunk_cache_rebuild": false,
+  "owt_chunk_cache_write_batch": 4096,
+  "owt_exact_repeat_per_chunk": 64,
+  "online_chunk_shuffle": false,
+  "online_chunk_shuffle_buffer": 10000,
+  "openwebtext_split": "train_minus_100k",
+  "detokenizer": "auto",
+  "resolved_detokenizer": null,
+  "num_workers": 0,
+  "latest_every": 500,
+  "resume_path": ""
+}
+step=100 epoch=100/500 epoch_step=1/1 micro_steps=100 elapsed=8.2s lr=2.000000e-03 loss=6.7057 loss_recon=6.7057 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0995 corrupt_frac=1.0000 acc_corrupt=0.0995 loss_corrupt=6.7057 wrong_frac=0.7915 init_acc_corrupt=0.1159 acc_corrupt_t_0p0_0p2=0.0488 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.1333 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.2777 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.3853 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=1.0988 out_g_norm=1.0104 loss_all=6.4460 init_gold_top10=0.2110 init_gold_top100=0.5461 rollout_applied_pos_frac=1.0000 init_acc_rollout_applied=0.1169 init_acc_rollout_kept=0.0000 logit_acc_rollout_applied=0.1051 logit_acc_rollout_kept=0.0000
+step=200 epoch=200/500 epoch_step=1/1 micro_steps=200 elapsed=7.5s lr=2.000000e-03 loss=6.0920 loss_recon=6.0920 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1116 corrupt_frac=1.0000 acc_corrupt=0.1116 loss_corrupt=6.0920 wrong_frac=0.7892 init_acc_corrupt=0.1190 acc_corrupt_t_0p0_0p2=0.0551 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.1512 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.2945 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.4229 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=3.3334 out_g_norm=1.4060 acc_corrupt_t_0p8_1p0=0.4766 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.8240 init_gold_top10=0.2049 init_gold_top100=0.5963 rollout_applied_pos_frac=1.0000 init_acc_rollout_applied=0.1107 init_acc_rollout_kept=0.0000 logit_acc_rollout_applied=0.1096 logit_acc_rollout_kept=0.0000
+step=300 epoch=300/500 epoch_step=1/1 micro_steps=300 elapsed=7.4s lr=2.000000e-03 loss=5.5560 loss_recon=5.5560 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1210 corrupt_frac=1.0000 acc_corrupt=0.1210 loss_corrupt=5.5560 wrong_frac=0.7935 init_acc_corrupt=0.1153 acc_corrupt_t_0p0_0p2=0.0590 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.1694 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.3234 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.4773 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=5.2201 out_g_norm=0.7125 acc_corrupt_t_0p8_1p0=0.6380 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.2826 init_gold_top10=0.2209 init_gold_top100=0.6553 rollout_applied_pos_frac=1.0000 init_acc_rollout_applied=0.1227 init_acc_rollout_kept=0.0000 logit_acc_rollout_applied=0.1303 logit_acc_rollout_kept=0.0000
+step=400 epoch=400/500 epoch_step=1/1 micro_steps=400 elapsed=7.3s lr=2.000000e-03 loss=4.9781 loss_recon=4.9781 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1483 corrupt_frac=1.0000 acc_corrupt=0.1483 loss_corrupt=4.9781 wrong_frac=0.7917 init_acc_corrupt=0.1181 acc_corrupt_t_0p0_0p2=0.0642 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.2056 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.4439 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=6.9063 out_g_norm=0.4180 acc_corrupt_t_0p6_0p8=0.6502 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.7422 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.7369 init_gold_top10=0.2113 init_gold_top100=0.7779 rollout_applied_pos_frac=1.0000 init_acc_rollout_applied=0.1016 init_acc_rollout_kept=0.0000 logit_acc_rollout_applied=0.1530 logit_acc_rollout_kept=0.0000
+step=500 epoch=500/500 epoch_step=1/1 micro_steps=500 elapsed=7.3s lr=2.000000e-03 loss=4.1805 loss_recon=4.1805 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1890 corrupt_frac=1.0000 acc_corrupt=0.1890 loss_corrupt=4.1805 wrong_frac=0.7928 init_acc_corrupt=0.1178 acc_corrupt_t_0p0_0p2=0.0758 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.2839 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.5403 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.7033 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.8555 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=8.4044 out_g_norm=0.4606 loss_all=3.6550 init_gold_top10=0.2562 init_gold_top100=0.9090 rollout_applied_pos_frac=1.0000 init_acc_rollout_applied=0.1274 init_acc_rollout_kept=0.0000 logit_acc_rollout_applied=0.2250 logit_acc_rollout_kept=0.0000
+[rollin-focused] eval config=rollin_p100_s4_i32 step=500
+[eval-decode-acc] train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused step=500 soft=none
+[decode] max_len=256 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused::none": {
+      "run": "train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused",
+      "checkpoint": "runs/train8_rollin_focused_len256_rollin_p100_s4_i32_20260517_1733focused/step_0000500.pt",
+      "ckpt_step": 500,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.0469970703125,
+      "token_acc_min": 0.02734375,
+      "token_acc_max": 0.078125,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        5,
+        5,
+        0,
+        3,
+        0,
+        3,
+        0,
+        7,
+        3,
+        0,
+        3,
+        5,
+        0,
+        3,
+        2,
+        2,
+        3,
+        3,
+        7,
+        3,
+        7,
+        7,
+        2,
+        2,
+        2,
+        3,
+        7,
+        0,
+        2,
+        3,
+        3,
+        3,
+        7,
+        2,
+        7,
+        3,
+        7,
+        3,
+        0,
+        3,
+        3,
+        5,
+        2,
+        0,
+        5,
+        2,
+        7,
+        2,
+        3,
+        3,
+        3,
+        3,
+        7,
+        3,
+        7,
+        3,
+        3,
+        3,
+        2,
+        3,
+        3,
+        7,
+        0,
+        0
+      ],
+      "best_token_acc": [
+        0.046875,
+        0.0546875,
+        0.02734375,
+        0.04296875,
+        0.05859375,
+        0.05078125,
+        0.03125,
+        0.046875,
+        0.05859375,
+        0.03125,
+        0.05078125,
+        0.03125,
+        0.046875,
+        0.0625,
+        0.0625,
+        0.046875,
+        0.0390625,
+        0.0390625,
+        0.046875,
+        0.04296875,
+        0.078125,
+        0.02734375,
+        0.03125,
+        0.03125,
+        0.078125,
+        0.0703125,
+        0.02734375,
+        0.0390625,
+        0.0546875,
+        0.03125,
+        0.03515625,
+        0.04296875,
+        0.078125,
+        0.05078125,
+        0.05078125,
+        0.03515625,
+        0.04296875,
+        0.03515625,
+        0.03515625,
+        0.0703125,
+        0.07421875,
+        0.06640625,
+        0.03515625,
+        0.046875,
+        0.0390625,
+        0.03125,
+        0.0703125,
+        0.0625,
+        0.0546875,
+        0.0625,
+        0.0390625,
+        0.0546875,
+        0.03515625,
+        0.04296875,
+        0.04296875,
+        0.0546875,
+        0.03515625,
+        0.03125,
+        0.02734375,
+        0.046875,
+        0.03125,
+        0.06640625,
+        0.0546875,
+        0.0390625
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=rollin_p100_s4_i32 ckpt_step=500 views=256000 token_acc=0.0470 exact=0/64 exact_refs=0 hits=[]
+[rollin-focused] train config=rollin_p50_s8_i64 from=0 to=500 rollout=0.50/s8/i64/temp1.45
+[launch] gpt2 cached OWT soft-endpoint m/n pilot
+[launch] run_name=train8_rollin_focused_len256_rollin_p50_s8_i64_20260517_1733focused
+[launch] save_dir=runs/train8_rollin_focused_len256_rollin_p50_s8_i64_20260517_1733focused
+[launch] n=256 m=0 clean_state_mode=onehot
+[launch] mask_mixture lowk=0.0 all=1.0
+[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=969
+[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1
+[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0
+[launch] mask_ratio=1.0->1.0
+[launch] mask_ratio_floor_schedule=none
+[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet
+[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids=
+[launch] rollout_train prob=0.50 steps=8 infer_steps=64 temp=1.45 corrupt_only=1 samplewise=1
+[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit exact_repeat_per_chunk=64
+NCCL version 2.25.1+cuda12.8
+{
+  "device": "cuda:0",
+  "rank": 0,
+  "world_size": 4,
+  "samples": "owt_cached_chunks:8",
+  "vocab_size": 969,
+  "tokenizer_vocab_size": 50257,
+  "save_dir": "runs/train8_rollin_focused_len256_rollin_p50_s8_i64_20260517_1733focused",
+  "batch_size": 128,
+  "grad_accum": 1,
+  "effective_batch_size": 512,
+  "global_batch_size": 512,
+  "lr_schedule": "constant_warmup",
+  "optimizer": "muon",
+  "epochs": 0.0,
+  "steps_per_epoch": 1,
+  "total_steps": 500,
+  "warmup_steps": 10,
+  "warmup_epochs": -1.0,
+  "min_lr": 0.0,
+  "weight_decay": 0.1,
+  "output_weight_decay": -1.0,
+  "adamw_param_groups": "nanogpt",
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_eps": 1e-08,
+  "muon_impl": "legacy",
+  "muon_momentum": 0.95,
+  "muon_ns_steps": 5,
+  "muon_update_scale": 1.0,
+  "muon_nesterov": false,
+  "muon_width_scale": false,
+  "muon_grouping": "legacy_dim_ge_2",
+  "muon_param_count": 1965440,
+  "muon_adam_param_count": 8192,
+  "muon_param_names": [
+    "vocab_embed.embedding",
+    "sigma_map.net.0.weight",
+    "sigma_map.net.2.weight",
+    "blocks.0.attn_qkv.weight",
+    "blocks.0.attn_out.weight",
+    "blocks.0.mlp.0.weight",
+    "blocks.0.mlp.2.weight",
+    "blocks.0.adaLN_modulation.weight",
+    "blocks.1.attn_qkv.weight",
+    "blocks.1.attn_out.weight",
+    "blocks.1.mlp.0.weight",
+    "blocks.1.mlp.2.weight",
+    "blocks.1.adaLN_modulation.weight",
+    "blocks.2.attn_qkv.weight",
+    "blocks.2.attn_out.weight",
+    "blocks.2.mlp.0.weight",
+    "blocks.2.mlp.2.weight",
+    "blocks.2.adaLN_modulation.weight",
+    "output_layer.linear.weight",
+    "output_layer.adaLN_modulation.weight"
+  ],
+  "muon_adam_param_names": [
+    "sigma_map.net.0.bias",
+    "sigma_map.net.2.bias",
+    "blocks.0.norm1.weight",
+    "blocks.0.norm2.weight",
+    "blocks.0.mlp.0.bias",
+    "blocks.0.mlp.2.bias",
+    "blocks.0.adaLN_modulation.bias",
+    "blocks.1.norm1.weight",
+    "blocks.1.norm2.weight",
+    "blocks.1.mlp.0.bias",
+    "blocks.1.mlp.2.bias",
+    "blocks.1.adaLN_modulation.bias",
+    "blocks.2.norm1.weight",
+    "blocks.2.norm2.weight",
+    "blocks.2.mlp.0.bias",
+    "blocks.2.mlp.2.bias",
+    "blocks.2.adaLN_modulation.bias",
+    "output_layer.norm_final.weight",
+    "output_layer.adaLN_modulation.bias"
+  ],
+  "muon_effective_nesterov": false,
+  "muon_effective_width_scale": false,
+  "muon_effective_weight_decay": 0.1,
+  "muon_adam_fallback_nesterov": false,
+  "muon_adam_fallback_weight_decay": 0.1,
+  "ema_decay": 0.9999,
+  "ema_start_step": 0,
+  "model_type": "ddit",
+  "ddit_mlp_type": "gelu",
+  "elf_num_time_tokens": 4,
+  "elf_num_model_mode_tokens": 0,
+  "qk_norm": true,
+  "output_bias": false,
+  "output_init_std": -1.0,
+  "norm_type": "rmsnorm",
+  "target_loss": "hard_ce",
+  "linear_soft_target_power": 1.0,
+  "linear_soft_target_min_conf": 0.0,
+  "linear_soft_target_max_conf": 1.0,
+  "t_sampling_mode": "logit_normal",
+  "t_sampling_power": 1.0,
+  "t_sampling_eps": 0.0001,
+  "t_sampling_logit_mean": -1.5,
+  "t_sampling_logit_std": 0.8,
+  "dual_t": true,
+  "corrupt_t_mode": "same",
+  "corrupt_min_t": 0.0,
+  "corrupt_max_t": 1.0,
+  "prefix_block_prob": 0.0,
+  "prefix_block_len": 128,
+  "mask_ratio_floor_schedule": "none",
+  "dirichlet_endpoint_mode": "categorical_dual_t",
+  "dirichlet_semantic_t_mode": "same",
+  "dirichlet_semantic_t_value": 0.0,
+  "dirichlet_semantic_t_curve": "linear",
+  "dirichlet_semantic_t_power": 1.0,
+  "endpoint_sequence_random_prob_alpha": 0.0,
+  "categorical_wrong_from_full_vocab": true,
+  "categorical_wrong_from_batch_valid_tokens": false,
+  "categorical_wrong_basin_token_ids": "",
+  "categorical_wrong_basin_prob": 0.0,
+  "categorical_wrong_unigram_prob": 0.0,
+  "categorical_wrong_uniform_prob": 0.0,
+  "categorical_wrong_prob_floor": 0.0,
+  "categorical_wrong_corpus_unigram_path": "",
+  "categorical_wrong_corpus_unigram_alpha": 1.0,
+  "categorical_wrong_basin_shared_prob": 0.0,
+  "categorical_wrong_unigram_shared_prob": 0.0,
+  "mask_mixture_original_prob": 0.0,
+  "mask_mixture_lowk_prob": 0.0,
+  "mask_mixture_lowcorrupt_prob": 0.0,
+  "mask_mixture_block_prob": 0.0,
+  "mask_mixture_all_prob": 1.0,
+  "mask_mixture_lowk_clean_tokens": "0",
+  "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_block_tokens": "64,128",
+  "simplex_bridge_sampler": "dirichlet",
+  "logistic_normal_sigma_min": 0.1,
+  "logistic_normal_sigma_max": 1.0,
+  "logistic_normal_tau_min": 1.0,
+  "logistic_normal_tau_max": 1.0,
+  "torch_compile": false,
+  "compile_mode": "max-autotune",
+  "state_format": "prob",
+  "meanflow_weight": 0.0,
+  "rollout_train_prob": 0.5,
+  "rollout_train_steps": 8,
+  "rollout_train_infer_steps": 64,
+  "rollout_train_temp": 1.45,
+  "rollout_train_max_gamma": 1.0,
+  "rollout_train_corrupt_only": true,
+  "rollout_train_samplewise": true,
+  "rollout_train_compute_always": false,
+  "bridge_noise_init": "logistic_normal",
+  "noise_sigma": -1.0,
+  "allow_tf32": true,
+  "activation_checkpointing": false,
+  "activation_checkpoint_interval": 1,
+  "activation_checkpoint_scope": "block",
+  "ddp_static_graph": false,
+  "ddp_gradient_as_bucket_view": true,
+  "blocking_data_transfer": false,
+  "dataloader_prefetch_factor": 4,
+  "full_train_stats": false,
+  "tokenized_hf": false,
+  "tokenized_pad_token": "pad",
+  "elf_conditional_hf": false,
+  "record_pad_truncate": false,
+  "record_add_eos": false,
+  "record_add_special_tokens": false,
+  "record_pad_token": "pad",
+  "record_shuffle_buffer": 10000,
+  "wrap": true,
+  "wrap_mode": "stream",
+  "wrap_record_buffer_size": 200,
+  "owt_cached_chunks": true,
+  "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit",
+  "owt_chunk_cache_rebuild": false,
+  "owt_chunk_cache_write_batch": 4096,
+  "owt_exact_repeat_per_chunk": 64,
+  "online_chunk_shuffle": false,
+  "online_chunk_shuffle_buffer": 10000,
+  "openwebtext_split": "train_minus_100k",
+  "detokenizer": "auto",
+  "resolved_detokenizer": null,
+  "num_workers": 0,
+  "latest_every": 500,
+  "resume_path": ""
+}
+step=100 epoch=100/500 epoch_step=1/1 micro_steps=100 elapsed=11.1s lr=2.000000e-03 loss=6.7066 loss_recon=6.7066 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0996 corrupt_frac=1.0000 acc_corrupt=0.0996 loss_corrupt=6.7066 wrong_frac=0.7915 init_acc_corrupt=0.1159 acc_corrupt_t_0p0_0p2=0.0486 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.1327 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.2812 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.4043 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=1.0999 out_g_norm=1.0063 loss_all=6.4484 init_gold_top10=0.2091 init_gold_top100=0.4879 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1131 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.1072 logit_acc_rollout_kept=0.1000
+step=200 epoch=200/500 epoch_step=1/1 micro_steps=200 elapsed=10.4s lr=2.000000e-03 loss=6.0959 loss_recon=6.0959 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4954 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1105 corrupt_frac=1.0000 acc_corrupt=0.1105 loss_corrupt=6.0959 wrong_frac=0.7892 init_acc_corrupt=0.1187 acc_corrupt_t_0p0_0p2=0.0551 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.1483 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.2930 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.4270 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=3.3156 out_g_norm=1.4051 acc_corrupt_t_0p8_1p0=0.4753 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.8272 init_gold_top10=0.2013 init_gold_top100=0.5202 rollout_applied_pos_frac=0.5234 init_acc_rollout_applied=0.0961 init_acc_rollout_kept=0.1255 logit_acc_rollout_applied=0.1021 logit_acc_rollout_kept=0.1155
+step=300 epoch=300/500 epoch_step=1/1 micro_steps=300 elapsed=10.4s lr=2.000000e-03 loss=5.5736 loss_recon=5.5736 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5003 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1198 corrupt_frac=1.0000 acc_corrupt=0.1198 loss_corrupt=5.5736 wrong_frac=0.7935 init_acc_corrupt=0.1146 acc_corrupt_t_0p0_0p2=0.0584 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.1671 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.3230 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.4747 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=5.1950 out_g_norm=0.7224 acc_corrupt_t_0p8_1p0=0.6680 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.3115 init_gold_top10=0.2068 init_gold_top100=0.5429 rollout_applied_pos_frac=0.4688 init_acc_rollout_applied=0.1298 init_acc_rollout_kept=0.1153 logit_acc_rollout_applied=0.1342 logit_acc_rollout_kept=0.1293
+step=400 epoch=400/500 epoch_step=1/1 micro_steps=400 elapsed=10.4s lr=2.000000e-03 loss=5.0145 loss_recon=5.0145 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5077 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1453 corrupt_frac=1.0000 acc_corrupt=0.1453 loss_corrupt=5.0145 wrong_frac=0.7917 init_acc_corrupt=0.1171 acc_corrupt_t_0p0_0p2=0.0634 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.1998 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.4370 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=6.8552 out_g_norm=0.4143 acc_corrupt_t_0p6_0p8=0.6432 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.7754 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.7813 init_gold_top10=0.2001 init_gold_top100=0.5698 rollout_applied_pos_frac=0.4297 init_acc_rollout_applied=0.0895 init_acc_rollout_kept=0.1080 logit_acc_rollout_applied=0.1349 logit_acc_rollout_kept=0.1588
+step=500 epoch=500/500 epoch_step=1/1 micro_steps=500 elapsed=10.4s lr=2.000000e-03 loss=4.2764 loss_recon=4.2764 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5024 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1819 corrupt_frac=1.0000 acc_corrupt=0.1819 loss_corrupt=4.2764 wrong_frac=0.7928 init_acc_corrupt=0.1165 acc_corrupt_t_0p0_0p2=0.0737 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.2704 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.5253 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.6928 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.8516 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=8.3760 out_g_norm=0.4541 loss_all=3.7745 init_gold_top10=0.2264 init_gold_top100=0.6832 rollout_applied_pos_frac=0.5156 init_acc_rollout_applied=0.1152 init_acc_rollout_kept=0.1360 logit_acc_rollout_applied=0.2051 logit_acc_rollout_kept=0.2191
+[rollin-focused] eval config=rollin_p50_s8_i64 step=500
+[eval-decode-acc] train8_rollin_focused_len256_rollin_p50_s8_i64_20260517_1733focused step=500 soft=none
+[decode] max_len=256 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_rollin_focused_len256_rollin_p50_s8_i64_20260517_1733focused::none": {
+      "run": "train8_rollin_focused_len256_rollin_p50_s8_i64_20260517_1733focused",
+      "checkpoint": "runs/train8_rollin_focused_len256_rollin_p50_s8_i64_20260517_1733focused/step_0000500.pt",
+      "ckpt_step": 500,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.04052734375,
+      "token_acc_min": 0.015625,
+      "token_acc_max": 0.078125,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        3,
+        5,
+        0,
+        3,
+        0,
+        7,
+        3,
+        1,
+        3,
+        6,
+        3,
+        0,
+        0,
+        3,
+        2,
+        2,
+        3,
+        3,
+        2,
+        6,
+        7,
+        2,
+        3,
+        0,
+        2,
+        3,
+        1,
+        5,
+        3,
+        0,
+        3,
+        3,
+        7,
+        0,
+        7,
+        7,
+        1,
+        3,
+        4,
+        3,
+        3,
+        6,
+        1,
+        7,
+        1,
+        2,
+        7,
+        3,
+        5,
+        3,
+        3,
+        1,
+        1,
+        1,
+        7,
+        3,
+        5,
+        3,
+        0,
+        2,
+        7,
+        7,
+        0,
+        0
+      ],
+      "best_token_acc": [
+        0.02734375,
+        0.046875,
+        0.0625,
+        0.05078125,
+        0.03125,
+        0.0390625,
+        0.03515625,
+        0.015625,
+        0.03515625,
+        0.02734375,
+        0.05078125,
+        0.0390625,
+        0.03515625,
+        0.04296875,
+        0.02734375,
+        0.04296875,
+        0.0390625,
+        0.04296875,
+        0.03125,
+        0.0234375,
+        0.078125,
+        0.0234375,
+        0.03125,
+        0.02734375,
+        0.01953125,
+        0.0546875,
+        0.02734375,
+        0.02734375,
+        0.02734375,
+        0.03515625,
+        0.05859375,
+        0.03125,
+        0.04296875,
+        0.03125,
+        0.06640625,
+        0.04296875,
+        0.03515625,
+        0.0625,
+        0.04296875,
+        0.05078125,
+        0.05078125,
+        0.03125,
+        0.046875,
+        0.046875,
+        0.0390625,
+        0.04296875,
+        0.0390625,
+        0.02734375,
+        0.04296875,
+        0.05859375,
+        0.05859375,
+        0.05078125,
+        0.03515625,
+        0.04296875,
+        0.046875,
+        0.05078125,
+        0.05078125,
+        0.03515625,
+        0.046875,
+        0.04296875,
+        0.046875,
+        0.03125,
+        0.046875,
+        0.01953125
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=rollin_p50_s8_i64 ckpt_step=500 views=256000 token_acc=0.0405 exact=0/64 exact_refs=0 hits=[]
+[rollin-focused] train config=rollin_p75_s8_i64 from=0 to=500 rollout=0.75/s8/i64/temp1.45
+[launch] gpt2 cached OWT soft-endpoint m/n pilot
+[launch] run_name=train8_rollin_focused_len256_rollin_p75_s8_i64_20260517_1733focused
+[launch] save_dir=runs/train8_rollin_focused_len256_rollin_p75_s8_i64_20260517_1733focused
+[launch] n=256 m=0 clean_state_mode=onehot
+[launch] mask_mixture lowk=0.0 all=1.0
+[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=969
+[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1
+[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0
+[launch] mask_ratio=1.0->1.0
+[launch] mask_ratio_floor_schedule=none
+[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet
+[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids=
+[launch] rollout_train prob=0.75 steps=8 infer_steps=64 temp=1.45 corrupt_only=1 samplewise=1
+[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit exact_repeat_per_chunk=64
+NCCL version 2.25.1+cuda12.8
+{
+  "device": "cuda:0",
+  "rank": 0,
+  "world_size": 4,
+  "samples": "owt_cached_chunks:8",
+  "vocab_size": 969,
+  "tokenizer_vocab_size": 50257,
+  "save_dir": "runs/train8_rollin_focused_len256_rollin_p75_s8_i64_20260517_1733focused",
+  "batch_size": 128,
+  "grad_accum": 1,
+  "effective_batch_size": 512,
+  "global_batch_size": 512,
+  "lr_schedule": "constant_warmup",
+  "optimizer": "muon",
+  "epochs": 0.0,
+  "steps_per_epoch": 1,
+  "total_steps": 500,
+  "warmup_steps": 10,
+  "warmup_epochs": -1.0,
+  "min_lr": 0.0,
+  "weight_decay": 0.1,
+  "output_weight_decay": -1.0,
+  "adamw_param_groups": "nanogpt",
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_eps": 1e-08,
+  "muon_impl": "legacy",
+  "muon_momentum": 0.95,
+  "muon_ns_steps": 5,
+  "muon_update_scale": 1.0,
+  "muon_nesterov": false,
+  "muon_width_scale": false,
+  "muon_grouping": "legacy_dim_ge_2",
+  "muon_param_count": 1965440,
+  "muon_adam_param_count": 8192,
+  "muon_param_names": [
+    "vocab_embed.embedding",
+    "sigma_map.net.0.weight",
+    "sigma_map.net.2.weight",
+    "blocks.0.attn_qkv.weight",
+    "blocks.0.attn_out.weight",
+    "blocks.0.mlp.0.weight",
+    "blocks.0.mlp.2.weight",
+    "blocks.0.adaLN_modulation.weight",
+    "blocks.1.attn_qkv.weight",
+    "blocks.1.attn_out.weight",
+    "blocks.1.mlp.0.weight",
+    "blocks.1.mlp.2.weight",
+    "blocks.1.adaLN_modulation.weight",
+    "blocks.2.attn_qkv.weight",
+    "blocks.2.attn_out.weight",
+    "blocks.2.mlp.0.weight",
+    "blocks.2.mlp.2.weight",
+    "blocks.2.adaLN_modulation.weight",
+    "output_layer.linear.weight",
+    "output_layer.adaLN_modulation.weight"
+  ],
+  "muon_adam_param_names": [
+    "sigma_map.net.0.bias",
+    "sigma_map.net.2.bias",
+    "blocks.0.norm1.weight",
+    "blocks.0.norm2.weight",
+    "blocks.0.mlp.0.bias",
+    "blocks.0.mlp.2.bias",
+    "blocks.0.adaLN_modulation.bias",
+    "blocks.1.norm1.weight",
+    "blocks.1.norm2.weight",
+    "blocks.1.mlp.0.bias",
+    "blocks.1.mlp.2.bias",
+    "blocks.1.adaLN_modulation.bias",
+    "blocks.2.norm1.weight",
+    "blocks.2.norm2.weight",
+    "blocks.2.mlp.0.bias",
+    "blocks.2.mlp.2.bias",
+    "blocks.2.adaLN_modulation.bias",
+    "output_layer.norm_final.weight",
+    "output_layer.adaLN_modulation.bias"
+  ],
+  "muon_effective_nesterov": false,
+  "muon_effective_width_scale": false,
+  "muon_effective_weight_decay": 0.1,
+  "muon_adam_fallback_nesterov": false,
+  "muon_adam_fallback_weight_decay": 0.1,
+  "ema_decay": 0.9999,
+  "ema_start_step": 0,
+  "model_type": "ddit",
+  "ddit_mlp_type": "gelu",
+  "elf_num_time_tokens": 4,
+  "elf_num_model_mode_tokens": 0,
+  "qk_norm": true,
+  "output_bias": false,
+  "output_init_std": -1.0,
+  "norm_type": "rmsnorm",
+  "target_loss": "hard_ce",
+  "linear_soft_target_power": 1.0,
+  "linear_soft_target_min_conf": 0.0,
+  "linear_soft_target_max_conf": 1.0,
+  "t_sampling_mode": "logit_normal",
+  "t_sampling_power": 1.0,
+  "t_sampling_eps": 0.0001,
+  "t_sampling_logit_mean": -1.5,
+  "t_sampling_logit_std": 0.8,
+  "dual_t": true,
+  "corrupt_t_mode": "same",
+  "corrupt_min_t": 0.0,
+  "corrupt_max_t": 1.0,
+  "prefix_block_prob": 0.0,
+  "prefix_block_len": 128,
+  "mask_ratio_floor_schedule": "none",
+  "dirichlet_endpoint_mode": "categorical_dual_t",
+  "dirichlet_semantic_t_mode": "same",
+  "dirichlet_semantic_t_value": 0.0,
+  "dirichlet_semantic_t_curve": "linear",
+  "dirichlet_semantic_t_power": 1.0,
+  "endpoint_sequence_random_prob_alpha": 0.0,
+  "categorical_wrong_from_full_vocab": true,
+  "categorical_wrong_from_batch_valid_tokens": false,
+  "categorical_wrong_basin_token_ids": "",
+  "categorical_wrong_basin_prob": 0.0,
+  "categorical_wrong_unigram_prob": 0.0,
+  "categorical_wrong_uniform_prob": 0.0,
+  "categorical_wrong_prob_floor": 0.0,
+  "categorical_wrong_corpus_unigram_path": "",
+  "categorical_wrong_corpus_unigram_alpha": 1.0,
+  "categorical_wrong_basin_shared_prob": 0.0,
+  "categorical_wrong_unigram_shared_prob": 0.0,
+  "mask_mixture_original_prob": 0.0,
+  "mask_mixture_lowk_prob": 0.0,
+  "mask_mixture_lowcorrupt_prob": 0.0,
+  "mask_mixture_block_prob": 0.0,
+  "mask_mixture_all_prob": 1.0,
+  "mask_mixture_lowk_clean_tokens": "0",
+  "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_block_tokens": "64,128",
+  "simplex_bridge_sampler": "dirichlet",
+  "logistic_normal_sigma_min": 0.1,
+  "logistic_normal_sigma_max": 1.0,
+  "logistic_normal_tau_min": 1.0,
+  "logistic_normal_tau_max": 1.0,
+  "torch_compile": false,
+  "compile_mode": "max-autotune",
+  "state_format": "prob",
+  "meanflow_weight": 0.0,
+  "rollout_train_prob": 0.75,
+  "rollout_train_steps": 8,
+  "rollout_train_infer_steps": 64,
+  "rollout_train_temp": 1.45,
+  "rollout_train_max_gamma": 1.0,
+  "rollout_train_corrupt_only": true,
+  "rollout_train_samplewise": true,
+  "rollout_train_compute_always": false,
+  "bridge_noise_init": "logistic_normal",
+  "noise_sigma": -1.0,
+  "allow_tf32": true,
+  "activation_checkpointing": false,
+  "activation_checkpoint_interval": 1,
+  "activation_checkpoint_scope": "block",
+  "ddp_static_graph": false,
+  "ddp_gradient_as_bucket_view": true,
+  "blocking_data_transfer": false,
+  "dataloader_prefetch_factor": 4,
+  "full_train_stats": false,
+  "tokenized_hf": false,
+  "tokenized_pad_token": "pad",
+  "elf_conditional_hf": false,
+  "record_pad_truncate": false,
+  "record_add_eos": false,
+  "record_add_special_tokens": false,
+  "record_pad_token": "pad",
+  "record_shuffle_buffer": 10000,
+  "wrap": true,
+  "wrap_mode": "stream",
+  "wrap_record_buffer_size": 200,
+  "owt_cached_chunks": true,
+  "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit",
+  "owt_chunk_cache_rebuild": false,
+  "owt_chunk_cache_write_batch": 4096,
+  "owt_exact_repeat_per_chunk": 64,
+  "online_chunk_shuffle": false,
+  "online_chunk_shuffle_buffer": 10000,
+  "openwebtext_split": "train_minus_100k",
+  "detokenizer": "auto",
+  "resolved_detokenizer": null,
+  "num_workers": 0,
+  "latest_every": 500,
+  "resume_path": ""
+}
+step=100 epoch=100/500 epoch_step=1/1 micro_steps=100 elapsed=11.2s lr=2.000000e-03 loss=6.7062 loss_recon=6.7062 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7551 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0995 corrupt_frac=1.0000 acc_corrupt=0.0995 loss_corrupt=6.7062 wrong_frac=0.7915 init_acc_corrupt=0.1159 acc_corrupt_t_0p0_0p2=0.0487 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.1328 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.2794 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.3953 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=1.0988 out_g_norm=1.0089 loss_all=6.4473 init_gold_top10=0.2094 init_gold_top100=0.5156 rollout_applied_pos_frac=0.7109 init_acc_rollout_applied=0.1260 init_acc_rollout_kept=0.0945 logit_acc_rollout_applied=0.1111 logit_acc_rollout_kept=0.0885
+step=200 epoch=200/500 epoch_step=1/1 micro_steps=200 elapsed=10.5s lr=2.000000e-03 loss=6.0950 loss_recon=6.0950 loss_meanflow=0.0000 mean_model_t=0.2108 mean_corrupt_t=0.2108 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7441 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1106 corrupt_frac=1.0000 acc_corrupt=0.1106 loss_corrupt=6.0950 wrong_frac=0.7892 init_acc_corrupt=0.1189 acc_corrupt_t_0p0_0p2=0.0550 corrupt_frac_t_0p0_0p2=0.5516 acc_corrupt_t_0p2_0p4=0.1488 corrupt_frac_t_0p2_0p4=0.3621 acc_corrupt_t_0p4_0p6=0.2932 corrupt_frac_t_0p4_0p6=0.0781 acc_corrupt_t_0p6_0p8=0.4252 corrupt_frac_t_0p6_0p8=0.0123 out_w_norm=3.3192 out_g_norm=1.4040 acc_corrupt_t_0p8_1p0=0.4740 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.8266 init_gold_top10=0.2031 init_gold_top100=0.5559 rollout_applied_pos_frac=0.7422 init_acc_rollout_applied=0.0931 init_acc_rollout_kept=0.1610 logit_acc_rollout_applied=0.1016 logit_acc_rollout_kept=0.1322
+step=300 epoch=300/500 epoch_step=1/1 micro_steps=300 elapsed=10.5s lr=2.000000e-03 loss=5.5671 loss_recon=5.5671 loss_meanflow=0.0000 mean_model_t=0.2067 mean_corrupt_t=0.2067 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7523 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1201 corrupt_frac=1.0000 acc_corrupt=0.1201 loss_corrupt=5.5671 wrong_frac=0.7935 init_acc_corrupt=0.1151 acc_corrupt_t_0p0_0p2=0.0585 corrupt_frac_t_0p0_0p2=0.5641 acc_corrupt_t_0p2_0p4=0.1676 corrupt_frac_t_0p2_0p4=0.3542 acc_corrupt_t_0p4_0p6=0.3231 corrupt_frac_t_0p4_0p6=0.0734 acc_corrupt_t_0p6_0p8=0.4789 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=5.1966 out_g_norm=0.7255 acc_corrupt_t_0p8_1p0=0.6445 corrupt_frac_t_0p8_1p0=0.0078 loss_all=5.2920 init_gold_top10=0.2141 init_gold_top100=0.6023 rollout_applied_pos_frac=0.7266 init_acc_rollout_applied=0.1141 init_acc_rollout_kept=0.1450 logit_acc_rollout_applied=0.1252 logit_acc_rollout_kept=0.1500
+step=400 epoch=400/500 epoch_step=1/1 micro_steps=400 elapsed=10.5s lr=2.000000e-03 loss=4.9962 loss_recon=4.9962 loss_meanflow=0.0000 mean_model_t=0.2085 mean_corrupt_t=0.2085 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7507 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1463 corrupt_frac=1.0000 acc_corrupt=0.1463 loss_corrupt=4.9962 wrong_frac=0.7917 init_acc_corrupt=0.1179 acc_corrupt_t_0p0_0p2=0.0639 corrupt_frac_t_0p0_0p2=0.5564 acc_corrupt_t_0p2_0p4=0.2015 corrupt_frac_t_0p2_0p4=0.3620 acc_corrupt_t_0p4_0p6=0.4385 corrupt_frac_t_0p4_0p6=0.0719 out_w_norm=6.8565 out_g_norm=0.4149 acc_corrupt_t_0p6_0p8=0.6449 corrupt_frac_t_0p6_0p8=0.0131 acc_corrupt_t_0p8_1p0=0.7031 corrupt_frac_t_0p8_1p0=0.0078 loss_all=4.7734 init_gold_top10=0.2049 init_gold_top100=0.6761 rollout_applied_pos_frac=0.7188 init_acc_rollout_applied=0.0819 init_acc_rollout_kept=0.1494 logit_acc_rollout_applied=0.1270 logit_acc_rollout_kept=0.2018
+step=500 epoch=500/500 epoch_step=1/1 micro_steps=500 elapsed=10.5s lr=2.000000e-03 loss=4.2104 loss_recon=4.2104 loss_meanflow=0.0000 mean_model_t=0.2071 mean_corrupt_t=0.2071 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.7552 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.1863 corrupt_frac=1.0000 acc_corrupt=0.1863 loss_corrupt=4.2104 wrong_frac=0.7928 init_acc_corrupt=0.1176 acc_corrupt_t_0p0_0p2=0.0757 corrupt_frac_t_0p0_0p2=0.5632 acc_corrupt_t_0p2_0p4=0.2778 corrupt_frac_t_0p2_0p4=0.3546 acc_corrupt_t_0p4_0p6=0.5334 corrupt_frac_t_0p4_0p6=0.0745 acc_corrupt_t_0p6_0p8=0.6991 corrupt_frac_t_0p6_0p8=0.0118 acc_corrupt_t_0p8_1p0=0.8594 corrupt_frac_t_0p8_1p0=0.0078 out_w_norm=8.3604 out_g_norm=0.4534 loss_all=3.6982 init_gold_top10=0.2438 init_gold_top100=0.7904 rollout_applied_pos_frac=0.7344 init_acc_rollout_applied=0.1312 init_acc_rollout_kept=0.1149 logit_acc_rollout_applied=0.2264 logit_acc_rollout_kept=0.2090
+[rollin-focused] eval config=rollin_p75_s8_i64 step=500
+[eval-decode-acc] train8_rollin_focused_len256_rollin_p75_s8_i64_20260517_1733focused step=500 soft=none
+[decode] max_len=256 generated=64/64
+{
+  "num_rows": 1,
+  "best_by_run": {
+    "train8_rollin_focused_len256_rollin_p75_s8_i64_20260517_1733focused::none": {
+      "run": "train8_rollin_focused_len256_rollin_p75_s8_i64_20260517_1733focused",
+      "checkpoint": "runs/train8_rollin_focused_len256_rollin_p75_s8_i64_20260517_1733focused/step_0000500.pt",
+      "ckpt_step": 500,
+      "endpoint_softening": "none",
+      "decode_rule": "flowmap",
+      "steps": 128,
+      "time_schedule": "logit_normal",
+      "model_t_mode": "post",
+      "final_from": "state",
+      "n_gen": 64,
+      "n_refs": 8,
+      "token_acc_mean": 0.05255126953125,
+      "token_acc_min": 0.01953125,
+      "token_acc_max": 0.08984375,
+      "exact_acc": 0.0,
+      "exact_count": 0,
+      "exact_ref_coverage": 0.0,
+      "exact_ref_count": 0,
+      "exact_ref_hits": [],
+      "best_ref_idx": [
+        5,
+        5,
+        0,
+        3,
+        0,
+        2,
+        0,
+        7,
+        0,
+        3,
+        0,
+        0,
+        0,
+        0,
+        5,
+        2,
+        0,
+        0,
+        5,
+        7,
+        7,
+        7,
+        7,
+        2,
+        2,
+        0,
+        5,
+        0,
+        1,
+        0,
+        3,
+        0,
+        7,
+        0,
+        7,
+        5,
+        7,
+        0,
+        0,
+        3,
+        3,
+        2,
+        4,
+        0,
+        0,
+        3,
+        7,
+        5,
+        5,
+        7,
+        3,
+        7,
+        0,
+        7,
+        7,
+        3,
+        5,
+        4,
+        0,
+        0,
+        7,
+        7,
+        0,
+        0
+      ],
+      "best_token_acc": [
+        0.06640625,
+        0.046875,
+        0.05859375,
+        0.0703125,
+        0.046875,
+        0.046875,
+        0.04296875,
+        0.05078125,
+        0.04296875,
+        0.03515625,
+        0.05078125,
+        0.0546875,
+        0.02734375,
+        0.04296875,
+        0.05078125,
+        0.06640625,
+        0.046875,
+        0.046875,
+        0.0390625,
+        0.05078125,
+        0.08984375,
+        0.0625,
+        0.05078125,
+        0.0390625,
+        0.05078125,
+        0.0703125,
+        0.04296875,
+        0.046875,
+        0.01953125,
+        0.05859375,
+        0.06640625,
+        0.04296875,
+        0.0703125,
+        0.0625,
+        0.04296875,
+        0.03515625,
+        0.0625,
+        0.05078125,
+        0.05078125,
+        0.0546875,
+        0.0859375,
+        0.03515625,
+        0.03125,
+        0.07421875,
+        0.05859375,
+        0.05078125,
+        0.08984375,
+        0.05078125,
+        0.05859375,
+        0.05078125,
+        0.0390625,
+        0.04296875,
+        0.03515625,
+        0.05859375,
+        0.04296875,
+        0.0546875,
+        0.05078125,
+        0.05078125,
+        0.05859375,
+        0.05078125,
+        0.05078125,
+        0.04296875,
+        0.05859375,
+        0.0859375
+      ]
+    }
+  },
+  "first_exact_by_run": {}
+}
+RESULT config=rollin_p75_s8_i64 ckpt_step=500 views=256000 token_acc=0.0526 exact=0/64 exact_refs=0 hits=[]
+[rollin-focused] train config=rollin_p50_s4_i32_temp1p0 from=0 to=500 rollout=0.50/s4/i32/temp1.0
+[launch] gpt2 cached OWT soft-endpoint m/n pilot
+[launch] run_name=train8_rollin_focused_len256_rollin_p50_s4_i32_temp1p0_20260517_1733focused
+[launch] save_dir=runs/train8_rollin_focused_len256_rollin_p50_s4_i32_temp1p0_20260517_1733focused
+[launch] n=256 m=0 clean_state_mode=onehot
+[launch] mask_mixture lowk=0.0 all=1.0
+[launch] model d=192 layers=3 heads=3 ff=768 vocab_override=969
+[launch] optimizer=muon muon_impl=legacy weight_decay=0.1 output_weight_decay=-1
+[launch] target_loss=hard_ce conf=0.0->1.0 power=1.0
+[launch] mask_ratio=1.0->1.0
+[launch] mask_ratio_floor_schedule=none
+[launch] dirichlet C=1.0->1024 endpoint=categorical_dual_t sampler=dirichlet
+[launch] wrong_mix seq_alpha=0.0 wrong_floor=0.0 unigram=0.0 uniform=0.0 basin=0.0 basin_ids=
+[launch] rollout_train prob=0.50 steps=4 infer_steps=32 temp=1.0 corrupt_only=1 samplewise=1
+[launch] cache=/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit exact_repeat_per_chunk=64
+NCCL version 2.25.1+cuda12.8
+{
+  "device": "cuda:0",
+  "rank": 0,
+  "world_size": 4,
+  "samples": "owt_cached_chunks:8",
+  "vocab_size": 969,
+  "tokenizer_vocab_size": 50257,
+  "save_dir": "runs/train8_rollin_focused_len256_rollin_p50_s4_i32_temp1p0_20260517_1733focused",
+  "batch_size": 128,
+  "grad_accum": 1,
+  "effective_batch_size": 512,
+  "global_batch_size": 512,
+  "lr_schedule": "constant_warmup",
+  "optimizer": "muon",
+  "epochs": 0.0,
+  "steps_per_epoch": 1,
+  "total_steps": 500,
+  "warmup_steps": 10,
+  "warmup_epochs": -1.0,
+  "min_lr": 0.0,
+  "weight_decay": 0.1,
+  "output_weight_decay": -1.0,
+  "adamw_param_groups": "nanogpt",
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_eps": 1e-08,
+  "muon_impl": "legacy",
+  "muon_momentum": 0.95,
+  "muon_ns_steps": 5,
+  "muon_update_scale": 1.0,
+  "muon_nesterov": false,
+  "muon_width_scale": false,
+  "muon_grouping": "legacy_dim_ge_2",
+  "muon_param_count": 1965440,
+  "muon_adam_param_count": 8192,
+  "muon_param_names": [
+    "vocab_embed.embedding",
+    "sigma_map.net.0.weight",
+    "sigma_map.net.2.weight",
+    "blocks.0.attn_qkv.weight",
+    "blocks.0.attn_out.weight",
+    "blocks.0.mlp.0.weight",
+    "blocks.0.mlp.2.weight",
+    "blocks.0.adaLN_modulation.weight",
+    "blocks.1.attn_qkv.weight",
+    "blocks.1.attn_out.weight",
+    "blocks.1.mlp.0.weight",
+    "blocks.1.mlp.2.weight",
+    "blocks.1.adaLN_modulation.weight",
+    "blocks.2.attn_qkv.weight",
+    "blocks.2.attn_out.weight",
+    "blocks.2.mlp.0.weight",
+    "blocks.2.mlp.2.weight",
+    "blocks.2.adaLN_modulation.weight",
+    "output_layer.linear.weight",
+    "output_layer.adaLN_modulation.weight"
+  ],
+  "muon_adam_param_names": [
+    "sigma_map.net.0.bias",
+    "sigma_map.net.2.bias",
+    "blocks.0.norm1.weight",
+    "blocks.0.norm2.weight",
+    "blocks.0.mlp.0.bias",
+    "blocks.0.mlp.2.bias",
+    "blocks.0.adaLN_modulation.bias",
+    "blocks.1.norm1.weight",
+    "blocks.1.norm2.weight",
+    "blocks.1.mlp.0.bias",
+    "blocks.1.mlp.2.bias",
+    "blocks.1.adaLN_modulation.bias",
+    "blocks.2.norm1.weight",
+    "blocks.2.norm2.weight",
+    "blocks.2.mlp.0.bias",
+    "blocks.2.mlp.2.bias",
+    "blocks.2.adaLN_modulation.bias",
+    "output_layer.norm_final.weight",
+    "output_layer.adaLN_modulation.bias"
+  ],
+  "muon_effective_nesterov": false,
+  "muon_effective_width_scale": false,
+  "muon_effective_weight_decay": 0.1,
+  "muon_adam_fallback_nesterov": false,
+  "muon_adam_fallback_weight_decay": 0.1,
+  "ema_decay": 0.9999,
+  "ema_start_step": 0,
+  "model_type": "ddit",
+  "ddit_mlp_type": "gelu",
+  "elf_num_time_tokens": 4,
+  "elf_num_model_mode_tokens": 0,
+  "qk_norm": true,
+  "output_bias": false,
+  "output_init_std": -1.0,
+  "norm_type": "rmsnorm",
+  "target_loss": "hard_ce",
+  "linear_soft_target_power": 1.0,
+  "linear_soft_target_min_conf": 0.0,
+  "linear_soft_target_max_conf": 1.0,
+  "t_sampling_mode": "logit_normal",
+  "t_sampling_power": 1.0,
+  "t_sampling_eps": 0.0001,
+  "t_sampling_logit_mean": -1.5,
+  "t_sampling_logit_std": 0.8,
+  "dual_t": true,
+  "corrupt_t_mode": "same",
+  "corrupt_min_t": 0.0,
+  "corrupt_max_t": 1.0,
+  "prefix_block_prob": 0.0,
+  "prefix_block_len": 128,
+  "mask_ratio_floor_schedule": "none",
+  "dirichlet_endpoint_mode": "categorical_dual_t",
+  "dirichlet_semantic_t_mode": "same",
+  "dirichlet_semantic_t_value": 0.0,
+  "dirichlet_semantic_t_curve": "linear",
+  "dirichlet_semantic_t_power": 1.0,
+  "endpoint_sequence_random_prob_alpha": 0.0,
+  "categorical_wrong_from_full_vocab": true,
+  "categorical_wrong_from_batch_valid_tokens": false,
+  "categorical_wrong_basin_token_ids": "",
+  "categorical_wrong_basin_prob": 0.0,
+  "categorical_wrong_unigram_prob": 0.0,
+  "categorical_wrong_uniform_prob": 0.0,
+  "categorical_wrong_prob_floor": 0.0,
+  "categorical_wrong_corpus_unigram_path": "",
+  "categorical_wrong_corpus_unigram_alpha": 1.0,
+  "categorical_wrong_basin_shared_prob": 0.0,
+  "categorical_wrong_unigram_shared_prob": 0.0,
+  "mask_mixture_original_prob": 0.0,
+  "mask_mixture_lowk_prob": 0.0,
+  "mask_mixture_lowcorrupt_prob": 0.0,
+  "mask_mixture_block_prob": 0.0,
+  "mask_mixture_all_prob": 1.0,
+  "mask_mixture_lowk_clean_tokens": "0",
+  "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
+  "mask_mixture_block_tokens": "64,128",
+  "simplex_bridge_sampler": "dirichlet",
+  "logistic_normal_sigma_min": 0.1,
+  "logistic_normal_sigma_max": 1.0,
+  "logistic_normal_tau_min": 1.0,
+  "logistic_normal_tau_max": 1.0,
+  "torch_compile": false,
+  "compile_mode": "max-autotune",
+  "state_format": "prob",
+  "meanflow_weight": 0.0,
+  "rollout_train_prob": 0.5,
+  "rollout_train_steps": 4,
+  "rollout_train_infer_steps": 32,
+  "rollout_train_temp": 1.0,
+  "rollout_train_max_gamma": 1.0,
+  "rollout_train_corrupt_only": true,
+  "rollout_train_samplewise": true,
+  "rollout_train_compute_always": false,
+  "bridge_noise_init": "logistic_normal",
+  "noise_sigma": -1.0,
+  "allow_tf32": true,
+  "activation_checkpointing": false,
+  "activation_checkpoint_interval": 1,
+  "activation_checkpoint_scope": "block",
+  "ddp_static_graph": false,
+  "ddp_gradient_as_bucket_view": true,
+  "blocking_data_transfer": false,
+  "dataloader_prefetch_factor": 4,
+  "full_train_stats": false,
+  "tokenized_hf": false,
+  "tokenized_pad_token": "pad",
+  "elf_conditional_hf": false,
+  "record_pad_truncate": false,
+  "record_add_eos": false,
+  "record_add_special_tokens": false,
+  "record_pad_token": "pad",
+  "record_shuffle_buffer": 10000,
+  "wrap": true,
+  "wrap_mode": "stream",
+  "wrap_record_buffer_size": 200,
+  "owt_cached_chunks": true,
+  "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len256_train8_compact_overfit",
+  "owt_chunk_cache_rebuild": false,
+  "owt_chunk_cache_write_batch": 4096,
+  "owt_exact_repeat_per_chunk": 64,
+  "online_chunk_shuffle": false,
+  "online_chunk_shuffle_buffer": 10000,
+  "openwebtext_split": "train_minus_100k",
+  "detokenizer": "auto",
+  "resolved_detokenizer": null,
+  "num_workers": 0,
+  "latest_every": 500,
+  "resume_path": ""
+}
+step=100 epoch=100/500 epoch_step=1/1 micro_steps=100 elapsed=7.8s lr=2.000000e-03 loss=6.7066 loss_recon=6.7066 loss_meanflow=0.0000 mean_model_t=0.2083 mean_corrupt_t=0.2083 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5128 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0996 corrupt_frac=1.0000 acc_corrupt=0.0996 loss_corrupt=6.7066 wrong_frac=0.7915 init_acc_corrupt=0.1159 acc_corrupt_t_0p0_0p2=0.0486 corrupt_frac_t_0p0_0p2=0.5559 acc_corrupt_t_0p2_0p4=0.1327 corrupt_frac_t_0p2_0p4=0.3589 acc_corrupt_t_0p4_0p6=0.2812 corrupt_frac_t_0p4_0p6=0.0773 acc_corrupt_t_0p6_0p8=0.4045 corrupt_frac_t_0p6_0p8=0.0121 out_w_norm=1.0998 out_g_norm=1.0065 loss_all=6.4484 init_gold_top10=0.2095 init_gold_top100=0.4890 rollout_applied_pos_frac=0.4844 init_acc_rollout_applied=0.1133 init_acc_rollout_kept=0.1206 logit_acc_rollout_applied=0.1074 logit_acc_rollout_kept=0.0999
+W0517 17:53:06.133000 250451 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
+W0517 17:53:06.134000 250451 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 250455 closing signal SIGTERM
+W0517 17:53:06.135000 250451 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 250456 closing signal SIGTERM
+W0517 17:53:06.135000 250451 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 250457 closing signal SIGTERM
+W0517 17:53:06.135000 250451 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 250458 closing signal SIGTERM
+Traceback (most recent call last):
+  File "<frozen runpy>", line 198, in _run_module_as_main
+  File "<frozen runpy>", line 88, in _run_code
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in <module>
+    main()
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main
+    run(args)
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run
+    elastic_launch(
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
+    result = agent.run()
+             ^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 137, in wrapper
+    result = f(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
+    result = self._invoke_run(role)
+             ^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
+    time.sleep(monitor_interval)
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
+    raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
+torch.distributed.elastic.multiprocessing.api.SignalException: Process 250451 got signal: 15

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/chmv2/configuration_chmv2.py

@@ -0,0 +1,117 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/chmv2/modular_chmv2.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_chmv2.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2026 Meta Platforms, Inc. and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Literal
+
+from huggingface_hub.dataclasses import strict
+
+from ...backbone_utils import consolidate_backbone_kwargs_to_config
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+from ..auto import AutoConfig
+
+
+@auto_docstring(checkpoint="facebook/dinov3-vitl16-chmv2-dpt-head")
+@strict
+class CHMv2Config(PreTrainedConfig):
+    r"""
+    backbone_config (`Union[dict, "PreTrainedConfig"]`, *optional*):
+        The configuration of the backbone model. Only DINOv3ViTConfig is currently supported.
+    patch_size (`int`, *optional*, defaults to 16):
+        The patch size used by the backbone vision transformer.
+    reassemble_factors (`list[float]`, *optional*, defaults to `[4, 2, 1, 0.5]`):
+        The up/downsampling factors of the reassemble layers.
+    post_process_channels (`list[int]`, *optional*, defaults to `[128, 256, 512, 1024]`):
+        The output channel sizes of the reassemble stage for each backbone feature level.
+    fusion_hidden_size (`int`, *optional*, defaults to 256):
+        The number of channels before fusion.
+    head_hidden_size (`int`, *optional*, defaults to 128):
+        The number of channels in the hidden layer of the depth estimation head.
+    number_output_channels (`int`, *optional*, defaults to 256):
+        Number of output channels for the CHMv2 head (number of depth bins).
+    readout_type (`str`, *optional*, defaults to `"project"`):
+        Type of readout operation for the CLS token. One of `["ignore", "add", "project"]`.
+    min_depth (`float`, *optional*, defaults to 0.001):
+        The minimum depth value for depth bin calculation.
+    max_depth (`float`, *optional*, defaults to 96.0):
+        The maximum depth value for depth bin calculation.
+    bins_strategy (`str`, *optional*, defaults to `"chmv2_mixlog"`):
+        The strategy for depth bins distribution. One of `["linear", "log", "chmv2_mixlog"]`.
+    norm_strategy (`str`, *optional*, defaults to `"chmv2_mixlog"`):
+        The normalization strategy for depth prediction. One of `["linear", "softmax", "sigmoid", "chmv2_mixlog"]`.
+
+    ```python
+    >>> from transformers import CHMv2Config, CHMv2ForDepthEstimation
+
+    >>> configuration = CHMv2Config()
+    >>> model = CHMv2ForDepthEstimation(configuration)
+    >>> configuration = model.config
+    ```
+    """
+
+    model_type = "chmv2"
+    sub_configs = {"backbone_config": AutoConfig}
+
+    backbone_config: dict | PreTrainedConfig | None = None
+    patch_size: int = 16
+    initializer_range: float = 0.02
+    reassemble_factors: list[float | int] | None = None
+    post_process_channels: list[int] | None = None
+    fusion_hidden_size: int = 256
+    head_hidden_size: int = 128
+    number_output_channels: int = 256
+    readout_type: str = "project"
+    min_depth: float = 0.001
+    max_depth: float = 96.0
+    bins_strategy: Literal["linear", "log", "chmv2_mixlog"] = "chmv2_mixlog"
+    norm_strategy: Literal["linear", "softmax", "sigmoid", "chmv2_mixlog"] = "chmv2_mixlog"
+
+    def __post_init__(self, **kwargs):
+        if self.reassemble_factors is None:
+            self.reassemble_factors = [4, 2, 1, 0.5]
+        if self.post_process_channels is None:
+            self.post_process_channels = [128, 256, 512, 1024]
+
+        default_config_kwargs = {
+            "image_size": 416,
+            "hidden_size": 1024,
+            "intermediate_size": 4096,
+            "num_attention_heads": 16,
+            "num_hidden_layers": 24,
+            "num_register_tokens": 4,
+            "key_bias": True,
+            "out_indices": [6, 12, 18, 24],
+            "reshape_hidden_states": True,
+            "apply_layernorm": True,
+            "layer_norm_eps": 1e-6,
+            "return_class_token": True,
+        }
+
+        self.backbone_config, kwargs = consolidate_backbone_kwargs_to_config(
+            backbone_config=self.backbone_config,
+            default_config_type="dinov3_vit",
+            default_config_kwargs=default_config_kwargs,
+            **kwargs,
+        )
+
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["CHMv2Config"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/chmv2/image_processing_chmv2.py

@@ -0,0 +1,405 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/chmv2/modular_chmv2.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_chmv2.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2026 Meta Platforms, Inc. and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from collections.abc import Iterable
+from typing import Union
+
+import torch
+import torch.nn.functional as F
+from torchvision.transforms.v2 import functional as tvF
+
+from ...image_processing_backends import TorchvisionBackend
+from ...image_processing_base import BatchFeature
+from ...image_transforms import group_images_by_shape, reorder_images
+from ...image_utils import ChannelDimension, ImageInput, PILImageResampling, SizeDict
+from ...modeling_outputs import DepthEstimatorOutput
+from ...processing_utils import ImagesKwargs, Unpack
+from ...utils import TensorType, auto_docstring, is_torch_available, requires_backends
+
+
+class CHMv2ImageProcessorKwargs(ImagesKwargs, total=False):
+    r"""
+    ensure_multiple_of (`int`, *optional*, defaults to 1):
+        If `do_resize` is `True`, the image is resized to a size that is a multiple of this value. Can be overridden
+        by `ensure_multiple_of` in `preprocess`.
+    keep_aspect_ratio (`bool`, *optional*, defaults to `False`):
+        If `True`, the image is resized to the largest possible size such that the aspect ratio is preserved. Can
+        be overridden by `keep_aspect_ratio` in `preprocess`.
+    do_reduce_labels (`bool`, *optional*, defaults to `self.do_reduce_labels`):
+        Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0
+        is used for background, and background itself is not included in all classes of a dataset (e.g.
+        ADE20k). The background label will be replaced by 255.
+    """
+
+    ensure_multiple_of: int
+    size_divisor: int
+    keep_aspect_ratio: bool
+    do_reduce_labels: bool
+
+
+def get_resize_output_image_size(
+    input_image: "torch.Tensor",
+    output_size: int | Iterable[int],
+    keep_aspect_ratio: bool,
+    multiple: int,
+) -> SizeDict:
+    def constrain_to_multiple_of(val, multiple, min_val=0, max_val=None):
+        x = round(val / multiple) * multiple
+
+        if max_val is not None and x > max_val:
+            x = math.floor(val / multiple) * multiple
+
+        if x < min_val:
+            x = math.ceil(val / multiple) * multiple
+
+        return x
+
+    input_height, input_width = input_image.shape[-2:]
+    output_height, output_width = output_size
+
+    # determine new height and width
+    scale_height = output_height / input_height
+    scale_width = output_width / input_width
+
+    if keep_aspect_ratio:
+        # scale as little as possible
+        if abs(1 - scale_width) < abs(1 - scale_height):
+            # fit width
+            scale_height = scale_width
+        else:
+            # fit height
+            scale_width = scale_height
+
+    new_height = constrain_to_multiple_of(scale_height * input_height, multiple=multiple)
+    new_width = constrain_to_multiple_of(scale_width * input_width, multiple=multiple)
+
+    return SizeDict(height=new_height, width=new_width)
+
+
+@auto_docstring
+class CHMv2ImageProcessor(TorchvisionBackend):
+    """PIL backend for CHMV2 with reduce_label support."""
+
+    valid_kwargs = CHMv2ImageProcessorKwargs
+    resample = PILImageResampling.BICUBIC
+    image_mean = [0.420, 0.411, 0.296]
+    image_std = [0.213, 0.156, 0.143]
+    size = {"height": 384, "width": 384}
+    default_to_square = True
+
+    # necessary for modular conversion
+    crop_size = None
+    do_resize = False
+    do_center_crop = None
+    do_rescale = True
+    do_normalize = True
+    do_reduce_labels = None
+    do_pad = True
+    rescale_factor = 1 / 255
+    ensure_multiple_of = 16
+    keep_aspect_ratio = True
+    size_divisor = 16
+
+    def __init__(self, **kwargs: Unpack[CHMv2ImageProcessorKwargs]):
+        super().__init__(**kwargs)
+
+    @auto_docstring
+    def preprocess(
+        self,
+        images: ImageInput,
+        segmentation_maps: ImageInput | None = None,
+        **kwargs: Unpack[CHMv2ImageProcessorKwargs],
+    ) -> BatchFeature:
+        r"""
+        segmentation_maps (`ImageInput`, *optional*):
+            The segmentation maps to preprocess.
+        """
+        return super().preprocess(images, segmentation_maps, **kwargs)
+
+    def _preprocess_image_like_inputs(
+        self,
+        images: ImageInput,
+        segmentation_maps: ImageInput | None,
+        do_convert_rgb: bool,
+        input_data_format: ChannelDimension,
+        return_tensors: str | TensorType | None,
+        device: Union[str, "torch.device"] | None = None,
+        **kwargs,
+    ) -> BatchFeature:
+        """Handle extra inputs beyond images."""
+        images = self._prepare_image_like_inputs(
+            images=images, do_convert_rgb=do_convert_rgb, input_data_format=input_data_format, device=device
+        )
+        images_kwargs = kwargs.copy()
+        images_kwargs["do_reduce_labels"] = False
+        data = {}
+        data["pixel_values"] = self._preprocess(images, **images_kwargs)
+
+        # Prepare segmentation maps if provided
+        if segmentation_maps is not None:
+            processed_segmentation_maps = self._prepare_image_like_inputs(
+                images=segmentation_maps,
+                expected_ndims=2,
+                do_convert_rgb=False,
+                input_data_format=ChannelDimension.FIRST,
+            )
+
+            # Process segmentation maps with do_normalize=False and do_rescale=False
+            segmentation_maps_kwargs = kwargs.copy()
+            segmentation_maps_kwargs.update({"do_normalize": False, "do_rescale": False})
+            processed_segmentation_maps = self._preprocess(
+                images=processed_segmentation_maps, **segmentation_maps_kwargs
+            )
+
+            # Convert to int64 and squeeze channel dimension
+            processed_segmentation_maps = [
+                processed_segmentation_map.squeeze(0).to(torch.int64)
+                for processed_segmentation_map in processed_segmentation_maps
+            ]
+            data["labels"] = processed_segmentation_maps
+
+        return BatchFeature(data=data, tensor_type=return_tensors)
+
+    def reduce_label(self, labels: list["torch.Tensor"]) -> list["torch.Tensor"]:
+        """Reduce label values by 1, replacing 0 with 255."""
+        for idx in range(len(labels)):
+            label = labels[idx]
+            label = torch.where(label == 0, torch.tensor(255, dtype=label.dtype, device=label.device), label)
+            label = label - 1
+            label = torch.where(label == 254, torch.tensor(255, dtype=label.dtype, device=label.device), label)
+            labels[idx] = label
+        return labels
+
+    def _preprocess(
+        self,
+        images: list["torch.Tensor"],
+        do_reduce_labels: bool,
+        do_resize: bool,
+        size: SizeDict,
+        resample: "PILImageResampling | tvF.InterpolationMode | int | None",
+        do_center_crop: bool,
+        crop_size: SizeDict,
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: float | list[float] | None,
+        image_std: float | list[float] | None,
+        keep_aspect_ratio: bool,
+        ensure_multiple_of: int | None,
+        do_pad: bool,
+        size_divisor: int | None,
+        disable_grouping: bool | None,
+        **kwargs,
+    ) -> BatchFeature:
+        """Custom preprocessing for CHMV2."""
+        if do_reduce_labels:
+            images = self.reduce_label(images)
+
+        # Group images by size for batched resizing
+        grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)
+        resized_images_grouped = {}
+        for shape, stacked_images in grouped_images.items():
+            if do_resize:
+                stacked_images = self.resize(
+                    image=stacked_images,
+                    size=size,
+                    resample=resample,
+                    ensure_multiple_of=ensure_multiple_of,
+                    keep_aspect_ratio=keep_aspect_ratio,
+                )
+            resized_images_grouped[shape] = stacked_images
+        resized_images = reorder_images(resized_images_grouped, grouped_images_index)
+
+        # Group images by size for further processing
+        # Needed in case do_resize is False, or resize returns images with different sizes
+        grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping)
+        processed_images_grouped = {}
+        for shape, stacked_images in grouped_images.items():
+            if do_center_crop:
+                stacked_images = self.center_crop(stacked_images, crop_size)
+            # Fused rescale and normalize
+            stacked_images = self.rescale_and_normalize(
+                stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std
+            )
+            if do_pad:
+                stacked_images = self.pad_image(stacked_images, size_divisor)
+            processed_images_grouped[shape] = stacked_images
+
+        processed_images = reorder_images(processed_images_grouped, grouped_images_index)
+
+        return processed_images
+
+    def post_process_semantic_segmentation(self, outputs, target_sizes: list[tuple] | None = None):
+        """
+        Converts the output of [`CHMv2ForSemanticSegmentation`] into semantic segmentation maps.
+
+        Args:
+            outputs ([`CHMv2ForSemanticSegmentation`]):
+                Raw outputs of the model.
+            target_sizes (`list[Tuple]` of length `batch_size`, *optional*):
+                List of tuples corresponding to the requested final size (height, width) of each prediction. If unset,
+                predictions will not be resized.
+
+        Returns:
+            semantic_segmentation: `list[torch.Tensor]` of length `batch_size`, where each item is a semantic
+            segmentation map of shape (height, width) corresponding to the target_sizes entry (if `target_sizes` is
+            specified). Each entry of each `torch.Tensor` correspond to a semantic class id.
+        """
+        if not is_torch_available():
+            raise ImportError("PyTorch is required for post_process_semantic_segmentation")
+
+        logits = outputs.logits
+
+        # Resize logits and compute semantic segmentation maps
+        if target_sizes is not None:
+            if len(logits) != len(target_sizes):
+                raise ValueError(
+                    "Make sure that you pass in as many target sizes as the batch dimension of the logits"
+                )
+
+            if isinstance(target_sizes, torch.Tensor):
+                target_sizes = target_sizes.numpy()
+
+            semantic_segmentation = []
+
+            for idx in range(len(logits)):
+                resized_logits = F.interpolate(
+                    logits[idx].unsqueeze(dim=0), size=target_sizes[idx], mode="bilinear", align_corners=False
+                )
+                semantic_map = resized_logits[0].argmax(dim=0)
+                semantic_segmentation.append(semantic_map)
+        else:
+            semantic_segmentation = logits.argmax(dim=1)
+            semantic_segmentation = [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0])]
+
+        return semantic_segmentation
+
+    def resize(
+        self,
+        image: "torch.Tensor",
+        size: SizeDict,
+        resample: "PILImageResampling | tvF.InterpolationMode | int | None",
+        antialias: bool = True,
+        ensure_multiple_of: int | None = 1,
+        keep_aspect_ratio: bool = False,
+    ) -> "torch.Tensor":
+        """
+        Resize an image to `(size["height"], size["width"])`.
+
+        Args:
+            image (`torch.Tensor`):
+                Image to resize.
+            size (`SizeDict`):
+                Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image.
+            interpolation (`InterpolationMode`, *optional*, defaults to `InterpolationMode.BILINEAR`):
+                `InterpolationMode` filter to use when resizing the image e.g. `InterpolationMode.BICUBIC`.
+            antialias (`bool`, *optional*, defaults to `True`):
+                Whether to use antialiasing when resizing the image
+            ensure_multiple_of (`int`, *optional*):
+                If `do_resize` is `True`, the image is resized to a size that is a multiple of this value
+            keep_aspect_ratio (`bool`, *optional*, defaults to `False`):
+                If `True`, and `do_resize` is `True`, the image is resized to the largest possible size such that the aspect ratio is preserved.
+
+        Returns:
+            `torch.Tensor`: The resized image.
+        """
+        if not size.height or not size.width:
+            raise ValueError(f"The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}")
+
+        output_size = get_resize_output_image_size(
+            image,
+            output_size=(size.height, size.width),
+            keep_aspect_ratio=keep_aspect_ratio,
+            multiple=ensure_multiple_of,
+        )
+        return super().resize(image, output_size, resample=resample, antialias=antialias)
+
+    def pad_image(
+        self,
+        image: "torch.Tensor",
+        size_divisor: int = 1,
+    ) -> "torch.Tensor":
+        r"""
+        Center pad a batch of images to be a multiple of `size_divisor`.
+
+        Args:
+            image (`torch.Tensor`):
+                Image to pad.  Can be a batch of images of dimensions (N, C, H, W) or a single image of dimensions (C, H, W).
+            size_divisor (`int`):
+                The width and height of the image will be padded to a multiple of this number.
+        """
+        height, width = image.shape[-2:]
+
+        def _get_pad(size, size_divisor):
+            new_size = math.ceil(size / size_divisor) * size_divisor
+            pad_size = new_size - size
+            pad_size_left = pad_size // 2
+            pad_size_right = pad_size - pad_size_left
+            return pad_size_left, pad_size_right
+
+        pad_top, pad_bottom = _get_pad(height, size_divisor)
+        pad_left, pad_right = _get_pad(width, size_divisor)
+        padding = (pad_left, pad_top, pad_right, pad_bottom)
+        return tvF.pad(image, padding)
+
+    def post_process_depth_estimation(
+        self,
+        outputs: "DepthEstimatorOutput",
+        target_sizes: TensorType | list[tuple[int, int]] | None | None = None,
+    ) -> list[dict[str, TensorType]]:
+        """
+        Converts the raw output of [`DepthEstimatorOutput`] into final depth predictions and depth PIL images.
+        Only supports PyTorch.
+
+        Args:
+            outputs ([`DepthEstimatorOutput`]):
+                Raw outputs of the model.
+            target_sizes (`TensorType` or `List[Tuple[int, int]]`, *optional*):
+                Tensor of shape `(batch_size, 2)` or list of tuples (`Tuple[int, int]`) containing the target size
+                (height, width) of each image in the batch. If left to None, predictions will not be resized.
+
+        Returns:
+            `List[Dict[str, TensorType]]`: A list of dictionaries of tensors representing the processed depth
+            predictions.
+        """
+        requires_backends(self, "torch")
+
+        predicted_depth = outputs.predicted_depth
+
+        if (target_sizes is not None) and (len(predicted_depth) != len(target_sizes)):
+            raise ValueError(
+                "Make sure that you pass in as many target sizes as the batch dimension of the predicted depth"
+            )
+
+        results = []
+        target_sizes = [None] * len(predicted_depth) if target_sizes is None else target_sizes
+        for depth, target_size in zip(predicted_depth, target_sizes):
+            if target_size is not None:
+                depth = torch.nn.functional.interpolate(
+                    depth[None, None, ...], size=target_size, mode="bilinear", align_corners=True
+                ).squeeze()
+
+            results.append({"predicted_depth": depth})
+
+        return results
+
+
+__all__ = ["CHMv2ImageProcessor"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/chmv2/modeling_chmv2.py

@@ -0,0 +1,434 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/chmv2/modular_chmv2.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_chmv2.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2026 Meta Platforms, Inc. and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import torch
+from torch import nn
+
+from ... import initialization as init
+from ...backbone_utils import load_backbone
+from ...modeling_outputs import DepthEstimatorOutput
+from ...modeling_utils import PreTrainedModel
+from ...processing_utils import Unpack
+from ...utils import TransformersKwargs, auto_docstring, can_return_tuple
+from .configuration_chmv2 import CHMv2Config
+
+
+def _get_backbone_hidden_size(config):
+    if config.backbone_config is not None and hasattr(config.backbone_config, "hidden_size"):
+        return config.backbone_config.hidden_size
+    else:
+        return config.hidden_size
+
+
+class CHMv2ReassembleLayer(nn.Module):
+    def __init__(self, config: CHMv2Config, channels: int, factor: int):
+        super().__init__()
+        # projection
+        hidden_size = _get_backbone_hidden_size(config)
+        self.projection = nn.Conv2d(in_channels=hidden_size, out_channels=channels, kernel_size=1)
+
+        # up/down sampling depending on factor
+        if factor > 1:
+            self.resize = nn.ConvTranspose2d(channels, channels, kernel_size=factor, stride=factor, padding=0)
+        elif factor == 1:
+            self.resize = nn.Identity()
+        elif factor < 1:
+            # so should downsample
+            self.resize = nn.Conv2d(channels, channels, kernel_size=3, stride=int(1 / factor), padding=1)
+
+    def forward(self, hidden_state):
+        hidden_state = self.projection(hidden_state)
+        hidden_state = self.resize(hidden_state)
+        return hidden_state
+
+
+class CHMv2ReassembleStage(nn.Module):
+    """
+    Reassemble stage that processes hidden states from the backbone into image-like feature
+    representations at various resolutions.
+    """
+
+    def __init__(self, config: CHMv2Config):
+        super().__init__()
+        self.config = config
+        self.readout_type = config.readout_type
+
+        self.layers = nn.ModuleList()
+        for out_channels, factor in zip(config.post_process_channels, config.reassemble_factors):
+            self.layers.append(
+                CHMv2ReassembleLayer(
+                    config=config,
+                    channels=out_channels,
+                    factor=factor,
+                )
+            )
+
+        hidden_size = _get_backbone_hidden_size(config)
+        if self.readout_type == "project":
+            self.readout_projects = nn.ModuleList()
+            for _ in range(len(self.layers)):
+                self.readout_projects.append(nn.Sequential(nn.Linear(2 * hidden_size, hidden_size), nn.GELU()))
+
+    def forward(self, hidden_states: list[torch.Tensor], patch_height=None, patch_width=None) -> list[torch.Tensor]:
+        out = []
+
+        for layer_idx, hidden_state in enumerate(hidden_states):
+            if isinstance(hidden_state, (tuple, list)) and len(hidden_state) == 2:
+                hidden_state, cls_token = hidden_state[0], hidden_state[1]
+                feature_shape = hidden_state.shape
+
+                if self.readout_type == "project":
+                    hidden_state = hidden_state.flatten(2).transpose(1, 2)
+                    readout = cls_token.unsqueeze(1).expand_as(hidden_state)
+                    hidden_state = self.readout_projects[layer_idx](torch.cat((hidden_state, readout), -1))
+                    hidden_state = hidden_state.permute(0, 2, 1).reshape(feature_shape)
+                elif self.readout_type == "add":
+                    hidden_state = hidden_state.flatten(2) + cls_token.unsqueeze(-1)
+                    hidden_state = hidden_state.reshape(feature_shape)
+            else:
+                if hidden_state.dim() == 3:
+                    hidden_state = hidden_state[:, 1:]
+                    batch_size, _, num_channels = hidden_state.shape
+                    hidden_state = hidden_state.reshape(batch_size, patch_height, patch_width, num_channels)
+                    hidden_state = hidden_state.permute(0, 3, 1, 2).contiguous()
+
+            hidden_state = self.layers[layer_idx](hidden_state)
+            out.append(hidden_state)
+
+        return out
+
+
+class CHMv2PreActResidualLayer(nn.Module):
+    """
+    ResidualConvUnit, pre-activate residual unit.
+
+    Args:
+        config (`[CHMv2Config]`):
+            Model configuration class defining the model architecture.
+    """
+
+    def __init__(self, config):
+        super().__init__()
+
+        self.activation1 = nn.ReLU()
+        self.convolution1 = nn.Conv2d(
+            config.fusion_hidden_size,
+            config.fusion_hidden_size,
+            kernel_size=3,
+            stride=1,
+            padding=1,
+            bias=True,
+        )
+
+        self.activation2 = nn.ReLU()
+        self.convolution2 = nn.Conv2d(
+            config.fusion_hidden_size,
+            config.fusion_hidden_size,
+            kernel_size=3,
+            stride=1,
+            padding=1,
+            bias=True,
+        )
+
+    def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
+        residual = hidden_state
+        hidden_state = self.activation1(hidden_state)
+        hidden_state = self.convolution1(hidden_state)
+        hidden_state = self.activation2(hidden_state)
+        hidden_state = self.convolution2(hidden_state)
+
+        return hidden_state + residual
+
+
+class CHMv2FeatureFusionLayer(nn.Module):
+    def __init__(self, config: CHMv2Config, is_first_layer: bool = False):
+        super().__init__()
+        self.is_first_layer = is_first_layer
+
+        self.projection = nn.Conv2d(config.fusion_hidden_size, config.fusion_hidden_size, kernel_size=1, bias=True)
+
+        if not is_first_layer:
+            self.residual_layer1 = CHMv2PreActResidualLayer(config)
+
+        self.residual_layer2 = CHMv2PreActResidualLayer(config)
+
+    def forward(self, hidden_state, residual=None, size=None):
+        if residual is not None and not self.is_first_layer:
+            if hidden_state.shape != residual.shape:
+                _, _, height, width = hidden_state.shape
+                residual = nn.functional.interpolate(
+                    residual, size=(height, width), mode="bilinear", align_corners=False
+                )
+            hidden_state = hidden_state + self.residual_layer1(residual)
+
+        hidden_state = self.residual_layer2(hidden_state)
+
+        modifier = {"scale_factor": 2} if size is None else {"size": size}
+
+        hidden_state = nn.functional.interpolate(
+            hidden_state,
+            **modifier,
+            mode="bilinear",
+            align_corners=True,
+        )
+
+        hidden_state = self.projection(hidden_state)
+
+        return hidden_state
+
+
+class CHMv2UpsampleConvHead(nn.Module):
+    """
+    Convolutional head with intermediate upsampling.
+
+    Architecture: Conv3x3 -> 2x bilinear upsample -> Conv3x3 -> ReLU -> Conv1x1.
+    """
+
+    def __init__(self, features, number_output_channels, n_hidden_channels=128):
+        super().__init__()
+        self.head = nn.ModuleList(
+            [
+                nn.Conv2d(features, features // 2, kernel_size=3, stride=1, padding=1),
+                nn.Upsample(scale_factor=2, mode="bilinear", align_corners=True),
+                nn.Conv2d(features // 2, n_hidden_channels, kernel_size=3, stride=1, padding=1),
+                nn.ReLU(),
+                nn.Conv2d(n_hidden_channels, number_output_channels, kernel_size=1, stride=1, padding=0),
+            ]
+        )
+
+    def forward(self, hidden_states):
+        for layer in self.head:
+            hidden_states = layer(hidden_states)
+        return hidden_states
+
+
+class CHMv2Head(nn.Module):
+    """
+    CHMv2 dense-prediction head adapted from DPT.
+
+    Integrates reassemble, projection convs, feature fusion, and UpConv depth head.
+    """
+
+    def __init__(self, config: CHMv2Config):
+        super().__init__()
+        self.config = config
+
+        self.reassemble_stage = CHMv2ReassembleStage(config)
+
+        self.convs = nn.ModuleList()
+        for channel in config.post_process_channels:
+            self.convs.append(nn.Conv2d(channel, config.fusion_hidden_size, kernel_size=3, padding=1, bias=False))
+
+        self.fusion_layers = nn.ModuleList()
+        for idx in range(len(config.post_process_channels)):
+            self.fusion_layers.append(CHMv2FeatureFusionLayer(config, is_first_layer=(idx == 0)))
+
+        self.conv_depth = CHMv2UpsampleConvHead(
+            features=config.fusion_hidden_size,
+            number_output_channels=config.number_output_channels,
+            n_hidden_channels=config.head_hidden_size,
+        )
+
+    def forward_features(self, hidden_states: list[torch.Tensor], patch_height: int, patch_width: int) -> torch.Tensor:
+        hidden_states = self.reassemble_stage(hidden_states, patch_height, patch_width)
+
+        features = [self.convs[i](feature) for i, feature in enumerate(hidden_states)]
+        features.reverse()
+
+        fused_hidden_state = self.fusion_layers[0](features[0])
+        for i in range(1, len(self.fusion_layers)):
+            fused_hidden_state = self.fusion_layers[i](fused_hidden_state, features[i])
+
+        return fused_hidden_state
+
+    def forward(self, hidden_states: list[torch.Tensor], patch_height: int, patch_width: int) -> torch.Tensor:
+        out = self.forward_features(hidden_states, patch_height, patch_width)
+        out = self.conv_depth(out)
+        return out
+
+
+class CHMv2FeaturesToDepth(nn.Module):
+    """Converts raw logits from the CHMv2 head into a depth map using depth bins."""
+
+    def __init__(self, config: CHMv2Config):
+        super().__init__()
+        self.min_depth = config.min_depth
+        self.max_depth = config.max_depth
+        self.bins_strategy = config.bins_strategy
+        self.norm_strategy = config.norm_strategy
+        self._mixlog_max_clamp_value = 1e-4
+        self._mixlog_eps_shift = 1e-8
+        self._mixlog_eps = 1e-12
+
+    def _create_mixlog_bins(self, n_bins: int, device: torch.device) -> torch.Tensor:
+        """
+        Creates mixed log bins interpolated between linear and log distributions.
+
+        The max_depth is divided by 8.0 internally; this scaling is reversed in
+        `_create_outputs_with_mixlog_norm` by multiplying by 8.0.
+        """
+        scaled_max_depth = self.max_depth / 8.0
+        linear = torch.linspace(self.min_depth, scaled_max_depth, n_bins, device=device)
+        log = torch.exp(
+            torch.linspace(
+                torch.log(torch.tensor(self.min_depth, device=device)),
+                torch.log(torch.tensor(scaled_max_depth, device=device)),
+                n_bins,
+                device=device,
+            )
+        )
+        interp_weight = torch.linspace(1.0, 0.0, n_bins, device=device)
+        bins = interp_weight * log + (1.0 - interp_weight) * linear
+        return bins
+
+    def _create_outputs_with_mixlog_norm(self, input: torch.Tensor, bins: torch.Tensor) -> torch.Tensor:
+        """Converts depth bin logits to depth values using mixlog normalization."""
+        logits = torch.relu(input)
+
+        min_per_sample = logits.amin(dim=1, keepdim=True)
+        shift = (-min_per_sample).clamp_min(0.0).clamp_max(self._mixlog_max_clamp_value) + self._mixlog_eps_shift
+        logits_pos = logits + shift
+
+        denom = logits_pos.sum(dim=1, keepdim=True)
+        denom = torch.nan_to_num(denom, nan=1.0, posinf=1.0, neginf=1.0).clamp_min(self._mixlog_eps)
+        weights = logits_pos / denom
+
+        bins_broadcast = bins.view(1, -1, 1, 1).clamp_min(self._mixlog_eps)
+        output = (weights * bins_broadcast).sum(dim=1, keepdim=True).clamp_min(self._mixlog_eps)
+
+        output = output * 8.0
+
+        return output
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        n_bins = x.shape[1]
+
+        if n_bins > 1:
+            if self.bins_strategy == "linear":
+                bins = torch.linspace(self.min_depth, self.max_depth, n_bins, device=x.device)
+            elif self.bins_strategy == "log":
+                bins = torch.linspace(
+                    torch.log(torch.tensor(self.min_depth)),
+                    torch.log(torch.tensor(self.max_depth)),
+                    n_bins,
+                    device=x.device,
+                )
+                bins = torch.exp(bins)
+            else:
+                bins = self._create_mixlog_bins(n_bins, x.device)
+
+            if self.norm_strategy in ["linear", "softmax", "sigmoid"]:
+                if self.norm_strategy == "linear":
+                    logit = torch.relu(x)
+                    eps = 0.1
+                    logit = logit + eps
+                    logit = logit / logit.sum(dim=1, keepdim=True)
+                elif self.norm_strategy == "softmax":
+                    logit = torch.softmax(x, dim=1)
+                else:
+                    logit = torch.sigmoid(x)
+                    logit = logit / logit.sum(dim=1, keepdim=True)
+                output = torch.einsum("ikmn,k->imn", [logit, bins]).unsqueeze(dim=1)
+            else:
+                output = self._create_outputs_with_mixlog_norm(x, bins)
+        else:
+            output = torch.relu(x) + self.min_depth
+
+        return output
+
+
+@auto_docstring
+class CHMv2PreTrainedModel(PreTrainedModel):
+    config: CHMv2Config
+    base_model_prefix = "chmv2"
+    main_input_name = "pixel_values"
+    input_modalities = ("image",)
+    supports_gradient_checkpointing = True
+    _supports_sdpa = True
+    _supports_flash_attn = True
+    _supports_flex_attn = True
+    _supports_attention_backend = True
+
+    def _init_weights(self, module) -> None:
+        super()._init_weights(module)
+        if isinstance(module, (nn.Linear, nn.Conv2d, nn.ConvTranspose2d)):
+            init.trunc_normal_(module.weight, mean=0.0, std=self.config.initializer_range)
+            if module.bias is not None:
+                init.zeros_(module.bias)
+
+
+@auto_docstring(
+    custom_intro="""
+    CHMv2 Model with a depth estimation head on top (consisting of convolutional layers) e.g. for canopy height
+    estimation.
+    """
+)
+class CHMv2ForDepthEstimation(CHMv2PreTrainedModel):
+    def __init__(self, config: CHMv2Config):
+        super().__init__(config)
+
+        self.backbone = load_backbone(config)
+        self.head = CHMv2Head(config)
+        self.features_to_depth = CHMv2FeaturesToDepth(config)
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.backbone.get_input_embeddings()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor,
+        labels: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> DepthEstimatorOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
+            Ground truth depth estimation maps for computing the loss.
+        """
+        loss = None
+        if labels is not None:
+            raise NotImplementedError("Training is not implemented yet")
+
+        _, _, height, width = pixel_values.shape
+        patch_size = self.config.patch_size
+        patch_height = height // patch_size
+        patch_width = width // patch_size
+
+        backbone_output = self.backbone(pixel_values, **kwargs)
+        intermediate_features = list(zip(backbone_output.feature_maps, backbone_output.cls_tokens))
+
+        head_output = self.head(intermediate_features, patch_height, patch_width)
+
+        predicted_depth = self.features_to_depth(head_output)
+        predicted_depth = predicted_depth.squeeze(dim=1)
+
+        return DepthEstimatorOutput(
+            loss=loss,
+            predicted_depth=predicted_depth,
+            hidden_states=backbone_output.hidden_states,
+            attentions=backbone_output.attentions,
+        )
+
+
+__all__ = ["CHMv2ForDepthEstimation", "CHMv2PreTrainedModel"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/clipseg/configuration_clipseg.py

@@ -0,0 +1,262 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/clipseg/modular_clipseg.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_clipseg.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2022 The OpenAI Team Authors and The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+@auto_docstring(checkpoint="CIDAS/clipseg-rd64")
+@strict
+class CLIPSegTextConfig(PreTrainedConfig):
+    r"""
+    Example:
+
+    ```python
+    >>> from transformers import CLIPSegTextConfig, CLIPSegTextModel
+
+    >>> # Initializing a CLIPSegTextConfig with CIDAS/clipseg-rd64 style configuration
+    >>> configuration = CLIPSegTextConfig()
+
+    >>> # Initializing a CLIPSegTextModel (with random weights) from the CIDAS/clipseg-rd64 style configuration
+    >>> model = CLIPSegTextModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "clipseg_text_model"
+    base_config_key = "text_config"
+
+    vocab_size: int = 49408
+    hidden_size: int = 512
+    intermediate_size: int = 2048
+    num_hidden_layers: int = 12
+    num_attention_heads: int = 8
+    max_position_embeddings: int = 77
+    hidden_act: str = "quick_gelu"
+    layer_norm_eps: float | None = 1e-5
+    attention_dropout: int | float | None = 0.0
+    initializer_range: float = 0.02
+    initializer_factor: float | None = 1.0
+
+    # This differs from `CLIPSegTokenizer`'s default and from openai/clipseg
+    # See https://github.com/huggingface/transformers/pull/24773#issuecomment-1632287538
+    pad_token_id: int | None = 1
+    bos_token_id: int | None = 49406
+    eos_token_id: int | list[int] | None = 49407
+
+    def validate_architecture(self):
+        """Part of `@strict`-powered validation. Validates the architecture of the config."""
+        if self.hidden_size % self.num_attention_heads != 0:
+            raise ValueError(
+                f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention "
+                f"heads ({self.num_attention_heads})."
+            )
+
+
+@auto_docstring(checkpoint="CIDAS/clipseg-rd64")
+@strict
+class CLIPSegVisionConfig(PreTrainedConfig):
+    r"""
+    Example:
+
+    ```python
+    >>> from transformers import CLIPSegVisionConfig, CLIPSegVisionModel
+
+    >>> # Initializing a CLIPSegVisionConfig with CIDAS/clipseg-rd64 style configuration
+    >>> configuration = CLIPSegVisionConfig()
+
+    >>> # Initializing a CLIPSegVisionModel (with random weights) from the CIDAS/clipseg-rd64 style configuration
+    >>> model = CLIPSegVisionModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "clipseg_vision_model"
+    base_config_key = "vision_config"
+
+    hidden_size: int = 768
+    intermediate_size: int = 3072
+    num_hidden_layers: int = 12
+    num_attention_heads: int = 12
+    num_channels: int = 3
+    image_size: int | list[int] | tuple[int, int] | None = 224
+    patch_size: int | list[int] | tuple[int, int] | None = 32
+    hidden_act: str = "quick_gelu"
+    layer_norm_eps: float = 1e-5
+    attention_dropout: int | float | None = 0.0
+    initializer_range: float = 0.02
+    initializer_factor: float = 1.0
+
+    def validate_architecture(self):
+        """Part of `@strict`-powered validation. Validates the architecture of the config."""
+        if self.hidden_size % self.num_attention_heads != 0:
+            raise ValueError(
+                f"The hidden size ({self.hidden_size}) is not a multiple of the number of attention "
+                f"heads ({self.num_attention_heads})."
+            )
+
+
+@auto_docstring(checkpoint="CIDAS/clipseg-rd64")
+@strict
+class CLIPSegConfig(PreTrainedConfig):
+    r"""
+    extract_layers (`list[int]`, *optional*, defaults to `[3, 6, 9]`):
+        Layers to extract when forwarding the query image through the frozen visual backbone of CLIP.
+    reduce_dim (`int`, *optional*, defaults to 64):
+        Dimensionality to reduce the CLIP vision embedding.
+    conditional_layer (`int`, *optional*, defaults to 0):
+        The layer to use of the Transformer encoder whose activations will be combined with the condition
+        embeddings using FiLM (Feature-wise Linear Modulation). If 0, the last layer is used.
+    use_complex_transposed_convolution (`bool`, *optional*, defaults to `False`):
+        Whether to use a more complex transposed convolution in the decoder, enabling more fine-grained
+        segmentation..
+
+    Example:
+
+    ```python
+    >>> from transformers import CLIPSegConfig, CLIPSegModel
+
+    >>> # Initializing a CLIPSegConfig with CIDAS/clipseg-rd64 style configuration
+    >>> configuration = CLIPSegConfig()
+
+    >>> # Initializing a CLIPSegModel (with random weights) from the CIDAS/clipseg-rd64 style configuration
+    >>> model = CLIPSegModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+
+    >>> # We can also initialize a CLIPSegConfig from a CLIPSegTextConfig and a CLIPSegVisionConfig
+
+    >>> # Initializing a CLIPSegText and CLIPSegVision configuration
+    >>> config_text = CLIPSegTextConfig()
+    >>> config_vision = CLIPSegVisionConfig()
+
+    >>> config = CLIPSegConfig(text_config=config_text, vision_config=config_vision)
+    ```"""
+
+    model_type = "clipseg"
+    sub_configs = {"text_config": CLIPSegTextConfig, "vision_config": CLIPSegVisionConfig}
+
+    text_config: dict | CLIPSegTextConfig | None = None
+    vision_config: dict | CLIPSegVisionConfig | None = None
+    projection_dim: int | None = 512
+    logit_scale_init_value: float | int | None = 2.6592
+    initializer_factor: float | None = 1.0
+
+    extract_layers: list[int] | tuple[int, ...] = (3, 6, 9)
+    reduce_dim: int = 64
+    decoder_num_attention_heads: int = 4
+    decoder_attention_dropout: float | int = 0.0
+    decoder_hidden_act: str = "quick_gelu"
+    decoder_intermediate_size: int = 2048
+    conditional_layer: int = 0
+    use_complex_transposed_convolution: bool = False
+
+    def __post_init__(self, **kwargs):
+        if self.text_config is None:
+            text_config = {}
+            logger.info("`text_config` is `None`. Initializing the `CLIPSegTextConfig` with default values.")
+        elif isinstance(self.text_config, CLIPSegTextConfig):
+            text_config = self.text_config.to_dict()
+        else:
+            text_config = self.text_config
+
+        if self.vision_config is None:
+            vision_config = {}
+            logger.info("`vision_config` is `None`. initializing the `CLIPSegVisionConfig` with default values.")
+        elif isinstance(self.vision_config, CLIPSegVisionConfig):
+            vision_config = self.vision_config.to_dict()
+        else:
+            vision_config = self.vision_config
+
+        # For backward compatibility check keyword args
+        # Instead of simply assigning `[text|vision]_config_dict` to `[text|vision]_config`, we use the values in
+        # `[text|vision]_config_dict` to update the values in `[text|vision]_config`. The values should be same in most
+        # cases, but we don't want to break anything regarding `_config_dict` that existed before commit `8827e1b2`.
+        text_config_dict = kwargs.pop("text_config_dict", None)
+        vision_config_dict = kwargs.pop("vision_config_dict", None)
+
+        if text_config_dict is not None:
+            # This is the complete result when using `text_config_dict`.
+            _text_config_dict = CLIPSegTextConfig(**text_config_dict).to_dict()
+
+            # Give a warning if the values exist in both `_text_config_dict` and `text_config` but being different.
+            for key, value in _text_config_dict.items():
+                if key in text_config and value != text_config[key] and key != "transformers_version":
+                    # If specified in `text_config_dict`
+                    if key in text_config_dict:
+                        message = (
+                            f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. "
+                            f'The value `text_config_dict["{key}"]` will be used instead.'
+                        )
+                    # If inferred from default argument values (just to be super careful)
+                    else:
+                        message = (
+                            f"`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The "
+                            f'value `text_config["{key}"]` will be overridden.'
+                        )
+                    logger.info(message)
+
+            # Update all values in `text_config` with the ones in `_text_config_dict`.
+            text_config.update(_text_config_dict)
+
+        if vision_config_dict is not None:
+            # This is the complete result when using `vision_config_dict`.
+            _vision_config_dict = CLIPSegVisionConfig(**vision_config_dict).to_dict()
+            # convert keys to string instead of integer
+            if "id2label" in _vision_config_dict:
+                _vision_config_dict["id2label"] = {
+                    str(key): value for key, value in _vision_config_dict["id2label"].items()
+                }
+
+            # Give a warning if the values exist in both `_vision_config_dict` and `vision_config` but being different.
+            for key, value in _vision_config_dict.items():
+                if key in vision_config and value != vision_config[key] and key != "transformers_version":
+                    # If specified in `vision_config_dict`
+                    if key in vision_config_dict:
+                        message = (
+                            f"`{key}` is found in both `vision_config_dict` and `vision_config` but with different "
+                            f'values. The value `vision_config_dict["{key}"]` will be used instead.'
+                        )
+                    # If inferred from default argument values (just to be super careful)
+                    else:
+                        message = (
+                            f"`vision_config_dict` is provided which will be used to initialize `CLIPVisionConfig`. "
+                            f'The value `vision_config["{key}"]` will be overridden.'
+                        )
+                    logger.info(message)
+
+            # Update all values in `vision_config` with the ones in `_vision_config_dict`.
+            vision_config.update(_vision_config_dict)
+
+        # Finally we can convert back our unified text/vision configs to `PretrainedConfig`
+        self.text_config = CLIPSegTextConfig(**text_config)
+        self.vision_config = CLIPSegVisionConfig(**vision_config)
+
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["CLIPSegConfig", "CLIPSegTextConfig", "CLIPSegVisionConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/clipseg/modular_clipseg.py

@@ -0,0 +1,681 @@
+# Copyright 2022 The OpenAI Team Authors and The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch CLIPSeg model."""
+
+import copy
+import math
+from dataclasses import dataclass
+from typing import Any
+
+import torch
+from huggingface_hub.dataclasses import strict
+from torch import nn
+
+from ... import initialization as init
+from ...modeling_outputs import BaseModelOutputWithPooling
+from ...processing_utils import Unpack
+from ...utils import ModelOutput, TransformersKwargs, auto_docstring
+from ...utils.generic import can_return_tuple, merge_with_config_defaults
+from ...utils.output_capturing import capture_outputs
+from ..clip.configuration_clip import CLIPConfig, CLIPTextConfig, CLIPVisionConfig
+from ..clip.modeling_clip import (
+    CLIPMLP,
+    CLIPAttention,
+    CLIPEncoder,
+    CLIPEncoderLayer,
+    CLIPModel,
+    CLIPOutput,
+    CLIPPreTrainedModel,
+    CLIPTextEmbeddings,
+    CLIPTextModel,
+    CLIPVisionEmbeddings,
+    CLIPVisionModel,
+)
+
+
+@auto_docstring(checkpoint="CIDAS/clipseg-rd64")
+@strict
+class CLIPSegTextConfig(CLIPTextConfig):
+    r"""
+    Example:
+
+    ```python
+    >>> from transformers import CLIPSegTextConfig, CLIPSegTextModel
+
+    >>> # Initializing a CLIPSegTextConfig with CIDAS/clipseg-rd64 style configuration
+    >>> configuration = CLIPSegTextConfig()
+
+    >>> # Initializing a CLIPSegTextModel (with random weights) from the CIDAS/clipseg-rd64 style configuration
+    >>> model = CLIPSegTextModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    projection_dim = AttributeError()
+
+
+@auto_docstring(checkpoint="CIDAS/clipseg-rd64")
+@strict
+class CLIPSegVisionConfig(CLIPVisionConfig):
+    r"""
+    Example:
+
+    ```python
+    >>> from transformers import CLIPSegVisionConfig, CLIPSegVisionModel
+
+    >>> # Initializing a CLIPSegVisionConfig with CIDAS/clipseg-rd64 style configuration
+    >>> configuration = CLIPSegVisionConfig()
+
+    >>> # Initializing a CLIPSegVisionModel (with random weights) from the CIDAS/clipseg-rd64 style configuration
+    >>> model = CLIPSegVisionModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    projection_dim = AttributeError()
+
+
+@auto_docstring(checkpoint="CIDAS/clipseg-rd64")
+@strict
+class CLIPSegConfig(CLIPConfig):
+    r"""
+    extract_layers (`list[int]`, *optional*, defaults to `[3, 6, 9]`):
+        Layers to extract when forwarding the query image through the frozen visual backbone of CLIP.
+    reduce_dim (`int`, *optional*, defaults to 64):
+        Dimensionality to reduce the CLIP vision embedding.
+    conditional_layer (`int`, *optional*, defaults to 0):
+        The layer to use of the Transformer encoder whose activations will be combined with the condition
+        embeddings using FiLM (Feature-wise Linear Modulation). If 0, the last layer is used.
+    use_complex_transposed_convolution (`bool`, *optional*, defaults to `False`):
+        Whether to use a more complex transposed convolution in the decoder, enabling more fine-grained
+        segmentation..
+
+    Example:
+
+    ```python
+    >>> from transformers import CLIPSegConfig, CLIPSegModel
+
+    >>> # Initializing a CLIPSegConfig with CIDAS/clipseg-rd64 style configuration
+    >>> configuration = CLIPSegConfig()
+
+    >>> # Initializing a CLIPSegModel (with random weights) from the CIDAS/clipseg-rd64 style configuration
+    >>> model = CLIPSegModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+
+    >>> # We can also initialize a CLIPSegConfig from a CLIPSegTextConfig and a CLIPSegVisionConfig
+
+    >>> # Initializing a CLIPSegText and CLIPSegVision configuration
+    >>> config_text = CLIPSegTextConfig()
+    >>> config_vision = CLIPSegVisionConfig()
+
+    >>> config = CLIPSegConfig(text_config=config_text, vision_config=config_vision)
+    ```"""
+
+    extract_layers: list[int] | tuple[int, ...] = (3, 6, 9)
+    reduce_dim: int = 64
+    decoder_num_attention_heads: int = 4
+    decoder_attention_dropout: float | int = 0.0
+    decoder_hidden_act: str = "quick_gelu"
+    decoder_intermediate_size: int = 2048
+    conditional_layer: int = 0
+    use_complex_transposed_convolution: bool = False
+
+
+class CLIPSegOutput(CLIPOutput):
+    pass
+
+
+@auto_docstring
+@dataclass
+class CLIPSegDecoderOutput(ModelOutput):
+    r"""
+    logits (`torch.FloatTensor` of shape `(batch_size, height, width)`):
+        Classification scores for each pixel.
+    hidden_states (`tuple(torch.FloatTensor)`, *optional*,):
+        Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        Rreturned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`
+    attentions (`tuple(torch.FloatTensor)`, *optional*):
+        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+        heads. Returned when `output_attentions=True` is passed or when `config.output_attentions=True`
+    """
+
+    logits: torch.FloatTensor | None = None
+    hidden_states: tuple[torch.FloatTensor, ...] | None = None
+    attentions: tuple[torch.FloatTensor, ...] | None = None
+
+
+@auto_docstring
+@dataclass
+class CLIPSegImageSegmentationOutput(ModelOutput):
+    r"""
+    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
+        Binary cross entropy loss for segmentation.
+    logits (`torch.FloatTensor` of shape `(batch_size, height, width)`):
+        Classification scores for each pixel.
+    conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, projection_dim)`):
+        Conditional embeddings used for segmentation.
+    pooled_output (`torch.FloatTensor` of shape `(batch_size, embed_dim)`):
+        Pooled output of the [`CLIPSegVisionModel`].
+    vision_model_output (`BaseModelOutputWithPooling`):
+        The output of the [`CLIPSegVisionModel`].
+    decoder_output (`CLIPSegDecoderOutput`):
+        The output of the [`CLIPSegDecoder`].
+    """
+
+    loss: torch.FloatTensor | None = None
+    logits: torch.FloatTensor | None = None
+    conditional_embeddings: torch.FloatTensor | None = None
+    pooled_output: torch.FloatTensor | None = None
+    vision_model_output: BaseModelOutputWithPooling = None
+    decoder_output: CLIPSegDecoderOutput = None
+
+    def to_tuple(self) -> tuple[Any]:
+        return tuple(v.to_tuple() if isinstance(v, ModelOutput) else v for v in self.values())
+
+
+class CLIPSegVisionEmbeddings(CLIPVisionEmbeddings):
+    # Different default for `interpolate_pos_encoding` from CLIP
+    def forward(self, pixel_values: torch.FloatTensor, interpolate_pos_encoding=True) -> torch.Tensor:
+        super().forward(pixel_values, interpolate_pos_encoding)
+
+
+class CLIPSegTextEmbeddings(CLIPTextEmbeddings):
+    pass
+
+
+class CLIPSegAttention(CLIPAttention):
+    pass
+
+
+class CLIPSegMLP(CLIPMLP):
+    pass
+
+
+class CLIPSegEncoderLayer(CLIPEncoderLayer):
+    pass
+
+
+class CLIPSegDecoderLayer(CLIPEncoderLayer):
+    """
+    CLIPSeg decoder layer, which is identical to `CLIPSegEncoderLayer`, except that normalization is applied after
+    self-attention/MLP, rather than before.
+    """
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor,
+        **kwargs,
+    ) -> tuple[torch.FloatTensor]:
+        residual = hidden_states
+
+        hidden_states, _ = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            **kwargs,
+        )
+
+        hidden_states = residual + hidden_states
+        hidden_states = self.layer_norm1(hidden_states)
+
+        residual = hidden_states
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+        hidden_states = self.layer_norm2(hidden_states)
+
+        return hidden_states
+
+
+@auto_docstring
+class CLIPSegPreTrainedModel(CLIPPreTrainedModel):
+    _can_record_outputs = {
+        "hidden_states": [CLIPSegEncoderLayer, CLIPSegDecoderLayer],
+        "attentions": CLIPSegAttention,
+    }
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        factor = self.config.initializer_factor
+        if isinstance(module, CLIPSegTextEmbeddings):
+            init.normal_(module.token_embedding.weight, mean=0.0, std=factor * 0.02)
+            init.normal_(module.position_embedding.weight, mean=0.0, std=factor * 0.02)
+            init.copy_(module.position_ids, torch.arange(module.position_ids.shape[-1]).expand((1, -1)))
+        elif isinstance(module, CLIPSegVisionEmbeddings):
+            init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor)
+            init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor)
+            init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor)
+            init.copy_(module.position_ids, torch.arange(module.num_positions).expand((1, -1)))
+        elif isinstance(module, CLIPSegAttention):
+            in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
+            out_proj_std = (module.embed_dim**-0.5) * factor
+            init.normal_(module.q_proj.weight, std=in_proj_std)
+            init.normal_(module.k_proj.weight, std=in_proj_std)
+            init.normal_(module.v_proj.weight, std=in_proj_std)
+            init.normal_(module.out_proj.weight, std=out_proj_std)
+        elif isinstance(module, CLIPSegMLP):
+            in_proj_std = (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
+            fc_std = (2 * module.config.hidden_size) ** -0.5 * factor
+            init.normal_(module.fc1.weight, std=fc_std)
+            init.normal_(module.fc2.weight, std=in_proj_std)
+        elif isinstance(module, CLIPSegModel):
+            init.normal_(
+                module.text_projection.weight,
+                std=module.text_embed_dim**-0.5 * factor,
+            )
+            init.normal_(
+                module.visual_projection.weight,
+                std=module.vision_embed_dim**-0.5 * factor,
+            )
+
+        if isinstance(module, nn.LayerNorm):
+            init.zeros_(module.bias)
+            init.ones_(module.weight)
+        if isinstance(module, nn.Linear) and module.bias is not None:
+            init.zeros_(module.bias)
+
+
+class CLIPSegEncoder(CLIPEncoder):
+    pass
+
+
+class CLIPSegDecoder(CLIPSegPreTrainedModel):
+    def __init__(self, config: CLIPSegConfig):
+        super().__init__(config)
+
+        self.conditional_layer = config.conditional_layer
+
+        self.film_mul = nn.Linear(config.projection_dim, config.reduce_dim)
+        self.film_add = nn.Linear(config.projection_dim, config.reduce_dim)
+
+        if config.use_complex_transposed_convolution:
+            transposed_kernels = (config.vision_config.patch_size // 4, config.vision_config.patch_size // 4)
+
+            self.transposed_convolution = nn.Sequential(
+                nn.Conv2d(config.reduce_dim, config.reduce_dim, kernel_size=3, padding=1),
+                nn.ReLU(),
+                nn.ConvTranspose2d(
+                    config.reduce_dim,
+                    config.reduce_dim // 2,
+                    kernel_size=transposed_kernels[0],
+                    stride=transposed_kernels[0],
+                ),
+                nn.ReLU(),
+                nn.ConvTranspose2d(
+                    config.reduce_dim // 2, 1, kernel_size=transposed_kernels[1], stride=transposed_kernels[1]
+                ),
+            )
+        else:
+            self.transposed_convolution = nn.ConvTranspose2d(
+                config.reduce_dim, 1, config.vision_config.patch_size, stride=config.vision_config.patch_size
+            )
+
+        depth = len(config.extract_layers)
+        self.reduces = nn.ModuleList(
+            [nn.Linear(config.vision_config.hidden_size, config.reduce_dim) for _ in range(depth)]
+        )
+
+        decoder_config = copy.deepcopy(config.vision_config)
+        decoder_config.hidden_size = config.reduce_dim
+        decoder_config.num_attention_heads = config.decoder_num_attention_heads
+        decoder_config.intermediate_size = config.decoder_intermediate_size
+        decoder_config.hidden_act = "relu"
+        self.layers = nn.ModuleList([CLIPSegDecoderLayer(decoder_config) for _ in range(len(config.extract_layers))])
+
+        self.post_init()
+
+    @merge_with_config_defaults
+    @capture_outputs
+    @auto_docstring
+    def forward(
+        self,
+        hidden_states: tuple[torch.Tensor],
+        conditional_embeddings: torch.Tensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> CLIPSegDecoderOutput:
+        r"""
+        conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, config.projection_dim)`, *optional*):
+            The conditional embeddings for the query images. If provided, the model will use this instead of computing
+            the embeddings from the conditional_pixel_values.
+        """
+        activations = hidden_states[::-1]
+
+        output = None
+        for i, (activation, layer, reduce) in enumerate(zip(activations, self.layers, self.reduces)):
+            if output is not None:
+                output = reduce(activation) + output
+            else:
+                output = reduce(activation)
+
+            if i == self.conditional_layer:
+                output = self.film_mul(conditional_embeddings) * output.permute(1, 0, 2) + self.film_add(
+                    conditional_embeddings
+                )
+                output = output.permute(1, 0, 2)
+
+            output = layer(output, attention_mask=None, **kwargs)
+
+        output = output[:, 1:, :].transpose(1, 2)  # remove cls token and reshape to [batch_size, reduce_dim, seq_len]
+
+        size = int(math.sqrt(output.shape[2]))
+
+        batch_size = conditional_embeddings.shape[0]
+        output = output.view(batch_size, output.shape[1], size, size)
+
+        logits = self.transposed_convolution(output).squeeze(1)
+
+        return CLIPSegDecoderOutput(logits=logits)
+
+
+class CLIPSegTextModel(CLIPTextModel):
+    def forward(self, **super_kwargs) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        Examples:
+
+        ```python
+        >>> from transformers import AutoTokenizer, CLIPSegTextModel
+
+        >>> tokenizer = AutoTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined")
+        >>> model = CLIPSegTextModel.from_pretrained("CIDAS/clipseg-rd64-refined")
+
+        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
+
+        >>> outputs = model(**inputs)
+        >>> last_hidden_state = outputs.last_hidden_state
+        >>> pooled_output = outputs.pooler_output  # pooled (EOS token) states
+        ```"""
+        return super().forward(**super_kwargs)
+
+
+class CLIPSegVisionModel(CLIPVisionModel):
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor | None,
+        interpolate_pos_encoding: bool | None = True,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        Examples:
+
+        ```python
+        >>> import httpx
+        >>> from io import BytesIO
+        >>> from PIL import Image
+        >>> from transformers import AutoProcessor, CLIPSegVisionModel
+
+        >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
+        >>> model = CLIPSegVisionModel.from_pretrained("CIDAS/clipseg-rd64-refined")
+
+        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+        >>> with httpx.stream("GET", url) as response:
+        ...     image = Image.open(BytesIO(response.read()))
+
+        >>> inputs = processor(images=image, return_tensors="pt")
+
+        >>> outputs = model(**inputs)
+        >>> last_hidden_state = outputs.last_hidden_state
+        >>> pooled_output = outputs.pooler_output  # pooled CLS states
+        ```"""
+        return super().forward(pixel_values, interpolate_pos_encoding, **kwargs)
+
+
+class CLIPSegModel(CLIPModel):
+    def get_text_features(self, **super_kwargs):
+        r"""
+        Examples:
+
+        ```python
+        >>> import torch
+        >>> from transformers import AutoTokenizer, CLIPSegModel
+
+        >>> tokenizer = AutoTokenizer.from_pretrained("CIDAS/clipseg-rd64-refined")
+        >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined")
+
+        >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
+        >>> with torch.inference_mode():
+        ...     text_features = model.get_text_features(**inputs)
+        ```"""
+        return super().get_text_features(**super_kwargs)
+
+    def get_image_features(
+        self,
+        pixel_values: torch.FloatTensor,
+        interpolate_pos_encoding: bool = True,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        Examples:
+
+        ```python
+        >>> import torch
+        >>> from transformers import AutoProcessor, CLIPSegModel
+        >>> from transformers.image_utils import load_image
+
+        >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
+        >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined")
+
+        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+        >>> image = load_image(url)
+
+        >>> inputs = processor(images=image, return_tensors="pt")
+
+        >>> with torch.inference_mode():
+        ...     image_features = model.get_image_features(**inputs)
+        ```"""
+        return super().get_image_features(pixel_values, interpolate_pos_encoding, **kwargs)
+
+    def forward(self, interpolate_pos_encoding: bool = True, **super_kwargs):
+        r"""
+        return_loss (`bool`, *optional*):
+            Whether or not to return the contrastive loss.
+
+        Examples:
+
+        ```python
+        >>> import torch
+        >>> from transformers import AutoProcessor, CLIPSegModel
+        >>> from transformers.image_utils import load_image
+
+        >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
+        >>> model = CLIPSegModel.from_pretrained("CIDAS/clipseg-rd64-refined")
+
+        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+        >>> image = load_image(url)
+
+        >>> inputs = processor(
+        ...     text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
+        ... )
+
+        >>> with torch.inference_mode():
+        ...     outputs = model(**inputs)
+        >>> logits_per_image = outputs.logits_per_image  # this is the image-text similarity score
+        >>> probs = logits_per_image.softmax(dim=1)  # we can take the softmax to get the label probabilities
+        ```"""
+        super().forward(interpolate_pos_encoding=interpolate_pos_encoding, **super_kwargs)
+
+
+@auto_docstring(
+    custom_intro="""
+    CLIPSeg model with a Transformer-based decoder on top for zero-shot and one-shot image segmentation.
+    """
+)
+class CLIPSegForImageSegmentation(CLIPSegPreTrainedModel):
+    config: CLIPSegConfig
+
+    def __init__(self, config: CLIPSegConfig):
+        super().__init__(config)
+        self.clip = CLIPSegModel(config)
+        self.extract_layers = config.extract_layers
+        self.decoder = CLIPSegDecoder(config)
+
+        self.post_init()
+
+    def get_conditional_embeddings(
+        self,
+        batch_size: int | None = None,
+        input_ids: torch.Tensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.Tensor | None = None,
+        conditional_pixel_values: torch.Tensor | None = None,
+    ) -> torch.FloatTensor:
+        if input_ids is not None:
+            # compute conditional embeddings from texts
+            if len(input_ids) != batch_size:
+                raise ValueError("Make sure to pass as many prompt texts as there are query images")
+            with torch.no_grad():
+                conditional_embeddings = self.clip.get_text_features(
+                    input_ids, attention_mask=attention_mask, position_ids=position_ids
+                ).pooler_output
+        elif conditional_pixel_values is not None:
+            # compute conditional embeddings from images
+            if len(conditional_pixel_values) != batch_size:
+                raise ValueError("Make sure to pass as many prompt images as there are query images")
+            with torch.no_grad():
+                conditional_embeddings = self.clip.get_image_features(conditional_pixel_values).pooler_output
+        else:
+            raise ValueError(
+                "Invalid conditional, should be either provided as `input_ids` or `conditional_pixel_values`"
+            )
+
+        return conditional_embeddings
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.FloatTensor | None = None,
+        pixel_values: torch.FloatTensor | None = None,
+        conditional_pixel_values: torch.FloatTensor | None = None,
+        conditional_embeddings: torch.FloatTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        interpolate_pos_encoding: bool = True,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | CLIPSegOutput:
+        r"""
+        conditional_pixel_values (`torch.FloatTensor`, *optional*):
+            The pixel values of the conditional images.
+        conditional_embeddings (`torch.FloatTensor` of shape `(batch_size, config.projection_dim)`, *optional*):
+            The conditional embeddings for the query images. If provided, the model will use this instead of computing
+            the embeddings from the conditional_pixel_values.
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+
+        Examples:
+
+        ```python
+        >>> import torch
+        >>> from transformers import AutoProcessor, CLIPSegForImageSegmentation
+        >>> from transformers.image_utils import load_image
+
+        >>> processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
+        >>> model = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined")
+
+        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+        >>> image = load_image(url)
+
+        >>> texts = ["a cat", "a remote", "a blanket"]
+        >>> inputs = processor(text=texts, images=[image] * len(texts), padding=True, return_tensors="pt")
+
+        >>> with torch.inference_mode():
+        ...     outputs = model(**inputs)
+
+        >>> logits = outputs.logits
+        >>> print(logits.shape)
+        torch.Size([3, 352, 352])
+        ```"""
+        # step 1: forward the query images through the frozen CLIP vision encoder
+        with torch.no_grad():
+            kwargs["output_hidden_states"] = True  # required to extract layers for the stages
+            vision_outputs = self.clip.get_image_features(
+                pixel_values=pixel_values,
+                interpolate_pos_encoding=interpolate_pos_encoding,
+                **kwargs,
+            )
+            pooled_output = vision_outputs.pooler_output
+
+            hidden_states = vision_outputs.hidden_states
+            # we add +1 here as the hidden states also include the initial embeddings
+            activations = [hidden_states[i + 1] for i in self.extract_layers]
+
+            # update vision_outputs
+            vision_outputs = BaseModelOutputWithPooling(
+                last_hidden_state=vision_outputs.last_hidden_state,
+                pooler_output=vision_outputs.pooler_output,
+                hidden_states=vision_outputs.hidden_states,
+                attentions=vision_outputs.attentions,
+            )
+
+        # step 2: compute conditional embeddings, either from text, images or an own provided embedding
+        if conditional_embeddings is None:
+            conditional_embeddings = self.get_conditional_embeddings(
+                batch_size=pixel_values.shape[0],
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                conditional_pixel_values=conditional_pixel_values,
+            )
+        else:
+            if conditional_embeddings.shape[0] != pixel_values.shape[0]:
+                raise ValueError(
+                    "Make sure to pass as many conditional embeddings as there are query images in the batch"
+                )
+            if conditional_embeddings.shape[1] != self.config.projection_dim:
+                raise ValueError(
+                    "Make sure that the feature dimension of the conditional embeddings matches"
+                    " `config.projection_dim`."
+                )
+
+        # step 3: forward both the pooled output and the activations through the lightweight decoder to predict masks
+        decoder_outputs = self.decoder(
+            activations,
+            conditional_embeddings,
+            **kwargs,
+        )
+        logits = decoder_outputs.logits
+
+        loss = None
+        if labels is not None:
+            # move labels to the correct device to enable PP
+            labels = labels.to(logits.device)
+            loss_fn = nn.BCEWithLogitsLoss()
+            loss = loss_fn(logits, labels)
+
+        return CLIPSegImageSegmentationOutput(
+            loss=loss,
+            logits=logits,
+            conditional_embeddings=conditional_embeddings,
+            pooled_output=pooled_output,
+            vision_model_output=vision_outputs,
+            decoder_output=decoder_outputs,
+        )
+
+
+__all__ = [
+    "CLIPSegConfig",
+    "CLIPSegTextConfig",
+    "CLIPSegVisionConfig",
+    "CLIPSegModel",
+    "CLIPSegPreTrainedModel",
+    "CLIPSegTextModel",
+    "CLIPSegVisionModel",
+    "CLIPSegForImageSegmentation",
+]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/clipseg/processing_clipseg.py

@@ -0,0 +1,88 @@
+# Copyright 2022 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Image/Text processor class for CLIPSeg
+"""
+
+from ...processing_utils import ProcessorMixin
+from ...tokenization_utils_base import BatchEncoding
+from ...utils import auto_docstring
+
+
+@auto_docstring
+class CLIPSegProcessor(ProcessorMixin):
+    def __init__(self, image_processor=None, tokenizer=None, **kwargs):
+        super().__init__(image_processor, tokenizer)
+
+    @auto_docstring
+    def __call__(self, text=None, images=None, visual_prompt=None, return_tensors=None, **kwargs):
+        r"""
+        visual_prompt (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`):
+            The visual prompt image or batch of images to be prepared. Each visual prompt image can be a PIL image,
+            NumPy array or PyTorch tensor. In case of a NumPy array/PyTorch tensor, each image should be of shape
+            (C, H, W), where C is a number of channels, H and W are image height and width.
+
+        Returns:
+            [`BatchEncoding`]: A [`BatchEncoding`] with the following fields:
+
+            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
+            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
+              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
+              `None`).
+            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
+        """
+        if text is None and visual_prompt is None and images is None:
+            raise ValueError("You have to specify either text, visual prompt or images.")
+
+        if text is not None and visual_prompt is not None:
+            raise ValueError("You have to specify exactly one type of prompt. Either text or visual prompt.")
+
+        output_kwargs = self._merge_kwargs(
+            self.valid_processor_kwargs, tokenizer_init_kwargs=self.tokenizer.init_kwargs, **kwargs
+        )
+
+        if text is not None:
+            encoding = self.tokenizer(text, return_tensors=return_tensors, **output_kwargs["text_kwargs"])
+
+        if visual_prompt is not None:
+            prompt_features = self.image_processor(
+                visual_prompt, return_tensors=return_tensors, **output_kwargs["images_kwargs"]
+            )
+
+        if images is not None:
+            image_features = self.image_processor(
+                images, return_tensors=return_tensors, **output_kwargs["images_kwargs"]
+            )
+
+        if visual_prompt is not None and images is not None:
+            encoding = {
+                "pixel_values": image_features.pixel_values,
+                "conditional_pixel_values": prompt_features.pixel_values,
+            }
+            return encoding
+        elif text is not None and images is not None:
+            encoding["pixel_values"] = image_features.pixel_values
+            return encoding
+        elif text is not None:
+            return encoding
+        elif visual_prompt is not None:
+            encoding = {
+                "conditional_pixel_values": prompt_features.pixel_values,
+            }
+            return encoding
+        else:
+            return BatchEncoding(data=dict(**image_features), tensor_type=return_tensors)
+
+
+__all__ = ["CLIPSegProcessor"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/ernie4_5_vl_moe/image_processing_pil_ernie4_5_vl_moe.py

@@ -0,0 +1,245 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/ernie4_5_vl_moe/modular_ernie4_5_vl_moe.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_ernie4_5_vl_moe.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2025 Baidu and HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+import numpy as np
+
+from ...image_processing_backends import PilBackend
+from ...image_processing_utils import BatchFeature
+from ...image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, ImageInput, PILImageResampling, SizeDict
+from ...processing_utils import ImagesKwargs, Unpack
+from ...utils import TensorType, auto_docstring, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class Ernie4_5_VLMoeImageProcessorKwargs(ImagesKwargs, total=False):
+    r"""
+    patch_size (`int`, *optional*, defaults to 14):
+        The spatial patch size of the vision encoder.
+    temporal_patch_size (`int`, *optional*):
+        The temporal patch size of the vision encoder. Unused in the image processor, only used for videos.
+    merge_size (`int`, *optional*, defaults to 2):
+        The merge size of the vision encoder to llm encoder.
+    """
+
+    patch_size: int
+    temporal_patch_size: int
+    merge_size: int
+
+
+def smart_resize(
+    height: int, width: int, factor: int = 28, min_pixels: int = 56 * 56, max_pixels: int = 14 * 14 * 4 * 1280
+):
+    """Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+
+    """
+    if max(height, width) / min(height, width) > 200:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}"
+        )
+    h_bar = round(height / factor) * factor
+    w_bar = round(width / factor) * factor
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = max(factor, math.floor(height / beta / factor) * factor)
+        w_bar = max(factor, math.floor(width / beta / factor) * factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = math.ceil(height * beta / factor) * factor
+        w_bar = math.ceil(width * beta / factor) * factor
+    return h_bar, w_bar
+
+
+@auto_docstring
+class Ernie4_5_VLMoeImageProcessorPil(PilBackend):
+    do_resize = True
+    resample = PILImageResampling.BICUBIC
+    size = {"shortest_edge": 56 * 56, "longest_edge": 28 * 28 * 6177}
+    default_to_square = False
+    do_rescale = True
+    rescale_factor = 1 / 255
+    do_normalize = True
+    image_mean = OPENAI_CLIP_MEAN
+    image_std = OPENAI_CLIP_STD
+    do_convert_rgb = True
+    patch_size = 14
+    temporal_patch_size = None  # Unused
+    merge_size = 2
+    valid_kwargs = Ernie4_5_VLMoeImageProcessorKwargs
+    model_input_names = ["pixel_values", "image_grid_thw"]
+
+    def __init__(self, **kwargs: Unpack[Ernie4_5_VLMoeImageProcessorKwargs]):
+        super().__init__(**kwargs)
+        if self.size is not None:
+            if not self.size.shortest_edge or not self.size.longest_edge:
+                raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.")
+
+    @auto_docstring
+    def preprocess(self, images: ImageInput, **kwargs: Unpack[Ernie4_5_VLMoeImageProcessorKwargs]) -> BatchFeature:
+        return super().preprocess(images, **kwargs)
+
+    def _standardize_kwargs(self, **kwargs) -> dict:
+        """
+        Update kwargs that need further processing before being validated
+        Can be overridden by subclasses to customize the processing of kwargs.
+        """
+        kwargs = super()._standardize_kwargs(**kwargs)
+        size = kwargs.get("size", self.size)
+        if not size.shortest_edge or not size.longest_edge:
+            raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.")
+
+        return kwargs
+
+    def _preprocess(
+        self,
+        images: list[np.ndarray],
+        do_resize: bool,
+        size: SizeDict,
+        resample: "PILImageResampling | None",
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: float | list[float] | None,
+        image_std: float | list[float] | None,
+        patch_size: int,
+        merge_size: int,
+        return_tensors: str | TensorType | None,
+        **kwargs,
+    ) -> BatchFeature:
+        """
+        Preprocess images one by one for PIL backend.
+        """
+        processed_images = []
+        processed_grids = []
+
+        for image in images:
+            height, width = image.shape[-2:]
+            if do_resize:
+                resized_height, resized_width = smart_resize(
+                    height=height,
+                    width=width,
+                    factor=patch_size * merge_size,
+                    min_pixels=size.shortest_edge,
+                    max_pixels=size.longest_edge,
+                )
+                image = self.resize(
+                    image,
+                    size=SizeDict(height=resized_height, width=resized_width),
+                    resample=resample,
+                )
+
+            # Rescale and normalize
+            if do_rescale:
+                image = self.rescale(image, rescale_factor)
+            if do_normalize:
+                image = self.normalize(image, image_mean, image_std)
+
+            # Ensure float32 for patch processing
+            image_array = np.asarray(image, dtype=np.float32)
+            if image_array.ndim == 3:  # (C, H, W)
+                image_array = np.expand_dims(image_array, axis=0)  # (1, C, H, W)
+            if image_array.ndim == 4:  # (B, C, H, W)
+                image_array = np.expand_dims(image_array, axis=1)  # (B, T=1, C, H, W)
+
+            resized_height, resized_width = image_array.shape[-2:]
+            batch_size, grid_t, channel = image_array.shape[:3]
+            grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
+
+            patches = image_array.reshape(
+                batch_size,
+                grid_t,
+                channel,
+                grid_h // merge_size,
+                merge_size,
+                patch_size,
+                grid_w // merge_size,
+                merge_size,
+                patch_size,
+            )
+            # Reorder dimensions to group grid and patch information for subsequent flattening.
+            # [batch, grid_t, grid_h/merge, grid_w/merge, merge, merge, channel, patch, patch]
+            patches = np.transpose(patches, (0, 1, 3, 6, 4, 7, 2, 5, 8))
+
+            flatten_patches = patches.reshape(
+                batch_size,
+                grid_t * grid_h * grid_w,
+                channel * patch_size * patch_size,
+            )
+
+            # Remove batch dimension and append: shape is (seq_len, hidden_dim)
+            processed_images.append(flatten_patches.squeeze(0))
+            processed_grids.append([grid_t, grid_h, grid_w])
+
+        # Concatenate all images along sequence dimension: (total_seq_len, hidden_dim)
+        pixel_values = np.concatenate(processed_images, axis=0)
+        image_grid_thw = np.array(processed_grids)
+
+        return BatchFeature(
+            data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}, tensor_type=return_tensors
+        )
+
+    def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None):
+        """
+        A utility that returns number of image patches for a given image size.
+
+        Note: Do not remove this method! It is used by vLLM to infer the number of patches and placeholders
+        without an image input.
+
+        Args:
+            height (`int`):
+                Height of the input image.
+            width (`int`):
+                Width of the input image.
+            images_kwargs (`dict`, *optional*)
+                Any kwargs to override defaults of the image processor.
+        Returns:
+            `int`: Number of image patches per image.
+        """
+        min_pixels = self.size["shortest_edge"]
+        max_pixels = self.size["longest_edge"]
+        patch_size = images_kwargs.get("patch_size", self.patch_size)
+        merge_size = images_kwargs.get("merge_size", self.merge_size)
+
+        factor = patch_size * merge_size
+        resized_height, resized_width = smart_resize(
+            height, width, factor, min_pixels=min_pixels, max_pixels=max_pixels
+        )
+        grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
+        return grid_h * grid_w
+
+
+class Ernie4_5_VL_MoeImageProcessorPil(Ernie4_5_VLMoeImageProcessorPil):
+    def __init__(self, *args, **kwargs):
+        logger.warning_once(
+            "`Ernie4_5_VL_MoeImageProcessorPil` is deprecated; please use `Ernie4_5_VLMoeImageProcessorPil` instead.",
+        )
+        super().__init__(*args, **kwargs)
+
+
+__all__ = ["Ernie4_5_VL_MoeImageProcessorPil", "Ernie4_5_VLMoeImageProcessorPil"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/sam3_tracker/__init__.py

@@ -0,0 +1,29 @@
+# Copyright 2025 the HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_sam3_tracker import *
+    from .modeling_sam3_tracker import *
+    from .processing_sam3_tracker import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/sam3_tracker/configuration_sam3_tracker.py

@@ -0,0 +1,164 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/sam3_tracker/modular_sam3_tracker.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_sam3_tracker.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2025 the HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+from ..auto import CONFIG_MAPPING, AutoConfig
+
+
+@auto_docstring(checkpoint="facebook/sam3")
+@strict
+class Sam3TrackerPromptEncoderConfig(PreTrainedConfig):
+    r"""
+    mask_input_channels (`int`, *optional*, defaults to 16):
+        The number of channels to be fed to the `MaskDecoder` module.
+    num_point_embeddings (`int`, *optional*, defaults to 4):
+        The number of point embeddings to be used.
+    scale (`float`, *optional*, defaults to 1):
+        The scale factor for the prompt encoder.
+    """
+
+    base_config_key = "prompt_encoder_config"
+
+    hidden_size: int = 256
+
+    image_size: int | list[int] | tuple[int, int] = 1008
+    patch_size: int | list[int] | tuple[int, int] = 14
+    mask_input_channels: int = 16
+    num_point_embeddings: int = 4
+    hidden_act: str = "gelu"
+    layer_norm_eps: float = 1e-6
+    scale: int = 1
+
+
+@auto_docstring(checkpoint="facebook/sam3")
+@strict
+class Sam3TrackerMaskDecoderConfig(PreTrainedConfig):
+    r"""
+    mlp_dim (`int`, *optional*, defaults to 2048):
+        The dimension of the MLP in the two-way transformer.
+    attention_downsample_rate (`int`, *optional*, defaults to 2):
+        The downsample rate for the attention layers.
+    num_multimask_outputs (`int`, *optional*, defaults to 3):
+        The number of multimask outputs.
+    iou_head_depth (`int`, *optional*, defaults to 3):
+        The depth of the IoU head.
+    iou_head_hidden_dim (`int`, *optional*, defaults to 256):
+        The hidden dimension of the IoU head.
+    dynamic_multimask_via_stability (`bool`, *optional*, defaults to `True`):
+        Whether to use dynamic multimask via stability.
+    dynamic_multimask_stability_delta (`float`, *optional*, defaults to 0.05):
+        The stability delta for the dynamic multimask.
+    dynamic_multimask_stability_thresh (`float`, *optional*, defaults to 0.98):
+        The stability threshold for the dynamic multimask.
+    """
+
+    base_config_key = "mask_decoder_config"
+
+    hidden_size: int = 256
+    hidden_act: str = "gelu"
+    mlp_dim: int = 2048
+    num_hidden_layers: int = 2
+    num_attention_heads: int = 8
+    attention_downsample_rate: int = 2
+    num_multimask_outputs: int = 3
+    iou_head_depth: int = 3
+    iou_head_hidden_dim: int = 256
+    dynamic_multimask_via_stability: bool = True
+    dynamic_multimask_stability_delta: float = 0.05
+    dynamic_multimask_stability_thresh: float = 0.98
+
+
+@auto_docstring(checkpoint="facebook/sam3")
+@strict
+class Sam3TrackerConfig(PreTrainedConfig):
+    r"""
+    prompt_encoder_config (Union[`dict`, `Sam3TrackerPromptEncoderConfig`], *optional*):
+        Dictionary of configuration options used to initialize [`Sam3TrackerPromptEncoderConfig`].
+    mask_decoder_config (Union[`dict`, `Sam3TrackerMaskDecoderConfig`], *optional*):
+        Dictionary of configuration options used to initialize [`Sam3TrackerMaskDecoderConfig`].
+
+    Example:
+
+    ```python
+    >>> from transformers import (
+    ...     Sam3TrackerVisionConfig,
+    ...     Sam3TrackerPromptEncoderConfig,
+    ...     Sam3TrackerMaskDecoderConfig,
+    ...     Sam3TrackerModel,
+    ... )
+
+    >>> # Initializing a Sam3TrackerConfig with `"facebook/sam3_tracker.1_hiera_tiny"` style configuration
+    >>> configuration = Sam3TrackerConfig()
+
+    >>> # Initializing a Sam3TrackerModel (with random weights) from the `"facebook/sam3_tracker.1_hiera_tiny"` style configuration
+    >>> model = Sam3TrackerModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+
+    >>> # We can also initialize a Sam3TrackerConfig from a Sam3TrackerVisionConfig, Sam3TrackerPromptEncoderConfig, and Sam3TrackerMaskDecoderConfig
+    >>> # Initializing SAM3_TRACKER vision encoder, memory attention, and memory encoder configurations
+    >>> vision_config = Sam3TrackerVisionConfig()
+    >>> prompt_encoder_config = Sam3TrackerPromptEncoderConfig()
+    >>> mask_decoder_config = Sam3TrackerMaskDecoderConfig()
+
+    >>> config = Sam3TrackerConfig(vision_config, prompt_encoder_config, mask_decoder_config)
+    ```
+    """
+
+    model_type = "sam3_tracker"
+    sub_configs = {
+        "vision_config": AutoConfig,
+        "prompt_encoder_config": Sam3TrackerPromptEncoderConfig,
+        "mask_decoder_config": Sam3TrackerMaskDecoderConfig,
+    }
+
+    vision_config: dict | PreTrainedConfig | None = None
+    prompt_encoder_config: dict | PreTrainedConfig | None = None
+    mask_decoder_config: dict | PreTrainedConfig | None = None
+    initializer_range: float = 0.02
+
+    def __post_init__(self, **kwargs):
+        if isinstance(self.vision_config, dict):
+            self.vision_config["model_type"] = self.vision_config.get("model_type", "sam3_vision_model")
+            self.vision_config = CONFIG_MAPPING[self.vision_config["model_type"]](**self.vision_config)
+        elif self.vision_config is None:
+            self.vision_config = CONFIG_MAPPING["sam3_vision_model"](
+                backbone_feature_sizes=[[288, 288], [144, 144], [72, 72]]
+            )
+
+        if isinstance(self.prompt_encoder_config, dict):
+            self.prompt_encoder_config = Sam3TrackerPromptEncoderConfig(**self.prompt_encoder_config)
+        elif self.prompt_encoder_config is None:
+            self.prompt_encoder_config = Sam3TrackerPromptEncoderConfig()
+
+        if isinstance(self.mask_decoder_config, dict):
+            self.mask_decoder_config = Sam3TrackerMaskDecoderConfig(**self.mask_decoder_config)
+        elif self.mask_decoder_config is None:
+            self.mask_decoder_config = Sam3TrackerMaskDecoderConfig()
+
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["Sam3TrackerConfig", "Sam3TrackerPromptEncoderConfig", "Sam3TrackerMaskDecoderConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/sam3_tracker/modeling_sam3_tracker.py

@@ -0,0 +1,1106 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/sam3_tracker/modular_sam3_tracker.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_sam3_tracker.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2025 the HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from collections.abc import Callable
+from dataclasses import dataclass
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import Tensor
+
+from ... import initialization as init
+from ...activations import ACT2FN
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
+from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from ...processing_utils import Unpack
+from ...utils import ModelOutput, auto_docstring, can_return_tuple, logging
+from ...utils.generic import TransformersKwargs, is_flash_attention_requested, merge_with_config_defaults
+from ...utils.output_capturing import OutputRecorder, capture_outputs
+from ..auto import AutoModel
+from .configuration_sam3_tracker import Sam3TrackerConfig, Sam3TrackerMaskDecoderConfig, Sam3TrackerPromptEncoderConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+@auto_docstring(custom_intro="Base class for the Sam3Tracker model's output.")
+@dataclass
+class Sam3TrackerImageSegmentationOutput(ModelOutput):
+    r"""
+    iou_scores (`torch.FloatTensor` of shape `(batch_size, point_batch_size, num_masks)`):
+        The Intersection over Union (IoU) scores of the predicted masks.
+    pred_masks (`torch.FloatTensor` of shape `(batch_size, point_batch_size, num_masks, height, width)`):
+        The predicted low-resolution masks. This is an alias for `low_res_masks`. These masks need to be post-processed
+        by the processor to be brought to the original image size.
+    object_score_logits (`torch.FloatTensor` of shape `(batch_size, point_batch_size, 1)`):
+        Logits for the object score, indicating if an object is present.
+    image_embeddings (`tuple(torch.FloatTensor)`):
+        The features from the FPN, which are used by the mask decoder. This is a tuple of `torch.FloatTensor` where each
+        tensor has shape `(batch_size, channels, height, width)`.
+    vision_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True`):
+        Tuple of `torch.FloatTensor` (one for the output of each stage) of shape `(batch_size, height, width, hidden_size)`.
+        Hidden-states of the vision model at the output of each stage.
+    vision_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True`):
+        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+        Attentions weights of the vision model.
+    mask_decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True`):
+        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+        Attentions weights of the mask decoder.
+    """
+
+    iou_scores: torch.FloatTensor | None = None
+    pred_masks: torch.FloatTensor | None = None
+    object_score_logits: torch.FloatTensor | None = None
+    image_embeddings: tuple[torch.FloatTensor, ...] = None
+    vision_hidden_states: tuple[torch.FloatTensor, ...] | None = None
+    vision_attentions: tuple[torch.FloatTensor, ...] | None = None
+    mask_decoder_attentions: tuple[torch.FloatTensor, ...] | None = None
+
+
+class Sam3TrackerFeedForward(nn.Module):
+    def __init__(
+        self,
+        input_dim: int,
+        hidden_dim: int,
+        output_dim: int,
+        num_layers: int,
+        activation: str = "relu",
+        sigmoid_output: bool = False,
+    ):
+        super().__init__()
+        self.num_layers = num_layers
+        self.activation = ACT2FN[activation]
+        self.proj_in = nn.Linear(input_dim, hidden_dim)
+        self.proj_out = nn.Linear(hidden_dim, output_dim)
+        self.layers = nn.ModuleList([nn.Linear(hidden_dim, hidden_dim) for _ in range(num_layers - 2)])
+        self.sigmoid_output = sigmoid_output
+
+    def forward(self, hidden_states):
+        hidden_states = self.proj_in(hidden_states)
+        hidden_states = self.activation(hidden_states)
+        for layer in self.layers:
+            hidden_states = self.activation(layer(hidden_states))
+
+        hidden_states = self.proj_out(hidden_states)
+        if self.sigmoid_output:
+            hidden_states = F.sigmoid(hidden_states)
+        return hidden_states
+
+
+@auto_docstring(
+    custom_intro="""
+    Segment Anything Model 3 (SAM 3) for generating segmentation masks, given an input image and
+    input points and labels, boxes, or masks.
+    """
+)
+class Sam3TrackerPreTrainedModel(PreTrainedModel):
+    config_class = Sam3TrackerConfig
+    base_model_prefix = "sam3_tracker"
+    main_input_name = "pixel_values"
+    input_modalities = ("image",)
+    _supports_sdpa = True
+    _supports_flash_attn = True
+    _supports_attention_backend = True
+    _keys_to_ignore_on_load_unexpected = [
+        r"^memory_.*",
+        r"^mask_downsample.*",
+        r"^object_pointer_proj.*",
+        r"^temporal_positional_encoding_projection_layer.*",
+        "no_memory_positional_encoding",
+        "no_object_pointer",
+        "occlusion_spatial_embedding_parameter",
+    ]
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        super()._init_weights(module)
+        if isinstance(module, Sam3TrackerModel):
+            if module.no_memory_embedding is not None:
+                init.zeros_(module.no_memory_embedding)
+        elif isinstance(module, Sam3TrackerPositionalEmbedding):
+            init.normal_(module.positional_embedding, std=module.scale)
+
+
+class Sam3TrackerPositionalEmbedding(nn.Module):
+    def __init__(self, config: Sam3TrackerPromptEncoderConfig):
+        super().__init__()
+        self.scale = config.scale
+        positional_embedding = self.scale * torch.randn((2, config.hidden_size // 2))
+        self.register_buffer("positional_embedding", positional_embedding)
+
+    def forward(self, input_coords, input_shape=None):
+        """Positionally encode points that are normalized to [0,1]."""
+        coordinates = input_coords.clone()
+
+        if input_shape is not None:
+            coordinates[:, :, :, 0] = coordinates[:, :, :, 0] / input_shape[1]
+            coordinates[:, :, :, 1] = coordinates[:, :, :, 1] / input_shape[0]
+        coordinates.to(torch.float32)
+
+        # assuming coords are in [0, 1]^2 square and have d_1 x ... x d_n x 2 shape
+        coordinates = 2 * coordinates - 1
+        coordinates = coordinates.to(self.positional_embedding.dtype)
+        coordinates = coordinates @ self.positional_embedding
+        coordinates = 2 * np.pi * coordinates
+        # outputs d_1 x ... x d_n x channel shape
+        return torch.cat([torch.sin(coordinates), torch.cos(coordinates)], dim=-1)
+
+
+class Sam3TrackerMaskEmbedding(nn.Module):
+    def __init__(self, config: Sam3TrackerPromptEncoderConfig):
+        super().__init__()
+        self.mask_input_channels = config.mask_input_channels // 4
+        self.activation = ACT2FN[config.hidden_act]
+        self.conv1 = nn.Conv2d(1, self.mask_input_channels, kernel_size=2, stride=2)
+        self.conv2 = nn.Conv2d(self.mask_input_channels, config.mask_input_channels, kernel_size=2, stride=2)
+        self.conv3 = nn.Conv2d(config.mask_input_channels, config.hidden_size, kernel_size=1)
+        self.layer_norm1 = Sam3TrackerLayerNorm(
+            self.mask_input_channels, eps=config.layer_norm_eps, data_format="channels_first"
+        )
+        self.layer_norm2 = Sam3TrackerLayerNorm(
+            self.mask_input_channels * 4, eps=config.layer_norm_eps, data_format="channels_first"
+        )
+
+    def forward(self, masks):
+        hidden_states = self.conv1(masks)
+        hidden_states = self.layer_norm1(hidden_states)
+        hidden_states = self.activation(hidden_states)
+
+        hidden_states = self.conv2(hidden_states)
+        hidden_states = self.layer_norm2(hidden_states)
+        hidden_states = self.activation(hidden_states)
+        dense_embeddings = self.conv3(hidden_states)
+        return dense_embeddings
+
+
+class Sam3TrackerPromptEncoder(nn.Module):
+    def __init__(self, config: Sam3TrackerPromptEncoderConfig):
+        super().__init__()
+        self.shared_embedding = Sam3TrackerPositionalEmbedding(config)
+        self.mask_embed = Sam3TrackerMaskEmbedding(config)
+        self.no_mask_embed = nn.Embedding(1, config.hidden_size)
+
+        self.image_embedding_size = (config.image_size // config.patch_size, config.image_size // config.patch_size)
+        self.mask_input_size = (4 * config.image_size // config.patch_size, 4 * config.image_size // config.patch_size)
+        self.input_image_size = config.image_size
+
+        self.point_embed = nn.Embedding(config.num_point_embeddings, config.hidden_size)
+        self.hidden_size = config.hidden_size
+        self.not_a_point_embed = nn.Embedding(1, config.hidden_size)
+
+    def _embed_points(self, points: torch.Tensor, labels: torch.Tensor, pad: bool) -> torch.Tensor:
+        """Embeds point prompts."""
+        points = points + 0.5  # Shift to center of pixel
+        if pad:
+            points = torch.nn.functional.pad(points, (0, 0, 0, 1), mode="constant", value=0)
+            labels = torch.nn.functional.pad(labels, (0, 1), mode="constant", value=-1)
+        input_shape = (self.input_image_size, self.input_image_size)
+        point_embedding = self.shared_embedding(points, input_shape)
+
+        # torch.where and expanding the labels tensor is required by the ONNX export
+        point_embedding = torch.where(labels[..., None] == -1, self.not_a_point_embed.weight, point_embedding)
+
+        # This is required for the ONNX export. The dtype, device need to be explicitly
+        # specified as otherwise torch.onnx.export interprets as double
+        point_embedding = torch.where(
+            labels[..., None] != -10,
+            point_embedding,
+            torch.zeros_like(point_embedding),
+        )
+
+        # Add point embeddings for labels >= 0
+        point_embedding = point_embedding + self.point_embed(labels.clamp(min=0)) * (labels >= 0).unsqueeze(-1)
+
+        return point_embedding
+
+    def _embed_boxes(self, boxes: torch.Tensor) -> torch.Tensor:
+        """Embeds box prompts."""
+        boxes = boxes + 0.5  # Shift to center of pixel
+        coords = boxes.view(*boxes.shape[:2], 2, 2)
+        # add padding point for consistency with the original implementation
+        coords = torch.nn.functional.pad(coords, (0, 0, 0, 1), mode="constant", value=0)
+        corner_embedding = self.shared_embedding(coords, (self.input_image_size, self.input_image_size))
+        corner_embedding[:, :, 0, :] += self.point_embed.weight[2]
+        corner_embedding[:, :, 1, :] += self.point_embed.weight[3]
+        corner_embedding[:, :, 2, :] = self.not_a_point_embed.weight.expand_as(corner_embedding[:, :, 2, :])
+        return corner_embedding
+
+    def forward(
+        self,
+        input_points: tuple[torch.Tensor, torch.Tensor] | None,
+        input_labels: torch.Tensor | None,
+        input_boxes: torch.Tensor | None,
+        input_masks: torch.Tensor | None,
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """
+        Embeds different types of prompts, returning both sparse and dense embeddings.
+
+        Args:
+            points (`torch.Tensor`, *optional*):
+                point coordinates and labels to embed.
+            boxes (`torch.Tensor`, *optional*):
+                boxes to embed
+            masks (`torch.Tensor`, *optional*):
+                masks to embed
+        """
+        sparse_embeddings = None
+        batch_size = 1
+        if input_points is not None:
+            batch_size = input_points.shape[0]
+            if input_labels is None:
+                raise ValueError("If points are provided, labels must also be provided.")
+            point_embeddings = self._embed_points(input_points, input_labels, pad=(input_boxes is None))
+            sparse_embeddings = point_embeddings
+        if input_boxes is not None:
+            batch_size = input_boxes.shape[0]
+            box_embeddings = self._embed_boxes(input_boxes)
+            if sparse_embeddings is None:
+                sparse_embeddings = box_embeddings
+            else:
+                sparse_embeddings = torch.cat([sparse_embeddings, box_embeddings], dim=2)
+        if input_masks is not None:
+            dense_embeddings = self.mask_embed(input_masks)
+        else:
+            dense_embeddings = self.no_mask_embed.weight.reshape(1, -1, 1, 1).expand(
+                batch_size, -1, self.image_embedding_size[0], self.image_embedding_size[1]
+            )
+
+        return sparse_embeddings, dense_embeddings
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    scaling: float,
+    dropout: float = 0.0,
+    **kwargs,
+):
+    attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling
+    if attention_mask is not None:
+        attn_weights = attn_weights + attention_mask
+
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query.dtype)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+    attn_output = torch.matmul(attn_weights, value)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+class Sam3TrackerAttention(nn.Module):
+    """
+    SAM3_TRACKER's attention layer that allows for downscaling the size of the embedding after projection to queries, keys, and
+    values.
+    """
+
+    def __init__(self, config, downsample_rate=None):
+        super().__init__()
+        downsample_rate = config.attention_downsample_rate if downsample_rate is None else downsample_rate
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.internal_dim = config.hidden_size // downsample_rate
+        self.num_attention_heads = config.num_attention_heads
+        self.head_dim = self.internal_dim // config.num_attention_heads
+        self.scaling = self.head_dim**-0.5
+        self.is_causal = False
+
+        self.q_proj = nn.Linear(self.hidden_size, self.internal_dim)
+        self.k_proj = nn.Linear(self.hidden_size, self.internal_dim)
+        self.v_proj = nn.Linear(self.hidden_size, self.internal_dim)
+        self.o_proj = nn.Linear(self.internal_dim, self.hidden_size)
+
+    def forward(
+        self,
+        query: torch.Tensor,
+        key: torch.Tensor,
+        value: torch.Tensor,
+        attention_similarity: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        # Input projections
+        batch_size, point_batch_size = query.shape[:2]
+        new_shape = (batch_size * point_batch_size, -1, self.num_attention_heads, self.head_dim)
+
+        query = self.q_proj(query).view(*new_shape).transpose(1, 2)
+        key = self.k_proj(key).view(*new_shape).transpose(1, 2)
+        value = self.v_proj(value).view(*new_shape).transpose(1, 2)
+
+        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, eager_attention_forward
+        )
+
+        if is_flash_attention_requested(self.config) and attention_similarity is not None:
+            # Target guided masks are represented as float masks and are incompatible with Flash Attention
+            # Fallback to SDPA for this call only so the rest of the model can still benefit from FA
+            attention_interface = ALL_ATTENTION_FUNCTIONS["sdpa"]
+            logger.warning_once(
+                "Falling back to SDPA for target-guided attention because "
+                "Flash Attention does not support additive bias masks."
+            )
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query,
+            key,
+            value,
+            attention_mask=attention_similarity,
+            dropout=0.0,
+            scaling=self.scaling,
+            is_causal=self.is_causal,
+            **kwargs,
+        )
+
+        attn_output = attn_output.reshape(
+            batch_size, point_batch_size, -1, self.num_attention_heads * self.head_dim
+        ).contiguous()
+        attn_output = self.o_proj(attn_output)
+
+        return attn_output, attn_weights
+
+
+class Sam3TrackerTwoWayAttentionBlock(GradientCheckpointingLayer):
+    def __init__(self, config: Sam3TrackerMaskDecoderConfig, skip_first_layer_pe: bool = False):
+        """
+        A transformer block with four layers:
+            (1) self-attention of sparse inputs (2) cross attention of sparse inputs -> dense inputs (3) mlp block on
+            sparse inputs (4) cross attention of dense inputs -> sparse inputs
+
+        Arguments:
+            config (`Sam3TrackerMaskDecoderConfig`):
+                The configuration file used to instantiate the block
+            attention_downsample_rate (*optionalk*, int, defaults to 2):
+                The downsample ratio of the block used to reduce the inner dim of the attention.
+            skip_first_layer_pe (*optional*, bool, defaults to `False`):
+                Whether or not to skip the addition of the query_point_embedding on the first layer.
+        """
+        super().__init__()
+        self.self_attn = Sam3TrackerAttention(config, downsample_rate=1)
+        self.layer_norm1 = nn.LayerNorm(config.hidden_size)
+
+        self.cross_attn_token_to_image = Sam3TrackerAttention(config)
+        self.layer_norm2 = nn.LayerNorm(config.hidden_size)
+
+        self.mlp = Sam3TrackerFeedForward(
+            config.hidden_size, config.mlp_dim, config.hidden_size, num_layers=config.num_hidden_layers
+        )
+        self.layer_norm3 = nn.LayerNorm(config.hidden_size)
+
+        self.layer_norm4 = nn.LayerNorm(config.hidden_size)
+        self.cross_attn_image_to_token = Sam3TrackerAttention(config)
+
+        self.skip_first_layer_pe = skip_first_layer_pe
+
+    def forward(
+        self,
+        queries: Tensor,
+        keys: Tensor,
+        query_point_embedding: Tensor,
+        key_point_embedding: Tensor,
+        attention_similarity: Tensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ):
+        # Self attention block
+        if self.skip_first_layer_pe:
+            queries, _ = self.self_attn(query=queries, key=queries, value=queries)
+        else:
+            query = queries + query_point_embedding
+            attn_out, _ = self.self_attn(query=query, key=query, value=queries)
+            queries = queries + attn_out
+        queries = self.layer_norm1(queries)
+
+        # Cross attention block, tokens attending to image embedding
+        query = queries + query_point_embedding
+        key = keys + key_point_embedding
+
+        attn_out, _ = self.cross_attn_token_to_image(
+            query=query, key=key, value=keys, attention_similarity=attention_similarity
+        )
+        queries = queries + attn_out
+
+        queries = self.layer_norm2(queries)
+
+        # MLP block
+        mlp_out = self.mlp(queries)
+        queries = queries + mlp_out
+        queries = self.layer_norm3(queries)
+
+        # Cross attention block, image embedding attending to tokens
+        query = queries + query_point_embedding
+        key = keys + key_point_embedding
+
+        attn_out, _ = self.cross_attn_image_to_token(query=key, key=query, value=queries)
+        keys = keys + attn_out
+
+        keys = self.layer_norm4(keys)
+        return queries, keys, attn_out
+
+
+class Sam3TrackerTwoWayTransformer(nn.Module):
+    def __init__(self, config: Sam3TrackerMaskDecoderConfig):
+        super().__init__()
+        self.config = config
+
+        self.num_hidden_layers = config.num_hidden_layers
+        self.layers = nn.ModuleList()
+
+        for i in range(self.num_hidden_layers):
+            self.layers.append(Sam3TrackerTwoWayAttentionBlock(config, skip_first_layer_pe=(i == 0)))
+
+        self.final_attn_token_to_image = Sam3TrackerAttention(config)
+        self.layer_norm_final_attn = nn.LayerNorm(config.hidden_size)
+
+    def forward(
+        self,
+        point_embeddings: Tensor,
+        image_embeddings: Tensor,
+        image_positional_embeddings: Tensor,
+        attention_similarity: Tensor,
+        target_embedding=None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutput:
+        if image_embeddings is None:
+            raise ValueError("You have to specify an image_embedding")
+
+        image_embeddings = image_embeddings.flatten(2).transpose(1, 2).unsqueeze(1)
+        image_positional_embeddings = image_positional_embeddings.flatten(2).transpose(1, 2).unsqueeze(1)
+
+        # Prepare queries
+        queries = point_embeddings
+        keys = image_embeddings
+
+        # Apply transformer blocks and final layernorm
+        for layer in self.layers:
+            if target_embedding is not None:
+                queries += target_embedding
+
+            queries, keys, _ = layer(
+                queries=queries,
+                keys=keys,
+                query_point_embedding=point_embeddings,
+                key_point_embedding=image_positional_embeddings,
+                attention_similarity=attention_similarity,
+                **kwargs,
+            )
+        # Apply the final attention layer from the points to the image
+        query = queries + point_embeddings
+        key = keys + image_positional_embeddings
+
+        attn_out, _ = self.final_attn_token_to_image(query=query, key=key, value=keys)
+
+        queries = queries + attn_out
+        queries = self.layer_norm_final_attn(queries)
+        return queries, keys
+
+
+class Sam3TrackerLayerNorm(nn.LayerNorm):
+    r"""LayerNorm that supports two data formats: channels_last (default) or channels_first.
+    The ordering of the dimensions in the inputs. channels_last corresponds to inputs with shape (batch_size, height,
+    width, channels) while channels_first corresponds to inputs with shape (batch_size, channels, height, width).
+    """
+
+    def __init__(self, normalized_shape, *, eps=1e-6, data_format="channels_last", **kwargs):
+        super().__init__(normalized_shape, eps=eps, **kwargs)
+        if data_format not in ["channels_last", "channels_first"]:
+            raise NotImplementedError(f"Unsupported data format: {data_format}")
+        self.data_format = data_format
+
+    def forward(self, features: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            features: Tensor of shape (batch_size, channels, height, width) OR (batch_size, height, width, channels)
+        """
+        if self.data_format == "channels_first":
+            features = features.permute(0, 2, 3, 1)
+            features = super().forward(features)
+            features = features.permute(0, 3, 1, 2)
+        else:
+            features = super().forward(features)
+        return features
+
+
+class Sam3TrackerMaskDecoder(nn.Module):
+    def __init__(self, config: Sam3TrackerMaskDecoderConfig):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+
+        self.num_multimask_outputs = config.num_multimask_outputs
+        self.num_mask_tokens = config.num_multimask_outputs + 1
+
+        self.iou_token = nn.Embedding(1, self.hidden_size)
+        self.mask_tokens = nn.Embedding(self.num_mask_tokens, self.hidden_size)
+
+        self.transformer = Sam3TrackerTwoWayTransformer(config)
+
+        # should we create a new class for this?
+        self.upscale_conv1 = nn.ConvTranspose2d(self.hidden_size, self.hidden_size // 4, kernel_size=2, stride=2)
+        self.upscale_conv2 = nn.ConvTranspose2d(self.hidden_size // 4, self.hidden_size // 8, kernel_size=2, stride=2)
+        self.upscale_layer_norm = Sam3TrackerLayerNorm(self.hidden_size // 4, data_format="channels_first")
+        self.activation = nn.GELU()
+
+        mlps_list = []
+        for _ in range(self.num_mask_tokens):
+            mlps_list += [Sam3TrackerFeedForward(self.hidden_size, self.hidden_size, self.hidden_size // 8, 3)]
+        self.output_hypernetworks_mlps = nn.ModuleList(mlps_list)
+        self.iou_prediction_head = Sam3TrackerFeedForward(
+            self.hidden_size,
+            config.iou_head_hidden_dim,
+            self.num_mask_tokens,
+            config.iou_head_depth,
+            sigmoid_output=True,
+        )
+
+        self.conv_s0 = nn.Conv2d(config.hidden_size, config.hidden_size // 8, kernel_size=1, stride=1)
+        self.conv_s1 = nn.Conv2d(config.hidden_size, config.hidden_size // 4, kernel_size=1, stride=1)
+
+        self.obj_score_token = nn.Embedding(1, self.hidden_size)
+        self.pred_obj_score_head = Sam3TrackerFeedForward(self.hidden_size, self.hidden_size, 1, 3)
+
+        self.dynamic_multimask_via_stability = config.dynamic_multimask_via_stability
+        self.dynamic_multimask_stability_delta = config.dynamic_multimask_stability_delta
+        self.dynamic_multimask_stability_thresh = config.dynamic_multimask_stability_thresh
+
+    def forward(
+        self,
+        image_embeddings: torch.Tensor,
+        image_positional_embeddings: torch.Tensor,
+        sparse_prompt_embeddings: torch.Tensor,
+        dense_prompt_embeddings: torch.Tensor,
+        multimask_output: bool,
+        high_resolution_features: list[torch.Tensor],
+        attention_similarity: torch.Tensor | None = None,
+        target_embedding: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
+        """
+        Predict masks given image and prompt embeddings.
+
+        Args:
+            image_embeddings (`torch.Tensor`):
+                The embeddings from the image encoder.
+            image_positional_embeddings (`torch.Tensor`):
+                Positional encoding with the shape of image_embeddings.
+            sparse_prompt_embeddings (`torch.Tensor`):
+                The embeddings of the points and boxes.
+            dense_prompt_embeddings (`torch.Tensor`):
+                The embeddings of the mask inputs.
+            multimask_output (`bool`):
+                Whether to return multiple masks or a single mask.
+            high_resolution_features (`list[torch.Tensor]`, *optional*):
+                The high-resolution features from the vision encoder.
+            attention_similarity (`torch.Tensor`, *optional*):
+                The attention similarity tensor.
+            target_embedding (`torch.Tensor`, *optional*):
+                The target embedding.
+        """
+        batch_size, num_channels, height, width = image_embeddings.shape
+        point_batch_size = sparse_prompt_embeddings.shape[1]
+        # Concatenate output tokens
+        output_tokens = torch.cat(
+            [
+                self.obj_score_token.weight,
+                self.iou_token.weight,
+                self.mask_tokens.weight,
+            ],
+            dim=0,
+        )
+        output_tokens = output_tokens.repeat(batch_size, point_batch_size, 1, 1)
+
+        if sparse_prompt_embeddings.shape[0] != 0:
+            tokens = torch.cat((output_tokens, sparse_prompt_embeddings), dim=2)
+        else:
+            tokens = output_tokens
+        point_embeddings = tokens.to(self.iou_token.weight.dtype)
+
+        # Expand per-image data in batch direction to be per-mask
+        image_embeddings = image_embeddings + dense_prompt_embeddings
+        image_embeddings = image_embeddings.repeat_interleave(point_batch_size, dim=0)
+        image_positional_embeddings = image_positional_embeddings.repeat_interleave(point_batch_size, 0)
+        # Run the transformer
+        point_embeddings, image_embeddings = self.transformer(
+            point_embeddings=point_embeddings,
+            image_embeddings=image_embeddings,
+            image_positional_embeddings=image_positional_embeddings,
+            attention_similarity=attention_similarity,
+            target_embedding=target_embedding,
+            **kwargs,
+        )
+        iou_token_out = point_embeddings[:, :, 1, :]
+        mask_tokens_out = point_embeddings[:, :, 2 : (2 + self.num_mask_tokens), :]
+
+        # Upscale mask embeddings and predict masks using the mask tokens
+        image_embeddings = image_embeddings.transpose(2, 3).view(
+            batch_size * point_batch_size, num_channels, height, width
+        )
+
+        feat_s0, feat_s1 = high_resolution_features
+        feat_s0 = feat_s0.repeat_interleave(point_batch_size, dim=0)
+        feat_s1 = feat_s1.repeat_interleave(point_batch_size, dim=0)
+        upscaled_embedding = self.upscale_conv1(image_embeddings) + feat_s1
+        upscaled_embedding = self.activation(self.upscale_layer_norm(upscaled_embedding))
+        upscaled_embedding = self.activation(self.upscale_conv2(upscaled_embedding) + feat_s0)
+
+        hyper_in_list: list[torch.Tensor] = []
+        for i in range(self.num_mask_tokens):
+            current_mlp = self.output_hypernetworks_mlps[i]
+            hyper_in_list += [current_mlp(mask_tokens_out[:, :, i, :])]
+        hyper_in = torch.stack(hyper_in_list, dim=2)
+
+        _, num_channels, height, width = upscaled_embedding.shape
+        upscaled_embedding = upscaled_embedding.view(batch_size, point_batch_size, num_channels, height * width)
+        masks = (hyper_in @ upscaled_embedding).view(batch_size, point_batch_size, -1, height, width)
+
+        # Generate mask quality predictions
+        iou_pred = self.iou_prediction_head(iou_token_out)
+        object_score_logits = self.pred_obj_score_head(point_embeddings[:, :, 0, :])
+
+        # Select the correct mask or masks for output
+        if multimask_output:
+            mask_slice = slice(1, None)
+            masks = masks[:, :, mask_slice, :, :]
+            iou_pred = iou_pred[:, :, mask_slice]
+        elif self.dynamic_multimask_via_stability and not self.training:
+            mask_slice = slice(0, 1)
+            masks, iou_pred = self._dynamic_multimask_via_stability(masks, iou_pred)
+        else:
+            mask_slice = slice(0, 1)
+            masks = masks[:, :, mask_slice, :, :]
+            iou_pred = iou_pred[:, :, mask_slice]
+
+        sam_tokens_out = mask_tokens_out[:, :, mask_slice]  # [b, 3, c] shape
+
+        return masks, iou_pred, sam_tokens_out, object_score_logits
+
+    def _get_stability_scores(self, mask_logits):
+        """
+        Compute stability scores of the mask logits based on the IoU between upper and
+        lower thresholds.
+        """
+        mask_logits = mask_logits.flatten(-2)
+        stability_delta = self.dynamic_multimask_stability_delta
+        area_i = torch.sum(mask_logits > stability_delta, dim=-1).float()
+        area_u = torch.sum(mask_logits > -stability_delta, dim=-1).float()
+        stability_scores = torch.where(area_u > 0, area_i / area_u, 1.0)
+        return stability_scores
+
+    def _dynamic_multimask_via_stability(self, all_mask_logits, all_iou_scores):
+        """
+        When outputting a single mask, if the stability score from the current single-mask
+        output (based on output token 0) falls below a threshold, we instead select from
+        multi-mask outputs (based on output token 1~3) the mask with the highest predicted
+        IoU score. This is intended to ensure a valid mask for both clicking and tracking.
+        """
+        # The best mask from multimask output tokens (1~3)
+        multimask_logits = all_mask_logits[:, :, 1:, :, :]
+        multimask_iou_scores = all_iou_scores[:, :, 1:]
+        best_scores_inds = torch.argmax(multimask_iou_scores, dim=-1)  # [B, P]
+        best_scores_inds_expanded = best_scores_inds.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1)
+        best_scores_inds_expanded = best_scores_inds_expanded.expand(
+            -1, -1, 1, multimask_logits.size(-2), multimask_logits.size(-1)
+        )
+        best_multimask_logits = torch.gather(multimask_logits, 2, best_scores_inds_expanded)  # [B, P, 1, H, W]
+        best_multimask_iou_scores = torch.gather(multimask_iou_scores, 2, best_scores_inds.unsqueeze(-1))  # [B, P, 1]
+
+        # The mask from singlemask output token 0 and its stability score
+        singlemask_logits = all_mask_logits[:, :, 0:1, :, :]
+        singlemask_iou_scores = all_iou_scores[:, :, 0:1]
+        stability_scores = self._get_stability_scores(singlemask_logits)
+        is_stable = stability_scores >= self.dynamic_multimask_stability_thresh
+
+        # Dynamically fall back to best multimask output upon low stability scores.
+        mask_logits_out = torch.where(
+            is_stable[..., None, None].expand_as(singlemask_logits),
+            singlemask_logits,
+            best_multimask_logits,
+        )
+        iou_scores_out = torch.where(
+            is_stable.expand_as(singlemask_iou_scores),
+            singlemask_iou_scores,
+            best_multimask_iou_scores,
+        )
+        return mask_logits_out, iou_scores_out
+
+
+@auto_docstring(custom_intro="Base class for the vision encoder's outputs.")
+@dataclass
+class Sam3TrackerVisionEncoderOutput(BaseModelOutputWithPooling):
+    r"""
+    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, height, width, hidden_size)`):
+        Sequence of hidden-states at the output of the last layer of the model.
+    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+        Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+        one for the output of each stage) of shape `(batch_size, height, width, hidden_size)`. Hidden-states of the
+        model at the output of each stage.
+    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+        sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+        the self-attention heads.
+    fpn_hidden_states (`tuple(torch.FloatTensor)`):
+        Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape
+        `(batch_size, hidden_size, height, width)`. Feature maps from the Feature Pyramid Network neck.
+    fpn_position_encoding (`tuple(torch.FloatTensor)`):
+        Tuple of `torch.FloatTensor` (one for each feature level, from high to low resolution) of shape
+        `(batch_size, hidden_size, height, width)`. Positional encodings corresponding to the `fpn_hidden_states`.
+    """
+
+    fpn_hidden_states: torch.FloatTensor | None = None
+    fpn_position_encoding: torch.FloatTensor | None = None
+
+
+@auto_docstring(
+    custom_intro="""
+    Segment Anything Model 2 (SAM 2) for generating segmentation masks, given an input image and
+    input points and labels, boxes, or masks.
+    """
+)
+class Sam3TrackerModel(Sam3TrackerPreTrainedModel):
+    input_modalities = ("image", "text")
+    _can_record_outputs = {"mask_decoder_attentions": OutputRecorder(Sam3TrackerTwoWayAttentionBlock, index=2)}
+    _tied_weights_keys = {}
+    _base_model_prefix = "tracker_model"
+    _keys_to_ignore_on_load_unexpected = [
+        r"^detector_model.",
+        r"^memory_.*",
+        r"^mask_downsample.*",
+        r"^object_pointer_proj.*",
+        r"^temporal_positional_encoding_projection_layer.*",
+        "no_memory_positional_encoding",
+        "no_object_pointer",
+        "occlusion_spatial_embedding_parameter",
+    ]
+
+    def __init__(self, config: Sam3TrackerConfig):
+        # loading from a sam3_video config
+        if hasattr(config, "tracker_config") and config.tracker_config is not None:
+            if isinstance(config.tracker_config, dict):
+                config.tracker_config = Sam3TrackerConfig(**config.tracker_config)
+            config = config.tracker_config
+        super().__init__(config)
+        self.shared_image_embedding = Sam3TrackerPositionalEmbedding(config.prompt_encoder_config)
+        self.vision_encoder = AutoModel.from_config(config.vision_config)
+        self.prompt_encoder = Sam3TrackerPromptEncoder(config.prompt_encoder_config)
+        # The module using it is not a PreTrainedModel subclass so we need this
+        config.mask_decoder_config._attn_implementation = config._attn_implementation
+        self.mask_decoder = Sam3TrackerMaskDecoder(config.mask_decoder_config)
+
+        self.backbone_feature_sizes = config.vision_config.backbone_feature_sizes
+        # a single token to indicate no memory embedding from previous frames
+        self.hidden_dim = config.vision_config.fpn_hidden_size
+        self.no_memory_embedding = torch.nn.Parameter(torch.zeros(1, 1, self.hidden_dim))
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.vision_encoder.get_input_embeddings()
+
+    def get_image_wide_positional_embeddings(self) -> torch.Tensor:
+        size = self.prompt_encoder.image_embedding_size
+        target_device = self.shared_image_embedding.positional_embedding.device
+        target_dtype = self.shared_image_embedding.positional_embedding.dtype
+        grid = torch.ones(size, device=target_device, dtype=target_dtype)
+        y_embed = grid.cumsum(dim=0) - 0.5
+        x_embed = grid.cumsum(dim=1) - 0.5
+        y_embed = y_embed / size[0]
+        x_embed = x_embed / size[1]
+
+        positional_embedding = self.shared_image_embedding(torch.stack([x_embed, y_embed], dim=-1))
+        return positional_embedding.permute(2, 0, 1).unsqueeze(0)  # channel x height x width
+
+    @torch.no_grad()
+    def get_image_embeddings(
+        self,
+        pixel_values: torch.FloatTensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> list[torch.Tensor]:
+        r"""
+        Returns the image embeddings by passing the pixel values through the vision encoder.
+
+        Args:
+            pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
+                Input pixel values
+        """
+        batch_size = pixel_values.shape[0]
+        image_outputs = self.get_image_features(pixel_values, return_dict=True, **kwargs)
+        feature_maps = image_outputs.fpn_hidden_states
+
+        # add no memory embedding to the last feature map
+        feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding
+
+        # reshape feature maps to the same shape as the backbone feature sizes
+        image_embeddings = [
+            feat.permute(1, 2, 0).view(batch_size, -1, *feat_size)
+            for feat, feat_size in zip(feature_maps, self.backbone_feature_sizes)
+        ]
+
+        return image_embeddings
+
+    @torch.no_grad()
+    def get_prompt_embeddings(
+        self,
+        input_points: torch.FloatTensor | None = None,
+        input_labels: torch.LongTensor | None = None,
+        input_boxes: torch.FloatTensor | None = None,
+        input_masks: torch.LongTensor | None = None,
+    ):
+        r"""
+        Returns the prompt embeddings by passing the input points, labels, boxes and masks through the prompt encoder.
+
+        Args:
+            input_points (`torch.FloatTensor` of shape `(batch_size, point_batch_size, num_points_per_image, 2)`):
+                Optional input points for the prompt encoder. The padding of the point is automatically done by the
+                processor. `point_batch_size` refers to the number of masks that we want the model to predict per
+                point. The model will output `point_batch_size` times 3 masks in total.
+            input_labels (`torch.LongTensor` of shape `(batch_size, point_batch_size, num_points_per_image)`):
+                Optional input labels for the prompt encoder. The padding of the labels is automatically done by the
+                processor, or can be fed by the user.
+            input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes_per_image, 4)`):
+                Optional input boxes for the prompt encoder. The padding of the boxes is automatically done by the
+                processor. users can also pass manually the input boxes.
+            input_masks (`torch.LongTensor` of shape `(batch_size, image_size, image_size)`):
+                Optional input masks for the prompt encoder.
+        """
+        prompt_output = self.prompt_encoder(
+            input_points=input_points,
+            input_labels=input_labels,
+            input_boxes=input_boxes,
+            input_masks=input_masks,
+        )
+        return prompt_output
+
+    @merge_with_config_defaults
+    @capture_outputs
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor | None = None,
+        input_points: torch.FloatTensor | None = None,
+        input_labels: torch.LongTensor | None = None,
+        input_boxes: torch.FloatTensor | None = None,
+        input_masks: torch.LongTensor | None = None,
+        image_embeddings: torch.FloatTensor | None = None,
+        multimask_output: bool = True,
+        attention_similarity: torch.FloatTensor | None = None,
+        target_embedding: torch.FloatTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> Sam3TrackerImageSegmentationOutput:
+        r"""
+        input_points (`torch.FloatTensor` of shape `(batch_size, num_points, 2)`):
+            Input 2D spatial points, this is used by the prompt encoder to encode the prompt. Generally yields to much
+            better results. The points can be obtained by passing a list of list of list to the processor that will
+            create corresponding `torch` tensors of dimension 4. The first dimension is the image batch size, the
+            second dimension is the point batch size (i.e. how many segmentation masks do we want the model to predict
+            per input point), the third dimension is the number of points per segmentation mask (it is possible to pass
+            multiple points for a single mask), and the last dimension is the x (vertical) and y (horizontal)
+            coordinates of the point. If a different number of points is passed either for each image, or for each
+            mask, the processor will create "PAD" points that will correspond to the (0, 0) coordinate, and the
+            computation of the embedding will be skipped for these points using the labels.
+        input_labels (`torch.LongTensor` of shape `(batch_size, point_batch_size, num_points)`):
+            Input labels for the points, this is used by the prompt encoder to encode the prompt. According to the
+            official implementation, there are 3 types of labels
+
+            - `1`: the point is a point that contains the object of interest
+            - `0`: the point is a point that does not contain the object of interest
+            - `-1`: the point corresponds to the background
+
+            We added the label:
+
+            - `-10`: the point is a padding point, thus should be ignored by the prompt encoder
+
+            The padding labels should be automatically done by the processor.
+        input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes, 4)`):
+            Input boxes for the points, this is used by the prompt encoder to encode the prompt. Generally yields to
+            much better generated masks. The boxes can be obtained by passing a list of list of list to the processor,
+            that will generate a `torch` tensor, with each dimension corresponding respectively to the image batch
+            size, the number of boxes per image and the coordinates of the top left and bottom right point of the box.
+            In the order (`x1`, `y1`, `x2`, `y2`):
+
+            - `x1`: the x coordinate of the top left point of the input box
+            - `y1`: the y coordinate of the top left point of the input box
+            - `x2`: the x coordinate of the bottom right point of the input box
+            - `y2`: the y coordinate of the bottom right point of the input box
+        input_masks (`torch.FloatTensor` of shape `(batch_size, image_size, image_size)`):
+            SAM model also accepts segmentation masks as input. The mask will be embedded by the prompt encoder to
+            generate a corresponding embedding, that will be fed later on to the mask decoder. These masks needs to be
+            manually fed by the user, and they need to be of shape (`batch_size`, `image_size`, `image_size`).
+        image_embeddings (`torch.FloatTensor` of shape `(batch_size, output_channels, window_size, window_size)`):
+            Image embeddings, this is used by the mask decoder to generate masks and iou scores. For more memory
+            efficient computation, users can first retrieve the image embeddings using the `get_image_embeddings`
+            method, and then feed them to the `forward` method instead of feeding the `pixel_values`.
+        multimask_output (`bool`, *optional*):
+            In the original implementation and paper, the model always outputs 3 masks per image (or per point / per
+            bounding box if relevant). However, it is possible to just output a single mask, that corresponds to the
+            "best" mask, by specifying `multimask_output=False`.
+        attention_similarity (`torch.FloatTensor`, *optional*):
+            Attention similarity tensor, to be provided to the mask decoder for target-guided attention in case the
+            model is used for personalization as introduced in [PerSAM](https://huggingface.co/papers/2305.03048).
+        target_embedding (`torch.FloatTensor`, *optional*):
+            Embedding of the target concept, to be provided to the mask decoder for target-semantic prompting in case
+            the model is used for personalization as introduced in [PerSAM](https://huggingface.co/papers/2305.03048).
+
+        Example:
+
+        ```python
+        >>> from PIL import Image
+        >>> import httpx
+        >>> from io import BytesIO
+        >>> from transformers import AutoModel, AutoProcessor
+
+        >>> model = AutoModel.from_pretrained("danelcsb/sam3_tracker.1_hiera_tiny")
+        >>> processor = AutoProcessor.from_pretrained("danelcsb/sam3_tracker.1_hiera_tiny")
+
+        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/sam-car.png"
+        >>> with httpx.stream("GET", url) as response:
+        ...     raw_image = Image.open(BytesIO(response.read())).convert("RGB")
+        >>> input_points = [[[400, 650]]]  # 2D location of a window on the car
+        >>> inputs = processor(images=raw_image, input_points=input_points, return_tensors="pt")
+
+        >>> # Get segmentation mask
+        >>> outputs = model(**inputs)
+
+        >>> # Postprocess masks
+        >>> masks = processor.post_process_masks(
+        ...     outputs.pred_masks, inputs["original_sizes"]
+        ... )
+        ```
+        """
+        if not ((pixel_values is None) ^ (image_embeddings is None)):
+            raise ValueError("Exactly one of pixel_values or image_embeddings must be provided.")
+        if input_points is not None and input_boxes is not None:
+            if input_points.shape[1] != input_boxes.shape[1]:
+                raise ValueError(
+                    f"You should provide as many bounding boxes as input points per box. Got {input_points.shape[1]} and {input_boxes.shape[1]}."
+                )
+
+        image_positional_embeddings = self.get_image_wide_positional_embeddings()
+        # repeat with batch size
+        batch_size = pixel_values.shape[0] if pixel_values is not None else image_embeddings[-1].shape[0]
+        image_positional_embeddings = image_positional_embeddings.repeat(batch_size, 1, 1, 1)
+
+        vision_attentions = None
+        vision_hidden_states = None
+
+        if pixel_values is not None:
+            image_outputs: Sam3TrackerVisionEncoderOutput = self.get_image_features(
+                pixel_values, return_dict=True, **kwargs
+            )
+            feature_maps = image_outputs.fpn_hidden_states
+            vision_hidden_states = image_outputs.hidden_states
+            vision_attentions = image_outputs.attentions
+
+            # add no memory embedding to the last feature map
+            feature_maps[-1] = feature_maps[-1] + self.no_memory_embedding
+
+            # reshape feature maps to the same shape as the backbone feature sizes
+            image_embeddings = [
+                feat.permute(1, 2, 0).view(batch_size, -1, *feat_size)
+                for feat, feat_size in zip(feature_maps, self.backbone_feature_sizes)
+            ]
+
+        if input_points is not None and input_labels is None:
+            input_labels = torch.ones_like(input_points[:, :, :, 0], dtype=torch.int, device=input_points.device)
+
+        if input_points is None and input_boxes is None:
+            # If no points are provide, pad with an empty point (with label -1)
+            input_points = torch.zeros(
+                batch_size, 1, 1, 2, dtype=image_embeddings[-1].dtype, device=image_embeddings[-1].device
+            )
+            input_labels = -torch.ones(batch_size, 1, 1, dtype=torch.int32, device=image_embeddings[-1].device)
+
+        if input_masks is not None:
+            # If mask_inputs is provided, downsize it into low-res mask input if needed
+            # and feed it as a dense mask prompt into the SAM mask encoder
+            if input_masks.shape[-2:] != self.prompt_encoder.mask_input_size:
+                input_masks = F.interpolate(
+                    input_masks.float(),
+                    size=self.prompt_encoder.mask_input_size,
+                    align_corners=False,
+                    mode="bilinear",
+                    antialias=True,  # use antialias for downsampling
+                ).to(input_masks.dtype)
+
+        sparse_embeddings, dense_embeddings = self.prompt_encoder(
+            input_points=input_points,
+            input_labels=input_labels,
+            input_boxes=input_boxes,
+            input_masks=input_masks,
+        )
+        low_res_multimasks, iou_scores, _, object_score_logits = self.mask_decoder(
+            image_embeddings=image_embeddings[-1],
+            image_positional_embeddings=image_positional_embeddings,
+            sparse_prompt_embeddings=sparse_embeddings,
+            dense_prompt_embeddings=dense_embeddings,
+            multimask_output=multimask_output,
+            high_resolution_features=image_embeddings[:-1],
+            attention_similarity=attention_similarity,
+            target_embedding=target_embedding,
+            **kwargs,
+        )
+
+        return Sam3TrackerImageSegmentationOutput(
+            iou_scores=iou_scores,
+            pred_masks=low_res_multimasks,
+            object_score_logits=object_score_logits,
+            image_embeddings=image_embeddings,
+            vision_hidden_states=vision_hidden_states,
+            vision_attentions=vision_attentions,
+        )
+
+    @can_return_tuple
+    @auto_docstring
+    def get_image_features(
+        self,
+        pixel_values: torch.FloatTensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | Sam3TrackerVisionEncoderOutput:
+        r"""
+        pixel_values (`torch.FloatTensor`):
+            Input pixel values of shape `(batch_size, num_channels, height, width)`.
+        """
+        vision_outputs: Sam3TrackerVisionEncoderOutput = self.vision_encoder(pixel_values, return_dict=True, **kwargs)
+
+        feature_maps = vision_outputs.fpn_hidden_states
+        feature_maps_position_embeddings = vision_outputs.fpn_position_encoding
+
+        # precompute projected level 0 and level 1 features in SAM decoder
+        # to avoid running it again on every SAM click
+        feature_maps = list(feature_maps)
+        feature_maps[0] = self.mask_decoder.conv_s0(feature_maps[0])
+        feature_maps[1] = self.mask_decoder.conv_s1(feature_maps[1])
+
+        # flatten NxCxHxW to HWxNxC
+        feature_maps = [feature_map.flatten(2).permute(2, 0, 1) for feature_map in feature_maps]
+        feature_maps_position_embeddings = [
+            feature_maps_position_embeddings.flatten(2).permute(2, 0, 1)
+            for feature_maps_position_embeddings in feature_maps_position_embeddings
+        ]
+        vision_outputs.fpn_hidden_states = feature_maps
+        vision_outputs.fpn_position_encoding = feature_maps_position_embeddings
+
+        return vision_outputs
+
+
+__all__ = ["Sam3TrackerModel", "Sam3TrackerPreTrainedModel"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/sam3_tracker/modular_sam3_tracker.py

@@ -0,0 +1,233 @@
+# Copyright 2025 the HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import torch
+from huggingface_hub.dataclasses import strict
+
+from ... import initialization as init
+from ...configuration_utils import PreTrainedConfig
+from ...modeling_utils import PreTrainedModel
+from ...utils import auto_docstring
+from ..auto import CONFIG_MAPPING, AutoModel
+from ..sam2.configuration_sam2 import (
+    Sam2Config,
+    Sam2MaskDecoderConfig,
+    Sam2PromptEncoderConfig,
+)
+from ..sam2.modeling_sam2 import (
+    Sam2Attention,
+    Sam2FeedForward,
+    Sam2ImageSegmentationOutput,
+    Sam2LayerNorm,
+    Sam2MaskDecoder,
+    Sam2MaskEmbedding,
+    Sam2Model,
+    Sam2PositionalEmbedding,
+    Sam2PreTrainedModel,
+    Sam2PromptEncoder,
+    Sam2TwoWayAttentionBlock,
+    Sam2TwoWayTransformer,
+)
+from ..sam2.processing_sam2 import Sam2Processor
+
+
+@auto_docstring(checkpoint="facebook/sam3")
+@strict
+class Sam3TrackerPromptEncoderConfig(Sam2PromptEncoderConfig):
+    r"""
+    mask_input_channels (`int`, *optional*, defaults to 16):
+        The number of channels to be fed to the `MaskDecoder` module.
+    num_point_embeddings (`int`, *optional*, defaults to 4):
+        The number of point embeddings to be used.
+    scale (`float`, *optional*, defaults to 1):
+        The scale factor for the prompt encoder.
+    """
+
+    base_config_key = "prompt_encoder_config"
+
+    image_size: int | list[int] | tuple[int, int] = 1008
+    patch_size: int | list[int] | tuple[int, int] = 14
+
+
+class Sam3TrackerProcessor(Sam2Processor):
+    pass
+
+
+@auto_docstring(checkpoint="facebook/sam3")
+@strict
+class Sam3TrackerMaskDecoderConfig(Sam2MaskDecoderConfig):
+    pass
+
+
+@auto_docstring(checkpoint="facebook/sam3")
+@strict
+class Sam3TrackerConfig(Sam2Config):
+    r"""
+    prompt_encoder_config (Union[`dict`, `Sam3TrackerPromptEncoderConfig`], *optional*):
+        Dictionary of configuration options used to initialize [`Sam3TrackerPromptEncoderConfig`].
+    mask_decoder_config (Union[`dict`, `Sam3TrackerMaskDecoderConfig`], *optional*):
+        Dictionary of configuration options used to initialize [`Sam3TrackerMaskDecoderConfig`].
+
+    Example:
+
+    ```python
+    >>> from transformers import (
+    ...     Sam3TrackerVisionConfig,
+    ...     Sam3TrackerPromptEncoderConfig,
+    ...     Sam3TrackerMaskDecoderConfig,
+    ...     Sam3TrackerModel,
+    ... )
+
+    >>> # Initializing a Sam3TrackerConfig with `"facebook/sam3_tracker.1_hiera_tiny"` style configuration
+    >>> configuration = Sam3TrackerConfig()
+
+    >>> # Initializing a Sam3TrackerModel (with random weights) from the `"facebook/sam3_tracker.1_hiera_tiny"` style configuration
+    >>> model = Sam3TrackerModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+
+    >>> # We can also initialize a Sam3TrackerConfig from a Sam3TrackerVisionConfig, Sam3TrackerPromptEncoderConfig, and Sam3TrackerMaskDecoderConfig
+    >>> # Initializing SAM3_TRACKER vision encoder, memory attention, and memory encoder configurations
+    >>> vision_config = Sam3TrackerVisionConfig()
+    >>> prompt_encoder_config = Sam3TrackerPromptEncoderConfig()
+    >>> mask_decoder_config = Sam3TrackerMaskDecoderConfig()
+
+    >>> config = Sam3TrackerConfig(vision_config, prompt_encoder_config, mask_decoder_config)
+    ```
+    """
+
+    def __post_init__(self, **kwargs):
+        if isinstance(self.vision_config, dict):
+            self.vision_config["model_type"] = self.vision_config.get("model_type", "sam3_vision_model")
+            self.vision_config = CONFIG_MAPPING[self.vision_config["model_type"]](**self.vision_config)
+        elif self.vision_config is None:
+            self.vision_config = CONFIG_MAPPING["sam3_vision_model"](
+                backbone_feature_sizes=[[288, 288], [144, 144], [72, 72]]
+            )
+
+        if isinstance(self.prompt_encoder_config, dict):
+            self.prompt_encoder_config = Sam3TrackerPromptEncoderConfig(**self.prompt_encoder_config)
+        elif self.prompt_encoder_config is None:
+            self.prompt_encoder_config = Sam3TrackerPromptEncoderConfig()
+
+        if isinstance(self.mask_decoder_config, dict):
+            self.mask_decoder_config = Sam3TrackerMaskDecoderConfig(**self.mask_decoder_config)
+        elif self.mask_decoder_config is None:
+            self.mask_decoder_config = Sam3TrackerMaskDecoderConfig()
+
+        PreTrainedConfig.__post_init__(**kwargs)
+
+
+class Sam3TrackerImageSegmentationOutput(Sam2ImageSegmentationOutput):
+    pass
+
+
+class Sam3TrackerFeedForward(Sam2FeedForward):
+    pass
+
+
+@auto_docstring(
+    custom_intro="""
+    Segment Anything Model 3 (SAM 3) for generating segmentation masks, given an input image and
+    input points and labels, boxes, or masks.
+    """
+)
+class Sam3TrackerPreTrainedModel(Sam2PreTrainedModel):
+    @torch.no_grad()
+    def _init_weights(self, module):
+        PreTrainedModel._init_weights(module)
+        if isinstance(module, Sam3TrackerModel):
+            if module.no_memory_embedding is not None:
+                init.zeros_(module.no_memory_embedding)
+        elif isinstance(module, Sam3TrackerPositionalEmbedding):
+            init.normal_(module.positional_embedding, std=module.scale)
+
+
+class Sam3TrackerPositionalEmbedding(Sam2PositionalEmbedding):
+    pass
+
+
+class Sam3TrackerMaskEmbedding(Sam2MaskEmbedding):
+    pass
+
+
+class Sam3TrackerPromptEncoder(Sam2PromptEncoder):
+    pass
+
+
+class Sam3TrackerAttention(Sam2Attention):
+    pass
+
+
+class Sam3TrackerTwoWayAttentionBlock(Sam2TwoWayAttentionBlock):
+    pass
+
+
+class Sam3TrackerTwoWayTransformer(Sam2TwoWayTransformer):
+    pass
+
+
+class Sam3TrackerLayerNorm(Sam2LayerNorm):
+    pass
+
+
+class Sam3TrackerMaskDecoder(Sam2MaskDecoder):
+    pass
+
+
+class Sam3TrackerModel(Sam2Model):
+    _base_model_prefix = "tracker_model"
+    _keys_to_ignore_on_load_unexpected = [
+        r"^detector_model.",
+        r"^memory_.*",
+        r"^mask_downsample.*",
+        r"^object_pointer_proj.*",
+        r"^temporal_positional_encoding_projection_layer.*",
+        "no_memory_positional_encoding",
+        "no_object_pointer",
+        "occlusion_spatial_embedding_parameter",
+    ]
+
+    def __init__(self, config: Sam3TrackerConfig):
+        # loading from a sam3_video config
+        if hasattr(config, "tracker_config") and config.tracker_config is not None:
+            if isinstance(config.tracker_config, dict):
+                config.tracker_config = Sam3TrackerConfig(**config.tracker_config)
+            config = config.tracker_config
+        Sam3TrackerPreTrainedModel.__init__(config)
+        self.shared_image_embedding = Sam3TrackerPositionalEmbedding(config.prompt_encoder_config)
+        self.vision_encoder = AutoModel.from_config(config.vision_config)
+        self.prompt_encoder = Sam3TrackerPromptEncoder(config.prompt_encoder_config)
+        # The module using it is not a PreTrainedModel subclass so we need this
+        config.mask_decoder_config._attn_implementation = config._attn_implementation
+        self.mask_decoder = Sam3TrackerMaskDecoder(config.mask_decoder_config)
+
+        self.backbone_feature_sizes = config.vision_config.backbone_feature_sizes
+        # a single token to indicate no memory embedding from previous frames
+        self.hidden_dim = config.vision_config.fpn_hidden_size
+        self.no_memory_embedding = torch.nn.Parameter(torch.zeros(1, 1, self.hidden_dim))
+
+        self.post_init()
+
+
+__all__ = [
+    "Sam3TrackerConfig",
+    "Sam3TrackerPromptEncoderConfig",
+    "Sam3TrackerMaskDecoderConfig",
+    "Sam3TrackerProcessor",
+    "Sam3TrackerModel",
+    "Sam3TrackerPreTrainedModel",
+]

LTA_openwebtext_dualt/mini_owt_logdirichlet/logs/debug_langflowalg_learnedembed_single_gpu_20260530_213823.log

@@ -0,0 +1,56 @@
+[W530 21:38:29.679686368 Utils.hpp:136] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
+NCCL version 2.25.1+cuda12.8
+[rank0]:[W530 21:38:30.182058535 Utils.hpp:111] Warning: Environment variable NCCL_ASYNC_ERROR_HANDLING is deprecated; use TORCH_NCCL_ASYNC_ERROR_HANDLING instead (function operator())
+Muon: 57 2D params; Nesterov-AdamW: 78 other params
+{
+  "cache_path": "cache/owt_t5_llmclean_qwen36_35b_articlefull_pack1023_10k_appendeos1.pt",
+  "tokenizer_path": "/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json",
+  "out_dir": "runs/debug_langflowalg_learnedembed_single_gpu_20260530_213823",
+  "subset_size": 16,
+  "resume": "",
+  "steps": 1,
+  "batch_size": 1,
+  "grad_accum": 1,
+  "num_workers": 0,
+  "lr": 3.90625e-06,
+  "blr": 0.001,
+  "min_lr": 0.0,
+  "lr_schedule": "constant",
+  "warmup_steps": 8,
+  "warmup_epochs": 0.5,
+  "optimizer": "muon",
+  "weight_decay": 0.0,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.95,
+  "adam_eps": 1e-08,
+  "grad_clip": 1.0,
+  "log_every": 1,
+  "save_every": 1,
+  "dim": 768,
+  "layers": 12,
+  "heads": 12,
+  "mlp_dim": 3072,
+  "time_tokens": 4,
+  "token_embed_init": "random",
+  "token_embed_model_path": "/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small",
+  "token_embed_dim": 768,
+  "freeze_token_embed": 0,
+  "concat_self_cond": 1,
+  "self_cond_prob": 0.5,
+  "use_bias": 1,
+  "gamma_loc": 4.723,
+  "gamma_scale": 0.852,
+  "gamma_cutoff": 1e-05,
+  "seed": 1234,
+  "loader_batches_per_rank": 16,
+  "optimizer_steps_per_epoch": 16,
+  "steps_per_epoch": 16,
+  "effective_batch_size": 1
+}
+[data] cache=cache/owt_t5_llmclean_qwen36_35b_articlefull_pack1023_10k_appendeos1.pt rows=16 length=1024 vocab=32100 bos=1:</s> eos=1:</s>
+[langflow_style] gamma=Gumbel(loc=4.723, scale=0.852) cutoff=1e-05 range=[2.6412,14.5320] use_bias=True self_cond_prob=0.5 concat=True
+[optim] optimizer=muon lr=3.906250e-06 blr=1.000000e-03 effective_batch=1 warmup_steps=8 lr_schedule=constant
+[embed] init=random dim=768 freeze=False model_path=/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small
+step=1 loss=10.5276 gamma=4.1065 self_cond_use=0.000 lr=4.883e-07 pos0_bos_p=0.0001 last_eos_p=0.0000
+[rank0]:[W530 21:38:32.536296296 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[exit] 2026-05-30T21:38:34+00:00 rc=0 run=debug_langflowalg_learnedembed_single_gpu_20260530_213823