diff --git a/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0018000_logistic_normal_t1p45.log b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0018000_logistic_normal_t1p45.log
new file mode 100644
index 0000000000000000000000000000000000000000..38735b80c5b4d3038532526b0332ead7e1fccf99
--- /dev/null
+++ b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0018000_logistic_normal_t1p45.log
@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_00:14:28 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0018000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0018000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0018000.pt
+[ckpt] step=18000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0018000.pt",
+  "step": 18000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 37.642507957083765,
+    "nll_per_token": 3.628133942600674,
+    "tokens": 30091,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 53.23995326170723,
+    "nll_per_token": 3.9748091156472474,
+    "tokens": 24722,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.1327908839169742,
+    "unique_tokens": 1552,
+    "token_count": 32768,
+    "distinct_1": 0.04736328125,
+    "distinct_2": 0.24351008858267717,
+    "top_token_mass": 0.271881103515625
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0018000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_00:15:56 done step_0018000
diff --git a/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0025000_logistic_normal_t1p45.log b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0025000_logistic_normal_t1p45.log
new file mode 100644
index 0000000000000000000000000000000000000000..1043d770c95e68282d31fe9af4a90d6331f97c14
--- /dev/null
+++ b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0025000_logistic_normal_t1p45.log
@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_00:53:13 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0025000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0025000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0025000.pt
+[ckpt] step=25000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0025000.pt",
+  "step": 25000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 31.231424512837485,
+    "nll_per_token": 3.441424783858224,
+    "tokens": 37226,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 43.50436377481887,
+    "nll_per_token": 3.772861249725761,
+    "tokens": 30767,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.79152059307615,
+    "unique_tokens": 1705,
+    "token_count": 32768,
+    "distinct_1": 0.052032470703125,
+    "distinct_2": 0.2902005413385827,
+    "top_token_mass": 0.07025146484375
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0025000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_00:54:41 done step_0025000
diff --git a/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0032000_logistic_normal_t1p45.log b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0032000_logistic_normal_t1p45.log
new file mode 100644
index 0000000000000000000000000000000000000000..a598626a0909a3d8ea1a936285ef34c9343a4e04
--- /dev/null
+++ b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0032000_logistic_normal_t1p45.log
@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_01:32:43 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0032000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0032000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0032000.pt
+[ckpt] step=32000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0032000.pt",
+  "step": 32000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 33.014796153697105,
+    "nll_per_token": 3.496955829273304,
+    "tokens": 36426,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 46.797879118519774,
+    "nll_per_token": 3.8458378839163636,
+    "tokens": 29987,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.7235357273924485,
+    "unique_tokens": 2030,
+    "token_count": 32768,
+    "distinct_1": 0.06195068359375,
+    "distinct_2": 0.3199741633858268,
+    "top_token_mass": 0.106903076171875
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0032000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_01:34:10 done step_0032000
diff --git a/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0039000_logistic_normal_t1p45.log b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0039000_logistic_normal_t1p45.log
new file mode 100644
index 0000000000000000000000000000000000000000..20545fab63595adb2183cea3425eefaeb3c50e6c
--- /dev/null
+++ b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0039000_logistic_normal_t1p45.log
@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_02:11:30 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0039000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0039000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0039000.pt
+[ckpt] step=39000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0039000.pt",
+  "step": 39000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 33.76559505090009,
+    "nll_per_token": 3.519442386176717,
+    "tokens": 32221,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 46.10090126017552,
+    "nll_per_token": 3.830832499923769,
+    "tokens": 26646,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.309203124839359,
+    "unique_tokens": 1626,
+    "token_count": 32768,
+    "distinct_1": 0.04962158203125,
+    "distinct_2": 0.2572896161417323,
+    "top_token_mass": 0.211181640625
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0039000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_02:12:57 done step_0039000
diff --git a/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0049000_logistic_normal_t1p45.log b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0049000_logistic_normal_t1p45.log
new file mode 100644
index 0000000000000000000000000000000000000000..5144a45abd9b6dc722c91add254f130507291fcb
--- /dev/null
+++ b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0049000_logistic_normal_t1p45.log
@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_03:07:26 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0049000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0049000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0049000.pt
+[ckpt] step=49000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0049000.pt",
+  "step": 49000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 35.48283825980655,
+    "nll_per_token": 3.569049150290528,
+    "tokens": 34732,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 52.17503986958083,
+    "nll_per_token": 3.9546042171140185,
+    "tokens": 28366,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.558572845970964,
+    "unique_tokens": 2054,
+    "token_count": 32768,
+    "distinct_1": 0.06268310546875,
+    "distinct_2": 0.30804010826771655,
+    "top_token_mass": 0.152191162109375
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0049000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_03:08:54 done step_0049000
diff --git a/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0077000_logistic_normal_t1p45.log b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0077000_logistic_normal_t1p45.log
new file mode 100644
index 0000000000000000000000000000000000000000..cb1b800818e9f752902a6e4ffb199f7ce0798759
--- /dev/null
+++ b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0077000_logistic_normal_t1p45.log
@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_05:43:25 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0077000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0077000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0077000.pt
+[ckpt] step=77000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0077000.pt",
+  "step": 77000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 31.90594878776546,
+    "nll_per_token": 3.462792474780215,
+    "tokens": 37167,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 46.30221403143558,
+    "nll_per_token": 3.8351897792023526,
+    "tokens": 30416,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.770319501431757,
+    "unique_tokens": 2137,
+    "token_count": 32768,
+    "distinct_1": 0.065216064453125,
+    "distinct_2": 0.33744463582677164,
+    "top_token_mass": 0.094970703125
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0077000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_05:44:53 done step_0077000
diff --git a/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0103000_logistic_normal_t1p45.log b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0103000_logistic_normal_t1p45.log
new file mode 100644
index 0000000000000000000000000000000000000000..a75eb8081dadadfd04b97cdc59f267255d53e419
--- /dev/null
+++ b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0103000_logistic_normal_t1p45.log
@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_08:09:06 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0103000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0103000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0103000.pt
+[ckpt] step=103000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0103000.pt",
+  "step": 103000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 30.371408722233586,
+    "nll_per_token": 3.413501663304038,
+    "tokens": 36546,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 41.969304283345274,
+    "nll_per_token": 3.7369385006854787,
+    "tokens": 30362,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.698089692782413,
+    "unique_tokens": 2344,
+    "token_count": 32768,
+    "distinct_1": 0.071533203125,
+    "distinct_2": 0.36540354330708663,
+    "top_token_mass": 0.0777587890625
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0103000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_08:10:34 done step_0103000
diff --git a/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0108000_logistic_normal_t1p45.log b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0108000_logistic_normal_t1p45.log
new file mode 100644
index 0000000000000000000000000000000000000000..045897b813954714db3fd299311ba1c564f82075
--- /dev/null
+++ b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0108000_logistic_normal_t1p45.log
@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_08:36:48 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0108000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0108000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0108000.pt
+[ckpt] step=108000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0108000.pt",
+  "step": 108000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 34.88124452485366,
+    "nll_per_token": 3.551949278589281,
+    "tokens": 34859,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 44.9734082467488,
+    "nll_per_token": 3.8060713872536174,
+    "tokens": 29578,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.532442503084375,
+    "unique_tokens": 2417,
+    "token_count": 32768,
+    "distinct_1": 0.073760986328125,
+    "distinct_2": 0.3453494094488189,
+    "top_token_mass": 0.121063232421875
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0108000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_08:38:16 done step_0108000
diff --git a/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0124000_logistic_normal_t1p45.log b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0124000_logistic_normal_t1p45.log
new file mode 100644
index 0000000000000000000000000000000000000000..661a2b48c46d97220884c9d7677986b8f38a87be
--- /dev/null
+++ b/LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0124000_logistic_normal_t1p45.log
@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_10:05:45 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0124000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0124000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0124000.pt
+[ckpt] step=124000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0124000.pt",
+  "step": 124000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 33.066483898805664,
+    "nll_per_token": 3.498520198353581,
+    "tokens": 34129,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 41.22609368102926,
+    "nll_per_token": 3.7190713976517773,
+    "tokens": 28994,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.5810572585206812,
+    "unique_tokens": 2244,
+    "token_count": 32768,
+    "distinct_1": 0.0684814453125,
+    "distinct_2": 0.3511011318897638,
+    "top_token_mass": 0.14215087890625
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0124000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_10:07:13 done step_0124000
diff --git a/LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_bernoulliwrong_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_193815.log b/LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_bernoulliwrong_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_193815.log
new file mode 100644
index 0000000000000000000000000000000000000000..57d348be7f8f9cf69c72caea5620869155084e34
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_bernoulliwrong_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_193815.log
@@ -0,0 +1,1047 @@
+W0526 19:38:17.504000 10232 torch/distributed/run.py:792] 
+W0526 19:38:17.504000 10232 torch/distributed/run.py:792] *****************************************
+W0526 19:38:17.504000 10232 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0526 19:38:17.504000 10232 torch/distributed/run.py:792] *****************************************
+[rank7]:[W526 19:38:20.869101078 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7]  using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank5]:[W526 19:38:20.958802773 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5]  using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank6]:[W526 19:38:20.984765569 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6]  using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank4]:[W526 19:38:20.016969469 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4]  using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank1]:[W526 19:38:20.061529572 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1]  using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank3]:[W526 19:38:20.064048691 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3]  using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank2]:[W526 19:38:20.066039787 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2]  using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[data] loaded_cache=cache/owt_t5_payload1022_appendeos1.pt seen=8013769 kept=2860537 dropped=5153232
+[rank0]:[W526 19:38:26.844431666 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 0]  using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+t-20260527033744-p2gmq-worker-0:10299:10299 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10299:10299 [0] NCCL INFO Bootstrap: Using eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10299:10299 [0] NCCL INFO cudaDriverVersion 12080
+t-20260527033744-p2gmq-worker-0:10299:10299 [0] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527033744-p2gmq-worker-0:10299:10299 [0] NCCL INFO Comm config Blocking set to 1
+t-20260527033744-p2gmq-worker-0:10303:10303 [4] NCCL INFO cudaDriverVersion 12080
+t-20260527033744-p2gmq-worker-0:10303:10303 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10303:10303 [4] NCCL INFO Bootstrap: Using eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10303:10303 [4] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527033744-p2gmq-worker-0:10301:10301 [2] NCCL INFO cudaDriverVersion 12080
+t-20260527033744-p2gmq-worker-0:10301:10301 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10301:10301 [2] NCCL INFO Bootstrap: Using eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10301:10301 [2] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527033744-p2gmq-worker-0:10303:10303 [4] NCCL INFO Comm config Blocking set to 1
+t-20260527033744-p2gmq-worker-0:10300:10300 [1] NCCL INFO cudaDriverVersion 12080
+t-20260527033744-p2gmq-worker-0:10300:10300 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10301:10301 [2] NCCL INFO Comm config Blocking set to 1
+t-20260527033744-p2gmq-worker-0:10300:10300 [1] NCCL INFO Bootstrap: Using eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10300:10300 [1] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527033744-p2gmq-worker-0:10300:10300 [1] NCCL INFO Comm config Blocking set to 1
+t-20260527033744-p2gmq-worker-0:10304:10304 [5] NCCL INFO cudaDriverVersion 12080
+t-20260527033744-p2gmq-worker-0:10304:10304 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10305:10305 [6] NCCL INFO cudaDriverVersion 12080
+t-20260527033744-p2gmq-worker-0:10305:10305 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10304:10304 [5] NCCL INFO Bootstrap: Using eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10304:10304 [5] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527033744-p2gmq-worker-0:10305:10305 [6] NCCL INFO Bootstrap: Using eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10305:10305 [6] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527033744-p2gmq-worker-0:10302:10302 [3] NCCL INFO cudaDriverVersion 12080
+t-20260527033744-p2gmq-worker-0:10302:10302 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10306:10306 [7] NCCL INFO cudaDriverVersion 12080
+t-20260527033744-p2gmq-worker-0:10306:10306 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10302:10302 [3] NCCL INFO Bootstrap: Using eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10302:10302 [3] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527033744-p2gmq-worker-0:10306:10306 [7] NCCL INFO Bootstrap: Using eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10306:10306 [7] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527033744-p2gmq-worker-0:10304:10304 [5] NCCL INFO Comm config Blocking set to 1
+t-20260527033744-p2gmq-worker-0:10305:10305 [6] NCCL INFO Comm config Blocking set to 1
+t-20260527033744-p2gmq-worker-0:10302:10302 [3] NCCL INFO Comm config Blocking set to 1
+t-20260527033744-p2gmq-worker-0:10306:10306 [7] NCCL INFO Comm config Blocking set to 1
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Using network IBext_v9
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO Using network IBext_v9
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO Using network IBext_v9
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO Using network IBext_v9
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO Using network IBext_v9
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.80.13<0>
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO Using network IBext_v9
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO Using network IBext_v9
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO Using network IBext_v9
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO ncclCommInitRankConfig comm 0xbca8730 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x81d76eb54545f141 - Init START
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO ncclCommInitRankConfig comm 0xa635a50 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x81d76eb54545f141 - Init START
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO ncclCommInitRankConfig comm 0xa796fd0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x81d76eb54545f141 - Init START
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO ncclCommInitRankConfig comm 0xa82cfe0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x81d76eb54545f141 - Init START
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO ncclCommInitRankConfig comm 0xb251190 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x81d76eb54545f141 - Init START
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO ncclCommInitRankConfig comm 0xa2b40f0 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x81d76eb54545f141 - Init START
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO ncclCommInitRankConfig comm 0x98885e0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x81d76eb54545f141 - Init START
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO ncclCommInitRankConfig comm 0x967bd00 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x81d76eb54545f141 - Init START
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO Bootstrap timings total 0.001070 (create 0.000018, send 0.000065, recv 0.000079, ring 0.000659, delay 0.000001)
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO Bootstrap timings total 0.002082 (create 0.000021, send 0.000078, recv 0.000030, ring 0.000652, delay 0.000000)
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO Bootstrap timings total 0.001592 (create 0.000020, send 0.000070, recv 0.000565, ring 0.000680, delay 0.000000)
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Bootstrap timings total 0.084657 (create 0.000022, send 0.000068, recv 0.083127, ring 0.000123, delay 0.000001)
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO Bootstrap timings total 0.006136 (create 0.000021, send 0.000065, recv 0.003071, ring 0.002550, delay 0.000001)
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO Bootstrap timings total 0.000563 (create 0.000022, send 0.000107, recv 0.000059, ring 0.000083, delay 0.000001)
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO Bootstrap timings total 0.003130 (create 0.000020, send 0.000071, recv 0.002628, ring 0.000132, delay 0.000001)
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO Bootstrap timings total 0.067727 (create 0.000023, send 0.000061, recv 0.061688, ring 0.001636, delay 0.000001)
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO MNNVL busId 0x6b020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO MNNVL busId 0x71020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO MNNVL busId 0x69020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO MNNVL busId 0x65040 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO MNNVL busId 0x73020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO MNNVL busId 0x75020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO MNNVL busId 0x67020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO MNNVL busId 0x6f020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO Setting affinity for GPU 4 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO Setting affinity for GPU 5 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO NVLS multicast support is available on dev 5
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO NVLS multicast support is available on dev 4
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO Setting affinity for GPU 3 to 03ffffff,ffffffff,ffffffff
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO NVLS multicast support is available on dev 3
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO Setting affinity for GPU 1 to 03ffffff,ffffffff,ffffffff
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Setting affinity for GPU 0 to 03ffffff,ffffffff,ffffffff
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO Setting affinity for GPU 2 to 03ffffff,ffffffff,ffffffff
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO NVLS multicast support is available on dev 1
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO NVLS multicast support is available on dev 0
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO Setting affinity for GPU 6 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO NVLS multicast support is available on dev 6
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO Setting affinity for GPU 7 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO NVLS multicast support is available on dev 7
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO NVLS multicast support is available on dev 2
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO comm 0x98885e0 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO comm 0xa2b40f0 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO comm 0xbca8730 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO comm 0x967bd00 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO comm 0xa82cfe0 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO comm 0xa796fd0 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO comm 0xb251190 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO comm 0xa635a50 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO P2P Chunksize set to 524288
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO P2P Chunksize set to 524288
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO P2P Chunksize set to 524288
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO P2P Chunksize set to 524288
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO P2P Chunksize set to 524288
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO P2P Chunksize set to 524288
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO P2P Chunksize set to 524288
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO P2P Chunksize set to 524288
+t-20260527033744-p2gmq-worker-0:10303:10475 [4] NCCL INFO [Proxy Service] Device 4 CPU core 92
+t-20260527033744-p2gmq-worker-0:10303:10476 [4] NCCL INFO [Proxy Service UDS] Device 4 CPU core 94
+t-20260527033744-p2gmq-worker-0:10300:10477 [1] NCCL INFO [Proxy Service] Device 1 CPU core 68
+t-20260527033744-p2gmq-worker-0:10300:10478 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 71
+t-20260527033744-p2gmq-worker-0:10302:10479 [3] NCCL INFO [Proxy Service] Device 3 CPU core 2
+t-20260527033744-p2gmq-worker-0:10302:10480 [3] NCCL INFO [Proxy Service UDS] Device 3 CPU core 4
+t-20260527033744-p2gmq-worker-0:10306:10481 [7] NCCL INFO [Proxy Service] Device 7 CPU core 134
+t-20260527033744-p2gmq-worker-0:10306:10482 [7] NCCL INFO [Proxy Service UDS] Device 7 CPU core 136
+t-20260527033744-p2gmq-worker-0:10304:10483 [5] NCCL INFO [Proxy Service] Device 5 CPU core 152
+t-20260527033744-p2gmq-worker-0:10304:10484 [5] NCCL INFO [Proxy Service UDS] Device 5 CPU core 155
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Check P2P Type intraNodeP2pSupport 1 directMode 0
+t-20260527033744-p2gmq-worker-0:10299:10485 [0] NCCL INFO [Proxy Service] Device 0 CPU core 86
+t-20260527033744-p2gmq-worker-0:10299:10486 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 84
+t-20260527033744-p2gmq-worker-0:10301:10487 [2] NCCL INFO [Proxy Service] Device 2 CPU core 2
+t-20260527033744-p2gmq-worker-0:10301:10488 [2] NCCL INFO [Proxy Service UDS] Device 2 CPU core 4
+t-20260527033744-p2gmq-worker-0:10305:10489 [6] NCCL INFO [Proxy Service] Device 6 CPU core 134
+t-20260527033744-p2gmq-worker-0:10305:10490 [6] NCCL INFO [Proxy Service UDS] Device 6 CPU core 136
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO CC Off, workFifoBytes 1048576
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO ncclCommInitRankConfig comm 0xa796fd0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x81d76eb54545f141 - Init COMPLETE
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO ncclCommInitRankConfig comm 0xbca8730 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x81d76eb54545f141 - Init COMPLETE
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO ncclCommInitRankConfig comm 0x967bd00 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x81d76eb54545f141 - Init COMPLETE
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO ncclCommInitRankConfig comm 0xa82cfe0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x81d76eb54545f141 - Init COMPLETE
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO ncclCommInitRankConfig comm 0xa635a50 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x81d76eb54545f141 - Init COMPLETE
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO ncclCommInitRankConfig comm 0x98885e0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x81d76eb54545f141 - Init COMPLETE
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO ncclCommInitRankConfig comm 0xb251190 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x81d76eb54545f141 - Init COMPLETE
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO ncclCommInitRankConfig comm 0xa2b40f0 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x81d76eb54545f141 - Init COMPLETE
+t-20260527033744-p2gmq-worker-0:10304:10399 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 2.21 (kernels 0.19, alloc 1.04, bootstrap 0.01, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.39, rest 0.03)
+t-20260527033744-p2gmq-worker-0:10299:10395 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 2.23 (kernels 0.18, alloc 1.00, bootstrap 0.08, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.39, rest 0.03)
+t-20260527033744-p2gmq-worker-0:10306:10402 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 2.21 (kernels 0.19, alloc 1.05, bootstrap 0.00, allgathers 0.00, topo 0.54, graphs 0.01, connections 0.39, rest 0.03)
+t-20260527033744-p2gmq-worker-0:10305:10400 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 2.21 (kernels 0.19, alloc 1.05, bootstrap 0.00, allgathers 0.00, topo 0.54, graphs 0.01, connections 0.38, rest 0.03)
+t-20260527033744-p2gmq-worker-0:10302:10401 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 2.21 (kernels 0.19, alloc 1.04, bootstrap 0.00, allgathers 0.00, topo 0.53, graphs 0.01, connections 0.39, rest 0.03)
+t-20260527033744-p2gmq-worker-0:10303:10396 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 2.21 (kernels 0.18, alloc 1.00, bootstrap 0.07, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.40, rest 0.02)
+t-20260527033744-p2gmq-worker-0:10300:10398 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 2.21 (kernels 0.19, alloc 1.05, bootstrap 0.00, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.40, rest 0.02)
+t-20260527033744-p2gmq-worker-0:10301:10397 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 2.21 (kernels 0.19, alloc 1.05, bootstrap 0.00, allgathers 0.00, topo 0.54, graphs 0.01, connections 0.39, rest 0.03)
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527033744-p2gmq-worker-0:10301:10495 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527033744-p2gmq-worker-0:10300:10498 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527033744-p2gmq-worker-0:10299:10497 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527033744-p2gmq-worker-0:10306:10496 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527033744-p2gmq-worker-0:10305:10494 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527033744-p2gmq-worker-0:10304:10493 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527033744-p2gmq-worker-0:10302:10491 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527033744-p2gmq-worker-0:10303:10492 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+{
+  "data_path": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext",
+  "tokenizer_path": "/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json",
+  "out_dir": "runs/mini_owt_fit_t5_bernoulliwrong_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_193815",
+  "text_column": "text",
+  "subset_size": 0,
+  "payload_len": 1022,
+  "append_eos": 1,
+  "log_skips": 20,
+  "cache_path": "cache/owt_t5_payload1022_appendeos1.pt",
+  "rebuild_cache": 0,
+  "online_data": 0,
+  "online_buffer_size": 8192,
+  "steps": 1000000,
+  "batch_size": 32,
+  "grad_accum": 2,
+  "lr": 0.0003,
+  "log_every": 50,
+  "save_every": 1000,
+  "dim": 768,
+  "layers": 12,
+  "heads": 12,
+  "mlp_dim": 3072,
+  "time_tokens": 4,
+  "abs_pos": 1,
+  "rope": 1,
+  "c_min": 1.0,
+  "c_max": 1024.0,
+  "seed": 1234
+}
+[data] rows=2860537 length=1024 vocab=32100 seen=8013769 dropped=5153232 kept=2860537 bos=1:</s> eos=1:</s>
+[head] ['</s>', '▁Port', '-', 'au', '-', 'Pri', 'nce', ',', '▁Haiti', '▁(', 'C', 'NN', ')', '▁--', '▁Earth', 'qua']
+[tail] ['▁magnitude', '▁earthquake', '▁flat', 't', 'ened', '▁Haiti', "'", 's', '▁capital', '▁city', '▁Tuesday', '▁afternoon', ',', '▁', 'affecting', '</s>']
+t-20260527033744-p2gmq-worker-0:10303:10611 [4] NCCL INFO NVLS comm 0xa635a50 headRank 4 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527033744-p2gmq-worker-0:10306:10610 [7] NCCL INFO NVLS comm 0x967bd00 headRank 7 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527033744-p2gmq-worker-0:10302:10612 [3] NCCL INFO NVLS comm 0xb251190 headRank 3 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527033744-p2gmq-worker-0:10300:10613 [1] NCCL INFO NVLS comm 0xa2b40f0 headRank 1 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527033744-p2gmq-worker-0:10304:10615 [5] NCCL INFO NVLS comm 0xa796fd0 headRank 5 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527033744-p2gmq-worker-0:10299:10616 [0] NCCL INFO NVLS comm 0xbca8730 headRank 0 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527033744-p2gmq-worker-0:10301:10614 [2] NCCL INFO NVLS comm 0x98885e0 headRank 2 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527033744-p2gmq-worker-0:10305:10617 [6] NCCL INFO NVLS comm 0xa82cfe0 headRank 6 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+step=50 loss=7.2234 {'pos0_bos_p': 0.8410916328430176, 'pos0_bos_top1': 4, 'last_eos_p': 0.8400060534477234, 'last_eos_top1': 4}
+step=100 loss=7.1883 {'pos0_bos_p': 0.9984087347984314, 'pos0_bos_top1': 4, 'last_eos_p': 0.9984415173530579, 'last_eos_top1': 4}
+step=150 loss=6.9457 {'pos0_bos_p': 0.9928956627845764, 'pos0_bos_top1': 4, 'last_eos_p': 0.9935449361801147, 'last_eos_top1': 4}
+step=200 loss=6.3220 {'pos0_bos_p': 0.9183393120765686, 'pos0_bos_top1': 4, 'last_eos_p': 0.9329309463500977, 'last_eos_top1': 4}
+step=250 loss=6.2433 {'pos0_bos_p': 0.9493728280067444, 'pos0_bos_top1': 4, 'last_eos_p': 0.9485777616500854, 'last_eos_top1': 4}
+step=300 loss=5.6348 {'pos0_bos_p': 0.9721682071685791, 'pos0_bos_top1': 4, 'last_eos_p': 0.9564447999000549, 'last_eos_top1': 4}
+step=350 loss=5.6769 {'pos0_bos_p': 0.9754027724266052, 'pos0_bos_top1': 4, 'last_eos_p': 0.9603887796401978, 'last_eos_top1': 4}
+step=400 loss=5.5343 {'pos0_bos_p': 0.9783049821853638, 'pos0_bos_top1': 4, 'last_eos_p': 0.9703626036643982, 'last_eos_top1': 4}
+step=450 loss=4.9934 {'pos0_bos_p': 0.9895748496055603, 'pos0_bos_top1': 4, 'last_eos_p': 0.9838179349899292, 'last_eos_top1': 4}
+step=500 loss=4.8684 {'pos0_bos_p': 0.9932355284690857, 'pos0_bos_top1': 4, 'last_eos_p': 0.9845588803291321, 'last_eos_top1': 4}
+step=550 loss=4.3846 {'pos0_bos_p': 0.9940125346183777, 'pos0_bos_top1': 4, 'last_eos_p': 0.9890598058700562, 'last_eos_top1': 4}
+step=600 loss=4.7291 {'pos0_bos_p': 0.9955198764801025, 'pos0_bos_top1': 4, 'last_eos_p': 0.9914763569831848, 'last_eos_top1': 4}
+step=650 loss=5.1061 {'pos0_bos_p': 0.9957165122032166, 'pos0_bos_top1': 4, 'last_eos_p': 0.9913386106491089, 'last_eos_top1': 4}
+step=700 loss=4.1143 {'pos0_bos_p': 0.9963093400001526, 'pos0_bos_top1': 4, 'last_eos_p': 0.9930015802383423, 'last_eos_top1': 4}
+step=750 loss=4.6373 {'pos0_bos_p': 0.9967343807220459, 'pos0_bos_top1': 4, 'last_eos_p': 0.9927859902381897, 'last_eos_top1': 4}
+step=800 loss=4.0914 {'pos0_bos_p': 0.9970274567604065, 'pos0_bos_top1': 4, 'last_eos_p': 0.9939413070678711, 'last_eos_top1': 4}
+step=850 loss=4.2351 {'pos0_bos_p': 0.9973978996276855, 'pos0_bos_top1': 4, 'last_eos_p': 0.9940692186355591, 'last_eos_top1': 4}
+step=900 loss=4.3954 {'pos0_bos_p': 0.9975166320800781, 'pos0_bos_top1': 4, 'last_eos_p': 0.9944340586662292, 'last_eos_top1': 4}
+step=950 loss=3.7541 {'pos0_bos_p': 0.9966287016868591, 'pos0_bos_top1': 4, 'last_eos_p': 0.9937399625778198, 'last_eos_top1': 4}
+step=1000 loss=3.8816 {'pos0_bos_p': 0.9978798627853394, 'pos0_bos_top1': 4, 'last_eos_p': 0.9957528114318848, 'last_eos_top1': 4}
+step=1050 loss=3.6337 {'pos0_bos_p': 0.9981043338775635, 'pos0_bos_top1': 4, 'last_eos_p': 0.9964368343353271, 'last_eos_top1': 4}
+step=1100 loss=4.4077 {'pos0_bos_p': 0.9982441663742065, 'pos0_bos_top1': 4, 'last_eos_p': 0.996547281742096, 'last_eos_top1': 4}
+step=1150 loss=4.5099 {'pos0_bos_p': 0.9985087513923645, 'pos0_bos_top1': 4, 'last_eos_p': 0.9973356127738953, 'last_eos_top1': 4}
+step=1200 loss=4.2448 {'pos0_bos_p': 0.9981324076652527, 'pos0_bos_top1': 4, 'last_eos_p': 0.9965559244155884, 'last_eos_top1': 4}
+step=1250 loss=4.1320 {'pos0_bos_p': 0.9981480836868286, 'pos0_bos_top1': 4, 'last_eos_p': 0.9966681599617004, 'last_eos_top1': 4}
+step=1300 loss=4.3440 {'pos0_bos_p': 0.9985987544059753, 'pos0_bos_top1': 4, 'last_eos_p': 0.9977434873580933, 'last_eos_top1': 4}
+step=1350 loss=4.1208 {'pos0_bos_p': 0.9981060028076172, 'pos0_bos_top1': 4, 'last_eos_p': 0.9972333312034607, 'last_eos_top1': 4}
+step=1400 loss=4.2679 {'pos0_bos_p': 0.998889148235321, 'pos0_bos_top1': 4, 'last_eos_p': 0.9982118606567383, 'last_eos_top1': 4}
+step=1450 loss=3.7389 {'pos0_bos_p': 0.9988946318626404, 'pos0_bos_top1': 4, 'last_eos_p': 0.9979343414306641, 'last_eos_top1': 4}
+step=1500 loss=3.6648 {'pos0_bos_p': 0.998691737651825, 'pos0_bos_top1': 4, 'last_eos_p': 0.9977328777313232, 'last_eos_top1': 4}
+step=1550 loss=3.6402 {'pos0_bos_p': 0.9979167580604553, 'pos0_bos_top1': 4, 'last_eos_p': 0.9969798922538757, 'last_eos_top1': 4}
+step=1600 loss=3.9200 {'pos0_bos_p': 0.9980452060699463, 'pos0_bos_top1': 4, 'last_eos_p': 0.9971649050712585, 'last_eos_top1': 4}
+step=1650 loss=3.9901 {'pos0_bos_p': 0.9981250166893005, 'pos0_bos_top1': 4, 'last_eos_p': 0.9973676800727844, 'last_eos_top1': 4}
+step=1700 loss=3.9617 {'pos0_bos_p': 0.9982129335403442, 'pos0_bos_top1': 4, 'last_eos_p': 0.9972752928733826, 'last_eos_top1': 4}
+step=1750 loss=4.5169 {'pos0_bos_p': 0.9986346364021301, 'pos0_bos_top1': 4, 'last_eos_p': 0.9979058504104614, 'last_eos_top1': 4}
+step=1800 loss=3.7237 {'pos0_bos_p': 0.9985482096672058, 'pos0_bos_top1': 4, 'last_eos_p': 0.9979708790779114, 'last_eos_top1': 4}
+step=1850 loss=3.7146 {'pos0_bos_p': 0.9987298846244812, 'pos0_bos_top1': 4, 'last_eos_p': 0.9981417655944824, 'last_eos_top1': 4}
+step=1900 loss=3.9051 {'pos0_bos_p': 0.9988681077957153, 'pos0_bos_top1': 4, 'last_eos_p': 0.9983444213867188, 'last_eos_top1': 4}
+step=1950 loss=4.0935 {'pos0_bos_p': 0.9986370205879211, 'pos0_bos_top1': 4, 'last_eos_p': 0.9979957342147827, 'last_eos_top1': 4}
+step=2000 loss=3.7086 {'pos0_bos_p': 0.9986200332641602, 'pos0_bos_top1': 4, 'last_eos_p': 0.9981788396835327, 'last_eos_top1': 4}
+step=2050 loss=3.8178 {'pos0_bos_p': 0.9989047050476074, 'pos0_bos_top1': 4, 'last_eos_p': 0.9984803795814514, 'last_eos_top1': 4}
+step=2100 loss=4.0149 {'pos0_bos_p': 0.9989466071128845, 'pos0_bos_top1': 4, 'last_eos_p': 0.9986082911491394, 'last_eos_top1': 4}
+step=2150 loss=3.6350 {'pos0_bos_p': 0.9990571141242981, 'pos0_bos_top1': 4, 'last_eos_p': 0.9987095594406128, 'last_eos_top1': 4}
+step=2200 loss=4.0369 {'pos0_bos_p': 0.9991350769996643, 'pos0_bos_top1': 4, 'last_eos_p': 0.9987767338752747, 'last_eos_top1': 4}
+step=2250 loss=3.9253 {'pos0_bos_p': 0.9990781545639038, 'pos0_bos_top1': 4, 'last_eos_p': 0.9987303614616394, 'last_eos_top1': 4}
+step=2300 loss=3.7598 {'pos0_bos_p': 0.9992414712905884, 'pos0_bos_top1': 4, 'last_eos_p': 0.9989332556724548, 'last_eos_top1': 4}
+step=2350 loss=3.5755 {'pos0_bos_p': 0.9992740750312805, 'pos0_bos_top1': 4, 'last_eos_p': 0.9989602565765381, 'last_eos_top1': 4}
+step=2400 loss=3.3700 {'pos0_bos_p': 0.9991924166679382, 'pos0_bos_top1': 4, 'last_eos_p': 0.9988082647323608, 'last_eos_top1': 4}
+step=2450 loss=3.6399 {'pos0_bos_p': 0.9992557168006897, 'pos0_bos_top1': 4, 'last_eos_p': 0.998906135559082, 'last_eos_top1': 4}
+step=2500 loss=3.9027 {'pos0_bos_p': 0.9992291927337646, 'pos0_bos_top1': 4, 'last_eos_p': 0.9988633394241333, 'last_eos_top1': 4}
+step=2550 loss=3.7713 {'pos0_bos_p': 0.9994018077850342, 'pos0_bos_top1': 4, 'last_eos_p': 0.999119222164154, 'last_eos_top1': 4}
+step=2600 loss=3.9680 {'pos0_bos_p': 0.9993995428085327, 'pos0_bos_top1': 4, 'last_eos_p': 0.9991395473480225, 'last_eos_top1': 4}
+step=2650 loss=3.7705 {'pos0_bos_p': 0.999397873878479, 'pos0_bos_top1': 4, 'last_eos_p': 0.9991519451141357, 'last_eos_top1': 4}
+step=2700 loss=3.4142 {'pos0_bos_p': 0.9994989633560181, 'pos0_bos_top1': 4, 'last_eos_p': 0.9992842078208923, 'last_eos_top1': 4}
+step=2750 loss=3.2250 {'pos0_bos_p': 0.9994999170303345, 'pos0_bos_top1': 4, 'last_eos_p': 0.9993199110031128, 'last_eos_top1': 4}
+step=2800 loss=4.0023 {'pos0_bos_p': 0.9995357990264893, 'pos0_bos_top1': 4, 'last_eos_p': 0.9993797540664673, 'last_eos_top1': 4}
+step=2850 loss=3.2594 {'pos0_bos_p': 0.9995520710945129, 'pos0_bos_top1': 4, 'last_eos_p': 0.9993582367897034, 'last_eos_top1': 4}
+step=2900 loss=3.7606 {'pos0_bos_p': 0.9995681643486023, 'pos0_bos_top1': 4, 'last_eos_p': 0.9993762373924255, 'last_eos_top1': 4}
+step=2950 loss=3.6813 {'pos0_bos_p': 0.9996504783630371, 'pos0_bos_top1': 4, 'last_eos_p': 0.9995319843292236, 'last_eos_top1': 4}
+step=3000 loss=3.2954 {'pos0_bos_p': 0.999609649181366, 'pos0_bos_top1': 4, 'last_eos_p': 0.9994325041770935, 'last_eos_top1': 4}
+step=3050 loss=3.5907 {'pos0_bos_p': 0.999663233757019, 'pos0_bos_top1': 4, 'last_eos_p': 0.9995342493057251, 'last_eos_top1': 4}
+step=3100 loss=3.3557 {'pos0_bos_p': 0.9996403455734253, 'pos0_bos_top1': 4, 'last_eos_p': 0.99949049949646, 'last_eos_top1': 4}
+step=3150 loss=3.6463 {'pos0_bos_p': 0.9996751546859741, 'pos0_bos_top1': 4, 'last_eos_p': 0.9995593428611755, 'last_eos_top1': 4}
+step=3200 loss=3.1684 {'pos0_bos_p': 0.9997177720069885, 'pos0_bos_top1': 4, 'last_eos_p': 0.9996047616004944, 'last_eos_top1': 4}
+step=3250 loss=3.1097 {'pos0_bos_p': 0.9997259974479675, 'pos0_bos_top1': 4, 'last_eos_p': 0.9996604919433594, 'last_eos_top1': 4}
+step=3300 loss=3.6621 {'pos0_bos_p': 0.9997088313102722, 'pos0_bos_top1': 4, 'last_eos_p': 0.9996312856674194, 'last_eos_top1': 4}
+step=3350 loss=3.6563 {'pos0_bos_p': 0.9996923208236694, 'pos0_bos_top1': 4, 'last_eos_p': 0.9996121525764465, 'last_eos_top1': 4}
+step=3400 loss=3.3509 {'pos0_bos_p': 0.9997097849845886, 'pos0_bos_top1': 4, 'last_eos_p': 0.9996292591094971, 'last_eos_top1': 4}
+step=3450 loss=3.7093 {'pos0_bos_p': 0.9997443556785583, 'pos0_bos_top1': 4, 'last_eos_p': 0.9996951818466187, 'last_eos_top1': 4}
+step=3500 loss=3.4635 {'pos0_bos_p': 0.9997255206108093, 'pos0_bos_top1': 4, 'last_eos_p': 0.999683141708374, 'last_eos_top1': 4}
+step=3550 loss=3.5433 {'pos0_bos_p': 0.9997650980949402, 'pos0_bos_top1': 4, 'last_eos_p': 0.999745786190033, 'last_eos_top1': 4}
+step=3600 loss=3.0142 {'pos0_bos_p': 0.9997095465660095, 'pos0_bos_top1': 4, 'last_eos_p': 0.9996802806854248, 'last_eos_top1': 4}
+step=3650 loss=3.5797 {'pos0_bos_p': 0.9997252821922302, 'pos0_bos_top1': 4, 'last_eos_p': 0.9996744394302368, 'last_eos_top1': 4}
+step=3700 loss=3.7214 {'pos0_bos_p': 0.9997599720954895, 'pos0_bos_top1': 4, 'last_eos_p': 0.9997411370277405, 'last_eos_top1': 4}
+step=3750 loss=3.5048 {'pos0_bos_p': 0.9997593760490417, 'pos0_bos_top1': 4, 'last_eos_p': 0.9997323155403137, 'last_eos_top1': 4}
+step=3800 loss=3.6710 {'pos0_bos_p': 0.9997710585594177, 'pos0_bos_top1': 4, 'last_eos_p': 0.9997499585151672, 'last_eos_top1': 4}
+step=3850 loss=3.4358 {'pos0_bos_p': 0.9997665286064148, 'pos0_bos_top1': 4, 'last_eos_p': 0.9997515082359314, 'last_eos_top1': 4}
+step=3900 loss=3.8056 {'pos0_bos_p': 0.9997884631156921, 'pos0_bos_top1': 4, 'last_eos_p': 0.9997743964195251, 'last_eos_top1': 4}
+step=3950 loss=3.1501 {'pos0_bos_p': 0.9997512698173523, 'pos0_bos_top1': 4, 'last_eos_p': 0.9997517466545105, 'last_eos_top1': 4}
+step=4000 loss=3.3060 {'pos0_bos_p': 0.9997738003730774, 'pos0_bos_top1': 4, 'last_eos_p': 0.9997718930244446, 'last_eos_top1': 4}
+step=4050 loss=3.3812 {'pos0_bos_p': 0.9997988343238831, 'pos0_bos_top1': 4, 'last_eos_p': 0.9997935891151428, 'last_eos_top1': 4}
+step=4100 loss=3.5699 {'pos0_bos_p': 0.9998095631599426, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998045563697815, 'last_eos_top1': 4}
+step=4150 loss=3.4591 {'pos0_bos_p': 0.9998354911804199, 'pos0_bos_top1': 4, 'last_eos_p': 0.999833345413208, 'last_eos_top1': 4}
+step=4200 loss=3.1666 {'pos0_bos_p': 0.9998096823692322, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998087286949158, 'last_eos_top1': 4}
+step=4250 loss=2.9324 {'pos0_bos_p': 0.9998120665550232, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998136162757874, 'last_eos_top1': 4}
+step=4300 loss=3.6598 {'pos0_bos_p': 0.9998341798782349, 'pos0_bos_top1': 4, 'last_eos_p': 0.99983811378479, 'last_eos_top1': 4}
+step=4350 loss=3.0817 {'pos0_bos_p': 0.9998291730880737, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998339414596558, 'last_eos_top1': 4}
+step=4400 loss=3.6275 {'pos0_bos_p': 0.9998666048049927, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998701810836792, 'last_eos_top1': 4}
+step=4450 loss=3.5226 {'pos0_bos_p': 0.9998575448989868, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998630285263062, 'last_eos_top1': 4}
+step=4500 loss=2.7634 {'pos0_bos_p': 0.999840497970581, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998425245285034, 'last_eos_top1': 4}
+step=4550 loss=3.9466 {'pos0_bos_p': 0.9998674392700195, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998728036880493, 'last_eos_top1': 4}
+step=4600 loss=3.4404 {'pos0_bos_p': 0.999875545501709, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998801946640015, 'last_eos_top1': 4}
+step=4650 loss=3.3415 {'pos0_bos_p': 0.9998675584793091, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998750686645508, 'last_eos_top1': 4}
+step=4700 loss=3.5761 {'pos0_bos_p': 0.9998691082000732, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998751878738403, 'last_eos_top1': 4}
+step=4750 loss=2.9369 {'pos0_bos_p': 0.9998711347579956, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998776912689209, 'last_eos_top1': 4}
+step=4800 loss=3.4183 {'pos0_bos_p': 0.9998832941055298, 'pos0_bos_top1': 4, 'last_eos_p': 0.9998886585235596, 'last_eos_top1': 4}
+step=4850 loss=3.4858 {'pos0_bos_p': 0.9999125003814697, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999165534973145, 'last_eos_top1': 4}
+step=4900 loss=3.2695 {'pos0_bos_p': 0.9999016523361206, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999042749404907, 'last_eos_top1': 4}
+step=4950 loss=3.4123 {'pos0_bos_p': 0.999909520149231, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999121427536011, 'last_eos_top1': 4}
+step=5000 loss=2.9828 {'pos0_bos_p': 0.9999160766601562, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999188184738159, 'last_eos_top1': 4}
+step=5050 loss=3.5923 {'pos0_bos_p': 0.9999237060546875, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999263286590576, 'last_eos_top1': 4}
+step=5100 loss=3.4871 {'pos0_bos_p': 0.9999173879623413, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999191761016846, 'last_eos_top1': 4}
+step=5150 loss=3.3358 {'pos0_bos_p': 0.9999274015426636, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999287128448486, 'last_eos_top1': 4}
+step=5200 loss=3.4354 {'pos0_bos_p': 0.9999254941940308, 'pos0_bos_top1': 4, 'last_eos_p': 0.999925971031189, 'last_eos_top1': 4}
+step=5250 loss=3.2880 {'pos0_bos_p': 0.999921441078186, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999212026596069, 'last_eos_top1': 4}
+step=5300 loss=3.9513 {'pos0_bos_p': 0.9999237060546875, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999239444732666, 'last_eos_top1': 4}
+step=5350 loss=3.0495 {'pos0_bos_p': 0.9999375343322754, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999377727508545, 'last_eos_top1': 4}
+step=5400 loss=3.1802 {'pos0_bos_p': 0.9999467134475708, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999468326568604, 'last_eos_top1': 4}
+step=5450 loss=3.1848 {'pos0_bos_p': 0.9999454021453857, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999454021453857, 'last_eos_top1': 4}
+step=5500 loss=3.4556 {'pos0_bos_p': 0.999945878982544, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999464750289917, 'last_eos_top1': 4}
+step=5550 loss=3.3164 {'pos0_bos_p': 0.9999372959136963, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999377727508545, 'last_eos_top1': 4}
+step=5600 loss=2.9475 {'pos0_bos_p': 0.999947190284729, 'pos0_bos_top1': 4, 'last_eos_p': 0.999948263168335, 'last_eos_top1': 4}
+step=5650 loss=3.1460 {'pos0_bos_p': 0.9999421834945679, 'pos0_bos_top1': 4, 'last_eos_p': 0.999942421913147, 'last_eos_top1': 4}
+step=5700 loss=3.6651 {'pos0_bos_p': 0.9999533891677856, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999527931213379, 'last_eos_top1': 4}
+step=5750 loss=2.9873 {'pos0_bos_p': 0.9999501705169678, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999489784240723, 'last_eos_top1': 4}
+step=5800 loss=3.3522 {'pos0_bos_p': 0.999947190284729, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999457597732544, 'last_eos_top1': 4}
+step=5850 loss=2.9468 {'pos0_bos_p': 0.9999487400054932, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999464750289917, 'last_eos_top1': 4}
+step=5900 loss=3.6862 {'pos0_bos_p': 0.9999557733535767, 'pos0_bos_top1': 4, 'last_eos_p': 0.999954342842102, 'last_eos_top1': 4}
+step=5950 loss=3.3077 {'pos0_bos_p': 0.9999551773071289, 'pos0_bos_top1': 4, 'last_eos_p': 0.999954104423523, 'last_eos_top1': 4}
+step=6000 loss=3.3930 {'pos0_bos_p': 0.999958872795105, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999582767486572, 'last_eos_top1': 4}
+step=6050 loss=3.5630 {'pos0_bos_p': 0.9999504089355469, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999487400054932, 'last_eos_top1': 4}
+step=6100 loss=3.5562 {'pos0_bos_p': 0.9999568462371826, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999555349349976, 'last_eos_top1': 4}
+step=6150 loss=2.9524 {'pos0_bos_p': 0.9999557733535767, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999547004699707, 'last_eos_top1': 4}
+step=6200 loss=3.1013 {'pos0_bos_p': 0.9999512434005737, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999501705169678, 'last_eos_top1': 4}
+step=6250 loss=2.7028 {'pos0_bos_p': 0.9999622106552124, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999620914459229, 'last_eos_top1': 4}
+step=6300 loss=3.0460 {'pos0_bos_p': 0.9999589920043945, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999587535858154, 'last_eos_top1': 4}
+step=6350 loss=3.1969 {'pos0_bos_p': 0.9999592304229736, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999582767486572, 'last_eos_top1': 4}
+step=6400 loss=3.0778 {'pos0_bos_p': 0.9999592304229736, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999587535858154, 'last_eos_top1': 4}
+step=6450 loss=3.2028 {'pos0_bos_p': 0.9999657869338989, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999656677246094, 'last_eos_top1': 4}
+step=6500 loss=3.6699 {'pos0_bos_p': 0.9999644756317139, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999648332595825, 'last_eos_top1': 4}
+step=6550 loss=3.3859 {'pos0_bos_p': 0.9999617338180542, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999611377716064, 'last_eos_top1': 4}
+step=6600 loss=3.2728 {'pos0_bos_p': 0.999963641166687, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999639987945557, 'last_eos_top1': 4}
+step=6650 loss=3.0634 {'pos0_bos_p': 0.9999688863754272, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999685287475586, 'last_eos_top1': 4}
+step=6700 loss=2.6626 {'pos0_bos_p': 0.9999634027481079, 'pos0_bos_top1': 4, 'last_eos_p': 0.999963641166687, 'last_eos_top1': 4}
+step=6750 loss=3.7541 {'pos0_bos_p': 0.9999659061431885, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999659061431885, 'last_eos_top1': 4}
+step=6800 loss=2.8614 {'pos0_bos_p': 0.9999741315841675, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999738931655884, 'last_eos_top1': 4}
+step=6850 loss=3.2824 {'pos0_bos_p': 0.9999712705612183, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999715089797974, 'last_eos_top1': 4}
+step=6900 loss=2.9090 {'pos0_bos_p': 0.9999692440032959, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999696016311646, 'last_eos_top1': 4}
+step=6950 loss=3.1401 {'pos0_bos_p': 0.9999703168869019, 'pos0_bos_top1': 4, 'last_eos_p': 0.999970555305481, 'last_eos_top1': 4}
+step=7000 loss=3.2636 {'pos0_bos_p': 0.9999717473983765, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999719858169556, 'last_eos_top1': 4}
+step=7050 loss=3.2060 {'pos0_bos_p': 0.9999775886535645, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999778270721436, 'last_eos_top1': 4}
+step=7100 loss=3.2560 {'pos0_bos_p': 0.9999727010726929, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999731779098511, 'last_eos_top1': 4}
+step=7150 loss=3.4647 {'pos0_bos_p': 0.9999791383743286, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999793767929077, 'last_eos_top1': 4}
+step=7200 loss=3.8580 {'pos0_bos_p': 0.9999758005142212, 'pos0_bos_top1': 4, 'last_eos_p': 0.999976396560669, 'last_eos_top1': 4}
+step=7250 loss=3.3906 {'pos0_bos_p': 0.9999749660491943, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999749660491943, 'last_eos_top1': 4}
+step=7300 loss=3.1115 {'pos0_bos_p': 0.9999725818634033, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999723434448242, 'last_eos_top1': 4}
+step=7350 loss=3.0173 {'pos0_bos_p': 0.9999804496765137, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999805688858032, 'last_eos_top1': 4}
+step=7400 loss=3.2709 {'pos0_bos_p': 0.9999783039093018, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999781847000122, 'last_eos_top1': 4}
+step=7450 loss=2.9318 {'pos0_bos_p': 0.9999773502349854, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999774694442749, 'last_eos_top1': 4}
+step=7500 loss=3.1039 {'pos0_bos_p': 0.9999746084213257, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999747276306152, 'last_eos_top1': 4}
+step=7550 loss=3.0932 {'pos0_bos_p': 0.9999783039093018, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999783039093018, 'last_eos_top1': 4}
+step=7600 loss=3.1335 {'pos0_bos_p': 0.9999788999557495, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999790191650391, 'last_eos_top1': 4}
+step=7650 loss=3.3918 {'pos0_bos_p': 0.999974250793457, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999744892120361, 'last_eos_top1': 4}
+step=7700 loss=3.7219 {'pos0_bos_p': 0.9999736547470093, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999736547470093, 'last_eos_top1': 4}
+step=7750 loss=2.9208 {'pos0_bos_p': 0.9999749660491943, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999743700027466, 'last_eos_top1': 4}
+step=7800 loss=2.7325 {'pos0_bos_p': 0.999976634979248, 'pos0_bos_top1': 4, 'last_eos_p': 0.999976634979248, 'last_eos_top1': 4}
+step=7850 loss=2.9562 {'pos0_bos_p': 0.9999722242355347, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999722242355347, 'last_eos_top1': 4}
+step=7900 loss=2.5984 {'pos0_bos_p': 0.9999765157699585, 'pos0_bos_top1': 4, 'last_eos_p': 0.999976396560669, 'last_eos_top1': 4}
+step=7950 loss=3.4669 {'pos0_bos_p': 0.9999806880950928, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999808073043823, 'last_eos_top1': 4}
+step=8000 loss=2.7450 {'pos0_bos_p': 0.9999809265136719, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999808073043823, 'last_eos_top1': 4}
+step=8050 loss=3.6576 {'pos0_bos_p': 0.9999754428863525, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999759197235107, 'last_eos_top1': 4}
+step=8100 loss=3.1435 {'pos0_bos_p': 0.9999786615371704, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999783039093018, 'last_eos_top1': 4}
+step=8150 loss=3.1076 {'pos0_bos_p': 0.9999769926071167, 'pos0_bos_top1': 4, 'last_eos_p': 0.999976634979248, 'last_eos_top1': 4}
+step=8200 loss=3.2689 {'pos0_bos_p': 0.9999756813049316, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999755620956421, 'last_eos_top1': 4}
+step=8250 loss=3.4085 {'pos0_bos_p': 0.9999812841415405, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999815225601196, 'last_eos_top1': 4}
+step=8300 loss=2.8337 {'pos0_bos_p': 0.9999768733978271, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999767541885376, 'last_eos_top1': 4}
+step=8350 loss=3.4514 {'pos0_bos_p': 0.999982476234436, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999822378158569, 'last_eos_top1': 4}
+step=8400 loss=3.3135 {'pos0_bos_p': 0.9999812841415405, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999812841415405, 'last_eos_top1': 4}
+step=8450 loss=3.1703 {'pos0_bos_p': 0.9999841451644897, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999840259552002, 'last_eos_top1': 4}
+step=8500 loss=3.2390 {'pos0_bos_p': 0.999985933303833, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999861717224121, 'last_eos_top1': 4}
+step=8550 loss=3.3350 {'pos0_bos_p': 0.999987006187439, 'pos0_bos_top1': 4, 'last_eos_p': 0.999987006187439, 'last_eos_top1': 4}
+step=8600 loss=3.9497 {'pos0_bos_p': 0.9999842643737793, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999843835830688, 'last_eos_top1': 4}
+step=8650 loss=2.9913 {'pos0_bos_p': 0.999988317489624, 'pos0_bos_top1': 4, 'last_eos_p': 0.999988317489624, 'last_eos_top1': 4}
+step=8700 loss=3.3607 {'pos0_bos_p': 0.9999873638153076, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999871253967285, 'last_eos_top1': 4}
+step=8750 loss=3.3274 {'pos0_bos_p': 0.9999866485595703, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999866485595703, 'last_eos_top1': 4}
+step=8800 loss=2.5924 {'pos0_bos_p': 0.9999871253967285, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999872446060181, 'last_eos_top1': 4}
+step=8850 loss=2.6373 {'pos0_bos_p': 0.9999874830245972, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999879598617554, 'last_eos_top1': 4}
+step=8900 loss=3.2003 {'pos0_bos_p': 0.9999886751174927, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999887943267822, 'last_eos_top1': 4}
+step=8950 loss=2.7473 {'pos0_bos_p': 0.9999890327453613, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999892711639404, 'last_eos_top1': 4}
+step=9000 loss=2.9785 {'pos0_bos_p': 0.9999861717224121, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999865293502808, 'last_eos_top1': 4}
+step=9050 loss=3.0745 {'pos0_bos_p': 0.9999853372573853, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999854564666748, 'last_eos_top1': 4}
+step=9100 loss=3.5311 {'pos0_bos_p': 0.9999837875366211, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999840259552002, 'last_eos_top1': 4}
+step=9150 loss=3.8067 {'pos0_bos_p': 0.999985933303833, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999860525131226, 'last_eos_top1': 4}
+step=9200 loss=3.5018 {'pos0_bos_p': 0.9999867677688599, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999873638153076, 'last_eos_top1': 4}
+step=9250 loss=2.4462 {'pos0_bos_p': 0.9999878406524658, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999878406524658, 'last_eos_top1': 4}
+step=9300 loss=3.7870 {'pos0_bos_p': 0.99998939037323, 'pos0_bos_top1': 4, 'last_eos_p': 0.99998939037323, 'last_eos_top1': 4}
+step=9350 loss=2.9020 {'pos0_bos_p': 0.9999884366989136, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999886751174927, 'last_eos_top1': 4}
+step=9400 loss=3.4380 {'pos0_bos_p': 0.9999864101409912, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999865293502808, 'last_eos_top1': 4}
+step=9450 loss=3.1199 {'pos0_bos_p': 0.9999867677688599, 'pos0_bos_top1': 4, 'last_eos_p': 0.999987006187439, 'last_eos_top1': 4}
+step=9500 loss=3.2013 {'pos0_bos_p': 0.9999850988388062, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999853372573853, 'last_eos_top1': 4}
+step=9550 loss=3.0353 {'pos0_bos_p': 0.9999877214431763, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999878406524658, 'last_eos_top1': 4}
+step=9600 loss=3.1640 {'pos0_bos_p': 0.9999897480010986, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999898672103882, 'last_eos_top1': 4}
+step=9650 loss=2.5325 {'pos0_bos_p': 0.99998939037323, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999895095825195, 'last_eos_top1': 4}
+step=9700 loss=3.0431 {'pos0_bos_p': 0.9999878406524658, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999877214431763, 'last_eos_top1': 4}
+step=9750 loss=2.5544 {'pos0_bos_p': 0.9999895095825195, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999895095825195, 'last_eos_top1': 4}
+step=9800 loss=3.4270 {'pos0_bos_p': 0.999990701675415, 'pos0_bos_top1': 4, 'last_eos_p': 0.999990701675415, 'last_eos_top1': 4}
+step=9850 loss=3.1366 {'pos0_bos_p': 0.9999912977218628, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999912977218628, 'last_eos_top1': 4}
+step=9900 loss=3.2963 {'pos0_bos_p': 0.9999905824661255, 'pos0_bos_top1': 4, 'last_eos_p': 0.999990701675415, 'last_eos_top1': 4}
+step=9950 loss=3.4279 {'pos0_bos_p': 0.9999902248382568, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999903440475464, 'last_eos_top1': 4}
+step=10000 loss=2.5380 {'pos0_bos_p': 0.9999902248382568, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999899864196777, 'last_eos_top1': 4}
+step=10050 loss=3.3771 {'pos0_bos_p': 0.9999924898147583, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999924898147583, 'last_eos_top1': 4}
+step=10100 loss=2.8628 {'pos0_bos_p': 0.9999933242797852, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999934434890747, 'last_eos_top1': 4}
+step=10150 loss=3.0367 {'pos0_bos_p': 0.9999926090240479, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999926090240479, 'last_eos_top1': 4}
+step=10200 loss=2.9719 {'pos0_bos_p': 0.9999939203262329, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999940395355225, 'last_eos_top1': 4}
+step=10250 loss=3.1192 {'pos0_bos_p': 0.9999939203262329, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999939203262329, 'last_eos_top1': 4}
+step=10300 loss=3.1450 {'pos0_bos_p': 0.9999896287918091, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999897480010986, 'last_eos_top1': 4}
+step=10350 loss=3.5001 {'pos0_bos_p': 0.9999791383743286, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999792575836182, 'last_eos_top1': 4}
+step=10400 loss=3.1251 {'pos0_bos_p': 0.9999822378158569, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999828338623047, 'last_eos_top1': 4}
+step=10450 loss=3.5845 {'pos0_bos_p': 0.999982476234436, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999830722808838, 'last_eos_top1': 4}
+step=10500 loss=3.6046 {'pos0_bos_p': 0.9999842643737793, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999847412109375, 'last_eos_top1': 4}
+step=10550 loss=3.5883 {'pos0_bos_p': 0.9999858140945435, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999860525131226, 'last_eos_top1': 4}
+step=10600 loss=3.3920 {'pos0_bos_p': 0.9999873638153076, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999876022338867, 'last_eos_top1': 4}
+step=10650 loss=3.7092 {'pos0_bos_p': 0.9999887943267822, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999889135360718, 'last_eos_top1': 4}
+step=10700 loss=3.2553 {'pos0_bos_p': 0.999990701675415, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999908208847046, 'last_eos_top1': 4}
+step=10750 loss=3.2148 {'pos0_bos_p': 0.9999897480010986, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999902248382568, 'last_eos_top1': 4}
+step=10800 loss=3.0847 {'pos0_bos_p': 0.9999921321868896, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999923706054688, 'last_eos_top1': 4}
+step=10850 loss=2.8792 {'pos0_bos_p': 0.9999912977218628, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999915361404419, 'last_eos_top1': 4}
+step=10900 loss=3.1619 {'pos0_bos_p': 0.9999924898147583, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999924898147583, 'last_eos_top1': 4}
+step=10950 loss=3.3963 {'pos0_bos_p': 0.9999934434890747, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999934434890747, 'last_eos_top1': 4}
+step=11000 loss=3.0068 {'pos0_bos_p': 0.9999923706054688, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999923706054688, 'last_eos_top1': 4}
+step=11050 loss=3.6583 {'pos0_bos_p': 0.999992847442627, 'pos0_bos_top1': 4, 'last_eos_p': 0.999992847442627, 'last_eos_top1': 4}
+step=11100 loss=2.1552 {'pos0_bos_p': 0.9999934434890747, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999934434890747, 'last_eos_top1': 4}
+step=11150 loss=3.1897 {'pos0_bos_p': 0.9999938011169434, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999936819076538, 'last_eos_top1': 4}
+step=11200 loss=3.2513 {'pos0_bos_p': 0.999995231628418, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999951124191284, 'last_eos_top1': 4}
+step=11250 loss=2.9944 {'pos0_bos_p': 0.9999926090240479, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999923706054688, 'last_eos_top1': 4}
+step=11300 loss=3.1619 {'pos0_bos_p': 0.9999939203262329, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999939203262329, 'last_eos_top1': 4}
+step=11350 loss=3.0796 {'pos0_bos_p': 0.9999935626983643, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999935626983643, 'last_eos_top1': 4}
+step=11400 loss=3.0805 {'pos0_bos_p': 0.9999939203262329, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999940395355225, 'last_eos_top1': 4}
+step=11450 loss=3.8792 {'pos0_bos_p': 0.9999939203262329, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999939203262329, 'last_eos_top1': 4}
+step=11500 loss=2.8649 {'pos0_bos_p': 0.9999943971633911, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999946355819702, 'last_eos_top1': 4}
+step=11550 loss=3.3171 {'pos0_bos_p': 0.9999953508377075, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999953508377075, 'last_eos_top1': 4}
+step=11600 loss=3.6034 {'pos0_bos_p': 0.9999955892562866, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999955892562866, 'last_eos_top1': 4}
+step=11650 loss=3.3873 {'pos0_bos_p': 0.9999951124191284, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999951124191284, 'last_eos_top1': 4}
+step=11700 loss=3.0638 {'pos0_bos_p': 0.9999953508377075, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999953508377075, 'last_eos_top1': 4}
+step=11750 loss=3.0671 {'pos0_bos_p': 0.9999959468841553, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999959468841553, 'last_eos_top1': 4}
+step=11800 loss=3.0673 {'pos0_bos_p': 0.9999955892562866, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999955892562866, 'last_eos_top1': 4}
+step=11850 loss=3.7292 {'pos0_bos_p': 0.9999960660934448, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999961853027344, 'last_eos_top1': 4}
+step=11900 loss=3.1611 {'pos0_bos_p': 0.9999960660934448, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999960660934448, 'last_eos_top1': 4}
+step=11950 loss=2.4943 {'pos0_bos_p': 0.9999966621398926, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999967813491821, 'last_eos_top1': 4}
+step=12000 loss=3.4734 {'pos0_bos_p': 0.9999951124191284, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999958276748657, 'last_eos_top1': 4}
+step=12050 loss=2.9444 {'pos0_bos_p': 0.9999957084655762, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999959468841553, 'last_eos_top1': 4}
+step=12100 loss=3.6102 {'pos0_bos_p': 0.9999961853027344, 'pos0_bos_top1': 4, 'last_eos_p': 0.999996542930603, 'last_eos_top1': 4}
+step=12150 loss=3.1850 {'pos0_bos_p': 0.9999970197677612, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999971389770508, 'last_eos_top1': 4}
+step=12200 loss=2.8358 {'pos0_bos_p': 0.9999974966049194, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999977350234985, 'last_eos_top1': 4}
+step=12250 loss=2.8296 {'pos0_bos_p': 0.9999977350234985, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999978542327881, 'last_eos_top1': 4}
+step=12300 loss=2.7134 {'pos0_bos_p': 0.9999977350234985, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999979734420776, 'last_eos_top1': 4}
+step=12350 loss=2.5272 {'pos0_bos_p': 0.9999977350234985, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999978542327881, 'last_eos_top1': 4}
+step=12400 loss=3.4596 {'pos0_bos_p': 0.9999977350234985, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999977350234985, 'last_eos_top1': 4}
+step=12450 loss=3.3807 {'pos0_bos_p': 0.9999979734420776, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999980926513672, 'last_eos_top1': 4}
+step=12500 loss=2.9183 {'pos0_bos_p': 0.9999977350234985, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999979734420776, 'last_eos_top1': 4}
+step=12550 loss=2.9635 {'pos0_bos_p': 0.9999908208847046, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999915361404419, 'last_eos_top1': 4}
+step=12600 loss=2.9001 {'pos0_bos_p': 0.999981164932251, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999830722808838, 'last_eos_top1': 4}
+step=12650 loss=3.0810 {'pos0_bos_p': 0.9999814033508301, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999829530715942, 'last_eos_top1': 4}
+step=12700 loss=3.3929 {'pos0_bos_p': 0.99997878074646, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999812841415405, 'last_eos_top1': 4}
+step=12750 loss=3.3298 {'pos0_bos_p': 0.999987006187439, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999885559082031, 'last_eos_top1': 4}
+step=12800 loss=3.3554 {'pos0_bos_p': 0.9999864101409912, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999884366989136, 'last_eos_top1': 4}
+step=12850 loss=3.0740 {'pos0_bos_p': 0.9999895095825195, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999904632568359, 'last_eos_top1': 4}
+step=12900 loss=2.4365 {'pos0_bos_p': 0.9999890327453613, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999897480010986, 'last_eos_top1': 4}
+step=12950 loss=3.3601 {'pos0_bos_p': 0.9999871253967285, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999884366989136, 'last_eos_top1': 4}
+step=13000 loss=3.2112 {'pos0_bos_p': 0.9999856948852539, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999856948852539, 'last_eos_top1': 4}
+step=13050 loss=3.8935 {'pos0_bos_p': 0.9999772310256958, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999785423278809, 'last_eos_top1': 4}
+step=13100 loss=2.6351 {'pos0_bos_p': 0.9999843835830688, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999852180480957, 'last_eos_top1': 4}
+step=13150 loss=3.4912 {'pos0_bos_p': 0.9999892711639404, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999903440475464, 'last_eos_top1': 4}
+step=13200 loss=3.3603 {'pos0_bos_p': 0.9999909400939941, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999915361404419, 'last_eos_top1': 4}
+step=13250 loss=3.2354 {'pos0_bos_p': 0.9999905824661255, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999916553497314, 'last_eos_top1': 4}
+step=13300 loss=3.1649 {'pos0_bos_p': 0.9999924898147583, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999927282333374, 'last_eos_top1': 4}
+step=13350 loss=2.9994 {'pos0_bos_p': 0.9999867677688599, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999889135360718, 'last_eos_top1': 4}
+step=13400 loss=3.3962 {'pos0_bos_p': 0.9999896287918091, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999926090240479, 'last_eos_top1': 4}
+step=13450 loss=2.5004 {'pos0_bos_p': 0.9999909400939941, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999935626983643, 'last_eos_top1': 4}
+step=13500 loss=2.9199 {'pos0_bos_p': 0.9999866485595703, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999886751174927, 'last_eos_top1': 4}
+step=13550 loss=3.1094 {'pos0_bos_p': 0.9999922513961792, 'pos0_bos_top1': 4, 'last_eos_p': 0.999992847442627, 'last_eos_top1': 4}
+step=13600 loss=3.1377 {'pos0_bos_p': 0.9999895095825195, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999912977218628, 'last_eos_top1': 4}
+step=13650 loss=3.1727 {'pos0_bos_p': 0.999993085861206, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999935626983643, 'last_eos_top1': 4}
+step=13700 loss=3.1430 {'pos0_bos_p': 0.99998939037323, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999896287918091, 'last_eos_top1': 4}
+step=13750 loss=2.9664 {'pos0_bos_p': 0.9999914169311523, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999897480010986, 'last_eos_top1': 4}
+step=13800 loss=2.6662 {'pos0_bos_p': 0.9999871253967285, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999884366989136, 'last_eos_top1': 4}
+step=13850 loss=2.7174 {'pos0_bos_p': 0.9999865293502808, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999880790710449, 'last_eos_top1': 4}
+step=13900 loss=3.7482 {'pos0_bos_p': 0.9999946355819702, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999948740005493, 'last_eos_top1': 4}
+step=13950 loss=3.0528 {'pos0_bos_p': 0.9999936819076538, 'pos0_bos_top1': 4, 'last_eos_p': 0.999994158744812, 'last_eos_top1': 4}
+step=14000 loss=2.9118 {'pos0_bos_p': 0.9999943971633911, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999946355819702, 'last_eos_top1': 4}
+step=14050 loss=2.3416 {'pos0_bos_p': 0.9999954700469971, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999958276748657, 'last_eos_top1': 4}
+step=14100 loss=3.2455 {'pos0_bos_p': 0.9999951124191284, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999954700469971, 'last_eos_top1': 4}
+step=14150 loss=3.6813 {'pos0_bos_p': 0.9999984502792358, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999984502792358, 'last_eos_top1': 4}
+step=14200 loss=1.9912 {'pos0_bos_p': 0.9999982118606567, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999984502792358, 'last_eos_top1': 4}
+step=14250 loss=3.2737 {'pos0_bos_p': 0.9999980926513672, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999980926513672, 'last_eos_top1': 4}
+step=14300 loss=3.0220 {'pos0_bos_p': 0.9999982118606567, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999982118606567, 'last_eos_top1': 4}
+step=14350 loss=3.2554 {'pos0_bos_p': 0.9999974966049194, 'pos0_bos_top1': 4, 'last_eos_p': 0.999997615814209, 'last_eos_top1': 4}
+step=14400 loss=2.8861 {'pos0_bos_p': 0.9999973773956299, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999973773956299, 'last_eos_top1': 4}
+step=14450 loss=3.1287 {'pos0_bos_p': 0.999997615814209, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999977350234985, 'last_eos_top1': 4}
+step=14500 loss=3.2466 {'pos0_bos_p': 0.9999966621398926, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999970197677612, 'last_eos_top1': 4}
+step=14550 loss=3.5160 {'pos0_bos_p': 0.9999970197677612, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999971389770508, 'last_eos_top1': 4}
+step=14600 loss=2.8267 {'pos0_bos_p': 0.9999970197677612, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999973773956299, 'last_eos_top1': 4}
+step=14650 loss=3.6816 {'pos0_bos_p': 0.9999974966049194, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999980926513672, 'last_eos_top1': 4}
+step=14700 loss=3.2690 {'pos0_bos_p': 0.9999971389770508, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999973773956299, 'last_eos_top1': 4}
+step=14750 loss=3.3172 {'pos0_bos_p': 0.999996542930603, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999966621398926, 'last_eos_top1': 4}
+step=14800 loss=3.0613 {'pos0_bos_p': 0.9999971389770508, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999973773956299, 'last_eos_top1': 4}
+step=14850 loss=2.2396 {'pos0_bos_p': 0.9999947547912598, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999935626983643, 'last_eos_top1': 4}
+step=14900 loss=3.0842 {'pos0_bos_p': 0.9999921321868896, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999932050704956, 'last_eos_top1': 4}
+step=14950 loss=3.0816 {'pos0_bos_p': 0.9999932050704956, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999939203262329, 'last_eos_top1': 4}
+step=15000 loss=3.1802 {'pos0_bos_p': 0.9999959468841553, 'pos0_bos_top1': 4, 'last_eos_p': 0.999996542930603, 'last_eos_top1': 4}
+step=15050 loss=2.8430 {'pos0_bos_p': 0.9999945163726807, 'pos0_bos_top1': 4, 'last_eos_p': 0.999995231628418, 'last_eos_top1': 4}
+step=15100 loss=3.4582 {'pos0_bos_p': 0.9999951124191284, 'pos0_bos_top1': 4, 'last_eos_p': 0.999996542930603, 'last_eos_top1': 4}
+step=15150 loss=3.6280 {'pos0_bos_p': 0.9999945163726807, 'pos0_bos_top1': 4, 'last_eos_p': 0.999995231628418, 'last_eos_top1': 4}
+step=15200 loss=2.9045 {'pos0_bos_p': 0.9999946355819702, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999953508377075, 'last_eos_top1': 4}
+step=15250 loss=3.0303 {'pos0_bos_p': 0.9999951124191284, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999957084655762, 'last_eos_top1': 4}
+step=15300 loss=3.3811 {'pos0_bos_p': 0.9999970197677612, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999973773956299, 'last_eos_top1': 4}
+step=15350 loss=3.0546 {'pos0_bos_p': 0.9999818801879883, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999880790710449, 'last_eos_top1': 4}
+step=15400 loss=2.9834 {'pos0_bos_p': 0.9999791383743286, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999856948852539, 'last_eos_top1': 4}
+step=15450 loss=2.9911 {'pos0_bos_p': 0.9999855756759644, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999901056289673, 'last_eos_top1': 4}
+step=15500 loss=3.4293 {'pos0_bos_p': 0.9999879598617554, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999921321868896, 'last_eos_top1': 4}
+step=15550 loss=3.6341 {'pos0_bos_p': 0.9999881982803345, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999911785125732, 'last_eos_top1': 4}
+step=15600 loss=3.0889 {'pos0_bos_p': 0.9999926090240479, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999942779541016, 'last_eos_top1': 4}
+step=15650 loss=3.8378 {'pos0_bos_p': 0.99998939037323, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999920129776001, 'last_eos_top1': 4}
+step=15700 loss=3.0675 {'pos0_bos_p': 0.9999926090240479, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999946355819702, 'last_eos_top1': 4}
+step=15750 loss=2.7422 {'pos0_bos_p': 0.9999927282333374, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999945163726807, 'last_eos_top1': 4}
+step=15800 loss=2.7224 {'pos0_bos_p': 0.9999916553497314, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999933242797852, 'last_eos_top1': 4}
+step=15850 loss=2.6200 {'pos0_bos_p': 0.9999921321868896, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999945163726807, 'last_eos_top1': 4}
+step=15900 loss=3.2028 {'pos0_bos_p': 0.999993085861206, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999946355819702, 'last_eos_top1': 4}
+step=15950 loss=3.0344 {'pos0_bos_p': 0.999993085861206, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999947547912598, 'last_eos_top1': 4}
+step=16000 loss=3.1517 {'pos0_bos_p': 0.9999933242797852, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999947547912598, 'last_eos_top1': 4}
+step=16050 loss=3.4683 {'pos0_bos_p': 0.9999936819076538, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999949932098389, 'last_eos_top1': 4}
+step=16100 loss=2.5095 {'pos0_bos_p': 0.9999940395355225, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999954700469971, 'last_eos_top1': 4}
+step=16150 loss=2.9377 {'pos0_bos_p': 0.9999949932098389, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999967813491821, 'last_eos_top1': 4}
+step=16200 loss=2.8756 {'pos0_bos_p': 0.9999947547912598, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999959468841553, 'last_eos_top1': 4}
+step=16250 loss=2.8224 {'pos0_bos_p': 0.9999948740005493, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999964237213135, 'last_eos_top1': 4}
+step=16300 loss=2.6659 {'pos0_bos_p': 0.9999953508377075, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999964237213135, 'last_eos_top1': 4}
+step=16350 loss=3.1355 {'pos0_bos_p': 0.999995231628418, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999966621398926, 'last_eos_top1': 4}
+step=16400 loss=3.5562 {'pos0_bos_p': 0.9999951124191284, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999964237213135, 'last_eos_top1': 4}
+step=16450 loss=3.1294 {'pos0_bos_p': 0.9999958276748657, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999969005584717, 'last_eos_top1': 4}
+step=16500 loss=3.3263 {'pos0_bos_p': 0.9999973773956299, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999982118606567, 'last_eos_top1': 4}
+step=16550 loss=3.1862 {'pos0_bos_p': 0.999996542930603, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999970197677612, 'last_eos_top1': 4}
+step=16600 loss=2.5213 {'pos0_bos_p': 0.999996542930603, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999971389770508, 'last_eos_top1': 4}
+step=16650 loss=2.9718 {'pos0_bos_p': 0.9999966621398926, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999973773956299, 'last_eos_top1': 4}
+step=16700 loss=2.7939 {'pos0_bos_p': 0.9999958276748657, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999969005584717, 'last_eos_top1': 4}
+step=16750 loss=3.0191 {'pos0_bos_p': 0.999996542930603, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999971389770508, 'last_eos_top1': 4}
+step=16800 loss=2.7966 {'pos0_bos_p': 0.9999964237213135, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999970197677612, 'last_eos_top1': 4}
+step=16850 loss=2.8075 {'pos0_bos_p': 0.9999969005584717, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999974966049194, 'last_eos_top1': 4}
+step=16900 loss=2.9395 {'pos0_bos_p': 0.9999974966049194, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999982118606567, 'last_eos_top1': 4}
+step=16950 loss=3.1904 {'pos0_bos_p': 0.9999969005584717, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999973773956299, 'last_eos_top1': 4}
+step=17000 loss=3.0445 {'pos0_bos_p': 0.9999948740005493, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999958276748657, 'last_eos_top1': 4}
+step=17050 loss=3.3680 {'pos0_bos_p': 0.9999938011169434, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999949932098389, 'last_eos_top1': 4}
+step=17100 loss=2.9631 {'pos0_bos_p': 0.9999939203262329, 'pos0_bos_top1': 4, 'last_eos_p': 0.999995231628418, 'last_eos_top1': 4}
+step=17150 loss=2.7308 {'pos0_bos_p': 0.9999943971633911, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999957084655762, 'last_eos_top1': 4}
+step=17200 loss=2.8417 {'pos0_bos_p': 0.9999946355819702, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999958276748657, 'last_eos_top1': 4}
+step=17250 loss=3.1893 {'pos0_bos_p': 0.9999949932098389, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999959468841553, 'last_eos_top1': 4}
+step=17300 loss=2.6262 {'pos0_bos_p': 0.9999949932098389, 'pos0_bos_top1': 4, 'last_eos_p': 0.999996542930603, 'last_eos_top1': 4}
+step=17350 loss=2.6558 {'pos0_bos_p': 0.9999958276748657, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999971389770508, 'last_eos_top1': 4}
+step=17400 loss=3.0016 {'pos0_bos_p': 0.9999953508377075, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999969005584717, 'last_eos_top1': 4}
+step=17450 loss=3.2920 {'pos0_bos_p': 0.9999969005584717, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999978542327881, 'last_eos_top1': 4}
+step=17500 loss=2.2271 {'pos0_bos_p': 0.9999958276748657, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999971389770508, 'last_eos_top1': 4}
+step=17550 loss=3.1431 {'pos0_bos_p': 0.9999959468841553, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999970197677612, 'last_eos_top1': 4}
+step=17600 loss=3.2386 {'pos0_bos_p': 0.9999966621398926, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999980926513672, 'last_eos_top1': 4}
+step=17650 loss=2.9758 {'pos0_bos_p': 0.9999973773956299, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999984502792358, 'last_eos_top1': 4}
+step=17700 loss=2.9014 {'pos0_bos_p': 0.9999967813491821, 'pos0_bos_top1': 4, 'last_eos_p': 0.999997615814209, 'last_eos_top1': 4}
+step=17750 loss=2.7461 {'pos0_bos_p': 0.9999964237213135, 'pos0_bos_top1': 4, 'last_eos_p': 0.999997615814209, 'last_eos_top1': 4}
+step=17800 loss=3.0900 {'pos0_bos_p': 0.9999969005584717, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999982118606567, 'last_eos_top1': 4}
+step=17850 loss=2.7298 {'pos0_bos_p': 0.9999970197677612, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999978542327881, 'last_eos_top1': 4}
+step=17900 loss=2.8426 {'pos0_bos_p': 0.9999972581863403, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999978542327881, 'last_eos_top1': 4}
+step=17950 loss=2.6699 {'pos0_bos_p': 0.9999972581863403, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999978542327881, 'last_eos_top1': 4}
+step=18000 loss=3.6658 {'pos0_bos_p': 0.9999970197677612, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999977350234985, 'last_eos_top1': 4}
+step=18050 loss=3.4583 {'pos0_bos_p': 0.999997615814209, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999982118606567, 'last_eos_top1': 4}
+step=18100 loss=2.5417 {'pos0_bos_p': 0.999997615814209, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999982118606567, 'last_eos_top1': 4}
+step=18150 loss=2.7194 {'pos0_bos_p': 0.999997615814209, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999984502792358, 'last_eos_top1': 4}
+step=18200 loss=2.7169 {'pos0_bos_p': 0.9999980926513672, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999986886978149, 'last_eos_top1': 4}
+step=18250 loss=2.4709 {'pos0_bos_p': 0.9999980926513672, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999985694885254, 'last_eos_top1': 4}
+step=18300 loss=3.0065 {'pos0_bos_p': 0.9999974966049194, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999980926513672, 'last_eos_top1': 4}
+step=18350 loss=2.9949 {'pos0_bos_p': 0.9999977350234985, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999984502792358, 'last_eos_top1': 4}
+step=18400 loss=2.7156 {'pos0_bos_p': 0.9999933242797852, 'pos0_bos_top1': 4, 'last_eos_p': 0.999995231628418, 'last_eos_top1': 4}
+step=18450 loss=2.9166 {'pos0_bos_p': 0.9999961853027344, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999972581863403, 'last_eos_top1': 4}
+step=18500 loss=3.0075 {'pos0_bos_p': 0.9999963045120239, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999977350234985, 'last_eos_top1': 4}
+step=18550 loss=2.4571 {'pos0_bos_p': 0.999995231628418, 'pos0_bos_top1': 4, 'last_eos_p': 0.999997615814209, 'last_eos_top1': 4}
+step=18600 loss=2.6807 {'pos0_bos_p': 0.9999946355819702, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999963045120239, 'last_eos_top1': 4}
+step=18650 loss=2.9289 {'pos0_bos_p': 0.9999912977218628, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999943971633911, 'last_eos_top1': 4}
+step=18700 loss=2.8296 {'pos0_bos_p': 0.9999892711639404, 'pos0_bos_top1': 4, 'last_eos_p': 0.999992847442627, 'last_eos_top1': 4}
+step=18750 loss=2.8464 {'pos0_bos_p': 0.9999916553497314, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999948740005493, 'last_eos_top1': 4}
+step=18800 loss=3.3807 {'pos0_bos_p': 0.9999926090240479, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999949932098389, 'last_eos_top1': 4}
+step=18850 loss=3.3787 {'pos0_bos_p': 0.9999920129776001, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999949932098389, 'last_eos_top1': 4}
+step=18900 loss=3.1856 {'pos0_bos_p': 0.999995231628418, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999969005584717, 'last_eos_top1': 4}
+step=18950 loss=2.7915 {'pos0_bos_p': 0.9999957084655762, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999972581863403, 'last_eos_top1': 4}
+step=19000 loss=3.1707 {'pos0_bos_p': 0.999994158744812, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999967813491821, 'last_eos_top1': 4}
+step=19050 loss=3.0780 {'pos0_bos_p': 0.9999969005584717, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999978542327881, 'last_eos_top1': 4}
+step=19100 loss=2.8519 {'pos0_bos_p': 0.9999955892562866, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999973773956299, 'last_eos_top1': 4}
+step=19150 loss=2.7222 {'pos0_bos_p': 0.9999957084655762, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999974966049194, 'last_eos_top1': 4}
+step=19200 loss=3.2865 {'pos0_bos_p': 0.9999959468841553, 'pos0_bos_top1': 4, 'last_eos_p': 0.999997615814209, 'last_eos_top1': 4}
+step=19250 loss=2.9816 {'pos0_bos_p': 0.9999964237213135, 'pos0_bos_top1': 4, 'last_eos_p': 0.999997615814209, 'last_eos_top1': 4}
+step=19300 loss=3.7326 {'pos0_bos_p': 0.9999966621398926, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999978542327881, 'last_eos_top1': 4}
+step=19350 loss=2.8540 {'pos0_bos_p': 0.9999980926513672, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999985694885254, 'last_eos_top1': 4}
+step=19400 loss=3.3185 {'pos0_bos_p': 0.9999974966049194, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999984502792358, 'last_eos_top1': 4}
+step=19450 loss=2.7277 {'pos0_bos_p': 0.9999967813491821, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999980926513672, 'last_eos_top1': 4}
+step=19500 loss=2.7165 {'pos0_bos_p': 0.9999969005584717, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999978542327881, 'last_eos_top1': 4}
+step=19550 loss=3.0601 {'pos0_bos_p': 0.9999970197677612, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999979734420776, 'last_eos_top1': 4}
+step=19600 loss=2.7457 {'pos0_bos_p': 0.9999972581863403, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999985694885254, 'last_eos_top1': 4}
+step=19650 loss=3.2114 {'pos0_bos_p': 0.9999959468841553, 'pos0_bos_top1': 4, 'last_eos_p': 0.999997615814209, 'last_eos_top1': 4}
+step=19700 loss=3.3399 {'pos0_bos_p': 0.9999948740005493, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999971389770508, 'last_eos_top1': 4}
+step=19750 loss=3.3003 {'pos0_bos_p': 0.9999963045120239, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999977350234985, 'last_eos_top1': 4}
+step=19800 loss=3.2280 {'pos0_bos_p': 0.9999970197677612, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999983310699463, 'last_eos_top1': 4}
+step=19850 loss=2.5845 {'pos0_bos_p': 0.999996542930603, 'pos0_bos_top1': 4, 'last_eos_p': 0.999997615814209, 'last_eos_top1': 4}
+step=19900 loss=3.5082 {'pos0_bos_p': 0.9999969005584717, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999978542327881, 'last_eos_top1': 4}
+step=19950 loss=2.4196 {'pos0_bos_p': 0.9999970197677612, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999978542327881, 'last_eos_top1': 4}
+step=20000 loss=2.9098 {'pos0_bos_p': 0.9999973773956299, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999980926513672, 'last_eos_top1': 4}
+step=20050 loss=2.7848 {'pos0_bos_p': 0.9999978542327881, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999985694885254, 'last_eos_top1': 4}
+step=20100 loss=3.2965 {'pos0_bos_p': 0.9999977350234985, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999985694885254, 'last_eos_top1': 4}
+step=20150 loss=3.0238 {'pos0_bos_p': 0.9999977350234985, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999985694885254, 'last_eos_top1': 4}
+step=20200 loss=2.9086 {'pos0_bos_p': 0.9999979734420776, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999988079071045, 'last_eos_top1': 4}
+step=20250 loss=3.0761 {'pos0_bos_p': 0.9999980926513672, 'pos0_bos_top1': 4, 'last_eos_p': 0.999998927116394, 'last_eos_top1': 4}
+step=20300 loss=2.9579 {'pos0_bos_p': 0.9999980926513672, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=20350 loss=2.2977 {'pos0_bos_p': 0.9999984502792358, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=20400 loss=2.9252 {'pos0_bos_p': 0.9999978542327881, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999985694885254, 'last_eos_top1': 4}
+step=20450 loss=3.0928 {'pos0_bos_p': 0.9999980926513672, 'pos0_bos_top1': 4, 'last_eos_p': 0.999998927116394, 'last_eos_top1': 4}
+step=20500 loss=2.7516 {'pos0_bos_p': 0.9999979734420776, 'pos0_bos_top1': 4, 'last_eos_p': 0.999998927116394, 'last_eos_top1': 4}
+step=20550 loss=2.5973 {'pos0_bos_p': 0.9999979734420776, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999985694885254, 'last_eos_top1': 4}
+step=20600 loss=2.9459 {'pos0_bos_p': 0.9999983310699463, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=20650 loss=3.2676 {'pos0_bos_p': 0.9999985694885254, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=20700 loss=3.0876 {'pos0_bos_p': 0.9999984502792358, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=20750 loss=2.7219 {'pos0_bos_p': 0.9999983310699463, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=20800 loss=3.2814 {'pos0_bos_p': 0.9999980926513672, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=20850 loss=2.4268 {'pos0_bos_p': 0.9999985694885254, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=20900 loss=2.6273 {'pos0_bos_p': 0.999998927116394, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=20950 loss=3.0174 {'pos0_bos_p': 0.9999986886978149, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=21000 loss=2.6212 {'pos0_bos_p': 0.9999990463256836, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=21050 loss=3.2589 {'pos0_bos_p': 0.9999991655349731, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999994039535522, 'last_eos_top1': 4}
+step=21100 loss=3.1442 {'pos0_bos_p': 0.9999991655349731, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999994039535522, 'last_eos_top1': 4}
+step=21150 loss=3.0941 {'pos0_bos_p': 0.9999991655349731, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999994039535522, 'last_eos_top1': 4}
+step=21200 loss=2.5845 {'pos0_bos_p': 0.9999992847442627, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999994039535522, 'last_eos_top1': 4}
+step=21250 loss=3.1657 {'pos0_bos_p': 0.9999991655349731, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999994039535522, 'last_eos_top1': 4}
+step=21300 loss=3.1461 {'pos0_bos_p': 0.9999991655349731, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999994039535522, 'last_eos_top1': 4}
+step=21350 loss=3.0316 {'pos0_bos_p': 0.9999991655349731, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999994039535522, 'last_eos_top1': 4}
+step=21400 loss=3.2284 {'pos0_bos_p': 0.9999974966049194, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999982118606567, 'last_eos_top1': 4}
+step=21450 loss=3.1770 {'pos0_bos_p': 0.9999831914901733, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999902248382568, 'last_eos_top1': 4}
+step=21500 loss=2.9939 {'pos0_bos_p': 0.9999868869781494, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999914169311523, 'last_eos_top1': 4}
+step=21550 loss=3.0536 {'pos0_bos_p': 0.999988317489624, 'pos0_bos_top1': 4, 'last_eos_p': 0.999992847442627, 'last_eos_top1': 4}
+step=21600 loss=3.3052 {'pos0_bos_p': 0.9999873638153076, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999924898147583, 'last_eos_top1': 4}
+step=21650 loss=3.0425 {'pos0_bos_p': 0.9999892711639404, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999939203262329, 'last_eos_top1': 4}
+step=21700 loss=2.6944 {'pos0_bos_p': 0.9999886751174927, 'pos0_bos_top1': 4, 'last_eos_p': 0.999993085861206, 'last_eos_top1': 4}
+step=21750 loss=2.9377 {'pos0_bos_p': 0.9999915361404419, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999946355819702, 'last_eos_top1': 4}
+step=21800 loss=2.6082 {'pos0_bos_p': 0.9999911785125732, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999945163726807, 'last_eos_top1': 4}
+step=21850 loss=3.1535 {'pos0_bos_p': 0.9999912977218628, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999942779541016, 'last_eos_top1': 4}
+step=21900 loss=3.1449 {'pos0_bos_p': 0.9999916553497314, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999946355819702, 'last_eos_top1': 4}
+step=21950 loss=2.4984 {'pos0_bos_p': 0.9999932050704956, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999963045120239, 'last_eos_top1': 4}
+step=22000 loss=2.8317 {'pos0_bos_p': 0.999994158744812, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999966621398926, 'last_eos_top1': 4}
+step=22050 loss=3.1049 {'pos0_bos_p': 0.999994158744812, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999966621398926, 'last_eos_top1': 4}
+step=22100 loss=2.8098 {'pos0_bos_p': 0.9999964237213135, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999977350234985, 'last_eos_top1': 4}
+step=22150 loss=3.3967 {'pos0_bos_p': 0.9999951124191284, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999972581863403, 'last_eos_top1': 4}
+step=22200 loss=2.5200 {'pos0_bos_p': 0.9999954700469971, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999972581863403, 'last_eos_top1': 4}
+step=22250 loss=3.1435 {'pos0_bos_p': 0.9999949932098389, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999973773956299, 'last_eos_top1': 4}
+step=22300 loss=2.6392 {'pos0_bos_p': 0.9999949932098389, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999974966049194, 'last_eos_top1': 4}
+step=22350 loss=3.4139 {'pos0_bos_p': 0.9999774694442749, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999897480010986, 'last_eos_top1': 4}
+step=22400 loss=3.1491 {'pos0_bos_p': 0.9999816417694092, 'pos0_bos_top1': 4, 'last_eos_p': 0.999990701675415, 'last_eos_top1': 4}
+step=22450 loss=2.5685 {'pos0_bos_p': 0.9999860525131226, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999916553497314, 'last_eos_top1': 4}
+step=22500 loss=3.0591 {'pos0_bos_p': 0.9999808073043823, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999908208847046, 'last_eos_top1': 4}
+step=22550 loss=2.5578 {'pos0_bos_p': 0.9999874830245972, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999923706054688, 'last_eos_top1': 4}
+step=22600 loss=3.2552 {'pos0_bos_p': 0.9999891519546509, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999940395355225, 'last_eos_top1': 4}
+step=22650 loss=2.7054 {'pos0_bos_p': 0.9999916553497314, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999946355819702, 'last_eos_top1': 4}
+step=22700 loss=3.0849 {'pos0_bos_p': 0.9999886751174927, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999940395355225, 'last_eos_top1': 4}
+step=22750 loss=2.9980 {'pos0_bos_p': 0.9999912977218628, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999951124191284, 'last_eos_top1': 4}
+step=22800 loss=2.6145 {'pos0_bos_p': 0.999990701675415, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999945163726807, 'last_eos_top1': 4}
+step=22850 loss=2.6088 {'pos0_bos_p': 0.9999922513961792, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999948740005493, 'last_eos_top1': 4}
+step=22900 loss=2.6382 {'pos0_bos_p': 0.9999946355819702, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999966621398926, 'last_eos_top1': 4}
+step=22950 loss=3.2917 {'pos0_bos_p': 0.999991774559021, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999953508377075, 'last_eos_top1': 4}
+step=23000 loss=3.1064 {'pos0_bos_p': 0.9999951124191284, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999964237213135, 'last_eos_top1': 4}
+step=23050 loss=2.3247 {'pos0_bos_p': 0.9999942779541016, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999963045120239, 'last_eos_top1': 4}
+step=23100 loss=3.0354 {'pos0_bos_p': 0.9999943971633911, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999963045120239, 'last_eos_top1': 4}
+step=23150 loss=3.2838 {'pos0_bos_p': 0.9999947547912598, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999963045120239, 'last_eos_top1': 4}
+step=23200 loss=2.8558 {'pos0_bos_p': 0.999995231628418, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999967813491821, 'last_eos_top1': 4}
+step=23250 loss=3.1582 {'pos0_bos_p': 0.999995231628418, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999964237213135, 'last_eos_top1': 4}
+step=23300 loss=3.5571 {'pos0_bos_p': 0.9999947547912598, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999964237213135, 'last_eos_top1': 4}
+step=23350 loss=3.3300 {'pos0_bos_p': 0.9999943971633911, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999961853027344, 'last_eos_top1': 4}
+step=23400 loss=3.1372 {'pos0_bos_p': 0.9999946355819702, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999964237213135, 'last_eos_top1': 4}
+step=23450 loss=3.3361 {'pos0_bos_p': 0.9999947547912598, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999969005584717, 'last_eos_top1': 4}
+step=23500 loss=3.2551 {'pos0_bos_p': 0.9999948740005493, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999970197677612, 'last_eos_top1': 4}
+step=23550 loss=2.7045 {'pos0_bos_p': 0.999995231628418, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999969005584717, 'last_eos_top1': 4}
+step=23600 loss=3.1944 {'pos0_bos_p': 0.9999951124191284, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999967813491821, 'last_eos_top1': 4}
+step=23650 loss=3.3145 {'pos0_bos_p': 0.9999963045120239, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999977350234985, 'last_eos_top1': 4}
+step=23700 loss=2.5692 {'pos0_bos_p': 0.9999954700469971, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999969005584717, 'last_eos_top1': 4}
+step=23750 loss=2.7647 {'pos0_bos_p': 0.9999961853027344, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999974966049194, 'last_eos_top1': 4}
+step=23800 loss=2.4500 {'pos0_bos_p': 0.9999969005584717, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999978542327881, 'last_eos_top1': 4}
+step=23850 loss=3.0078 {'pos0_bos_p': 0.9999964237213135, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999974966049194, 'last_eos_top1': 4}
+step=23900 loss=3.1958 {'pos0_bos_p': 0.9999964237213135, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999974966049194, 'last_eos_top1': 4}
+step=23950 loss=2.6673 {'pos0_bos_p': 0.9999977350234985, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999982118606567, 'last_eos_top1': 4}
+step=24000 loss=3.4957 {'pos0_bos_p': 0.9999979734420776, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999983310699463, 'last_eos_top1': 4}
+step=24050 loss=3.1268 {'pos0_bos_p': 0.9999990463256836, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=24100 loss=2.8265 {'pos0_bos_p': 0.9999990463256836, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=24150 loss=2.8904 {'pos0_bos_p': 0.9999986886978149, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=24200 loss=3.0995 {'pos0_bos_p': 0.9999988079071045, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=24250 loss=3.4637 {'pos0_bos_p': 0.999998927116394, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=24300 loss=2.5656 {'pos0_bos_p': 0.9999990463256836, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=24350 loss=3.0422 {'pos0_bos_p': 0.999998927116394, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=24400 loss=3.0555 {'pos0_bos_p': 0.9999990463256836, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=24450 loss=2.6055 {'pos0_bos_p': 0.999998927116394, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=24500 loss=2.8566 {'pos0_bos_p': 0.9999986886978149, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=24550 loss=2.9598 {'pos0_bos_p': 0.9999990463256836, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=24600 loss=3.1670 {'pos0_bos_p': 0.9999988079071045, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=24650 loss=3.3136 {'pos0_bos_p': 0.9999984502792358, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=24700 loss=2.8120 {'pos0_bos_p': 0.9999985694885254, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=24750 loss=3.1371 {'pos0_bos_p': 0.999998927116394, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=24800 loss=3.2671 {'pos0_bos_p': 0.9999988079071045, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=24850 loss=3.3720 {'pos0_bos_p': 0.999998927116394, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=24900 loss=2.6715 {'pos0_bos_p': 0.9999983310699463, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=24950 loss=2.8226 {'pos0_bos_p': 0.9999983310699463, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=25000 loss=3.0927 {'pos0_bos_p': 0.9999986886978149, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=25050 loss=2.7778 {'pos0_bos_p': 0.999998927116394, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
+step=25100 loss=2.7974 {'pos0_bos_p': 0.9999985694885254, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999991655349731, 'last_eos_top1': 4}
+step=25150 loss=2.5951 {'pos0_bos_p': 0.9999983310699463, 'pos0_bos_top1': 4, 'last_eos_p': 0.9999990463256836, 'last_eos_top1': 4}
diff --git a/LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_bernoulliwrong_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260527_063225.log b/LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_bernoulliwrong_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260527_063225.log
new file mode 100644
index 0000000000000000000000000000000000000000..cd690fc32529549b71bf0ef6494c09b9e64956e3
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_bernoulliwrong_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260527_063225.log
@@ -0,0 +1,167 @@
+W0527 06:32:27.026000 2692375 torch/distributed/run.py:792] 
+W0527 06:32:27.026000 2692375 torch/distributed/run.py:792] *****************************************
+W0527 06:32:27.026000 2692375 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0527 06:32:27.026000 2692375 torch/distributed/run.py:792] *****************************************
+[rank6]: Traceback (most recent call last):
+[rank6]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
+[rank6]:     main()
+[rank6]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
+[rank6]:     rank, world, device = setup_ddp()
+[rank6]:                           ^^^^^^^^^^^
+[rank6]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
+[rank6]:     torch.cuda.set_device(local_rank)
+[rank6]:   File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
+[rank6]:     torch._C._cuda_setDevice(device)
+[rank6]: RuntimeError: CUDA error: invalid device ordinal
+[rank6]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
+[rank6]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
+[rank6]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
+
+[rank3]: Traceback (most recent call last):
+[rank3]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
+[rank3]:     main()
+[rank3]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
+[rank3]:     rank, world, device = setup_ddp()
+[rank3]:                           ^^^^^^^^^^^
+[rank3]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
+[rank3]:     torch.cuda.set_device(local_rank)
+[rank3]:   File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
+[rank3]:     torch._C._cuda_setDevice(device)
+[rank3]: RuntimeError: CUDA error: invalid device ordinal
+[rank3]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
+[rank3]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
+[rank3]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
+
+[rank1]: Traceback (most recent call last):
+[rank1]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
+[rank1]:     main()
+[rank1]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
+[rank1]:     rank, world, device = setup_ddp()
+[rank1]:                           ^^^^^^^^^^^
+[rank1]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
+[rank1]:     torch.cuda.set_device(local_rank)
+[rank1]:   File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
+[rank1]:     torch._C._cuda_setDevice(device)
+[rank1]: RuntimeError: CUDA error: invalid device ordinal
+[rank1]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
+[rank1]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
+[rank1]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
+
+[rank7]: Traceback (most recent call last):
+[rank7]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
+[rank7]:     main()
+[rank7]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
+[rank7]:     rank, world, device = setup_ddp()
+[rank7]:                           ^^^^^^^^^^^
+[rank7]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
+[rank7]:     torch.cuda.set_device(local_rank)
+[rank7]:   File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
+[rank7]:     torch._C._cuda_setDevice(device)
+[rank7]: RuntimeError: CUDA error: invalid device ordinal
+[rank7]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
+[rank7]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
+[rank7]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
+
+[rank4]: Traceback (most recent call last):
+[rank4]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
+[rank4]:     main()
+[rank4]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
+[rank4]:     rank, world, device = setup_ddp()
+[rank4]:                           ^^^^^^^^^^^
+[rank4]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
+[rank4]:     torch.cuda.set_device(local_rank)
+[rank4]:   File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
+[rank4]:     torch._C._cuda_setDevice(device)
+[rank4]: RuntimeError: CUDA error: invalid device ordinal
+[rank4]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
+[rank4]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
+[rank4]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
+
+[rank2]: Traceback (most recent call last):
+[rank2]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
+[rank2]:     main()
+[rank2]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
+[rank2]:     rank, world, device = setup_ddp()
+[rank2]:                           ^^^^^^^^^^^
+[rank2]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
+[rank2]:     torch.cuda.set_device(local_rank)
+[rank2]:   File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
+[rank2]:     torch._C._cuda_setDevice(device)
+[rank2]: RuntimeError: CUDA error: invalid device ordinal
+[rank2]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
+[rank2]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
+[rank2]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
+
+[rank5]: Traceback (most recent call last):
+[rank5]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
+[rank5]:     main()
+[rank5]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
+[rank5]:     rank, world, device = setup_ddp()
+[rank5]:                           ^^^^^^^^^^^
+[rank5]:   File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
+[rank5]:     torch.cuda.set_device(local_rank)
+[rank5]:   File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
+[rank5]:     torch._C._cuda_setDevice(device)
+[rank5]: RuntimeError: CUDA error: invalid device ordinal
+[rank5]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
+[rank5]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
+[rank5]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
+
+[rank6]:[W527 06:32:28.179191492 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[rank3]:[W527 06:32:28.196687749 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[rank1]:[W527 06:32:28.200738161 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[rank4]:[W527 06:32:28.281200240 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[rank7]:[W527 06:32:28.289848591 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[rank2]:[W527 06:32:28.328670757 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[rank5]:[W527 06:32:28.342647483 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+W0527 06:32:28.756000 2692375 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 2692488 closing signal SIGTERM
+W0527 06:32:28.757000 2692375 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 2692490 closing signal SIGTERM
+W0527 06:32:28.757000 2692375 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 2692492 closing signal SIGTERM
+W0527 06:32:28.758000 2692375 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 2692493 closing signal SIGTERM
+W0527 06:32:28.759000 2692375 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 2692495 closing signal SIGTERM
+E0527 06:32:30.025000 2692375 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 1 (pid: 2692489) of binary: /usr/bin/python
+Traceback (most recent call last):
+  File "/usr/local/bin/torchrun", line 33, in <module>
+    sys.exit(load_entry_point('torch==2.7.0a0+ecf3bae40a.nv25.2', 'console_scripts', 'torchrun')())
+             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
+    return f(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main
+    run(args)
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run
+    elastic_launch(
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
+    return launch_agent(self._config, self._entrypoint, list(args))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent
+    raise ChildFailedError(
+torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
+============================================================
+train.py FAILED
+------------------------------------------------------------
+Failures:
+[1]:
+  time      : 2026-05-27_06:32:28
+  host      : localhost
+  rank      : 3 (local_rank: 3)
+  exitcode  : 1 (pid: 2692491)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+[2]:
+  time      : 2026-05-27_06:32:28
+  host      : localhost
+  rank      : 6 (local_rank: 6)
+  exitcode  : 1 (pid: 2692494)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+------------------------------------------------------------
+Root Cause (first observed failure):
+[0]:
+  time      : 2026-05-27_06:32:28
+  host      : localhost
+  rank      : 1 (local_rank: 1)
+  exitcode  : 1 (pid: 2692489)
+  error_file: <N/A>
+  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
+============================================================
diff --git a/LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_full_gbs512_8gpu_20260526_155049.log b/LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_full_gbs512_8gpu_20260526_155049.log
new file mode 100644
index 0000000000000000000000000000000000000000..6c2f74930b2fc36962ea81708fa02b1c48edcb0b
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_full_gbs512_8gpu_20260526_155049.log
@@ -0,0 +1,544 @@
+W0526 15:50:51.060000 10232 torch/distributed/run.py:792] 
+W0526 15:50:51.060000 10232 torch/distributed/run.py:792] *****************************************
+W0526 15:50:51.060000 10232 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0526 15:50:51.060000 10232 torch/distributed/run.py:792] *****************************************
+[rank7]:[W526 15:50:54.488272530 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7]  using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank1]:[W526 15:50:54.606918486 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1]  using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank2]:[W526 15:50:54.651901857 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2]  using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank5]:[W526 15:50:54.658253820 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5]  using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank4]:[W526 15:50:54.674422130 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4]  using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank6]:[W526 15:50:54.674769368 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6]  using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank3]:[W526 15:50:54.676109529 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3]  using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[data] loaded_cache=cache/owt_t5_payload1022_appendeos1.pt seen=8013769 kept=2860537 dropped=5153232
+[rank0]:[W526 15:51:00.641574510 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 0]  using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+t-20260526235016-fvc2m-worker-0:10299:10299 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10299:10299 [0] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10299:10299 [0] NCCL INFO cudaDriverVersion 12080
+t-20260526235016-fvc2m-worker-0:10299:10299 [0] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260526235016-fvc2m-worker-0:10299:10299 [0] NCCL INFO Comm config Blocking set to 1
+t-20260526235016-fvc2m-worker-0:10300:10300 [1] NCCL INFO cudaDriverVersion 12080
+t-20260526235016-fvc2m-worker-0:10300:10300 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10300:10300 [1] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10300:10300 [1] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260526235016-fvc2m-worker-0:10300:10300 [1] NCCL INFO Comm config Blocking set to 1
+t-20260526235016-fvc2m-worker-0:10303:10303 [4] NCCL INFO cudaDriverVersion 12080
+t-20260526235016-fvc2m-worker-0:10303:10303 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10302:10302 [3] NCCL INFO cudaDriverVersion 12080
+t-20260526235016-fvc2m-worker-0:10302:10302 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10303:10303 [4] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10302:10302 [3] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10302:10302 [3] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260526235016-fvc2m-worker-0:10303:10303 [4] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260526235016-fvc2m-worker-0:10306:10306 [7] NCCL INFO cudaDriverVersion 12080
+t-20260526235016-fvc2m-worker-0:10306:10306 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10306:10306 [7] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10306:10306 [7] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260526235016-fvc2m-worker-0:10305:10305 [6] NCCL INFO cudaDriverVersion 12080
+t-20260526235016-fvc2m-worker-0:10305:10305 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10304:10304 [5] NCCL INFO cudaDriverVersion 12080
+t-20260526235016-fvc2m-worker-0:10305:10305 [6] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10305:10305 [6] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260526235016-fvc2m-worker-0:10304:10304 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10303:10303 [4] NCCL INFO Comm config Blocking set to 1
+t-20260526235016-fvc2m-worker-0:10302:10302 [3] NCCL INFO Comm config Blocking set to 1
+t-20260526235016-fvc2m-worker-0:10304:10304 [5] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10304:10304 [5] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260526235016-fvc2m-worker-0:10301:10301 [2] NCCL INFO cudaDriverVersion 12080
+t-20260526235016-fvc2m-worker-0:10301:10301 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10301:10301 [2] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10301:10301 [2] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260526235016-fvc2m-worker-0:10306:10306 [7] NCCL INFO Comm config Blocking set to 1
+t-20260526235016-fvc2m-worker-0:10305:10305 [6] NCCL INFO Comm config Blocking set to 1
+t-20260526235016-fvc2m-worker-0:10304:10304 [5] NCCL INFO Comm config Blocking set to 1
+t-20260526235016-fvc2m-worker-0:10301:10301 [2] NCCL INFO Comm config Blocking set to 1
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO P2P plugin v9 IBext_v9
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO P2P plugin v9 IBext_v9
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO P2P plugin v9 IBext_v9
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO P2P plugin v9 IBext_v9
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO P2P plugin v9 IBext_v9
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO P2P plugin v9 IBext_v9
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO P2P plugin v9 IBext_v9
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO P2P plugin v9 IBext_v9
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO Using network IBext_v9
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO Using network IBext_v9
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Using network IBext_v9
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO Using network IBext_v9
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO Using network IBext_v9
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO Using network IBext_v9
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO Using network IBext_v9
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO Using network IBext_v9
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO ncclCommInitRankConfig comm 0xb143f10 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x153ec126bc8139c1 - Init START
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO ncclCommInitRankConfig comm 0xbd4dcd0 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x153ec126bc8139c1 - Init START
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO ncclCommInitRankConfig comm 0xa37e4e0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x153ec126bc8139c1 - Init START
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO ncclCommInitRankConfig comm 0x9d716c0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x153ec126bc8139c1 - Init START
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO ncclCommInitRankConfig comm 0x9b81db0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x153ec126bc8139c1 - Init START
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO ncclCommInitRankConfig comm 0x9b30c10 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x153ec126bc8139c1 - Init START
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO ncclCommInitRankConfig comm 0x9831810 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x153ec126bc8139c1 - Init START
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO ncclCommInitRankConfig comm 0xa99ebf0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x153ec126bc8139c1 - Init START
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Bootstrap timings total 0.009389 (create 0.000021, send 0.000076, recv 0.007843, ring 0.001136, delay 0.000001)
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO Bootstrap timings total 0.004700 (create 0.000020, send 0.000074, recv 0.000078, ring 0.004231, delay 0.000001)
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO Bootstrap timings total 0.006253 (create 0.000025, send 0.000072, recv 0.001602, ring 0.000124, delay 0.000001)
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO Bootstrap timings total 0.009861 (create 0.000025, send 0.000073, recv 0.004202, ring 0.001097, delay 0.000001)
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO Bootstrap timings total 0.000634 (create 0.000020, send 0.000071, recv 0.000088, ring 0.000198, delay 0.000001)
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO Bootstrap timings total 0.001625 (create 0.000020, send 0.000077, recv 0.000101, ring 0.001092, delay 0.000001)
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO Bootstrap timings total 0.005219 (create 0.000020, send 0.000072, recv 0.004652, ring 0.000152, delay 0.000001)
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO Bootstrap timings total 0.005755 (create 0.000022, send 0.000072, recv 0.000616, ring 0.004733, delay 0.000001)
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO MNNVL busId 0x75020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO MNNVL busId 0x65040 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO MNNVL busId 0x6f020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO MNNVL busId 0x73020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO MNNVL busId 0x71020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO MNNVL busId 0x69020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO MNNVL busId 0x67020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO MNNVL busId 0x6b020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO Setting affinity for GPU 7 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NVLS multicast support is available on dev 7
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO Setting affinity for GPU 2 to 03ffffff,ffffffff,ffffffff
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NVLS multicast support is available on dev 2
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO Setting affinity for GPU 1 to 03ffffff,ffffffff,ffffffff
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NVLS multicast support is available on dev 1
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Setting affinity for GPU 0 to 03ffffff,ffffffff,ffffffff
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO Setting affinity for GPU 5 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NVLS multicast support is available on dev 0
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NVLS multicast support is available on dev 5
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO Setting affinity for GPU 6 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NVLS multicast support is available on dev 6
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO Setting affinity for GPU 4 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO Setting affinity for GPU 3 to 03ffffff,ffffffff,ffffffff
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NVLS multicast support is available on dev 3
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NVLS multicast support is available on dev 4
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO comm 0xa37e4e0 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO comm 0x9831810 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO comm 0x9d716c0 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO comm 0xb143f10 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO comm 0x9b81db0 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO comm 0xbd4dcd0 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO comm 0x9b30c10 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO comm 0xa99ebf0 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO P2P Chunksize set to 524288
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO P2P Chunksize set to 524288
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO P2P Chunksize set to 524288
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO P2P Chunksize set to 524288
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO P2P Chunksize set to 524288
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO P2P Chunksize set to 524288
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO P2P Chunksize set to 524288
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO P2P Chunksize set to 524288
+t-20260526235016-fvc2m-worker-0:10305:10476 [6] NCCL INFO [Proxy Service UDS] Device 6 CPU core 94
+t-20260526235016-fvc2m-worker-0:10305:10475 [6] NCCL INFO [Proxy Service] Device 6 CPU core 92
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Check P2P Type intraNodeP2pSupport 1 directMode 0
+t-20260526235016-fvc2m-worker-0:10299:10477 [0] NCCL INFO [Proxy Service] Device 0 CPU core 2
+t-20260526235016-fvc2m-worker-0:10299:10478 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 4
+t-20260526235016-fvc2m-worker-0:10300:10479 [1] NCCL INFO [Proxy Service] Device 1 CPU core 59
+t-20260526235016-fvc2m-worker-0:10300:10480 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 60
+t-20260526235016-fvc2m-worker-0:10304:10481 [5] NCCL INFO [Proxy Service] Device 5 CPU core 138
+t-20260526235016-fvc2m-worker-0:10304:10482 [5] NCCL INFO [Proxy Service UDS] Device 5 CPU core 141
+t-20260526235016-fvc2m-worker-0:10301:10483 [2] NCCL INFO [Proxy Service] Device 2 CPU core 20
+t-20260526235016-fvc2m-worker-0:10301:10484 [2] NCCL INFO [Proxy Service UDS] Device 2 CPU core 22
+t-20260526235016-fvc2m-worker-0:10302:10485 [3] NCCL INFO [Proxy Service] Device 3 CPU core 82
+t-20260526235016-fvc2m-worker-0:10302:10486 [3] NCCL INFO [Proxy Service UDS] Device 3 CPU core 84
+t-20260526235016-fvc2m-worker-0:10303:10487 [4] NCCL INFO [Proxy Service] Device 4 CPU core 92
+t-20260526235016-fvc2m-worker-0:10303:10488 [4] NCCL INFO [Proxy Service UDS] Device 4 CPU core 94
+t-20260526235016-fvc2m-worker-0:10306:10489 [7] NCCL INFO [Proxy Service] Device 7 CPU core 116
+t-20260526235016-fvc2m-worker-0:10306:10490 [7] NCCL INFO [Proxy Service UDS] Device 7 CPU core 114
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO CC Off, workFifoBytes 1048576
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO ncclCommInitRankConfig comm 0xa99ebf0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x153ec126bc8139c1 - Init COMPLETE
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO ncclCommInitRankConfig comm 0x9b81db0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x153ec126bc8139c1 - Init COMPLETE
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO ncclCommInitRankConfig comm 0xbd4dcd0 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x153ec126bc8139c1 - Init COMPLETE
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 2.20 (kernels 0.21, alloc 1.04, bootstrap 0.00, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.37, rest 0.03)
+t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 2.22 (kernels 0.19, alloc 1.07, bootstrap 0.01, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.38, rest 0.02)
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 2.20 (kernels 0.19, alloc 1.06, bootstrap 0.01, allgathers 0.00, topo 0.53, graphs 0.01, connections 0.37, rest 0.03)
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO ncclCommInitRankConfig comm 0x9831810 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x153ec126bc8139c1 - Init COMPLETE
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO ncclCommInitRankConfig comm 0x9d716c0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x153ec126bc8139c1 - Init COMPLETE
+t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 2.20 (kernels 0.20, alloc 1.06, bootstrap 0.00, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.37, rest 0.03)
+t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 2.20 (kernels 0.19, alloc 1.06, bootstrap 0.01, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.37, rest 0.03)
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO ncclCommInitRankConfig comm 0x9b30c10 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x153ec126bc8139c1 - Init COMPLETE
+t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 2.20 (kernels 0.19, alloc 1.06, bootstrap 0.00, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.37, rest 0.03)
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO ncclCommInitRankConfig comm 0xa37e4e0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x153ec126bc8139c1 - Init COMPLETE
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO ncclCommInitRankConfig comm 0xb143f10 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x153ec126bc8139c1 - Init COMPLETE
+t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 2.20 (kernels 0.19, alloc 1.05, bootstrap 0.01, allgathers 0.00, topo 0.53, graphs 0.01, connections 0.38, rest 0.02)
+t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 2.20 (kernels 0.19, alloc 1.05, bootstrap 0.01, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.37, rest 0.03)
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+{
+  "data_path": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext",
+  "tokenizer_path": "/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json",
+  "out_dir": "runs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_full_gbs512_8gpu_20260526_155049",
+  "text_column": "text",
+  "subset_size": 0,
+  "payload_len": 1022,
+  "append_eos": 1,
+  "log_skips": 20,
+  "cache_path": "cache/owt_t5_payload1022_appendeos1.pt",
+  "rebuild_cache": 0,
+  "online_data": 0,
+  "online_buffer_size": 8192,
+  "steps": 20000,
+  "batch_size": 32,
+  "grad_accum": 2,
+  "lr": 0.0003,
+  "log_every": 50,
+  "save_every": 1000,
+  "dim": 768,
+  "layers": 12,
+  "heads": 12,
+  "mlp_dim": 3072,
+  "time_tokens": 4,
+  "abs_pos": 1,
+  "rope": 1,
+  "c_min": 1.0,
+  "c_max": 1024.0,
+  "seed": 1234
+}
+[data] rows=2860537 length=1024 vocab=32100 seen=8013769 dropped=5153232 kept=2860537 bos=1:</s> eos=1:</s>
+[head] ['</s>', '▁Port', '-', 'au', '-', 'Pri', 'nce', ',', '▁Haiti', '▁(', 'C', 'NN', ')', '▁--', '▁Earth', 'qua']
+[tail] ['▁magnitude', '▁earthquake', '▁flat', 't', 'ened', '▁Haiti', "'", 's', '▁capital', '▁city', '▁Tuesday', '▁afternoon', ',', '▁', 'affecting', '</s>']
+t-20260526235016-fvc2m-worker-0:10306:10674 [7] NCCL INFO NVLS comm 0x9b30c10 headRank 7 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260526235016-fvc2m-worker-0:10300:10675 [1] NCCL INFO NVLS comm 0x9831810 headRank 1 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260526235016-fvc2m-worker-0:10304:10676 [5] NCCL INFO NVLS comm 0xa99ebf0 headRank 5 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260526235016-fvc2m-worker-0:10301:10677 [2] NCCL INFO NVLS comm 0xb143f10 headRank 2 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260526235016-fvc2m-worker-0:10302:10678 [3] NCCL INFO NVLS comm 0x9d716c0 headRank 3 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260526235016-fvc2m-worker-0:10305:10679 [6] NCCL INFO NVLS comm 0xa37e4e0 headRank 6 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260526235016-fvc2m-worker-0:10303:10680 [4] NCCL INFO NVLS comm 0x9b81db0 headRank 4 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260526235016-fvc2m-worker-0:10299:10681 [0] NCCL INFO NVLS comm 0xbd4dcd0 headRank 0 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
diff --git a/LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_163925.log b/LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_163925.log
new file mode 100644
index 0000000000000000000000000000000000000000..1a0f827e7797337719d530c748df21f0f5d67154
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_163925.log
@@ -0,0 +1,597 @@
+W0526 16:39:26.907000 10232 torch/distributed/run.py:792] 
+W0526 16:39:26.907000 10232 torch/distributed/run.py:792] *****************************************
+W0526 16:39:26.907000 10232 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0526 16:39:26.907000 10232 torch/distributed/run.py:792] *****************************************
+[rank3]:[W526 16:39:30.312943735 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3]  using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank2]:[W526 16:39:30.393296581 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2]  using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank1]:[W526 16:39:30.412360668 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1]  using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank5]:[W526 16:39:30.485600324 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5]  using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank6]:[W526 16:39:30.506970342 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6]  using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank4]:[W526 16:39:30.515612102 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4]  using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[rank7]:[W526 16:39:30.523092350 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7]  using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+[data] loaded_cache=cache/owt_t5_payload1022_appendeos1.pt seen=8013769 kept=2860537 dropped=5153232
+[rank0]:[W526 16:39:36.458074028 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 0]  using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
+t-20260527003833-zv4xx-worker-0:10299:10299 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10299:10299 [0] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10299:10299 [0] NCCL INFO cudaDriverVersion 12080
+t-20260527003833-zv4xx-worker-0:10299:10299 [0] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527003833-zv4xx-worker-0:10299:10299 [0] NCCL INFO Comm config Blocking set to 1
+t-20260527003833-zv4xx-worker-0:10300:10300 [1] NCCL INFO cudaDriverVersion 12080
+t-20260527003833-zv4xx-worker-0:10300:10300 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10300:10300 [1] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10300:10300 [1] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527003833-zv4xx-worker-0:10304:10304 [5] NCCL INFO cudaDriverVersion 12080
+t-20260527003833-zv4xx-worker-0:10304:10304 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10303:10303 [4] NCCL INFO cudaDriverVersion 12080
+t-20260527003833-zv4xx-worker-0:10303:10303 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10300:10300 [1] NCCL INFO Comm config Blocking set to 1
+t-20260527003833-zv4xx-worker-0:10302:10302 [3] NCCL INFO cudaDriverVersion 12080
+t-20260527003833-zv4xx-worker-0:10302:10302 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10304:10304 [5] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10304:10304 [5] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527003833-zv4xx-worker-0:10303:10303 [4] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10303:10303 [4] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527003833-zv4xx-worker-0:10301:10301 [2] NCCL INFO cudaDriverVersion 12080
+t-20260527003833-zv4xx-worker-0:10301:10301 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10306:10306 [7] NCCL INFO cudaDriverVersion 12080
+t-20260527003833-zv4xx-worker-0:10306:10306 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10305:10305 [6] NCCL INFO cudaDriverVersion 12080
+t-20260527003833-zv4xx-worker-0:10305:10305 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10302:10302 [3] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10302:10302 [3] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527003833-zv4xx-worker-0:10301:10301 [2] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10306:10306 [7] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10305:10305 [6] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10301:10301 [2] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527003833-zv4xx-worker-0:10306:10306 [7] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527003833-zv4xx-worker-0:10305:10305 [6] NCCL INFO NCCL version 2.25.1+cuda12.8
+t-20260527003833-zv4xx-worker-0:10303:10303 [4] NCCL INFO Comm config Blocking set to 1
+t-20260527003833-zv4xx-worker-0:10304:10304 [5] NCCL INFO Comm config Blocking set to 1
+t-20260527003833-zv4xx-worker-0:10302:10302 [3] NCCL INFO Comm config Blocking set to 1
+t-20260527003833-zv4xx-worker-0:10306:10306 [7] NCCL INFO Comm config Blocking set to 1
+t-20260527003833-zv4xx-worker-0:10305:10305 [6] NCCL INFO Comm config Blocking set to 1
+t-20260527003833-zv4xx-worker-0:10301:10301 [2] NCCL INFO Comm config Blocking set to 1
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO P2P plugin v9 IBext_v9
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Using network IBext_v9
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO Using network IBext_v9
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO Using network IBext_v9
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO Using network IBext_v9
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO Using network IBext_v9
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO Using network IBext_v9
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO Using network IBext_v9
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO Using network IBext_v9
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO ncclCommInitRankConfig comm 0xabe1e00 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x71c0ab8013683c2b - Init START
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO ncclCommInitRankConfig comm 0xaec9340 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x71c0ab8013683c2b - Init START
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO ncclCommInitRankConfig comm 0xb193950 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x71c0ab8013683c2b - Init START
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO ncclCommInitRankConfig comm 0x97e19d0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x71c0ab8013683c2b - Init START
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO ncclCommInitRankConfig comm 0x95b4ea0 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x71c0ab8013683c2b - Init START
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO ncclCommInitRankConfig comm 0xa832730 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x71c0ab8013683c2b - Init START
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO ncclCommInitRankConfig comm 0xac3b760 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x71c0ab8013683c2b - Init START
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO ncclCommInitRankConfig comm 0xafdc9d0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x71c0ab8013683c2b - Init START
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO RAS client listening socket at ::1<28028>
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO Bootstrap timings total 0.086836 (create 0.000021, send 0.000066, recv 0.011557, ring 0.000081, delay 0.000001)
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO Bootstrap timings total 0.074335 (create 0.000019, send 0.000074, recv 0.000066, ring 0.000114, delay 0.000001)
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO Bootstrap timings total 0.074834 (create 0.000022, send 0.000067, recv 0.000539, ring 0.000113, delay 0.000001)
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO Bootstrap timings total 0.077938 (create 0.000021, send 0.000069, recv 0.003157, ring 0.000124, delay 0.000001)
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO Bootstrap timings total 0.075359 (create 0.000020, send 0.000067, recv 0.000093, ring 0.000128, delay 0.000001)
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO Bootstrap timings total 0.086289 (create 0.000024, send 0.000069, recv 0.008438, ring 0.000173, delay 0.000001)
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO Bootstrap timings total 0.077370 (create 0.000021, send 0.000067, recv 0.000052, ring 0.000165, delay 0.000001)
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Bootstrap timings total 0.300148 (create 0.000025, send 0.000079, recv 0.213971, ring 0.000176, delay 0.000001)
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO MNNVL busId 0x6f020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO MNNVL busId 0x71020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO MNNVL busId 0x6b020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO MNNVL busId 0x73020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO MNNVL busId 0x69020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO MNNVL busId 0x75020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO MNNVL busId 0x67020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO MNNVL busId 0x65040 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Setting affinity for GPU 0 to 03ffffff,ffffffff,ffffffff
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NVLS multicast support is available on dev 0
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO Setting affinity for GPU 5 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NVLS multicast support is available on dev 5
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO Setting affinity for GPU 6 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NVLS multicast support is available on dev 6
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO Setting affinity for GPU 7 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NVLS multicast support is available on dev 7
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO Setting affinity for GPU 4 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NVLS multicast support is available on dev 4
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO Setting affinity for GPU 2 to 03ffffff,ffffffff,ffffffff
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NVLS multicast support is available on dev 2
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO Setting affinity for GPU 1 to 03ffffff,ffffffff,ffffffff
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NVLS multicast support is available on dev 1
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO Setting affinity for GPU 3 to 03ffffff,ffffffff,ffffffff
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NVLS multicast support is available on dev 3
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO comm 0xa832730 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO comm 0xaec9340 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO comm 0xafdc9d0 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO comm 0xac3b760 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO comm 0x95b4ea0 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO comm 0xb193950 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO comm 0x97e19d0 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO comm 0xabe1e00 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO P2P Chunksize set to 524288
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO P2P Chunksize set to 524288
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO P2P Chunksize set to 524288
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO P2P Chunksize set to 524288
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO P2P Chunksize set to 524288
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO P2P Chunksize set to 524288
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO P2P Chunksize set to 524288
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO P2P Chunksize set to 524288
+t-20260527003833-zv4xx-worker-0:10303:10475 [4] NCCL INFO [Proxy Service] Device 4 CPU core 104
+t-20260527003833-zv4xx-worker-0:10303:10476 [4] NCCL INFO [Proxy Service UDS] Device 4 CPU core 107
+t-20260527003833-zv4xx-worker-0:10306:10477 [7] NCCL INFO [Proxy Service] Device 7 CPU core 108
+t-20260527003833-zv4xx-worker-0:10306:10478 [7] NCCL INFO [Proxy Service UDS] Device 7 CPU core 110
+t-20260527003833-zv4xx-worker-0:10305:10479 [6] NCCL INFO [Proxy Service] Device 6 CPU core 94
+t-20260527003833-zv4xx-worker-0:10305:10480 [6] NCCL INFO [Proxy Service UDS] Device 6 CPU core 96
+t-20260527003833-zv4xx-worker-0:10301:10481 [2] NCCL INFO [Proxy Service] Device 2 CPU core 2
+t-20260527003833-zv4xx-worker-0:10301:10482 [2] NCCL INFO [Proxy Service UDS] Device 2 CPU core 4
+t-20260527003833-zv4xx-worker-0:10300:10483 [1] NCCL INFO [Proxy Service] Device 1 CPU core 86
+t-20260527003833-zv4xx-worker-0:10300:10484 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 2
+t-20260527003833-zv4xx-worker-0:10304:10485 [5] NCCL INFO [Proxy Service] Device 5 CPU core 131
+t-20260527003833-zv4xx-worker-0:10304:10486 [5] NCCL INFO [Proxy Service UDS] Device 5 CPU core 132
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Check P2P Type intraNodeP2pSupport 1 directMode 0
+t-20260527003833-zv4xx-worker-0:10302:10487 [3] NCCL INFO [Proxy Service] Device 3 CPU core 2
+t-20260527003833-zv4xx-worker-0:10302:10488 [3] NCCL INFO [Proxy Service UDS] Device 3 CPU core 4
+t-20260527003833-zv4xx-worker-0:10299:10489 [0] NCCL INFO [Proxy Service] Device 0 CPU core 77
+t-20260527003833-zv4xx-worker-0:10299:10490 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 79
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO CC Off, workFifoBytes 1048576
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO ncclCommInitRankConfig comm 0x95b4ea0 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x71c0ab8013683c2b - Init COMPLETE
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO ncclCommInitRankConfig comm 0xabe1e00 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x71c0ab8013683c2b - Init COMPLETE
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO ncclCommInitRankConfig comm 0x97e19d0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x71c0ab8013683c2b - Init COMPLETE
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO ncclCommInitRankConfig comm 0xa832730 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x71c0ab8013683c2b - Init COMPLETE
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO ncclCommInitRankConfig comm 0xac3b760 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x71c0ab8013683c2b - Init COMPLETE
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO ncclCommInitRankConfig comm 0xaec9340 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x71c0ab8013683c2b - Init COMPLETE
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO ncclCommInitRankConfig comm 0xafdc9d0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x71c0ab8013683c2b - Init COMPLETE
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO ncclCommInitRankConfig comm 0xb193950 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x71c0ab8013683c2b - Init COMPLETE
+t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 2.18 (kernels 0.19, alloc 0.85, bootstrap 0.30, allgathers 0.02, topo 0.53, graphs 0.01, connections 0.25, rest 0.04)
+t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 2.16 (kernels 0.20, alloc 1.04, bootstrap 0.08, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.28, rest 0.02)
+t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 2.16 (kernels 0.20, alloc 1.03, bootstrap 0.08, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.27, rest 0.02)
+t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 2.16 (kernels 0.20, alloc 1.04, bootstrap 0.08, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.27, rest 0.02)
+t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 2.17 (kernels 0.20, alloc 1.03, bootstrap 0.09, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.26, rest 0.03)
+t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 2.16 (kernels 0.20, alloc 1.04, bootstrap 0.07, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.25, rest 0.04)
+t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 2.17 (kernels 0.22, alloc 1.02, bootstrap 0.07, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.28, rest 0.01)
+t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 2.17 (kernels 0.20, alloc 1.03, bootstrap 0.09, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.26, rest 0.03)
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM
+t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
+{
+  "data_path": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext",
+  "tokenizer_path": "/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json",
+  "out_dir": "runs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_163925",
+  "text_column": "text",
+  "subset_size": 0,
+  "payload_len": 1022,
+  "append_eos": 1,
+  "log_skips": 20,
+  "cache_path": "cache/owt_t5_payload1022_appendeos1.pt",
+  "rebuild_cache": 0,
+  "online_data": 0,
+  "online_buffer_size": 8192,
+  "steps": 1000000,
+  "batch_size": 32,
+  "grad_accum": 2,
+  "lr": 0.0003,
+  "log_every": 50,
+  "save_every": 1000,
+  "dim": 768,
+  "layers": 12,
+  "heads": 12,
+  "mlp_dim": 3072,
+  "time_tokens": 4,
+  "abs_pos": 1,
+  "rope": 1,
+  "c_min": 1.0,
+  "c_max": 1024.0,
+  "seed": 1234
+}
+[data] rows=2860537 length=1024 vocab=32100 seen=8013769 dropped=5153232 kept=2860537 bos=1:</s> eos=1:</s>
+[head] ['</s>', '▁Port', '-', 'au', '-', 'Pri', 'nce', ',', '▁Haiti', '▁(', 'C', 'NN', ')', '▁--', '▁Earth', 'qua']
+[tail] ['▁magnitude', '▁earthquake', '▁flat', 't', 'ened', '▁Haiti', "'", 's', '▁capital', '▁city', '▁Tuesday', '▁afternoon', ',', '▁', 'affecting', '</s>']
+t-20260527003833-zv4xx-worker-0:10304:10610 [5] NCCL INFO NVLS comm 0xaec9340 headRank 5 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527003833-zv4xx-worker-0:10302:10611 [3] NCCL INFO NVLS comm 0xac3b760 headRank 3 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527003833-zv4xx-worker-0:10299:10612 [0] NCCL INFO NVLS comm 0xabe1e00 headRank 0 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527003833-zv4xx-worker-0:10306:10613 [7] NCCL INFO NVLS comm 0x95b4ea0 headRank 7 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527003833-zv4xx-worker-0:10301:10615 [2] NCCL INFO NVLS comm 0x97e19d0 headRank 2 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527003833-zv4xx-worker-0:10300:10614 [1] NCCL INFO NVLS comm 0xb193950 headRank 1 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527003833-zv4xx-worker-0:10303:10616 [4] NCCL INFO NVLS comm 0xafdc9d0 headRank 4 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+t-20260527003833-zv4xx-worker-0:10305:10617 [6] NCCL INFO NVLS comm 0xa832730 headRank 6 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
+step=50 loss=7.2226 {'pos0_bos_p': 0.8895623087882996, 'pos0_bos_top1': 4, 'last_eos_p': 0.8887543082237244, 'last_eos_top1': 4}
+step=100 loss=7.1043 {'pos0_bos_p': 0.9955054521560669, 'pos0_bos_top1': 4, 'last_eos_p': 0.9959879517555237, 'last_eos_top1': 4}
+step=150 loss=6.0746 {'pos0_bos_p': 0.6761358380317688, 'pos0_bos_top1': 4, 'last_eos_p': 0.7208590507507324, 'last_eos_top1': 4}
+step=200 loss=5.6690 {'pos0_bos_p': 0.9043604731559753, 'pos0_bos_top1': 4, 'last_eos_p': 0.922759473323822, 'last_eos_top1': 4}
+step=250 loss=4.8215 {'pos0_bos_p': 0.933864951133728, 'pos0_bos_top1': 4, 'last_eos_p': 0.953918993473053, 'last_eos_top1': 4}
+step=300 loss=4.5186 {'pos0_bos_p': 0.9616466164588928, 'pos0_bos_top1': 4, 'last_eos_p': 0.9727951288223267, 'last_eos_top1': 4}
+step=350 loss=4.1555 {'pos0_bos_p': 0.965355396270752, 'pos0_bos_top1': 4, 'last_eos_p': 0.9713674783706665, 'last_eos_top1': 4}
+step=400 loss=3.4321 {'pos0_bos_p': 0.982333242893219, 'pos0_bos_top1': 4, 'last_eos_p': 0.984869122505188, 'last_eos_top1': 4}
+step=450 loss=3.5727 {'pos0_bos_p': 0.9869136214256287, 'pos0_bos_top1': 4, 'last_eos_p': 0.9893201589584351, 'last_eos_top1': 4}
+step=500 loss=3.4097 {'pos0_bos_p': 0.9890345335006714, 'pos0_bos_top1': 4, 'last_eos_p': 0.9915169477462769, 'last_eos_top1': 4}
+step=550 loss=2.9839 {'pos0_bos_p': 0.990917980670929, 'pos0_bos_top1': 4, 'last_eos_p': 0.9927908182144165, 'last_eos_top1': 4}
+step=600 loss=2.7384 {'pos0_bos_p': 0.9930053353309631, 'pos0_bos_top1': 4, 'last_eos_p': 0.9942381381988525, 'last_eos_top1': 4}
+step=650 loss=2.4446 {'pos0_bos_p': 0.993517279624939, 'pos0_bos_top1': 4, 'last_eos_p': 0.9944773316383362, 'last_eos_top1': 4}
+step=700 loss=2.3503 {'pos0_bos_p': 0.9943650960922241, 'pos0_bos_top1': 4, 'last_eos_p': 0.9950743317604065, 'last_eos_top1': 4}
+step=750 loss=2.9878 {'pos0_bos_p': 0.9950012564659119, 'pos0_bos_top1': 4, 'last_eos_p': 0.9952785968780518, 'last_eos_top1': 4}
+step=800 loss=2.6886 {'pos0_bos_p': 0.9956516623497009, 'pos0_bos_top1': 4, 'last_eos_p': 0.9956568479537964, 'last_eos_top1': 4}
+step=850 loss=2.6424 {'pos0_bos_p': 0.9948635697364807, 'pos0_bos_top1': 4, 'last_eos_p': 0.9943944215774536, 'last_eos_top1': 4}
+step=900 loss=2.3033 {'pos0_bos_p': 0.9968313574790955, 'pos0_bos_top1': 4, 'last_eos_p': 0.9964890480041504, 'last_eos_top1': 4}
+step=950 loss=2.7804 {'pos0_bos_p': 0.9972208738327026, 'pos0_bos_top1': 4, 'last_eos_p': 0.9968255758285522, 'last_eos_top1': 4}
+step=1000 loss=2.3661 {'pos0_bos_p': 0.9971387386322021, 'pos0_bos_top1': 4, 'last_eos_p': 0.9966385364532471, 'last_eos_top1': 4}
+step=1050 loss=2.2603 {'pos0_bos_p': 0.9974852800369263, 'pos0_bos_top1': 4, 'last_eos_p': 0.9969478249549866, 'last_eos_top1': 4}
+step=1100 loss=2.3556 {'pos0_bos_p': 0.9976263642311096, 'pos0_bos_top1': 4, 'last_eos_p': 0.9970782995223999, 'last_eos_top1': 4}
+step=1150 loss=2.5678 {'pos0_bos_p': 0.9978169202804565, 'pos0_bos_top1': 4, 'last_eos_p': 0.9971928000450134, 'last_eos_top1': 4}
+step=1200 loss=2.8455 {'pos0_bos_p': 0.9978358149528503, 'pos0_bos_top1': 4, 'last_eos_p': 0.9972167015075684, 'last_eos_top1': 4}
+step=1250 loss=1.9140 {'pos0_bos_p': 0.9979123473167419, 'pos0_bos_top1': 4, 'last_eos_p': 0.9973716735839844, 'last_eos_top1': 4}
+step=1300 loss=2.2091 {'pos0_bos_p': 0.9973376393318176, 'pos0_bos_top1': 4, 'last_eos_p': 0.9966146349906921, 'last_eos_top1': 4}
+step=1350 loss=1.5151 {'pos0_bos_p': 0.997620165348053, 'pos0_bos_top1': 4, 'last_eos_p': 0.9968921542167664, 'last_eos_top1': 4}
+step=1400 loss=1.9534 {'pos0_bos_p': 0.9971216320991516, 'pos0_bos_top1': 4, 'last_eos_p': 0.9958294034004211, 'last_eos_top1': 4}
+step=1450 loss=2.0907 {'pos0_bos_p': 0.9967049956321716, 'pos0_bos_top1': 4, 'last_eos_p': 0.995191216468811, 'last_eos_top1': 4}
+step=1500 loss=1.6090 {'pos0_bos_p': 0.9971720576286316, 'pos0_bos_top1': 4, 'last_eos_p': 0.99595707654953, 'last_eos_top1': 4}
+step=1550 loss=1.8623 {'pos0_bos_p': 0.9975792765617371, 'pos0_bos_top1': 4, 'last_eos_p': 0.9965457320213318, 'last_eos_top1': 4}
+step=1600 loss=1.6597 {'pos0_bos_p': 0.9975658655166626, 'pos0_bos_top1': 4, 'last_eos_p': 0.9966031312942505, 'last_eos_top1': 4}
+step=1650 loss=1.7848 {'pos0_bos_p': 0.9976959824562073, 'pos0_bos_top1': 4, 'last_eos_p': 0.9968312382698059, 'last_eos_top1': 4}
+step=1700 loss=1.9018 {'pos0_bos_p': 0.997767448425293, 'pos0_bos_top1': 4, 'last_eos_p': 0.9969239830970764, 'last_eos_top1': 4}
+step=1750 loss=1.6703 {'pos0_bos_p': 0.9975913763046265, 'pos0_bos_top1': 4, 'last_eos_p': 0.9966752529144287, 'last_eos_top1': 4}
+step=1800 loss=1.9970 {'pos0_bos_p': 0.9979997277259827, 'pos0_bos_top1': 4, 'last_eos_p': 0.9973554611206055, 'last_eos_top1': 4}
+step=1850 loss=1.5861 {'pos0_bos_p': 0.9977781176567078, 'pos0_bos_top1': 4, 'last_eos_p': 0.9969488978385925, 'last_eos_top1': 4}
+step=1900 loss=1.9109 {'pos0_bos_p': 0.9981799125671387, 'pos0_bos_top1': 4, 'last_eos_p': 0.9975284934043884, 'last_eos_top1': 4}
+step=1950 loss=2.0138 {'pos0_bos_p': 0.9982655644416809, 'pos0_bos_top1': 4, 'last_eos_p': 0.9977399110794067, 'last_eos_top1': 4}
+step=2000 loss=1.8339 {'pos0_bos_p': 0.9980721473693848, 'pos0_bos_top1': 4, 'last_eos_p': 0.9973933696746826, 'last_eos_top1': 4}
+step=2050 loss=1.6342 {'pos0_bos_p': 0.9986487030982971, 'pos0_bos_top1': 4, 'last_eos_p': 0.9981813430786133, 'last_eos_top1': 4}
+step=2100 loss=1.8772 {'pos0_bos_p': 0.9984073042869568, 'pos0_bos_top1': 4, 'last_eos_p': 0.9978986978530884, 'last_eos_top1': 4}
+step=2150 loss=1.7135 {'pos0_bos_p': 0.9985877275466919, 'pos0_bos_top1': 4, 'last_eos_p': 0.998180627822876, 'last_eos_top1': 4}
+step=2200 loss=1.5222 {'pos0_bos_p': 0.9986546039581299, 'pos0_bos_top1': 4, 'last_eos_p': 0.9982472658157349, 'last_eos_top1': 4}
+step=2250 loss=1.4951 {'pos0_bos_p': 0.9984161853790283, 'pos0_bos_top1': 4, 'last_eos_p': 0.9979872703552246, 'last_eos_top1': 4}
+step=2300 loss=1.3507 {'pos0_bos_p': 0.9987239241600037, 'pos0_bos_top1': 4, 'last_eos_p': 0.9984145164489746, 'last_eos_top1': 4}
+step=2350 loss=1.4153 {'pos0_bos_p': 0.9986024498939514, 'pos0_bos_top1': 4, 'last_eos_p': 0.9983096122741699, 'last_eos_top1': 4}
+step=2400 loss=1.8935 {'pos0_bos_p': 0.9989206790924072, 'pos0_bos_top1': 4, 'last_eos_p': 0.9986801743507385, 'last_eos_top1': 4}
+step=2450 loss=1.8997 {'pos0_bos_p': 0.998845100402832, 'pos0_bos_top1': 4, 'last_eos_p': 0.9985548853874207, 'last_eos_top1': 4}
+step=2500 loss=1.4746 {'pos0_bos_p': 0.9988980293273926, 'pos0_bos_top1': 4, 'last_eos_p': 0.9986351132392883, 'last_eos_top1': 4}
+step=2550 loss=1.6229 {'pos0_bos_p': 0.9988683462142944, 'pos0_bos_top1': 4, 'last_eos_p': 0.998468816280365, 'last_eos_top1': 4}
+step=2600 loss=1.4606 {'pos0_bos_p': 0.99871826171875, 'pos0_bos_top1': 4, 'last_eos_p': 0.9982940554618835, 'last_eos_top1': 4}
+step=2650 loss=1.8987 {'pos0_bos_p': 0.9986856579780579, 'pos0_bos_top1': 4, 'last_eos_p': 0.9983665347099304, 'last_eos_top1': 4}
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/configuration_data2vec_text.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/configuration_data2vec_text.py
new file mode 100644
index 0000000000000000000000000000000000000000..c26d419fe28358bad422a270b5f789752471f827
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/configuration_data2vec_text.py
@@ -0,0 +1,65 @@
+# Copyright 2022 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Data2VecText configuration"""
+
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+
+
+@auto_docstring(checkpoint="facebook/data2vec-text-base")
+@strict
+class Data2VecTextConfig(PreTrainedConfig):
+    r"""
+    Examples:
+
+    ```python
+    >>> from transformers import Data2VecTextConfig, Data2VecTextModel
+
+    >>> # Initializing a Data2VecText facebook/data2vec-text-base style configuration
+    >>> configuration = Data2VecTextConfig()
+
+    >>> # Initializing a model (with random weights) from the facebook/data2vec-text-base style configuration
+    >>> model = Data2VecTextModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "data2vec-text"
+
+    vocab_size: int = 30522
+    hidden_size: int = 768
+    num_hidden_layers: int = 12
+    num_attention_heads: int = 12
+    intermediate_size: int = 3072
+    hidden_act: str = "gelu"
+    hidden_dropout_prob: float | int = 0.1
+    attention_probs_dropout_prob: float | int = 0.1
+    max_position_embeddings: int = 512
+    type_vocab_size: int = 2
+    initializer_range: float = 0.02
+    layer_norm_eps: float = 1e-12
+    pad_token_id: int | None = 1
+    bos_token_id: int | None = 0
+    eos_token_id: int | list[int] | None = 2
+    use_cache: bool = True
+    classifier_dropout: float | int | None = None
+    is_decoder: bool = False
+    add_cross_attention: bool = False
+    tie_word_embeddings: bool = True
+
+
+__all__ = ["Data2VecTextConfig"]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_audio.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_audio.py
new file mode 100644
index 0000000000000000000000000000000000000000..bb3af4ecf25a56549c1acbb55736c87880cce45a
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_audio.py
@@ -0,0 +1,1324 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/data2vec/modular_data2vec_audio.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_data2vec_audio.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2022 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import warnings
+from collections.abc import Callable
+
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import CrossEntropyLoss
+
+from ... import initialization as init
+from ...activations import ACT2FN
+from ...integrations.deepspeed import is_deepspeed_zero3_enabled
+from ...integrations.fsdp import is_fsdp_managed_module
+from ...masking_utils import create_bidirectional_mask
+from ...modeling_flash_attention_utils import FlashAttentionKwargs
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import (
+    BaseModelOutput,
+    CausalLMOutput,
+    SequenceClassifierOutput,
+    TokenClassifierOutput,
+    Wav2Vec2BaseModelOutput,
+    XVectorOutput,
+)
+from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from ...processing_utils import Unpack
+from ...utils import TransformersKwargs, auto_docstring, is_peft_available
+from .configuration_data2vec_audio import Data2VecAudioConfig
+
+
+class Data2VecAudioConvLayer(GradientCheckpointingLayer):
+    def __init__(self, config, layer_id=0):
+        super().__init__()
+        self.in_conv_dim = config.conv_dim[layer_id - 1] if layer_id > 0 else 1
+        self.out_conv_dim = config.conv_dim[layer_id]
+
+        self.conv = nn.Conv1d(
+            self.in_conv_dim,
+            self.out_conv_dim,
+            kernel_size=config.conv_kernel[layer_id],
+            stride=config.conv_stride[layer_id],
+            bias=config.conv_bias,
+        )
+        self.layer_norm = nn.LayerNorm(self.out_conv_dim, elementwise_affine=True)
+        self.activation = ACT2FN[config.feat_extract_activation]
+
+    def forward(self, hidden_states):
+        hidden_states = self.conv(hidden_states)
+
+        hidden_states = hidden_states.transpose(-2, -1)
+        hidden_states = self.layer_norm(hidden_states)
+        hidden_states = hidden_states.transpose(-2, -1)
+
+        hidden_states = self.activation(hidden_states)
+        return hidden_states
+
+
+class Data2VecAudioPadLayer(nn.Module):
+    def __init__(self, num_conv_pos_embeddings):
+        super().__init__()
+        self.num_pad_remove = 1 if num_conv_pos_embeddings % 2 == 0 else 0
+
+    def forward(self, hidden_states):
+        if self.num_pad_remove > 0:
+            hidden_states = hidden_states[:, :, : -self.num_pad_remove]
+        return hidden_states
+
+
+class Data2VecAudioPositionalConvLayer(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.conv = nn.Conv1d(
+            config.hidden_size,
+            config.hidden_size,
+            kernel_size=config.conv_pos_kernel_size,
+            padding=config.conv_pos_kernel_size // 2,
+            groups=config.num_conv_pos_embedding_groups,
+        )
+
+        self.padding = Data2VecAudioPadLayer(config.conv_pos_kernel_size)
+        self.activation = ACT2FN[config.feat_extract_activation]
+        # no learnable parameters
+        self.layer_norm = nn.LayerNorm(config.hidden_size, elementwise_affine=False)
+
+    def forward(self, hidden_states):
+        hidden_states = self.conv(hidden_states)
+        hidden_states = self.padding(hidden_states)
+
+        hidden_states = hidden_states.transpose(1, 2)
+        hidden_states = self.layer_norm(hidden_states)
+        hidden_states = hidden_states.transpose(1, 2)
+        hidden_states = self.activation(hidden_states)
+        return hidden_states
+
+
+class Data2VecAudioPositionalConvEmbedding(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.layers = nn.ModuleList(
+            [Data2VecAudioPositionalConvLayer(config) for _ in range(config.num_conv_pos_embeddings)]
+        )
+
+    def forward(self, hidden_states):
+        hidden_states = hidden_states.transpose(1, 2)
+        for layer in self.layers:
+            hidden_states = layer(hidden_states)
+        hidden_states = hidden_states.transpose(1, 2)
+        return hidden_states
+
+
+class Data2VecAudioFeatureEncoder(nn.Module):
+    """Construct the features from raw audio waveform"""
+
+    def __init__(self, config):
+        super().__init__()
+        self.conv_layers = nn.ModuleList(
+            [Data2VecAudioConvLayer(config, layer_id=i) for i in range(config.num_feat_extract_layers)]
+        )
+        self.gradient_checkpointing = False
+        self._requires_grad = True
+
+    def _freeze_parameters(self):
+        for param in self.parameters():
+            param.requires_grad = False
+        self._requires_grad = False
+
+    def forward(self, input_values):
+        hidden_states = input_values[:, None]
+
+        # make sure hidden_states require grad for gradient_checkpointing
+        if self._requires_grad and self.training:
+            hidden_states.requires_grad = True
+
+        for conv_layer in self.conv_layers:
+            hidden_states = conv_layer(hidden_states)
+
+        return hidden_states
+
+
+class Data2VecAudioFeatureProjection(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.layer_norm = nn.LayerNorm(config.conv_dim[-1], eps=config.layer_norm_eps)
+        self.projection = nn.Linear(config.conv_dim[-1], config.hidden_size)
+        self.dropout = nn.Dropout(config.feat_proj_dropout)
+
+    def forward(self, hidden_states):
+        # non-projected hidden states are needed for quantization
+        norm_hidden_states = self.layer_norm(hidden_states)
+        hidden_states = self.projection(norm_hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        return hidden_states, norm_hidden_states
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    scaling: float | None = None,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    if scaling is None:
+        scaling = query.size(-1) ** -0.5
+
+    # Take the dot product between "query" and "key" to get the raw attention scores.
+    attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling
+
+    if attention_mask is not None:
+        attn_weights = attn_weights + attention_mask
+
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+
+    attn_output = torch.matmul(attn_weights, value)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+class Data2VecAudioAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(
+        self,
+        embed_dim: int,
+        num_heads: int,
+        dropout: float = 0.0,
+        is_decoder: bool = False,
+        bias: bool = True,
+        is_causal: bool = False,
+        config: Data2VecAudioConfig | None = None,
+    ):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.dropout = dropout
+        self.head_dim = embed_dim // num_heads
+        self.config = config
+
+        if (self.head_dim * num_heads) != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
+                f" and `num_heads`: {num_heads})."
+            )
+        self.scaling = self.head_dim**-0.5
+        self.is_decoder = is_decoder
+        self.is_causal = is_causal
+
+        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+        self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        key_value_states: torch.Tensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        output_attentions: bool | None = False,
+        # TODO: we need a refactor so that the different attention modules can get their specific kwargs
+        # ATM, we have mixed things encoder, decoder, and encoder-decoder attn
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]:
+        """Input shape: Batch x Time x Channel"""
+
+        # if key_value_states are provided this layer is used as a cross-attention layer
+        # for the decoder
+        is_cross_attention = key_value_states is not None
+
+        # determine input shapes
+        input_shape = hidden_states.shape[:-1]
+
+        hidden_shape = (*input_shape, -1, self.head_dim)
+
+        # get query proj
+        query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        current_states = key_value_states if is_cross_attention else hidden_states
+        kv_shape = (*current_states.shape[:-1], -1, self.head_dim)
+        key_states = self.k_proj(current_states).view(kv_shape).transpose(1, 2)
+        value_states = self.v_proj(current_states).view(kv_shape).transpose(1, 2)
+
+        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, eager_attention_forward
+        )
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            dropout=0.0 if not self.training else self.dropout,
+            scaling=self.scaling,
+            output_attentions=output_attentions,
+            **kwargs,
+        )
+
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        attn_output = self.out_proj(attn_output)
+
+        return attn_output, attn_weights, None
+
+
+class Data2VecAudioFeedForward(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.intermediate_dropout = nn.Dropout(config.activation_dropout)
+
+        self.intermediate_dense = nn.Linear(config.hidden_size, config.intermediate_size)
+        if isinstance(config.hidden_act, str):
+            self.intermediate_act_fn = ACT2FN[config.hidden_act]
+        else:
+            self.intermediate_act_fn = config.hidden_act
+
+        self.output_dense = nn.Linear(config.intermediate_size, config.hidden_size)
+        self.output_dropout = nn.Dropout(config.hidden_dropout)
+
+    def forward(self, hidden_states):
+        hidden_states = self.intermediate_dense(hidden_states)
+        hidden_states = self.intermediate_act_fn(hidden_states)
+        hidden_states = self.intermediate_dropout(hidden_states)
+
+        hidden_states = self.output_dense(hidden_states)
+        hidden_states = self.output_dropout(hidden_states)
+        return hidden_states
+
+
+class Data2VecAudioEncoderLayer(GradientCheckpointingLayer):
+    def __init__(self, config):
+        super().__init__()
+        self.attention = Data2VecAudioAttention(
+            embed_dim=config.hidden_size,
+            num_heads=config.num_attention_heads,
+            dropout=config.attention_dropout,
+            is_decoder=False,
+            config=config,
+        )
+
+        self.dropout = nn.Dropout(config.hidden_dropout)
+        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.feed_forward = Data2VecAudioFeedForward(config)
+        self.final_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+    def forward(self, hidden_states, attention_mask=None, output_attentions=False):
+        attn_residual = hidden_states
+        hidden_states, attn_weights, _ = self.attention(
+            hidden_states, attention_mask=attention_mask, output_attentions=output_attentions
+        )
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = attn_residual + hidden_states
+
+        hidden_states = self.layer_norm(hidden_states)
+        hidden_states = hidden_states + self.feed_forward(hidden_states)
+        hidden_states = self.final_layer_norm(hidden_states)
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        return outputs
+
+
+class Data2VecAudioEncoder(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.pos_conv_embed = Data2VecAudioPositionalConvEmbedding(config)
+        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout)
+        self.layers = nn.ModuleList([Data2VecAudioEncoderLayer(config) for _ in range(config.num_hidden_layers)])
+        self.gradient_checkpointing = False
+
+    def forward(
+        self,
+        hidden_states: torch.tensor,
+        attention_mask: torch.Tensor | None = None,
+        output_attentions: bool = False,
+        output_hidden_states: bool = False,
+        return_dict: bool = True,
+    ):
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attentions = () if output_attentions else None
+
+        if attention_mask is not None:
+            # make sure padded tokens output 0
+            expand_attention_mask = attention_mask.unsqueeze(-1).repeat(1, 1, hidden_states.shape[2])
+            hidden_states[~expand_attention_mask] = 0
+
+        attention_mask = create_bidirectional_mask(
+            config=self.config,
+            inputs_embeds=hidden_states,
+            attention_mask=attention_mask,
+        )
+
+        position_embeddings = self.pos_conv_embed(hidden_states)
+        hidden_states = hidden_states + position_embeddings.to(hidden_states.device)
+        hidden_states = self.layer_norm(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+
+        synced_gpus = is_deepspeed_zero3_enabled() or is_fsdp_managed_module(self)
+
+        for layer in self.layers:
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            # add LayerDrop (see https://huggingface.co/papers/1909.11556 for description)
+            dropout_probability = torch.rand([])
+
+            skip_the_layer = self.training and dropout_probability < self.config.layerdrop
+            if not skip_the_layer or synced_gpus:
+                # under fsdp or deepspeed zero3 all gpus must run in sync
+                layer_outputs = layer(
+                    hidden_states, attention_mask=attention_mask, output_attentions=output_attentions
+                )
+                hidden_states = layer_outputs[0]
+
+            if skip_the_layer:
+                layer_outputs = (None, None)
+
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (layer_outputs[1],)
+
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, all_hidden_states, all_self_attentions] if v is not None)
+        return BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+        )
+
+
+class Data2VecAudioAdapterLayer(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.conv = nn.Conv1d(
+            config.output_hidden_size,
+            2 * config.output_hidden_size,
+            config.adapter_kernel_size,
+            stride=config.adapter_stride,
+            padding=1,
+        )
+
+    def forward(self, hidden_states):
+        hidden_states = self.conv(hidden_states)
+        hidden_states = nn.functional.glu(hidden_states, dim=1)
+
+        return hidden_states
+
+
+class Data2VecAudioAdapter(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+
+        # feature dim might need to be down-projected
+        if config.output_hidden_size != config.hidden_size:
+            self.proj = nn.Linear(config.hidden_size, config.output_hidden_size)
+            self.proj_layer_norm = nn.LayerNorm(config.output_hidden_size)
+        else:
+            self.proj = self.proj_layer_norm = None
+
+        self.layers = nn.ModuleList(Data2VecAudioAdapterLayer(config) for _ in range(config.num_adapter_layers))
+        self.layerdrop = config.layerdrop
+
+    def forward(self, hidden_states):
+        # down project hidden_states if necessary
+        if self.proj is not None and self.proj_layer_norm is not None:
+            hidden_states = self.proj(hidden_states)
+            hidden_states = self.proj_layer_norm(hidden_states)
+
+        hidden_states = hidden_states.transpose(1, 2)
+
+        for layer in self.layers:
+            layerdrop_prob = np.random.random()
+            if not self.training or (layerdrop_prob > self.layerdrop):
+                hidden_states = layer(hidden_states)
+
+        hidden_states = hidden_states.transpose(1, 2)
+        return hidden_states
+
+
+@auto_docstring
+class Data2VecAudioPreTrainedModel(PreTrainedModel):
+    config: Data2VecAudioConfig
+    base_model_prefix = "data2vec_audio"
+    main_input_name = "input_values"
+    input_modalities = "audio"
+    supports_gradient_checkpointing = True
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    _supports_flex_attn = True
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        if isinstance(module, Data2VecAudioFeatureProjection):
+            k = math.sqrt(1 / module.projection.in_features)
+            init.uniform_(module.projection.weight, a=-k, b=k)
+            init.uniform_(module.projection.bias, a=-k, b=k)
+        elif isinstance(module, Data2VecAudioPositionalConvLayer):
+            init.constant_(module.conv.bias, 0)
+        elif isinstance(module, nn.Linear):
+            init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
+
+            if module.bias is not None:
+                init.zeros_(module.bias)
+        elif isinstance(module, (nn.LayerNorm, nn.GroupNorm)):
+            if module.bias is not None:
+                init.zeros_(module.bias)
+            if module.weight is not None:
+                init.ones_(module.weight)
+        elif isinstance(module, nn.Conv1d):
+            init.kaiming_normal_(module.weight)
+
+            if module.bias is not None:
+                k = math.sqrt(module.groups / (module.in_channels * module.kernel_size[0]))
+                init.uniform_(module.bias, a=-k, b=k)
+
+    def _get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor | int, add_adapter: bool | None = None):
+        """
+        Computes the output length of the convolutional layers
+        """
+
+        add_adapter = self.config.add_adapter if add_adapter is None else add_adapter
+
+        def _conv_out_length(input_length, kernel_size, stride):
+            # 1D convolutional layer output length formula taken
+            # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html
+            return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1
+
+        for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride):
+            input_lengths = _conv_out_length(input_lengths, kernel_size, stride)
+
+        if add_adapter:
+            for _ in range(self.config.num_adapter_layers):
+                input_lengths = _conv_out_length(input_lengths, 1, self.config.adapter_stride)
+
+        return input_lengths
+
+    def _get_feature_vector_attention_mask(
+        self, feature_vector_length: int, attention_mask: torch.LongTensor, add_adapter=None
+    ):
+        # Effectively attention_mask.sum(-1), but not inplace to be able to run
+        # on inference mode.
+        non_padded_lengths = attention_mask.cumsum(dim=-1)[:, -1]
+
+        output_lengths = self._get_feat_extract_output_lengths(non_padded_lengths, add_adapter=add_adapter)
+        output_lengths = output_lengths.to(torch.long)
+
+        batch_size = attention_mask.shape[0]
+
+        attention_mask = torch.zeros(
+            (batch_size, feature_vector_length), dtype=attention_mask.dtype, device=attention_mask.device
+        )
+        # these two operations makes sure that all values before the output lengths idxs are attended to
+        attention_mask[(torch.arange(attention_mask.shape[0], device=attention_mask.device), output_lengths - 1)] = 1
+        attention_mask = attention_mask.flip([-1]).cumsum(-1).flip([-1]).bool()
+        return attention_mask
+
+
+def _compute_mask_indices(
+    shape: tuple[int, int],
+    mask_prob: float,
+    mask_length: int,
+    attention_mask: torch.LongTensor | None = None,
+    min_masks: int = 0,
+) -> np.ndarray:
+    """
+    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
+    ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on
+    CPU as part of the preprocessing during training.
+
+    Args:
+        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
+               the first element is the batch size and the second element is the length of the axis to span.
+        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
+                    independently generated mask spans of length `mask_length` is computed by
+                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
+                    actual percentage will be smaller.
+        mask_length: size of the mask
+        min_masks: minimum number of masked spans
+        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
+                        each batch dimension.
+    """
+    batch_size, sequence_length = shape
+
+    if mask_length < 1:
+        raise ValueError("`mask_length` has to be bigger than 0.")
+
+    if mask_length > sequence_length:
+        raise ValueError(
+            f"`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: {mask_length}"
+            f" and `sequence_length`: {sequence_length}`"
+        )
+
+    # epsilon is used for probabilistic rounding
+    epsilon = np.random.rand(1).item()
+
+    def compute_num_masked_span(input_length):
+        """Given input length, compute how many spans should be masked"""
+        num_masked_span = int(mask_prob * input_length / mask_length + epsilon)
+        num_masked_span = max(num_masked_span, min_masks)
+
+        # make sure num masked span <= sequence_length
+        if num_masked_span * mask_length > sequence_length:
+            num_masked_span = sequence_length // mask_length
+
+        # make sure num_masked span is also <= input_length - (mask_length - 1)
+        if input_length - (mask_length - 1) < num_masked_span:
+            num_masked_span = max(input_length - (mask_length - 1), 0)
+
+        return num_masked_span
+
+    # compute number of masked spans in batch
+    input_lengths = (
+        attention_mask.detach().sum(-1).tolist()
+        if attention_mask is not None
+        else [sequence_length for _ in range(batch_size)]
+    )
+
+    # SpecAugment mask to fill
+    spec_aug_mask = np.zeros((batch_size, sequence_length), dtype=bool)
+    spec_aug_mask_idxs = []
+
+    max_num_masked_span = compute_num_masked_span(sequence_length)
+
+    if max_num_masked_span == 0:
+        return spec_aug_mask
+
+    for input_length in input_lengths:
+        # compute num of masked spans for this input
+        num_masked_span = compute_num_masked_span(input_length)
+
+        # get random indices to mask
+        spec_aug_mask_idx = np.random.choice(
+            np.arange(input_length - (mask_length - 1)), num_masked_span, replace=False
+        )
+
+        # pick first sampled index that will serve as a dummy index to pad vector
+        # to ensure same dimension for all batches due to probabilistic rounding
+        # Picking first sample just pads those vectors twice.
+        if len(spec_aug_mask_idx) == 0:
+            # this case can only happen if `input_length` is strictly smaller then
+            # `sequence_length` in which case the last token has to be a padding
+            # token which we can use as a dummy mask id
+            dummy_mask_idx = sequence_length - 1
+        else:
+            dummy_mask_idx = spec_aug_mask_idx[0]
+
+        spec_aug_mask_idx = np.concatenate(
+            [spec_aug_mask_idx, np.ones(max_num_masked_span - num_masked_span, dtype=np.int32) * dummy_mask_idx]
+        )
+        spec_aug_mask_idxs.append(spec_aug_mask_idx)
+
+    spec_aug_mask_idxs = np.array(spec_aug_mask_idxs)
+
+    # expand masked indices to masked spans
+    spec_aug_mask_idxs = np.broadcast_to(
+        spec_aug_mask_idxs[:, :, None], (batch_size, max_num_masked_span, mask_length)
+    )
+    spec_aug_mask_idxs = spec_aug_mask_idxs.reshape(batch_size, max_num_masked_span * mask_length)
+
+    # add offset to the starting indexes so that indexes now create a span
+    offsets = np.arange(mask_length)[None, None, :]
+    offsets = np.broadcast_to(offsets, (batch_size, max_num_masked_span, mask_length)).reshape(
+        batch_size, max_num_masked_span * mask_length
+    )
+    spec_aug_mask_idxs = spec_aug_mask_idxs + offsets
+
+    # ensure that we cannot have indices larger than sequence_length
+    if spec_aug_mask_idxs.max() > sequence_length - 1:
+        spec_aug_mask_idxs[spec_aug_mask_idxs > sequence_length - 1] = sequence_length - 1
+
+    # scatter indices to mask
+    np.put_along_axis(spec_aug_mask, spec_aug_mask_idxs, 1, -1)
+
+    return spec_aug_mask
+
+
+Data2VecAudioBaseModelOutput = Wav2Vec2BaseModelOutput
+
+
+@auto_docstring
+class Data2VecAudioModel(Data2VecAudioPreTrainedModel):
+    def __init__(self, config: Data2VecAudioConfig):
+        super().__init__(config)
+        self.config = config
+        self.feature_extractor = Data2VecAudioFeatureEncoder(config)
+        self.feature_projection = Data2VecAudioFeatureProjection(config)
+
+        # model only needs masking vector if mask prob is > 0.0
+        if config.mask_time_prob > 0.0 or config.mask_feature_prob > 0.0:
+            self.masked_spec_embed = nn.Parameter(torch.Tensor(config.hidden_size).uniform_())
+
+        self.encoder = Data2VecAudioEncoder(config)
+
+        self.adapter = Data2VecAudioAdapter(config) if config.add_adapter else None
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def freeze_feature_encoder(self):
+        """
+        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
+        not be updated during training.
+        """
+        self.feature_extractor._freeze_parameters()
+
+    def _mask_hidden_states(
+        self,
+        hidden_states: torch.FloatTensor,
+        mask_time_indices: torch.FloatTensor | None = None,
+        attention_mask: torch.LongTensor | None = None,
+    ):
+        """
+        Masks extracted features along time axis and/or along feature axis according to
+        [SpecAugment](https://huggingface.co/papers/1904.08779).
+        """
+
+        # `config.apply_spec_augment` can set masking to False
+        if not getattr(self.config, "apply_spec_augment", True):
+            return hidden_states
+
+        # generate indices & apply SpecAugment along time axis
+        batch_size, sequence_length, hidden_size = hidden_states.size()
+
+        if mask_time_indices is not None:
+            # apply SpecAugment along time axis with given mask_time_indices
+            hidden_states[mask_time_indices] = self.masked_spec_embed.to(hidden_states.dtype)
+        elif self.config.mask_time_prob > 0 and self.training:
+            mask_time_indices = _compute_mask_indices(
+                (batch_size, sequence_length),
+                mask_prob=self.config.mask_time_prob,
+                mask_length=self.config.mask_time_length,
+                attention_mask=attention_mask,
+                min_masks=self.config.mask_time_min_masks,
+            )
+            mask_time_indices = torch.tensor(mask_time_indices, device=hidden_states.device, dtype=torch.bool)
+            hidden_states[mask_time_indices] = self.masked_spec_embed.to(hidden_states.dtype)
+
+        if self.config.mask_feature_prob > 0 and self.training:
+            # generate indices & apply SpecAugment along feature axis
+            mask_feature_indices = _compute_mask_indices(
+                (batch_size, hidden_size),
+                mask_prob=self.config.mask_feature_prob,
+                mask_length=self.config.mask_feature_length,
+                min_masks=self.config.mask_feature_min_masks,
+            )
+            mask_feature_indices = torch.tensor(mask_feature_indices, device=hidden_states.device, dtype=torch.bool)
+            mask_feature_indices = mask_feature_indices[:, None].expand(-1, sequence_length, -1)
+            hidden_states[mask_feature_indices] = 0
+
+        return hidden_states
+
+    @auto_docstring
+    def forward(
+        self,
+        input_values: torch.Tensor | None,
+        attention_mask: torch.Tensor | None = None,
+        mask_time_indices: torch.FloatTensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        **kwargs,
+    ) -> tuple | Data2VecAudioBaseModelOutput:
+        r"""
+        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
+            masked extracted features in *config.proj_codevector_dim* space.
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        extract_features = self.feature_extractor(input_values)
+        extract_features = extract_features.transpose(1, 2)
+
+        if attention_mask is not None:
+            # compute reduced attention_mask corresponding to feature vectors
+            attention_mask = self._get_feature_vector_attention_mask(
+                extract_features.shape[1], attention_mask, add_adapter=False
+            )
+
+        hidden_states, extract_features = self.feature_projection(extract_features)
+        hidden_states = self._mask_hidden_states(
+            hidden_states, mask_time_indices=mask_time_indices, attention_mask=attention_mask
+        )
+
+        encoder_outputs = self.encoder(
+            hidden_states,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = encoder_outputs[0]
+
+        if self.adapter is not None:
+            hidden_states = self.adapter(hidden_states)
+
+        if not return_dict:
+            return (hidden_states, extract_features) + encoder_outputs[1:]
+
+        return Data2VecAudioBaseModelOutput(
+            last_hidden_state=hidden_states,
+            extract_features=extract_features,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+        )
+
+
+_HIDDEN_STATES_START_POSITION = 2
+
+
+@auto_docstring(
+    custom_intro="""
+    Data2VecAudio Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).
+    """
+)
+class Data2VecAudioForCTC(Data2VecAudioPreTrainedModel):
+    def __init__(self, config):
+        r"""
+        config ([`Data2VecAudioForCTC`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`]  method to load the model weights.
+        """
+        super().__init__(config)
+
+        self.data2vec_audio = Data2VecAudioModel(config)
+        self.dropout = nn.Dropout(config.final_dropout)
+
+        if config.vocab_size is None:
+            raise ValueError(
+                f"You are trying to instantiate {self.__class__} with a configuration that "
+                "does not define the vocabulary size of the language model head. Please "
+                "instantiate the model as follows: `Data2VecAudioForCTC.from_pretrained(..., vocab_size=vocab_size)`. "
+                "or define `vocab_size` of your model's configuration."
+            )
+        output_hidden_size = (
+            config.output_hidden_size if hasattr(config, "add_adapter") and config.add_adapter else config.hidden_size
+        )
+        self.lm_head = nn.Linear(output_hidden_size, config.vocab_size)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def freeze_feature_encoder(self):
+        """
+        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
+        not be updated during training.
+        """
+        self.data2vec_audio.feature_extractor._freeze_parameters()
+
+    @auto_docstring
+    def forward(
+        self,
+        input_values: torch.Tensor | None,
+        attention_mask: torch.Tensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        labels: torch.Tensor | None = None,
+        **kwargs,
+    ) -> tuple | CausalLMOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
+            Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
+            the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
+            All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
+            config.vocab_size - 1]`.
+        """
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        if labels is not None and labels.max() >= self.config.vocab_size:
+            raise ValueError(f"Label values must be <= vocab_size: {self.config.vocab_size}")
+
+        outputs = self.data2vec_audio(
+            input_values,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = outputs[0]
+        hidden_states = self.dropout(hidden_states)
+
+        logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            # retrieve loss input_lengths from attention_mask
+            attention_mask = (
+                attention_mask if attention_mask is not None else torch.ones_like(input_values, dtype=torch.long)
+            )
+            input_lengths = self._get_feat_extract_output_lengths(attention_mask.sum(-1)).to(torch.long)
+
+            # assuming that padded tokens are filled with -100
+            # when not being attended to
+            labels_mask = labels >= 0
+            target_lengths = labels_mask.sum(-1)
+            flattened_targets = labels.masked_select(labels_mask)
+
+            # ctc_loss doesn't support fp16
+            log_probs = nn.functional.log_softmax(logits, dim=-1, dtype=torch.float32).transpose(0, 1)
+
+            with torch.backends.cudnn.flags(enabled=False):
+                loss = nn.functional.ctc_loss(
+                    log_probs,
+                    flattened_targets,
+                    input_lengths,
+                    target_lengths,
+                    blank=self.config.pad_token_id,
+                    reduction=self.config.ctc_loss_reduction,
+                    zero_infinity=self.config.ctc_zero_infinity,
+                )
+
+        if not return_dict:
+            output = (logits,) + outputs[_HIDDEN_STATES_START_POSITION:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutput(
+            loss=loss, logits=logits, hidden_states=outputs.hidden_states, attentions=outputs.attentions
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    Data2VecAudio Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
+    SUPERB Keyword Spotting.
+    """
+)
+class Data2VecAudioForSequenceClassification(Data2VecAudioPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        if hasattr(config, "add_adapter") and config.add_adapter:
+            raise ValueError(
+                "Sequence classification does not support the use of Data2VecAudio adapters (config.add_adapter=True)"
+            )
+        self.data2vec_audio = Data2VecAudioModel(config)
+        num_layers = config.num_hidden_layers + 1  # transformer layers + input embeddings
+        if config.use_weighted_layer_sum:
+            self.layer_weights = nn.Parameter(torch.ones(num_layers) / num_layers)
+        self.projector = nn.Linear(config.hidden_size, config.classifier_proj_size)
+        self.classifier = nn.Linear(config.classifier_proj_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def freeze_feature_encoder(self):
+        """
+        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
+        not be updated during training.
+        """
+        self.data2vec_audio.feature_extractor._freeze_parameters()
+
+    def freeze_base_model(self):
+        """
+        Calling this function will disable the gradient computation for the base model so that its parameters will not
+        be updated during training. Only the classification head will be updated.
+        """
+        for param in self.data2vec_audio.parameters():
+            param.requires_grad = False
+
+    @auto_docstring
+    def forward(
+        self,
+        input_values: torch.Tensor | None,
+        attention_mask: torch.Tensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        labels: torch.Tensor | None = None,
+        **kwargs,
+    ) -> tuple | SequenceClassifierOutput:
+        r"""
+        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
+            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
+            into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
+            (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
+            To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
+            into a tensor of type `torch.FloatTensor`. See [`Data2VecAudioProcessor.__call__`] for details.
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+        output_hidden_states = True if self.config.use_weighted_layer_sum else output_hidden_states
+
+        outputs = self.data2vec_audio(
+            input_values,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        if self.config.use_weighted_layer_sum:
+            hidden_states = outputs[_HIDDEN_STATES_START_POSITION]
+            hidden_states = torch.stack(hidden_states, dim=1)
+            norm_weights = nn.functional.softmax(self.layer_weights, dim=-1)
+            hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(dim=1)
+        else:
+            hidden_states = outputs[0]
+
+        hidden_states = self.projector(hidden_states)
+        if attention_mask is None:
+            pooled_output = hidden_states.mean(dim=1)
+        else:
+            padding_mask = self._get_feature_vector_attention_mask(hidden_states.shape[1], attention_mask)
+            expand_padding_mask = padding_mask.unsqueeze(-1).repeat(1, 1, hidden_states.shape[2])
+            hidden_states[~expand_padding_mask] = 0.0
+            pooled_output = hidden_states.sum(dim=1) / padding_mask.sum(dim=1).view(-1, 1)
+
+        logits = self.classifier(pooled_output)
+
+        loss = None
+        if labels is not None:
+            loss_fct = CrossEntropyLoss()
+            loss = loss_fct(logits.view(-1, self.config.num_labels), labels.view(-1))
+
+        if not return_dict:
+            output = (logits,) + outputs[_HIDDEN_STATES_START_POSITION:]
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring
+class Data2VecAudioForAudioFrameClassification(Data2VecAudioPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        if hasattr(config, "add_adapter") and config.add_adapter:
+            raise ValueError(
+                "Audio frame classification does not support the use of Data2VecAudio adapters (config.add_adapter=True)"
+            )
+        self.data2vec_audio = Data2VecAudioModel(config)
+        num_layers = config.num_hidden_layers + 1  # transformer layers + input embeddings
+        if config.use_weighted_layer_sum:
+            self.layer_weights = nn.Parameter(torch.ones(num_layers) / num_layers)
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+        self.num_labels = config.num_labels
+
+        self.post_init()
+
+    def freeze_feature_encoder(self):
+        """
+        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
+        not be updated during training.
+        """
+        self.data2vec_audio.feature_extractor._freeze_parameters()
+
+    def freeze_base_model(self):
+        """
+        Calling this function will disable the gradient computation for the base model so that its parameters will not
+        be updated during training. Only the classification head will be updated.
+        """
+        for param in self.data2vec_audio.parameters():
+            param.requires_grad = False
+
+    @auto_docstring
+    def forward(
+        self,
+        input_values: torch.Tensor | None,
+        attention_mask: torch.Tensor | None = None,
+        labels: torch.Tensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        **kwargs,
+    ) -> tuple | TokenClassifierOutput:
+        r"""
+        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
+            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
+            into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
+            (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
+            To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
+            into a tensor of type `torch.FloatTensor`. See [`Data2VecAudioProcessor.__call__`] for details.
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+        output_hidden_states = True if self.config.use_weighted_layer_sum else output_hidden_states
+
+        outputs = self.data2vec_audio(
+            input_values,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        if self.config.use_weighted_layer_sum:
+            hidden_states = outputs[_HIDDEN_STATES_START_POSITION]
+            hidden_states = torch.stack(hidden_states, dim=1)
+            norm_weights = nn.functional.softmax(self.layer_weights, dim=-1)
+            hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(dim=1)
+        else:
+            hidden_states = outputs[0]
+
+        logits = self.classifier(hidden_states)
+
+        loss = None
+        if labels is not None:
+            loss_fct = CrossEntropyLoss()
+            loss = loss_fct(logits.view(-1, self.num_labels), torch.argmax(labels.view(-1, self.num_labels), axis=1))
+
+        if not return_dict:
+            output = (logits,) + outputs[_HIDDEN_STATES_START_POSITION:]
+            return output
+
+        return TokenClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+class AMSoftmaxLoss(nn.Module):
+    def __init__(self, input_dim, num_labels, scale=30.0, margin=0.4):
+        super().__init__()
+        self.scale = scale
+        self.margin = margin
+        self.num_labels = num_labels
+        self.weight = nn.Parameter(torch.randn(input_dim, num_labels), requires_grad=True)
+        self.loss = nn.CrossEntropyLoss()
+
+    def forward(self, hidden_states, labels):
+        labels = labels.flatten()
+        weight = nn.functional.normalize(self.weight, dim=0)
+        hidden_states = nn.functional.normalize(hidden_states, dim=1)
+        cos_theta = torch.mm(hidden_states, weight)
+        psi = cos_theta - self.margin
+
+        onehot = nn.functional.one_hot(labels, self.num_labels)
+        logits = self.scale * torch.where(onehot.bool(), psi, cos_theta)
+        loss = self.loss(logits, labels)
+
+        return loss
+
+
+class TDNNLayer(nn.Module):
+    def __init__(self, config, layer_id=0):
+        super().__init__()
+        self.in_conv_dim = config.tdnn_dim[layer_id - 1] if layer_id > 0 else config.tdnn_dim[layer_id]
+        self.out_conv_dim = config.tdnn_dim[layer_id]
+        self.kernel_size = config.tdnn_kernel[layer_id]
+        self.dilation = config.tdnn_dilation[layer_id]
+
+        self.kernel = nn.Linear(self.in_conv_dim * self.kernel_size, self.out_conv_dim)
+        self.activation = nn.ReLU()
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        if is_peft_available():
+            from peft.tuners.lora import LoraLayer
+
+        if is_peft_available():
+            if isinstance(self.kernel, LoraLayer):
+                warnings.warn(
+                    "Detected LoRA on TDNNLayer. LoRA weights won't be applied due to optimization. "
+                    "You should exclude TDNNLayer from LoRA's target modules.",
+                )
+
+        # for backward compatibility, we keep nn.Linear but call F.conv1d for speed up
+        hidden_states = hidden_states.transpose(1, 2)
+        weight = self.kernel.weight.view(self.out_conv_dim, self.kernel_size, self.in_conv_dim).transpose(1, 2)
+        hidden_states = nn.functional.conv1d(hidden_states, weight, self.kernel.bias, dilation=self.dilation)
+        hidden_states = hidden_states.transpose(1, 2)
+
+        hidden_states = self.activation(hidden_states)
+        return hidden_states
+
+
+@auto_docstring(
+    custom_intro="""
+    Data2VecAudio Model with an XVector feature extraction head on top for tasks like Speaker Verification.
+    """
+)
+class Data2VecAudioForXVector(Data2VecAudioPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.data2vec_audio = Data2VecAudioModel(config)
+        num_layers = config.num_hidden_layers + 1  # transformer layers + input embeddings
+        if config.use_weighted_layer_sum:
+            self.layer_weights = nn.Parameter(torch.ones(num_layers) / num_layers)
+        self.projector = nn.Linear(config.hidden_size, config.tdnn_dim[0])
+
+        tdnn_layers = [TDNNLayer(config, i) for i in range(len(config.tdnn_dim))]
+        self.tdnn = nn.ModuleList(tdnn_layers)
+
+        self.feature_extractor = nn.Linear(config.tdnn_dim[-1] * 2, config.xvector_output_dim)
+        self.classifier = nn.Linear(config.xvector_output_dim, config.xvector_output_dim)
+
+        self.objective = AMSoftmaxLoss(config.xvector_output_dim, config.num_labels)
+
+        self.post_init()
+
+    def freeze_feature_encoder(self):
+        """
+        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
+        not be updated during training.
+        """
+        self.data2vec_audio.feature_extractor._freeze_parameters()
+
+    def freeze_base_model(self):
+        """
+        Calling this function will disable the gradient computation for the base model so that its parameters will not
+        be updated during training. Only the classification head will be updated.
+        """
+        for param in self.data2vec_audio.parameters():
+            param.requires_grad = False
+
+    def _get_tdnn_output_lengths(self, input_lengths: torch.LongTensor | int):
+        """
+        Computes the output length of the TDNN layers
+        """
+
+        def _conv_out_length(input_length, kernel_size, stride):
+            # 1D convolutional layer output length formula taken
+            # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html
+            return (input_length - kernel_size) // stride + 1
+
+        for kernel_size in self.config.tdnn_kernel:
+            input_lengths = _conv_out_length(input_lengths, kernel_size, 1)
+
+        return input_lengths
+
+    @auto_docstring
+    def forward(
+        self,
+        input_values: torch.Tensor | None,
+        attention_mask: torch.Tensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        return_dict: bool | None = None,
+        labels: torch.Tensor | None = None,
+        **kwargs,
+    ) -> tuple | XVectorOutput:
+        r"""
+        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
+            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
+            into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
+            (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
+            To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
+            into a tensor of type `torch.FloatTensor`. See [`Data2VecAudioProcessor.__call__`] for details.
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+        output_hidden_states = True if self.config.use_weighted_layer_sum else output_hidden_states
+
+        outputs = self.data2vec_audio(
+            input_values,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        if self.config.use_weighted_layer_sum:
+            hidden_states = outputs[_HIDDEN_STATES_START_POSITION]
+            hidden_states = torch.stack(hidden_states, dim=1)
+            norm_weights = nn.functional.softmax(self.layer_weights, dim=-1)
+            hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(dim=1)
+        else:
+            hidden_states = outputs[0]
+
+        hidden_states = self.projector(hidden_states)
+
+        for tdnn_layer in self.tdnn:
+            hidden_states = tdnn_layer(hidden_states)
+
+        # Statistic Pooling
+        if attention_mask is None:
+            mean_features = hidden_states.mean(dim=1)
+            std_features = hidden_states.std(dim=1)
+        else:
+            feat_extract_output_lengths = self._get_feat_extract_output_lengths(attention_mask.sum(dim=1))
+            tdnn_output_lengths = self._get_tdnn_output_lengths(feat_extract_output_lengths)
+            mean_features = []
+            std_features = []
+            for i, length in enumerate(tdnn_output_lengths):
+                mean_features.append(hidden_states[i, :length].mean(dim=0))
+                std_features.append(hidden_states[i, :length].std(dim=0))
+            mean_features = torch.stack(mean_features)
+            std_features = torch.stack(std_features)
+        statistic_pooling = torch.cat([mean_features, std_features], dim=-1)
+
+        output_embeddings = self.feature_extractor(statistic_pooling)
+        logits = self.classifier(output_embeddings)
+
+        loss = None
+        if labels is not None:
+            loss = self.objective(logits, labels)
+
+        if not return_dict:
+            output = (logits, output_embeddings) + outputs[_HIDDEN_STATES_START_POSITION:]
+            return ((loss,) + output) if loss is not None else output
+
+        return XVectorOutput(
+            loss=loss,
+            logits=logits,
+            embeddings=output_embeddings,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+__all__ = [
+    "Data2VecAudioForAudioFrameClassification",
+    "Data2VecAudioForCTC",
+    "Data2VecAudioForSequenceClassification",
+    "Data2VecAudioForXVector",
+    "Data2VecAudioModel",
+    "Data2VecAudioPreTrainedModel",
+]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_text.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_text.py
new file mode 100644
index 0000000000000000000000000000000000000000..512431cb3b0a293d58d9133875818eaaee5b810a
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_text.py
@@ -0,0 +1,1208 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/data2vec/modular_data2vec_text.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_data2vec_text.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# Copyright 2022 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import Callable
+
+import torch
+import torch.nn as nn
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+
+from ... import initialization as init
+from ...activations import ACT2FN, gelu
+from ...cache_utils import Cache, DynamicCache, EncoderDecoderCache
+from ...generation import GenerationMixin
+from ...masking_utils import create_bidirectional_mask, create_causal_mask
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import (
+    BaseModelOutputWithPastAndCrossAttentions,
+    BaseModelOutputWithPoolingAndCrossAttentions,
+    CausalLMOutputWithCrossAttentions,
+    MaskedLMOutput,
+    MultipleChoiceModelOutput,
+    QuestionAnsweringModelOutput,
+    SequenceClassifierOutput,
+    TokenClassifierOutput,
+)
+from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from ...processing_utils import Unpack
+from ...pytorch_utils import apply_chunking_to_forward
+from ...utils import TransformersKwargs, auto_docstring, logging
+from ...utils.generic import can_return_tuple, merge_with_config_defaults
+from ...utils.output_capturing import capture_outputs
+from .configuration_data2vec_text import Data2VecTextConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+class Data2VecTextEmbeddings(nn.Module):
+    """Construct the embeddings from word, position and token_type embeddings."""
+
+    def __init__(self, config):
+        super().__init__()
+        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
+        self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)
+
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        # position_ids (1, len position emb) is contiguous in memory and exported when serialized
+        self.register_buffer(
+            "position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)), persistent=False
+        )
+        self.register_buffer(
+            "token_type_ids", torch.zeros(self.position_ids.size(), dtype=torch.long), persistent=False
+        )
+
+        self.padding_idx = config.pad_token_id
+        self.position_embeddings = nn.Embedding(
+            config.max_position_embeddings, config.hidden_size, padding_idx=self.padding_idx
+        )
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        past_key_values_length: int = 0,
+    ) -> torch.Tensor:
+        if position_ids is None:
+            if input_ids is not None:
+                # Create the position ids from the input token ids. Any padded tokens remain padded.
+                position_ids = self.create_position_ids_from_input_ids(
+                    input_ids, self.padding_idx, past_key_values_length
+                )
+            else:
+                position_ids = self.create_position_ids_from_inputs_embeds(inputs_embeds, self.padding_idx)
+
+        if input_ids is not None:
+            input_shape = input_ids.size()
+        else:
+            input_shape = inputs_embeds.size()[:-1]
+
+        batch_size, seq_length = input_shape
+
+        # Setting the token_type_ids to the registered buffer in constructor where it is all zeros, which usually occurs
+        # when its auto-generated, registered buffer helps users when tracing the model without passing token_type_ids, solves
+        # issue #5664
+        if token_type_ids is None:
+            if hasattr(self, "token_type_ids"):
+                # NOTE: We assume either pos ids to have bsz == 1 (broadcastable) or bsz == effective bsz (input_shape[0])
+                buffered_token_type_ids = self.token_type_ids.expand(position_ids.shape[0], -1)
+                buffered_token_type_ids = torch.gather(buffered_token_type_ids, dim=1, index=position_ids)
+                token_type_ids = buffered_token_type_ids.expand(batch_size, seq_length)
+            else:
+                token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=self.position_ids.device)
+
+        if inputs_embeds is None:
+            inputs_embeds = self.word_embeddings(input_ids)
+        token_type_embeddings = self.token_type_embeddings(token_type_ids)
+        embeddings = inputs_embeds + token_type_embeddings
+
+        position_embeddings = self.position_embeddings(position_ids)
+        embeddings = embeddings + position_embeddings
+
+        embeddings = self.LayerNorm(embeddings)
+        embeddings = self.dropout(embeddings)
+        return embeddings
+
+    @staticmethod
+    def create_position_ids_from_inputs_embeds(inputs_embeds, padding_idx):
+        """
+        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.
+
+        Args:
+            inputs_embeds: torch.Tensor
+
+        Returns: torch.Tensor
+        """
+        input_shape = inputs_embeds.size()[:-1]
+        sequence_length = input_shape[1]
+
+        position_ids = torch.arange(
+            padding_idx + 1, sequence_length + padding_idx + 1, dtype=torch.long, device=inputs_embeds.device
+        )
+        return position_ids.unsqueeze(0).expand(input_shape)
+
+    @staticmethod
+    def create_position_ids_from_input_ids(input_ids, padding_idx, past_key_values_length=0):
+        """
+        Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
+        are ignored. This is modified from fairseq's `utils.make_positions`.
+
+        Args:
+            x: torch.Tensor x:
+
+        Returns: torch.Tensor
+        """
+        # The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA.
+        mask = input_ids.ne(padding_idx).int()
+        incremental_indices = (torch.cumsum(mask, dim=1).type_as(mask) + past_key_values_length) * mask
+        return incremental_indices.long() + padding_idx
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    scaling: float | None = None,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    if scaling is None:
+        scaling = query.size(-1) ** -0.5
+
+    # Take the dot product between "query" and "key" to get the raw attention scores.
+    attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling
+
+    if attention_mask is not None:
+        attn_weights = attn_weights + attention_mask
+
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+
+    attn_output = torch.matmul(attn_weights, value)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+class Data2VecTextSelfAttention(nn.Module):
+    def __init__(self, config, is_causal=False, layer_idx=None):
+        super().__init__()
+        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
+            raise ValueError(
+                f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
+                f"heads ({config.num_attention_heads})"
+            )
+        self.config = config
+
+        self.num_attention_heads = config.num_attention_heads
+        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+        self.scaling = self.attention_head_size**-0.5
+
+        self.query = nn.Linear(config.hidden_size, self.all_head_size)
+        self.key = nn.Linear(config.hidden_size, self.all_head_size)
+        self.value = nn.Linear(config.hidden_size, self.all_head_size)
+
+        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
+
+        self.is_decoder = config.is_decoder
+        self.is_causal = is_causal
+        self.layer_idx = layer_idx
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.FloatTensor | None = None,
+        past_key_values: Cache | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor]:
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.attention_head_size)
+
+        # get all proj
+        query_layer = self.query(hidden_states).view(*hidden_shape).transpose(1, 2)
+        key_layer = self.key(hidden_states).view(*hidden_shape).transpose(1, 2)
+        value_layer = self.value(hidden_states).view(*hidden_shape).transpose(1, 2)
+
+        if past_key_values is not None:
+            # decoder-only data2vec_text can have a simple dynamic cache for example
+            current_past_key_values = past_key_values
+            if isinstance(past_key_values, EncoderDecoderCache):
+                current_past_key_values = past_key_values.self_attention_cache
+
+            # save all key/value_layer to cache to be re-used for fast auto-regressive generation
+            key_layer, value_layer = current_past_key_values.update(key_layer, value_layer, self.layer_idx)
+
+        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, eager_attention_forward
+        )
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_layer,
+            key_layer,
+            value_layer,
+            attention_mask,
+            dropout=0.0 if not self.training else self.dropout.p,
+            scaling=self.scaling,
+            **kwargs,
+        )
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        return attn_output, attn_weights
+
+
+class Data2VecTextCrossAttention(nn.Module):
+    def __init__(self, config, is_causal=False, layer_idx=None):
+        super().__init__()
+        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
+            raise ValueError(
+                f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
+                f"heads ({config.num_attention_heads})"
+            )
+        self.config = config
+
+        self.num_attention_heads = config.num_attention_heads
+        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+        self.scaling = self.attention_head_size**-0.5
+
+        self.query = nn.Linear(config.hidden_size, self.all_head_size)
+        self.key = nn.Linear(config.hidden_size, self.all_head_size)
+        self.value = nn.Linear(config.hidden_size, self.all_head_size)
+
+        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
+
+        self.is_causal = is_causal
+        self.layer_idx = layer_idx
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        encoder_hidden_states: torch.FloatTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        past_key_values: EncoderDecoderCache | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor]:
+        # determine input shapes
+        input_shape = hidden_states.shape[:-1]
+
+        hidden_shape = (*input_shape, -1, self.attention_head_size)
+
+        # get query proj
+        query_layer = self.query(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        is_updated = past_key_values.is_updated.get(self.layer_idx) if past_key_values is not None else False
+        if past_key_values is not None and is_updated:
+            # reuse k,v, cross_attentions
+            key_layer = past_key_values.cross_attention_cache.layers[self.layer_idx].keys
+            value_layer = past_key_values.cross_attention_cache.layers[self.layer_idx].values
+        else:
+            kv_shape = (*encoder_hidden_states.shape[:-1], -1, self.attention_head_size)
+            key_layer = self.key(encoder_hidden_states).view(kv_shape).transpose(1, 2)
+            value_layer = self.value(encoder_hidden_states).view(kv_shape).transpose(1, 2)
+
+            if past_key_values is not None:
+                # save all states to the cache
+                key_layer, value_layer = past_key_values.cross_attention_cache.update(
+                    key_layer, value_layer, self.layer_idx
+                )
+                # set flag that curr layer for cross-attn is already updated so we can re-use in subsequent calls
+                past_key_values.is_updated[self.layer_idx] = True
+
+        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, eager_attention_forward
+        )
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_layer,
+            key_layer,
+            value_layer,
+            attention_mask,
+            dropout=0.0 if not self.training else self.dropout.p,
+            scaling=self.scaling,
+            **kwargs,
+        )
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        return attn_output, attn_weights
+
+
+class Data2VecTextSelfOutput(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        return hidden_states
+
+
+class Data2VecTextAttention(nn.Module):
+    def __init__(self, config, is_causal=False, layer_idx=None, is_cross_attention=False):
+        super().__init__()
+        self.is_cross_attention = is_cross_attention
+        attention_class = Data2VecTextCrossAttention if is_cross_attention else Data2VecTextSelfAttention
+        self.self = attention_class(config, is_causal=is_causal, layer_idx=layer_idx)
+        self.output = Data2VecTextSelfOutput(config)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.FloatTensor | None = None,
+        encoder_hidden_states: torch.FloatTensor | None = None,
+        encoder_attention_mask: torch.FloatTensor | None = None,
+        past_key_values: Cache | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor]:
+        attention_mask = attention_mask if not self.is_cross_attention else encoder_attention_mask
+        attention_output, attn_weights = self.self(
+            hidden_states,
+            encoder_hidden_states=encoder_hidden_states,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            **kwargs,
+        )
+        attention_output = self.output(attention_output, hidden_states)
+        return attention_output, attn_weights
+
+
+class Data2VecTextIntermediate(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
+        if isinstance(config.hidden_act, str):
+            self.intermediate_act_fn = ACT2FN[config.hidden_act]
+        else:
+            self.intermediate_act_fn = config.hidden_act
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.intermediate_act_fn(hidden_states)
+        return hidden_states
+
+
+class Data2VecTextOutput(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
+        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.LayerNorm(hidden_states + input_tensor)
+        return hidden_states
+
+
+class Data2VecTextLayer(GradientCheckpointingLayer):
+    def __init__(self, config, layer_idx=None):
+        super().__init__()
+        self.chunk_size_feed_forward = config.chunk_size_feed_forward
+        self.seq_len_dim = 1
+        self.attention = Data2VecTextAttention(config, is_causal=config.is_decoder, layer_idx=layer_idx)
+        self.is_decoder = config.is_decoder
+        self.add_cross_attention = config.add_cross_attention
+        if self.add_cross_attention:
+            if not self.is_decoder:
+                raise ValueError(f"{self} should be used as a decoder model if cross attention is added")
+            self.crossattention = Data2VecTextAttention(
+                config,
+                is_causal=False,
+                layer_idx=layer_idx,
+                is_cross_attention=True,
+            )
+        self.intermediate = Data2VecTextIntermediate(config)
+        self.output = Data2VecTextOutput(config)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.FloatTensor | None = None,
+        encoder_hidden_states: torch.FloatTensor | None = None,
+        encoder_attention_mask: torch.FloatTensor | None = None,
+        past_key_values: Cache | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> torch.Tensor:
+        self_attention_output, _ = self.attention(
+            hidden_states,
+            attention_mask,
+            past_key_values=past_key_values,
+            **kwargs,
+        )
+        attention_output = self_attention_output
+
+        if self.is_decoder and encoder_hidden_states is not None:
+            if not hasattr(self, "crossattention"):
+                raise ValueError(
+                    f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers"
+                    " by setting `config.add_cross_attention=True`"
+                )
+
+            cross_attention_output, _ = self.crossattention(
+                self_attention_output,
+                None,  # attention_mask
+                encoder_hidden_states,
+                encoder_attention_mask,
+                past_key_values=past_key_values,
+                **kwargs,
+            )
+            attention_output = cross_attention_output
+
+        layer_output = apply_chunking_to_forward(
+            self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
+        )
+        return layer_output
+
+    def feed_forward_chunk(self, attention_output):
+        intermediate_output = self.intermediate(attention_output)
+        layer_output = self.output(intermediate_output, attention_output)
+        return layer_output
+
+
+@auto_docstring
+class Data2VecTextPreTrainedModel(PreTrainedModel):
+    config_class = Data2VecTextConfig
+    base_model_prefix = "data2vec_text"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Data2VecTextForTextEmbeddings", "Data2VecTextLayer"]
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    _supports_flex_attn = True
+    _supports_attention_backend = True
+    _can_record_outputs = {
+        "hidden_states": Data2VecTextLayer,
+        "attentions": Data2VecTextSelfAttention,
+        "cross_attentions": Data2VecTextCrossAttention,
+    }
+
+    def _init_weights(self, module):
+        super()._init_weights(module)
+        if isinstance(module, Data2VecTextEmbeddings):
+            init.copy_(module.position_ids, torch.arange(module.position_ids.shape[-1]).expand((1, -1)))
+            init.zeros_(module.token_type_ids)
+
+
+class Data2VecTextEncoder(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.layer = nn.ModuleList([Data2VecTextLayer(config, layer_idx=i) for i in range(config.num_hidden_layers)])
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.FloatTensor | None = None,
+        encoder_hidden_states: torch.FloatTensor | None = None,
+        encoder_attention_mask: torch.FloatTensor | None = None,
+        past_key_values: Cache | None = None,
+        use_cache: bool | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor] | BaseModelOutputWithPastAndCrossAttentions:
+        for i, layer_module in enumerate(self.layer):
+            hidden_states = layer_module(
+                hidden_states,
+                attention_mask,
+                encoder_hidden_states,  # as a positional argument for gradient checkpointing
+                encoder_attention_mask=encoder_attention_mask,
+                past_key_values=past_key_values,
+                **kwargs,
+            )
+
+        return BaseModelOutputWithPastAndCrossAttentions(
+            last_hidden_state=hidden_states,
+            past_key_values=past_key_values if use_cache else None,
+        )
+
+
+class Data2VecTextPooler(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.activation = nn.Tanh()
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        # We "pool" the model by simply taking the hidden state corresponding
+        # to the first token.
+        first_token_tensor = hidden_states[:, 0]
+        pooled_output = self.dense(first_token_tensor)
+        pooled_output = self.activation(pooled_output)
+        return pooled_output
+
+
+@auto_docstring
+class Data2VecTextModel(Data2VecTextPreTrainedModel):
+    _no_split_modules = ["Data2VecTextEmbeddings", "Data2VecTextLayer"]
+
+    def __init__(self, config, add_pooling_layer=True):
+        r"""
+        add_pooling_layer (bool, *optional*, defaults to `True`):
+            Whether to add a pooling layer
+        """
+        super().__init__(config)
+        self.config = config
+        self.gradient_checkpointing = False
+
+        self.embeddings = Data2VecTextEmbeddings(config)
+        self.encoder = Data2VecTextEncoder(config)
+
+        self.pooler = Data2VecTextPooler(config) if add_pooling_layer else None
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embeddings.word_embeddings
+
+    def set_input_embeddings(self, value):
+        self.embeddings.word_embeddings = value
+
+    @merge_with_config_defaults
+    @capture_outputs
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.Tensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        token_type_ids: torch.Tensor | None = None,
+        position_ids: torch.Tensor | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        encoder_hidden_states: torch.Tensor | None = None,
+        encoder_attention_mask: torch.Tensor | None = None,
+        past_key_values: Cache | None = None,
+        use_cache: bool | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor] | BaseModelOutputWithPoolingAndCrossAttentions:
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if self.config.is_decoder:
+            use_cache = use_cache if use_cache is not None else self.config.use_cache
+        else:
+            use_cache = False
+
+        if use_cache and past_key_values is None:
+            past_key_values = (
+                EncoderDecoderCache(DynamicCache(config=self.config), DynamicCache(config=self.config))
+                if encoder_hidden_states is not None or self.config.is_encoder_decoder
+                else DynamicCache(config=self.config)
+            )
+
+        past_key_values_length = past_key_values.get_seq_length() if past_key_values is not None else 0
+
+        embedding_output = self.embeddings(
+            input_ids=input_ids,
+            position_ids=position_ids,
+            token_type_ids=token_type_ids,
+            inputs_embeds=inputs_embeds,
+            past_key_values_length=past_key_values_length,
+        )
+
+        attention_mask, encoder_attention_mask = self._create_attention_masks(
+            attention_mask=attention_mask,
+            encoder_attention_mask=encoder_attention_mask,
+            embedding_output=embedding_output,
+            encoder_hidden_states=encoder_hidden_states,
+            past_key_values=past_key_values,
+        )
+
+        encoder_outputs = self.encoder(
+            embedding_output,
+            attention_mask=attention_mask,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            position_ids=position_ids,
+            **kwargs,
+        )
+        sequence_output = encoder_outputs.last_hidden_state
+        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
+
+        return BaseModelOutputWithPoolingAndCrossAttentions(
+            last_hidden_state=sequence_output,
+            pooler_output=pooled_output,
+            past_key_values=encoder_outputs.past_key_values,
+        )
+
+    def _create_attention_masks(
+        self,
+        attention_mask,
+        encoder_attention_mask,
+        embedding_output,
+        encoder_hidden_states,
+        past_key_values,
+    ):
+        if self.config.is_decoder:
+            attention_mask = create_causal_mask(
+                config=self.config,
+                inputs_embeds=embedding_output,
+                attention_mask=attention_mask,
+                past_key_values=past_key_values,
+            )
+        else:
+            attention_mask = create_bidirectional_mask(
+                config=self.config,
+                inputs_embeds=embedding_output,
+                attention_mask=attention_mask,
+            )
+
+        if encoder_attention_mask is not None:
+            encoder_attention_mask = create_bidirectional_mask(
+                config=self.config,
+                inputs_embeds=embedding_output,
+                attention_mask=encoder_attention_mask,
+                encoder_hidden_states=encoder_hidden_states,
+            )
+
+        return attention_mask, encoder_attention_mask
+
+
+class Data2VecTextLMHead(nn.Module):
+    """Data2VecText Head for masked language modeling."""
+
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+        self.decoder = nn.Linear(config.hidden_size, config.vocab_size)
+        self.bias = nn.Parameter(torch.zeros(config.vocab_size))
+
+    def forward(self, features, **kwargs):
+        x = self.dense(features)
+        x = gelu(x)
+        x = self.layer_norm(x)
+
+        # project back to size of vocabulary with bias
+        x = self.decoder(x)
+
+        return x
+
+
+class Data2VecTextClassificationHead(nn.Module):
+    """Head for sentence-level classification tasks."""
+
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        classifier_dropout = (
+            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
+        )
+        self.dropout = nn.Dropout(classifier_dropout)
+        self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
+
+    def forward(self, features, **kwargs):
+        x = features[:, 0, :]  # take <s> token (equiv. to [CLS])
+        x = self.dropout(x)
+        x = self.dense(x)
+        x = torch.tanh(x)
+        x = self.dropout(x)
+        x = self.out_proj(x)
+        return x
+
+
+@auto_docstring(
+    custom_intro="""
+    Data2VecText Model with a `language modeling` head on top for CLM fine-tuning.
+    """
+)
+class Data2VecTextForCausalLM(Data2VecTextPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {
+        "lm_head.decoder.weight": "data2vec_text.embeddings.word_embeddings.weight",
+        "lm_head.decoder.bias": "lm_head.bias",
+    }
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        if not config.is_decoder:
+            logger.warning("If you want to use `Data2VecTextLMHeadModel` as a standalone, add `is_decoder=True.`")
+
+        self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
+        self.lm_head = Data2VecTextLMHead(config)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_output_embeddings(self):
+        return self.lm_head.decoder
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head.decoder = new_embeddings
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        encoder_hidden_states: torch.FloatTensor | None = None,
+        encoder_attention_mask: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        past_key_values: tuple[tuple[torch.FloatTensor]] | None = None,
+        use_cache: bool | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | CausalLMOutputWithCrossAttentions:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
+            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
+            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+
+        Example:
+
+        ```python
+        >>> from transformers import AutoTokenizer, Data2VecTextForCausalLM, Data2VecTextConfig
+        >>> import torch
+
+        >>> tokenizer = AutoTokenizer.from_pretrained("facebook/data2vec-text-base")
+        >>> config = Data2VecTextConfig.from_pretrained("facebook/data2vec-text-base")
+        >>> config.is_decoder = True
+        >>> model = Data2VecTextForCausalLM.from_pretrained("facebook/data2vec-text-base", config=config)
+
+        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
+        >>> outputs = model(**inputs)
+
+        >>> prediction_logits = outputs.logits
+        ```"""
+        if labels is not None:
+            use_cache = False
+
+        outputs: BaseModelOutputWithPoolingAndCrossAttentions = self.data2vec_text(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            return_dict=True,
+            **kwargs,
+        )
+
+        hidden_states = outputs.last_hidden_state
+        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
+
+        return CausalLMOutputWithCrossAttentions(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            cross_attentions=outputs.cross_attentions,
+        )
+
+
+@auto_docstring
+class Data2VecTextForMaskedLM(Data2VecTextPreTrainedModel):
+    _tied_weights_keys = {
+        "lm_head.decoder.weight": "data2vec_text.embeddings.word_embeddings.weight",
+        "lm_head.decoder.bias": "lm_head.bias",
+    }
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        if config.is_decoder:
+            logger.warning(
+                "If you want to use `Data2VecTextForMaskedLM` make sure `config.is_decoder=False` for "
+                "bi-directional self-attention."
+            )
+
+        self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
+        self.lm_head = Data2VecTextLMHead(config)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_output_embeddings(self):
+        return self.lm_head.decoder
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head.decoder = new_embeddings
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        encoder_hidden_states: torch.FloatTensor | None = None,
+        encoder_attention_mask: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | MaskedLMOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+        """
+        outputs = self.data2vec_text(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            return_dict=True,
+            **kwargs,
+        )
+        sequence_output = outputs[0]
+        prediction_scores = self.lm_head(sequence_output)
+
+        masked_lm_loss = None
+        if labels is not None:
+            loss_fct = CrossEntropyLoss()
+
+            labels = labels.to(prediction_scores.device)
+            masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
+
+        return MaskedLMOutput(
+            loss=masked_lm_loss,
+            logits=prediction_scores,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    Data2VecText Model transformer with a sequence classification/regression head on top (a linear layer on top of the
+    pooled output) e.g. for GLUE tasks.
+    """
+)
+class Data2VecTextForSequenceClassification(Data2VecTextPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.config = config
+
+        self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
+        self.classifier = Data2VecTextClassificationHead(config)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | SequenceClassifierOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+        outputs = self.data2vec_text(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            return_dict=True,
+            **kwargs,
+        )
+        sequence_output = outputs[0]
+        logits = self.classifier(sequence_output)
+
+        loss = None
+        if labels is not None:
+            labels = labels.to(logits.device)
+
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(logits, labels)
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = BCEWithLogitsLoss()
+                loss = loss_fct(logits, labels)
+
+        return SequenceClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring
+class Data2VecTextForMultipleChoice(Data2VecTextPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.data2vec_text = Data2VecTextModel(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.classifier = nn.Linear(config.hidden_size, 1)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | MultipleChoiceModelOutput:
+        r"""
+        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
+            Indices of input sequence tokens in the vocabulary.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
+            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+            1]`:
+
+            - 0 corresponds to a *sentence A* token,
+            - 1 corresponds to a *sentence B* token.
+
+            [What are token type IDs?](../glossary#token-type-ids)
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+            `input_ids` above)
+        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.max_position_embeddings - 1]`.
+
+            [What are position IDs?](../glossary#position-ids)
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+        """
+        num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]
+
+        flat_input_ids = input_ids.view(-1, input_ids.size(-1)) if input_ids is not None else None
+        flat_position_ids = position_ids.view(-1, position_ids.size(-1)) if position_ids is not None else None
+        flat_token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None
+        flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None
+        flat_inputs_embeds = (
+            inputs_embeds.view(-1, inputs_embeds.size(-2), inputs_embeds.size(-1))
+            if inputs_embeds is not None
+            else None
+        )
+
+        outputs = self.data2vec_text(
+            flat_input_ids,
+            position_ids=flat_position_ids,
+            token_type_ids=flat_token_type_ids,
+            attention_mask=flat_attention_mask,
+            inputs_embeds=flat_inputs_embeds,
+            return_dict=True,
+            **kwargs,
+        )
+        pooled_output = outputs[1]
+
+        pooled_output = self.dropout(pooled_output)
+        logits = self.classifier(pooled_output)
+        reshaped_logits = logits.view(-1, num_choices)
+
+        loss = None
+        if labels is not None:
+            loss_fct = CrossEntropyLoss()
+
+            labels = labels.to(reshaped_logits.device)
+            loss = loss_fct(reshaped_logits, labels)
+
+        return MultipleChoiceModelOutput(
+            loss=loss,
+            logits=reshaped_logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring
+class Data2VecTextForTokenClassification(Data2VecTextPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+
+        self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
+        classifier_dropout = (
+            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
+        )
+        self.dropout = nn.Dropout(classifier_dropout)
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | TokenClassifierOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
+        """
+        outputs = self.data2vec_text(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            return_dict=True,
+            **kwargs,
+        )
+
+        sequence_output = outputs[0]
+
+        sequence_output = self.dropout(sequence_output)
+        logits = self.classifier(sequence_output)
+
+        loss = None
+        if labels is not None:
+            loss_fct = CrossEntropyLoss()
+
+            labels = labels.to(logits.device)
+            loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+
+        return TokenClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring
+class Data2VecTextForQuestionAnswering(Data2VecTextPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+
+        self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
+        self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        start_positions: torch.LongTensor | None = None,
+        end_positions: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | QuestionAnsweringModelOutput:
+        outputs = self.data2vec_text(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            return_dict=True,
+            **kwargs,
+        )
+
+        sequence_output = outputs[0]
+
+        logits = self.qa_outputs(sequence_output)
+        start_logits, end_logits = logits.split(1, dim=-1)
+        start_logits = start_logits.squeeze(-1).contiguous()
+        end_logits = end_logits.squeeze(-1).contiguous()
+
+        total_loss = None
+        if start_positions is not None and end_positions is not None:
+            # If we are on multi-GPU, split add a dimension
+            if len(start_positions.size()) > 1:
+                start_positions = start_positions.squeeze(-1)
+            if len(end_positions.size()) > 1:
+                end_positions = end_positions.squeeze(-1)
+            # sometimes the start/end positions are outside our model inputs, we ignore these terms
+            ignored_index = start_logits.size(1)
+            start_positions = start_positions.clamp(0, ignored_index)
+            end_positions = end_positions.clamp(0, ignored_index)
+
+            loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
+            start_loss = loss_fct(start_logits, start_positions)
+            end_loss = loss_fct(end_logits, end_positions)
+            total_loss = (start_loss + end_loss) / 2
+
+        return QuestionAnsweringModelOutput(
+            loss=total_loss,
+            start_logits=start_logits,
+            end_logits=end_logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+__all__ = [
+    "Data2VecTextForCausalLM",
+    "Data2VecTextForMaskedLM",
+    "Data2VecTextForMultipleChoice",
+    "Data2VecTextForQuestionAnswering",
+    "Data2VecTextForSequenceClassification",
+    "Data2VecTextForTokenClassification",
+    "Data2VecTextModel",
+    "Data2VecTextPreTrainedModel",
+]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_vision.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_vision.py
new file mode 100644
index 0000000000000000000000000000000000000000..b8d8d7721c591dd661c2f1ec743a908696531222
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_vision.py
@@ -0,0 +1,1214 @@
+# Copyright 2022 Meta Platforms and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch Data2VecVision model."""
+
+import collections.abc
+import math
+from dataclasses import dataclass
+from typing import Optional
+
+import torch
+from torch import nn
+from torch.nn import CrossEntropyLoss
+
+from ... import initialization as init
+from ...activations import ACT2FN
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import (
+    BaseModelOutput,
+    BaseModelOutputWithPooling,
+    ImageClassifierOutput,
+    SemanticSegmenterOutput,
+)
+from ...modeling_utils import PreTrainedModel
+from ...pytorch_utils import compile_compatible_method_lru_cache
+from ...utils import auto_docstring, logging, torch_int
+from .configuration_data2vec_vision import Data2VecVisionConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+@auto_docstring(
+    custom_intro="""
+    Class for outputs of [`Data2VecVisionModel`].
+    """
+)
+@dataclass
+# Copied from transformers.models.beit.modeling_beit.BeitModelOutputWithPooling with Beit->Data2VecVision
+class Data2VecVisionModelOutputWithPooling(BaseModelOutputWithPooling):
+    r"""
+    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
+        Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
+        *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
+        will be returned.
+    """
+
+
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitEmbeddings with Beit->Data2VecVision
+class Data2VecVisionEmbeddings(nn.Module):
+    """
+    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
+
+    """
+
+    def __init__(self, config: Data2VecVisionConfig) -> None:
+        super().__init__()
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
+        if config.use_mask_token:
+            self.mask_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
+        else:
+            self.mask_token = None
+        self.patch_embeddings = Data2VecVisionPatchEmbeddings(config)
+        self.patch_size = config.patch_size
+        self.image_size = (
+            config.image_size
+            if isinstance(config.image_size, collections.abc.Iterable)
+            else (config.image_size, config.image_size)
+        )
+        num_patches = self.patch_embeddings.num_patches
+        if config.use_absolute_position_embeddings:
+            self.position_embeddings = nn.Parameter(torch.zeros(1, num_patches + 1, config.hidden_size))
+        else:
+            self.position_embeddings = None
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    # Copied from transformers.models.vit.modeling_vit.ViTEmbeddings.interpolate_pos_encoding
+    def interpolate_pos_encoding(self, embeddings: torch.Tensor, height: int, width: int) -> torch.Tensor:
+        """
+        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
+        images. This method is also adapted to support torch.jit tracing.
+
+        Adapted from:
+        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
+        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
+        """
+
+        num_patches = embeddings.shape[1] - 1
+        num_positions = self.position_embeddings.shape[1] - 1
+
+        # always interpolate when tracing to ensure the exported model works for dynamic input shapes
+        if not torch.jit.is_tracing() and num_patches == num_positions and height == width:
+            return self.position_embeddings
+
+        class_pos_embed = self.position_embeddings[:, :1]
+        patch_pos_embed = self.position_embeddings[:, 1:]
+
+        dim = embeddings.shape[-1]
+
+        new_height = height // self.patch_size
+        new_width = width // self.patch_size
+
+        sqrt_num_positions = torch_int(num_positions**0.5)
+        patch_pos_embed = patch_pos_embed.reshape(1, sqrt_num_positions, sqrt_num_positions, dim)
+        patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2)
+
+        patch_pos_embed = nn.functional.interpolate(
+            patch_pos_embed,
+            size=(new_height, new_width),
+            mode="bicubic",
+            align_corners=False,
+        )
+
+        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
+
+        return torch.cat((class_pos_embed, patch_pos_embed), dim=1)
+
+    def forward(
+        self,
+        pixel_values: torch.Tensor,
+        bool_masked_pos: torch.BoolTensor | None = None,
+    ) -> torch.Tensor:
+        _, _, height, width = pixel_values.shape
+        embeddings, (patch_height, patch_width) = self.patch_embeddings(pixel_values)
+        batch_size, seq_len, _ = embeddings.size()
+
+        if bool_masked_pos is not None:
+            mask_tokens = self.mask_token.expand(batch_size, seq_len, -1)
+            # replace the masked visual tokens by mask_tokens
+            w = bool_masked_pos.unsqueeze(-1).type_as(mask_tokens)
+            embeddings = embeddings * (1 - w) + mask_tokens * w
+
+        cls_tokens = self.cls_token.expand(batch_size, -1, -1)
+        embeddings = torch.cat((cls_tokens, embeddings), dim=1)
+
+        if self.position_embeddings is not None:
+            embeddings = embeddings + self.interpolate_pos_encoding(embeddings, height, width)
+
+        embeddings = self.dropout(embeddings)
+
+        return embeddings, (patch_height, patch_width)
+
+
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitPatchEmbeddings with Beit->Data2VecVision
+class Data2VecVisionPatchEmbeddings(nn.Module):
+    """
+    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
+    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
+    Transformer.
+    """
+
+    def __init__(self, config):
+        super().__init__()
+        image_size, patch_size = config.image_size, config.patch_size
+        num_channels, hidden_size = config.num_channels, config.hidden_size
+
+        image_size = image_size if isinstance(image_size, collections.abc.Iterable) else (image_size, image_size)
+        patch_size = patch_size if isinstance(patch_size, collections.abc.Iterable) else (patch_size, patch_size)
+        num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
+        patch_shape = (image_size[0] // patch_size[0], image_size[1] // patch_size[1])
+        self.image_size = image_size
+        self.patch_size = patch_size
+        self.num_channels = num_channels
+        self.num_patches = num_patches
+        self.patch_shape = patch_shape
+
+        self.projection = nn.Conv2d(num_channels, hidden_size, kernel_size=patch_size, stride=patch_size)
+
+    def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
+        batch_size, num_channels, height, width = pixel_values.shape
+        if num_channels != self.num_channels:
+            raise ValueError(
+                "Make sure that the channel dimension of the pixel values match with the one set in the configuration."
+            )
+
+        embeddings = self.projection(pixel_values.to(self.projection.weight.dtype))
+        patch_height, patch_width = embeddings.shape[2], embeddings.shape[3]
+        embeddings = embeddings.flatten(2).transpose(1, 2)
+
+        return embeddings, (patch_height, patch_width)
+
+
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitSelfAttention with Beit->Data2VecVision
+class Data2VecVisionSelfAttention(nn.Module):
+    def __init__(self, config: Data2VecVisionConfig, window_size: tuple | None = None) -> None:
+        super().__init__()
+        self.config = config
+        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
+            raise ValueError(
+                f"The hidden size {config.hidden_size} is not a multiple of the number of attention "
+                f"heads {config.num_attention_heads}."
+            )
+
+        self.num_attention_heads = config.num_attention_heads
+        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+
+        self.query = nn.Linear(config.hidden_size, self.all_head_size)
+        self.key = nn.Linear(config.hidden_size, self.all_head_size, bias=False)
+        self.value = nn.Linear(config.hidden_size, self.all_head_size)
+
+        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
+
+        self.has_relative_position_bias = bool(window_size)
+        if self.has_relative_position_bias:
+            self.relative_position_bias = Data2VecVisionRelativePositionBias(config, window_size=window_size)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        output_attentions: bool = False,
+        relative_position_bias: torch.Tensor | None = None,
+        interpolate_pos_encoding: bool = False,
+        resolution: tuple[int] | None = None,
+    ) -> tuple[torch.Tensor] | tuple[torch.Tensor, torch.Tensor]:
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.attention_head_size)
+        query_layer = self.query(hidden_states).view(hidden_shape).transpose(1, 2)
+        key_layer = self.key(hidden_states).view(hidden_shape).transpose(1, 2)
+        value_layer = self.value(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
+
+        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
+
+        # Add relative position bias if present.
+        if self.has_relative_position_bias:
+            height, width = resolution
+            window_size = (height // self.config.patch_size, width // self.config.patch_size)
+            attention_scores = attention_scores + self.relative_position_bias(
+                window_size, interpolate_pos_encoding, dim_size=hidden_states.shape[1]
+            )
+
+        # Add shared relative position bias if provided.
+        if relative_position_bias is not None:
+            attention_scores = attention_scores + relative_position_bias
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = nn.functional.softmax(attention_scores, dim=-1)
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        attention_probs = self.dropout(attention_probs)
+
+        context_layer = torch.matmul(attention_probs, value_layer)
+
+        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
+        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
+        context_layer = context_layer.view(*new_context_layer_shape)
+
+        outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
+
+        return outputs
+
+
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitSdpaSelfAttention with Beit->Data2VecVision
+class Data2VecVisionSdpaSelfAttention(Data2VecVisionSelfAttention):
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        output_attentions: bool = False,
+        relative_position_bias: torch.Tensor | None = None,
+        interpolate_pos_encoding: bool = False,
+        resolution: tuple[int] | None = None,
+    ) -> tuple[torch.Tensor] | tuple[torch.Tensor, torch.Tensor]:
+        if output_attentions:
+            logger.warning_once(
+                f"{self.__class__.__name__} does not support `output_attentions=True`. The returned attention weights will "
+                "be `None`. If you want to get attention weights, please set `attn_implementation='eager'` when loading the model."
+            )
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.attention_head_size)
+        query_layer = self.query(hidden_states).view(hidden_shape).transpose(1, 2)
+        key_layer = self.key(hidden_states).view(hidden_shape).transpose(1, 2)
+        value_layer = self.value(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        attn_bias = None
+        if self.has_relative_position_bias:
+            height, width = resolution
+            window_size = (height // self.config.patch_size, width // self.config.patch_size)
+            attn_bias = self.relative_position_bias(
+                window_size, interpolate_pos_encoding, dim_size=hidden_states.shape[1]
+            )
+
+        # Add shared relative position bias if provided.
+        if relative_position_bias is not None:
+            if attn_bias is None:
+                attn_bias = relative_position_bias
+            else:
+                attn_bias += relative_position_bias
+
+        scaling = 1 / math.sqrt(self.attention_head_size)
+        context_layer = torch.nn.functional.scaled_dot_product_attention(
+            query_layer,
+            key_layer,
+            value_layer,
+            attn_mask=attn_bias,
+            dropout_p=self.config.attention_probs_dropout_prob if self.training else 0.0,
+            is_causal=False,
+            scale=scaling,
+        )
+        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
+        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
+        context_layer = context_layer.view(*new_context_layer_shape)
+        return context_layer, None
+
+
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitSelfOutput with Beit->Data2VecVision
+class Data2VecVisionSelfOutput(nn.Module):
+    """
+    The residual connection is defined in Data2VecVisionLayer instead of here (as is the case with other models), due to the
+    layernorm applied before each block.
+    """
+
+    def __init__(self, config: Data2VecVisionConfig) -> None:
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Tensor, gamma=None) -> torch.Tensor:
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+
+        return hidden_states
+
+
+DATA2VEC_VISION_SELF_ATTENTION_CLASSES = {
+    "eager": Data2VecVisionSelfAttention,
+    "sdpa": Data2VecVisionSdpaSelfAttention,
+}
+
+
+# Copied from tests.models.beit.modeling_beit.BeitAttention with Beit->Data2VecVision, BEIT->DATA2VEC_VISION
+class Data2VecVisionAttention(nn.Module):
+    def __init__(self, config: Data2VecVisionConfig, window_size: tuple | None = None) -> None:
+        super().__init__()
+        self.attention = DATA2VEC_VISION_SELF_ATTENTION_CLASSES[config._attn_implementation](
+            config, window_size=window_size
+        )
+        self.output = Data2VecVisionSelfOutput(config)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        output_attentions: bool = False,
+        relative_position_bias: Optional["Data2VecVisionRelativePositionBias"] = None,
+        interpolate_pos_encoding: bool = False,
+        resolution: tuple[int] | None = None,
+    ) -> tuple[torch.Tensor] | tuple[torch.Tensor, torch.Tensor]:
+        self_outputs = self.attention(
+            hidden_states, output_attentions, relative_position_bias, interpolate_pos_encoding, resolution
+        )
+
+        attention_output = self.output(self_outputs[0], hidden_states)
+
+        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
+        return outputs
+
+
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitIntermediate with Beit->Data2VecVision
+class Data2VecVisionIntermediate(nn.Module):
+    def __init__(self, config: Data2VecVisionConfig) -> None:
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
+        if isinstance(config.hidden_act, str):
+            self.intermediate_act_fn = ACT2FN[config.hidden_act]
+        else:
+            self.intermediate_act_fn = config.hidden_act
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.intermediate_act_fn(hidden_states)
+
+        return hidden_states
+
+
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitOutput with Beit->Data2VecVision
+class Data2VecVisionOutput(nn.Module):
+    def __init__(self, config: Data2VecVisionConfig) -> None:
+        super().__init__()
+        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.dense(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+
+        return hidden_states
+
+
+# Copied from transformers.models.swin.modular_swin.SwinDropPath with SwinDropPath->Data2VecVisionDropPath
+class Data2VecVisionDropPath(nn.Module):
+    """Stochastic depth (DropPath) per sample, for residual blocks.
+
+    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
+    <https://arxiv.org/abs/1603.09382>`_.
+    """
+
+    def __init__(self, drop_prob: float = 0.0) -> None:
+        super().__init__()
+        self.drop_prob = drop_prob
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        if self.drop_prob == 0.0 or not self.training:
+            return hidden_states
+        keep_prob = 1 - self.drop_prob
+        shape = (hidden_states.shape[0],) + (1,) * (hidden_states.ndim - 1)
+        random_tensor = torch.rand(shape, dtype=hidden_states.dtype, device=hidden_states.device)
+        random_tensor = torch.floor(random_tensor + keep_prob)
+        return hidden_states.div(keep_prob) * random_tensor
+
+    def extra_repr(self) -> str:
+        return f"p={self.drop_prob}"
+
+
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitLayer with Beit->Data2VecVision,BEiT->Data2VecVision
+class Data2VecVisionLayer(GradientCheckpointingLayer):
+    """This corresponds to the Block class in the timm implementation."""
+
+    def __init__(
+        self, config: Data2VecVisionConfig, window_size: tuple | None = None, drop_path_rate: float = 0.0
+    ) -> None:
+        super().__init__()
+        self.chunk_size_feed_forward = config.chunk_size_feed_forward
+        self.seq_len_dim = 1
+        self.attention = Data2VecVisionAttention(config, window_size=window_size)
+        self.intermediate = Data2VecVisionIntermediate(config)
+        self.output = Data2VecVisionOutput(config)
+        self.layernorm_before = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.drop_path = Data2VecVisionDropPath(drop_path_rate) if drop_path_rate > 0.0 else nn.Identity()
+        self.layernorm_after = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+
+        init_values = config.layer_scale_init_value
+        if init_values > 0:
+            self.lambda_1 = nn.Parameter(init_values * torch.ones(config.hidden_size), requires_grad=True)
+            self.lambda_2 = nn.Parameter(init_values * torch.ones(config.hidden_size), requires_grad=True)
+        else:
+            self.lambda_1, self.lambda_2 = None, None
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        output_attentions: bool = False,
+        relative_position_bias: torch.Tensor | None = None,
+        interpolate_pos_encoding: bool = False,
+        resolution: tuple[int, int] | None = None,
+    ) -> tuple[torch.Tensor] | tuple[torch.Tensor, torch.Tensor]:
+        self_attention_outputs = self.attention(
+            self.layernorm_before(hidden_states),  # in Data2VecVision, layernorm is applied before self-attention
+            output_attentions=output_attentions,
+            relative_position_bias=relative_position_bias,
+            interpolate_pos_encoding=interpolate_pos_encoding,
+            resolution=resolution,
+        )
+        attention_output = self_attention_outputs[0]
+        outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights
+
+        # apply lambda_1 if present
+        if self.lambda_1 is not None:
+            attention_output = self.lambda_1 * attention_output
+
+        # first residual connection
+        hidden_states = self.drop_path(attention_output) + hidden_states
+
+        # in Data2VecVision, layernorm is also applied after self-attention
+        layer_output = self.layernorm_after(hidden_states)
+
+        layer_output = self.intermediate(layer_output)
+        layer_output = self.output(layer_output)
+
+        if self.lambda_2 is not None:
+            layer_output = self.lambda_2 * layer_output
+
+        # second residual connection
+        layer_output = self.drop_path(layer_output) + hidden_states
+
+        outputs = (layer_output,) + outputs
+
+        return outputs
+
+
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitRelativePositionBias with Beit->Data2VecVision
+class Data2VecVisionRelativePositionBias(nn.Module):
+    def __init__(self, config: Data2VecVisionConfig, window_size: tuple) -> None:
+        super().__init__()
+        self.window_size = window_size
+        self.num_relative_distance = (2 * window_size[0] - 1) * (2 * window_size[1] - 1) + 3
+        self.relative_position_bias_table = nn.Parameter(
+            torch.zeros(self.num_relative_distance, config.num_attention_heads)
+        )  # 2*Wh-1 * 2*Ww-1, nH
+        # cls to token & token 2 cls & cls to cls
+
+    @staticmethod
+    @compile_compatible_method_lru_cache(maxsize=10)
+    def generate_relative_position_index(window_size: tuple[int, int]) -> torch.Tensor:
+        """
+        This method creates the relative position index, modified to support arbitrary window sizes,
+        as introduced in [MiDaS v3.1](https://huggingface.co/papers/2307.14460).
+        """
+        num_relative_distance = (2 * window_size[0] - 1) * (2 * window_size[1] - 1) + 3
+        # cls to token & token 2 cls & cls to cls
+        # get pair-wise relative position index for each token inside the window
+        window_area = window_size[0] * window_size[1]
+        grid = torch.meshgrid(torch.arange(window_size[0]), torch.arange(window_size[1]), indexing="ij")
+        coords = torch.stack(grid)  # 2, Wh, Ww
+        coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww
+        relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww
+        relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2
+        relative_coords[:, :, 0] += window_size[0] - 1  # shift to start from 0
+        relative_coords[:, :, 1] += window_size[1] - 1
+        relative_coords[:, :, 0] *= 2 * window_size[1] - 1
+        relative_position_index = torch.zeros(size=(window_area + 1,) * 2, dtype=relative_coords.dtype)
+        relative_position_index[1:, 1:] = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
+        relative_position_index[0, 0:] = num_relative_distance - 3
+        relative_position_index[0:, 0] = num_relative_distance - 2
+        relative_position_index[0, 0] = num_relative_distance - 1
+        return relative_position_index
+
+    def forward(self, window_size, interpolate_pos_encoding: bool = False, dim_size=None) -> torch.Tensor:
+        """
+        Modification of timm.models.beit.py: Attention._get_rel_pos_bias to support arbitrary window sizes.
+        """
+        old_height = 2 * self.window_size[0] - 1
+        old_width = 2 * self.window_size[1] - 1
+
+        new_height = 2 * window_size[0] - 1
+        new_width = 2 * window_size[1] - 1
+
+        old_relative_position_bias_table = self.relative_position_bias_table
+
+        old_num_relative_distance = self.num_relative_distance
+        new_num_relative_distance = new_height * new_width + 3
+
+        old_sub_table = old_relative_position_bias_table[: old_num_relative_distance - 3]
+
+        old_sub_table = old_sub_table.reshape(1, old_width, old_height, -1).permute(0, 3, 1, 2)
+        new_sub_table = nn.functional.interpolate(
+            old_sub_table, size=(torch_int(new_height), torch_int(new_width)), mode="bilinear"
+        )
+        new_sub_table = new_sub_table.permute(0, 2, 3, 1).reshape(new_num_relative_distance - 3, -1)
+
+        new_relative_position_bias_table = torch.cat(
+            [new_sub_table, old_relative_position_bias_table[old_num_relative_distance - 3 :]]
+        )
+
+        relative_position_index = self.generate_relative_position_index(window_size)
+        relative_position_bias = new_relative_position_bias_table[relative_position_index.view(-1)]
+
+        # patch_size*num_patches_height, patch_size*num_patches_width, num_attention_heads
+        relative_position_bias = relative_position_bias.view(
+            window_size[0] * window_size[1] + 1, window_size[0] * window_size[1] + 1, -1
+        )
+        # num_attention_heads, patch_size*num_patches_width, patch_size*num_patches_height
+        relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous()
+
+        if interpolate_pos_encoding:
+            relative_position_bias = nn.functional.interpolate(
+                relative_position_bias.unsqueeze(1),
+                size=(dim_size, dim_size),
+                mode="bilinear",
+                align_corners=False,
+            ).squeeze(1)
+
+        return relative_position_bias.unsqueeze(0)
+
+
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitEncoder with Beit->Data2VecVision
+class Data2VecVisionEncoder(nn.Module):
+    def __init__(self, config: Data2VecVisionConfig, window_size: tuple | None = None) -> None:
+        super().__init__()
+        self.config = config
+        self.has_relative_position_bias = config.use_shared_relative_position_bias
+        if self.has_relative_position_bias:
+            self.relative_position_bias = Data2VecVisionRelativePositionBias(config, window_size=window_size)
+
+        # stochastic depth decay rule
+        dpr = [x.item() for x in torch.linspace(0, config.drop_path_rate, config.num_hidden_layers, device="cpu")]
+        self.layer = nn.ModuleList(
+            [
+                Data2VecVisionLayer(
+                    config,
+                    window_size=window_size if config.use_relative_position_bias else None,
+                    drop_path_rate=dpr[i],
+                )
+                for i in range(config.num_hidden_layers)
+            ]
+        )
+        self.gradient_checkpointing = False
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        output_attentions: bool = False,
+        output_hidden_states: bool = False,
+        interpolate_pos_encoding: bool = False,
+        resolution: tuple[int, int] | None = None,
+        return_dict: bool = True,
+    ) -> tuple | BaseModelOutput:
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attentions = () if output_attentions else None
+
+        for i, layer_module in enumerate(self.layer):
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            if self.has_relative_position_bias:
+                height, width = resolution
+                window_size = (height // self.config.patch_size, width // self.config.patch_size)
+                relative_position_bias = self.relative_position_bias(
+                    window_size, interpolate_pos_encoding=interpolate_pos_encoding, dim_size=hidden_states.shape[1]
+                )
+            else:
+                relative_position_bias = None
+
+            layer_outputs = layer_module(
+                hidden_states,
+                output_attentions=output_attentions,
+                relative_position_bias=relative_position_bias,
+                interpolate_pos_encoding=interpolate_pos_encoding,
+                resolution=resolution,
+            )
+
+            hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (layer_outputs[1],)
+
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, all_hidden_states, all_self_attentions] if v is not None)
+        return BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+        )
+
+
+@auto_docstring
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitPreTrainedModel with Beit->Data2VecVision,beit->data2vec_vision
+class Data2VecVisionPreTrainedModel(PreTrainedModel):
+    config: Data2VecVisionConfig
+    base_model_prefix = "data2vec_vision"
+    input_modalities = ("image",)
+    main_input_name = "pixel_values"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Data2VecVisionLayer"]
+    _keys_to_ignore_on_load_unexpected = [r".*relative_position_index.*"]
+    _supports_sdpa = True
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        super()._init_weights(module)
+        if isinstance(module, Data2VecVisionEmbeddings):
+            init.zeros_(module.cls_token)
+            if module.mask_token is not None:
+                init.zeros_(module.mask_token)
+            if module.position_embeddings is not None:
+                init.zeros_(module.position_embeddings)
+        elif isinstance(module, Data2VecVisionRelativePositionBias):
+            init.zeros_(module.relative_position_bias_table)
+        elif isinstance(module, Data2VecVisionLayer):
+            if module.lambda_1 is not None:
+                init.constant_(module.lambda_1, self.config.layer_scale_init_value)
+                init.constant_(module.lambda_2, self.config.layer_scale_init_value)
+
+
+@auto_docstring
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitModel with BEIT->DATA2VEC_VISION,Beit->Data2VecVision,True->False
+class Data2VecVisionModel(Data2VecVisionPreTrainedModel):
+    def __init__(self, config: Data2VecVisionConfig, add_pooling_layer: bool = False) -> None:
+        r"""
+        add_pooling_layer (bool, *optional*, defaults to `False`):
+            Whether to add a pooling layer
+        """
+        super().__init__(config)
+        self.config = config
+
+        self.embeddings = Data2VecVisionEmbeddings(config)
+        self.encoder = Data2VecVisionEncoder(config, window_size=self.embeddings.patch_embeddings.patch_shape)
+
+        self.layernorm = (
+            nn.Identity() if config.use_mean_pooling else nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        )
+        self.pooler = Data2VecVisionPooler(config) if add_pooling_layer else None
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embeddings.patch_embeddings
+
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.Tensor,
+        bool_masked_pos: torch.BoolTensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        interpolate_pos_encoding: bool = False,
+        return_dict: bool | None = None,
+        **kwargs,
+    ) -> tuple | Data2VecVisionModelOutputWithPooling:
+        r"""
+        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
+            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+
+        embedding_output, _ = self.embeddings(pixel_values, bool_masked_pos=bool_masked_pos)
+        resolution = pixel_values.shape[2:]
+
+        encoder_outputs = self.encoder(
+            embedding_output,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            resolution=resolution,
+            return_dict=return_dict,
+            interpolate_pos_encoding=interpolate_pos_encoding,
+        )
+        sequence_output = encoder_outputs[0]
+        sequence_output = self.layernorm(sequence_output)
+        pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
+
+        if not return_dict:
+            head_outputs = (sequence_output, pooled_output) if pooled_output is not None else (sequence_output,)
+            return head_outputs + encoder_outputs[1:]
+
+        return Data2VecVisionModelOutputWithPooling(
+            last_hidden_state=sequence_output,
+            pooler_output=pooled_output,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+        )
+
+
+# Copied from transformers.models.beit.modeling_beit.BeitPooler with Beit->Data2VecVision
+class Data2VecVisionPooler(nn.Module):
+    def __init__(self, config: Data2VecVisionConfig) -> None:
+        super().__init__()
+        self.layernorm = (
+            nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) if config.use_mean_pooling else None
+        )
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        # Mean pool patch tokens with layernorm, or take the [CLS] token
+        return self.layernorm(hidden_states[:, 1:, :].mean(1)) if self.layernorm is not None else hidden_states[:, 0]
+
+
+@auto_docstring(
+    custom_intro="""
+    Data2VecVision Model transformer with an image classification head on top (a linear layer on top of the average of
+    the final hidden states of the patch tokens) e.g. for ImageNet.
+    """
+)
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitForImageClassification with BEIT->DATA2VEC_VISION,Beit->Data2VecVision,beit->data2vec_vision
+class Data2VecVisionForImageClassification(Data2VecVisionPreTrainedModel):
+    def __init__(self, config: Data2VecVisionConfig) -> None:
+        super().__init__(config)
+
+        self.num_labels = config.num_labels
+        self.data2vec_vision = Data2VecVisionModel(config, add_pooling_layer=True)
+
+        # Classifier head
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels) if config.num_labels > 0 else nn.Identity()
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.Tensor | None = None,
+        labels: torch.Tensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        interpolate_pos_encoding: bool = False,
+        return_dict: bool | None = None,
+        **kwargs,
+    ) -> tuple | ImageClassifierOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+        outputs = self.data2vec_vision(
+            pixel_values,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            interpolate_pos_encoding=interpolate_pos_encoding,
+            return_dict=return_dict,
+        )
+
+        pooled_output = outputs.pooler_output if return_dict else outputs[1]
+
+        logits = self.classifier(pooled_output)
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(labels, logits, self.config)
+
+        if not return_dict:
+            output = (logits,) + outputs[2:]
+            return ((loss,) + output) if loss is not None else output
+
+        return ImageClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+# Copied from transformers.models.beit.modeling_beit.BeitConvLayer with Beit->Data2VecVision
+class Data2VecVisionConvLayer(nn.Module):
+    def __init__(
+        self,
+        in_channels: int,
+        out_channels: int,
+        kernel_size: int | tuple[int, int] = 3,
+        stride: int = 1,
+        padding: int | tuple[int, int] | str = 0,
+        bias: bool = False,
+        dilation: int | tuple[int, int] = 1,
+        groups: int = 1,
+        activation: str = "relu",
+    ):
+        super().__init__()
+        self.convolution = nn.Conv2d(
+            in_channels=in_channels,
+            out_channels=out_channels,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            groups=groups,
+            bias=bias,
+        )
+        self.normalization = nn.BatchNorm2d(out_channels)
+        self.activation = ACT2FN[activation] if activation is not None else nn.Identity()
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.convolution(hidden_states)
+        hidden_states = self.normalization(hidden_states)
+        hidden_states = self.activation(hidden_states)
+        return hidden_states
+
+
+# Copied from transformers.models.beit.modeling_beit.BeitPyramidPoolingBlock with Beit->Data2VecVision
+class Data2VecVisionPyramidPoolingBlock(nn.Module):
+    def __init__(self, pool_scale: int, in_channels: int, channels: int) -> None:
+        super().__init__()
+        self.pooling = nn.AdaptiveAvgPool2d(pool_scale)
+        self.conv = Data2VecVisionConvLayer(in_channels, channels, kernel_size=1)
+
+    def forward(self, input: torch.Tensor, size: tuple[int, int]) -> torch.Tensor:
+        hidden_state = self.pooling(input)
+        hidden_state = self.conv(hidden_state)
+        hidden_state = nn.functional.interpolate(hidden_state, size=size, mode="bilinear", align_corners=False)
+        return hidden_state
+
+
+# Copied from transformers.models.beit.modeling_beit.BeitPyramidPoolingModule with Beit->Data2VecVision
+class Data2VecVisionPyramidPoolingModule(nn.Module):
+    """
+    Pyramid Pooling Module (PPM) used in PSPNet.
+
+    Args:
+        pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid
+            Module.
+        in_channels (int): Input channels.
+        channels (int): Channels after modules, before conv_seg.
+
+    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
+    """
+
+    def __init__(self, pool_scales: tuple[int, ...], in_channels: int, channels: int) -> None:
+        super().__init__()
+        self.pool_scales = pool_scales
+        self.in_channels = in_channels
+        self.channels = channels
+        self.blocks = nn.ModuleList(
+            [
+                Data2VecVisionPyramidPoolingBlock(pool_scale=pool_scale, in_channels=in_channels, channels=channels)
+                for pool_scale in pool_scales
+            ]
+        )
+
+    def forward(self, hidden_states: torch.Tensor) -> list[torch.Tensor]:
+        original_size = hidden_states.size()[2:]
+        return [block(hidden_states, size=original_size) for block in self.blocks]
+
+
+# Copied from transformers.models.beit.modeling_beit.BeitUperHead with Beit->Data2VecVision
+class Data2VecVisionUperHead(nn.Module):
+    """
+    Unified Perceptual Parsing for Scene Understanding. This head is the implementation of
+    [UPerNet](https://huggingface.co/papers/1807.10221).
+
+    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
+    """
+
+    def __init__(self, config: Data2VecVisionConfig) -> None:
+        super().__init__()
+
+        self.pool_scales = config.pool_scales  # e.g. (1, 2, 3, 6)
+        self.in_channels = [config.hidden_size] * 4  # e.g. [768, 768, 768, 768]
+        self.channels = config.hidden_size
+        self.classifier = nn.Conv2d(self.channels, config.num_labels, kernel_size=1)
+
+        # PSP Module
+        self.psp_modules = Data2VecVisionPyramidPoolingModule(
+            self.pool_scales,
+            self.in_channels[-1],
+            self.channels,
+        )
+        self.psp_bottleneck = Data2VecVisionConvLayer(
+            self.in_channels[-1] + len(self.pool_scales) * self.channels,
+            self.channels,
+            kernel_size=3,
+            padding=1,
+        )
+        # FPN Module
+        self.lateral_convs = nn.ModuleList()
+        self.fpn_convs = nn.ModuleList()
+        for in_channels in self.in_channels[:-1]:  # skip the top layer
+            self.lateral_convs.append(Data2VecVisionConvLayer(in_channels, self.channels, kernel_size=1))
+            self.fpn_convs.append(Data2VecVisionConvLayer(self.channels, self.channels, kernel_size=3, padding=1))
+
+        self.fpn_bottleneck = Data2VecVisionConvLayer(
+            len(self.in_channels) * self.channels,
+            self.channels,
+            kernel_size=3,
+            padding=1,
+        )
+
+    def psp_forward(self, hidden_states: list[torch.Tensor]) -> torch.Tensor:
+        hidden_state = hidden_states[-1]
+        hidden_state = torch.cat([hidden_state, *self.psp_modules(hidden_state)], dim=1)
+        return self.psp_bottleneck(hidden_state)
+
+    def forward(self, encoder_hidden_states: list[torch.Tensor]) -> torch.Tensor:
+        # build laterals
+        laterals = []
+        for lateral_conv, hidden_state in zip(self.lateral_convs, encoder_hidden_states):
+            laterals.append(lateral_conv(hidden_state))
+
+        laterals.append(self.psp_forward(encoder_hidden_states))
+
+        # build top-down path
+        used_backbone_levels = len(laterals)
+        for i in range(used_backbone_levels - 1, 0, -1):
+            prev_shape = laterals[i - 1].shape[2:]
+            laterals[i - 1] = laterals[i - 1] + nn.functional.interpolate(
+                laterals[i], size=prev_shape, mode="bilinear", align_corners=False
+            )
+
+        # build outputs
+        fpn_outs = []
+        for i in range(used_backbone_levels - 1):
+            fpn_outs.append(self.fpn_convs[i](laterals[i]))
+        # append psp feature
+        fpn_outs.append(laterals[-1])
+
+        for i in range(used_backbone_levels - 1, 0, -1):
+            fpn_outs[i] = nn.functional.interpolate(
+                fpn_outs[i], size=fpn_outs[0].shape[2:], mode="bilinear", align_corners=False
+            )
+        fpn_outs = torch.cat(fpn_outs, dim=1)
+        output = self.fpn_bottleneck(fpn_outs)
+        output = self.classifier(output)
+
+        return output
+
+
+# Copied from transformers.models.beit.modeling_beit.BeitFCNHead with Beit->Data2VecVision
+class Data2VecVisionFCNHead(nn.Module):
+    """
+    Fully Convolution Networks for Semantic Segmentation. This head is implemented of
+    [FCNNet](https://huggingface.co/papers/1411.4038>).
+
+    Args:
+        config (Data2VecVisionConfig): Configuration.
+        in_channels
+        kernel_size (int): The kernel size for convs in the head. Default: 3.
+        dilation (int): The dilation rate for convs in the head. Default: 1.
+
+
+    Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
+    """
+
+    def __init__(
+        self,
+        config: Data2VecVisionConfig,
+        in_index: int = 2,
+        kernel_size: int = 3,
+        dilation: int | tuple[int, int] = 1,
+    ) -> None:
+        super().__init__()
+        self.in_channels = config.hidden_size
+        self.channels = config.auxiliary_channels
+        self.num_convs = config.auxiliary_num_convs
+        self.concat_input = config.auxiliary_concat_input
+        self.in_index = in_index
+
+        conv_padding = (kernel_size // 2) * dilation
+        self.convs = nn.ModuleList()
+        if self.num_convs > 0:
+            self.convs.append(
+                Data2VecVisionConvLayer(
+                    self.in_channels, self.channels, kernel_size=kernel_size, padding=conv_padding, dilation=dilation
+                )
+            )
+            for _ in range(self.num_convs - 1):
+                self.convs.append(
+                    Data2VecVisionConvLayer(
+                        self.channels,
+                        self.channels,
+                        kernel_size=kernel_size,
+                        padding=conv_padding,
+                        dilation=dilation,
+                    )
+                )
+        if self.concat_input:
+            self.conv_cat = Data2VecVisionConvLayer(
+                self.in_channels + self.channels, self.channels, kernel_size=kernel_size, padding=kernel_size // 2
+            )
+
+        self.classifier = nn.Conv2d(self.channels, config.num_labels, kernel_size=1)
+
+    def forward(self, encoder_hidden_states: list[torch.Tensor]) -> torch.Tensor:
+        residual = encoder_hidden_states[self.in_index]
+        hidden_states = residual
+        for conv in self.convs:
+            hidden_states = conv(hidden_states)
+        if self.concat_input:
+            hidden_states = self.conv_cat(torch.cat([residual, hidden_states], dim=1))
+        hidden_states = self.classifier(hidden_states)
+        return hidden_states
+
+
+@auto_docstring
+# Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitForSemanticSegmentation with BEIT->DATA2VEC_VISION,Beit->Data2VecVision,microsoft/beit-base-finetuned-ade-640-640->facebook/data2vec-vision-base,beit->data2vec_vision
+class Data2VecVisionForSemanticSegmentation(Data2VecVisionPreTrainedModel):
+    def __init__(self, config: Data2VecVisionConfig) -> None:
+        super().__init__(config)
+
+        self.num_labels = config.num_labels
+        self.data2vec_vision = Data2VecVisionModel(config, add_pooling_layer=False)
+
+        # FPNs
+        if len(self.config.out_indices) != 4:
+            raise ValueError(
+                "Data2VecVisionForSemanticSegmentation requires config.out_indices to be a list of 4 integers, "
+                "specifying which features to use from the backbone. One can use [3, 5, 7, 11] in case of "
+                "a base-sized architecture."
+            )
+        self.fpn1 = nn.Sequential(
+            nn.ConvTranspose2d(config.hidden_size, config.hidden_size, kernel_size=2, stride=2),
+            nn.BatchNorm2d(config.hidden_size),
+            nn.GELU(),
+            nn.ConvTranspose2d(config.hidden_size, config.hidden_size, kernel_size=2, stride=2),
+        )
+        self.fpn2 = nn.Sequential(
+            nn.ConvTranspose2d(config.hidden_size, config.hidden_size, kernel_size=2, stride=2),
+        )
+        self.fpn3 = nn.Identity()
+        self.fpn4 = nn.MaxPool2d(kernel_size=2, stride=2)
+
+        # Semantic segmentation head(s)
+        self.decode_head = Data2VecVisionUperHead(config)
+        self.auxiliary_head = Data2VecVisionFCNHead(config) if config.use_auxiliary_head else None
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def compute_loss(self, logits, auxiliary_logits, labels):
+        # upsample logits to the images' original size
+        upsampled_logits = nn.functional.interpolate(
+            logits, size=labels.shape[-2:], mode="bilinear", align_corners=False
+        )
+        if auxiliary_logits is not None:
+            upsampled_auxiliary_logits = nn.functional.interpolate(
+                auxiliary_logits, size=labels.shape[-2:], mode="bilinear", align_corners=False
+            )
+        # compute weighted loss
+        loss_fct = CrossEntropyLoss(ignore_index=self.config.semantic_loss_ignore_index)
+        main_loss = loss_fct(upsampled_logits, labels)
+        loss = main_loss
+        if auxiliary_logits is not None:
+            auxiliary_loss = loss_fct(upsampled_auxiliary_logits, labels)
+            loss += self.config.auxiliary_loss_weight * auxiliary_loss
+
+        return loss
+
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.Tensor | None = None,
+        labels: torch.Tensor | None = None,
+        output_attentions: bool | None = None,
+        output_hidden_states: bool | None = None,
+        interpolate_pos_encoding: bool = False,
+        return_dict: bool | None = None,
+        **kwargs,
+    ) -> tuple | SemanticSegmenterOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
+            Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).
+
+        Examples:
+
+        ```python
+        >>> from transformers import AutoImageProcessor, Data2VecVisionForSemanticSegmentation
+        >>> from PIL import Image
+        >>> import httpx
+        >>> from io import BytesIO
+
+        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+        >>> with httpx.stream("GET", url) as response:
+        ...     image = Image.open(BytesIO(response.read()))
+
+        >>> image_processor = AutoImageProcessor.from_pretrained("facebook/data2vec-vision-base")
+        >>> model = Data2VecVisionForSemanticSegmentation.from_pretrained("facebook/data2vec-vision-base")
+
+        >>> inputs = image_processor(images=image, return_tensors="pt")
+        >>> outputs = model(**inputs)
+        >>> # logits are of shape (batch_size, num_labels, height, width)
+        >>> logits = outputs.logits
+        ```"""
+        return_dict = return_dict if return_dict is not None else self.config.return_dict
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+
+        if labels is not None and self.config.num_labels == 1:
+            raise ValueError("The number of labels should be greater than one")
+
+        outputs = self.data2vec_vision(
+            pixel_values,
+            output_attentions=output_attentions,
+            output_hidden_states=True,  # we need the intermediate hidden states
+            interpolate_pos_encoding=interpolate_pos_encoding,
+            return_dict=return_dict,
+        )
+
+        encoder_hidden_states = outputs.hidden_states if return_dict else outputs[1]
+
+        # only keep certain features, and reshape
+        # note that we do +1 as the encoder_hidden_states also includes the initial embeddings
+        features = [feature for idx, feature in enumerate(encoder_hidden_states) if idx + 1 in self.config.out_indices]
+        batch_size = pixel_values.shape[0]
+        patch_resolution = self.config.image_size // self.config.patch_size
+        features = [
+            x[:, 1:, :].permute(0, 2, 1).reshape(batch_size, -1, patch_resolution, patch_resolution) for x in features
+        ]
+
+        # apply FPNs
+        ops = [self.fpn1, self.fpn2, self.fpn3, self.fpn4]
+        for i in range(len(features)):
+            features[i] = ops[i](features[i])
+
+        logits = self.decode_head(features)
+
+        auxiliary_logits = None
+        if self.auxiliary_head is not None:
+            auxiliary_logits = self.auxiliary_head(features)
+
+        loss = None
+        if labels is not None:
+            loss = self.compute_loss(logits, auxiliary_logits, labels)
+
+        if not return_dict:
+            if output_hidden_states:
+                output = (logits,) + outputs[1:]
+            else:
+                output = (logits,) + outputs[2:]
+            return ((loss,) + output) if loss is not None else output
+
+        return SemanticSegmenterOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states if output_hidden_states else None,
+            attentions=outputs.attentions,
+        )
+
+
+__all__ = [
+    "Data2VecVisionForImageClassification",
+    "Data2VecVisionForSemanticSegmentation",
+    "Data2VecVisionModel",
+    "Data2VecVisionPreTrainedModel",
+]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modular_data2vec_audio.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modular_data2vec_audio.py
new file mode 100644
index 0000000000000000000000000000000000000000..a6f5dda725dadfae5749f43ebb9e4c5a4bf40a32
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modular_data2vec_audio.py
@@ -0,0 +1,272 @@
+# Copyright 2022 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch Data2VecText model."""
+
+import math
+
+import torch
+from torch import nn
+
+from ... import initialization as init
+from ...activations import ACT2FN
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import Wav2Vec2BaseModelOutput
+from ...modeling_utils import PreTrainedModel
+from ..wav2vec2.modeling_wav2vec2 import (
+    Wav2Vec2Adapter,
+    Wav2Vec2Encoder,
+    Wav2Vec2FeatureEncoder,
+    Wav2Vec2FeatureProjection,
+    Wav2Vec2ForAudioFrameClassification,
+    Wav2Vec2ForCTC,
+    Wav2Vec2ForSequenceClassification,
+    Wav2Vec2ForXVector,
+    Wav2Vec2Model,
+    Wav2Vec2PreTrainedModel,
+    Wav2Vec2SamePadLayer,
+)
+from .configuration_data2vec_audio import Data2VecAudioConfig
+
+
+class Data2VecAudioConvLayer(GradientCheckpointingLayer):
+    def __init__(self, config, layer_id=0):
+        super().__init__()
+        self.in_conv_dim = config.conv_dim[layer_id - 1] if layer_id > 0 else 1
+        self.out_conv_dim = config.conv_dim[layer_id]
+
+        self.conv = nn.Conv1d(
+            self.in_conv_dim,
+            self.out_conv_dim,
+            kernel_size=config.conv_kernel[layer_id],
+            stride=config.conv_stride[layer_id],
+            bias=config.conv_bias,
+        )
+        self.layer_norm = nn.LayerNorm(self.out_conv_dim, elementwise_affine=True)
+        self.activation = ACT2FN[config.feat_extract_activation]
+
+    def forward(self, hidden_states):
+        hidden_states = self.conv(hidden_states)
+
+        hidden_states = hidden_states.transpose(-2, -1)
+        hidden_states = self.layer_norm(hidden_states)
+        hidden_states = hidden_states.transpose(-2, -1)
+
+        hidden_states = self.activation(hidden_states)
+        return hidden_states
+
+
+class Data2VecAudioPadLayer(Wav2Vec2SamePadLayer):
+    pass
+
+
+class Data2VecAudioPositionalConvLayer(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.conv = nn.Conv1d(
+            config.hidden_size,
+            config.hidden_size,
+            kernel_size=config.conv_pos_kernel_size,
+            padding=config.conv_pos_kernel_size // 2,
+            groups=config.num_conv_pos_embedding_groups,
+        )
+
+        self.padding = Data2VecAudioPadLayer(config.conv_pos_kernel_size)
+        self.activation = ACT2FN[config.feat_extract_activation]
+        # no learnable parameters
+        self.layer_norm = nn.LayerNorm(config.hidden_size, elementwise_affine=False)
+
+    def forward(self, hidden_states):
+        hidden_states = self.conv(hidden_states)
+        hidden_states = self.padding(hidden_states)
+
+        hidden_states = hidden_states.transpose(1, 2)
+        hidden_states = self.layer_norm(hidden_states)
+        hidden_states = hidden_states.transpose(1, 2)
+        hidden_states = self.activation(hidden_states)
+        return hidden_states
+
+
+class Data2VecAudioPositionalConvEmbedding(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.layers = nn.ModuleList(
+            [Data2VecAudioPositionalConvLayer(config) for _ in range(config.num_conv_pos_embeddings)]
+        )
+
+    def forward(self, hidden_states):
+        hidden_states = hidden_states.transpose(1, 2)
+        for layer in self.layers:
+            hidden_states = layer(hidden_states)
+        hidden_states = hidden_states.transpose(1, 2)
+        return hidden_states
+
+
+class Data2VecAudioFeatureEncoder(Wav2Vec2FeatureEncoder):
+    def __init__(self, config):
+        nn.Module.__init__(self)
+        self.conv_layers = nn.ModuleList(
+            [Data2VecAudioConvLayer(config, layer_id=i) for i in range(config.num_feat_extract_layers)]
+        )
+        self.gradient_checkpointing = False
+        self._requires_grad = True
+
+
+class Data2VecAudioFeatureProjection(Wav2Vec2FeatureProjection):
+    pass
+
+
+class Data2VecAudioEncoder(Wav2Vec2Encoder):
+    pass
+
+
+class Data2VecAudioAdapter(Wav2Vec2Adapter):
+    pass
+
+
+class Data2VecAudioPreTrainedModel(PreTrainedModel, Wav2Vec2PreTrainedModel):
+    config: Data2VecAudioConfig
+    base_model_prefix = "data2vec_audio"
+    main_input_name = "input_values"
+    input_modalities = "audio"
+    supports_gradient_checkpointing = True
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    _supports_flex_attn = True
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        if isinstance(module, Data2VecAudioFeatureProjection):
+            k = math.sqrt(1 / module.projection.in_features)
+            init.uniform_(module.projection.weight, a=-k, b=k)
+            init.uniform_(module.projection.bias, a=-k, b=k)
+        elif isinstance(module, Data2VecAudioPositionalConvLayer):
+            init.constant_(module.conv.bias, 0)
+        elif isinstance(module, nn.Linear):
+            init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
+
+            if module.bias is not None:
+                init.zeros_(module.bias)
+        elif isinstance(module, (nn.LayerNorm, nn.GroupNorm)):
+            if module.bias is not None:
+                init.zeros_(module.bias)
+            if module.weight is not None:
+                init.ones_(module.weight)
+        elif isinstance(module, nn.Conv1d):
+            init.kaiming_normal_(module.weight)
+
+            if module.bias is not None:
+                k = math.sqrt(module.groups / (module.in_channels * module.kernel_size[0]))
+                init.uniform_(module.bias, a=-k, b=k)
+
+    def _get_adapters(self):
+        raise AttributeError("Not needed for Data2VecAudio")
+
+    def init_adapter_layers(self):
+        raise AttributeError("Not needed for Data2VecAudio")
+
+    def load_adapter(self):
+        raise AttributeError("Not needed for Data2VecAudio")
+
+
+Data2VecAudioBaseModelOutput = Wav2Vec2BaseModelOutput
+
+
+class Data2VecAudioModel(Data2VecAudioPreTrainedModel, Wav2Vec2Model):
+    def __init__(self, config: Data2VecAudioConfig):
+        Data2VecAudioPreTrainedModel.__init__(self, config)
+        self.config = config
+        self.feature_extractor = Data2VecAudioFeatureEncoder(config)
+        self.feature_projection = Data2VecAudioFeatureProjection(config)
+
+        # model only needs masking vector if mask prob is > 0.0
+        if config.mask_time_prob > 0.0 or config.mask_feature_prob > 0.0:
+            self.masked_spec_embed = nn.Parameter(torch.Tensor(config.hidden_size).uniform_())
+
+        self.encoder = Data2VecAudioEncoder(config)
+
+        self.adapter = Data2VecAudioAdapter(config) if config.add_adapter else None
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def freeze_feature_encoder(self):
+        """
+        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
+        not be updated during training.
+        """
+        self.feature_extractor._freeze_parameters()
+
+    def forward(self, **super_kwargs):
+        return super().forward(**super_kwargs)
+
+
+class Data2VecAudioForCTC(Data2VecAudioPreTrainedModel, Wav2Vec2ForCTC):
+    def __init__(self, config):
+        r"""
+        config ([`Data2VecAudioForCTC`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`]  method to load the model weights.
+        """
+        Data2VecAudioPreTrainedModel.__init__(self, config)
+
+        self.data2vec_audio = Data2VecAudioModel(config)
+        self.dropout = nn.Dropout(config.final_dropout)
+
+        if config.vocab_size is None:
+            raise ValueError(
+                f"You are trying to instantiate {self.__class__} with a configuration that "
+                "does not define the vocabulary size of the language model head. Please "
+                "instantiate the model as follows: `Data2VecAudioForCTC.from_pretrained(..., vocab_size=vocab_size)`. "
+                "or define `vocab_size` of your model's configuration."
+            )
+        output_hidden_size = (
+            config.output_hidden_size if hasattr(config, "add_adapter") and config.add_adapter else config.hidden_size
+        )
+        self.lm_head = nn.Linear(output_hidden_size, config.vocab_size)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def freeze_base_model(self):
+        raise AttributeError("Not needed for Data2VecAudio")
+
+    def tie_weights(self):
+        raise AttributeError("Not needed for Data2VecAudio")
+
+    def forward(self, **super_kwargs):
+        return super().forward(**super_kwargs)
+
+
+class Data2VecAudioForSequenceClassification(Wav2Vec2ForSequenceClassification):
+    pass
+
+
+class Data2VecAudioForAudioFrameClassification(Wav2Vec2ForAudioFrameClassification):
+    pass
+
+
+class Data2VecAudioForXVector(Wav2Vec2ForXVector):
+    pass
+
+
+__all__ = [
+    "Data2VecAudioForAudioFrameClassification",
+    "Data2VecAudioForCTC",
+    "Data2VecAudioForSequenceClassification",
+    "Data2VecAudioForXVector",
+    "Data2VecAudioModel",
+    "Data2VecAudioPreTrainedModel",
+]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modular_data2vec_text.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modular_data2vec_text.py
new file mode 100644
index 0000000000000000000000000000000000000000..1adc7abfcd2f4cbfb36e5db31a67c0b86967ea42
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modular_data2vec_text.py
@@ -0,0 +1,599 @@
+# Copyright 2022 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch Data2VecText model."""
+
+import torch
+import torch.nn as nn
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+
+from ... import initialization as init
+from ...generation import GenerationMixin
+from ...modeling_outputs import (
+    BaseModelOutputWithPoolingAndCrossAttentions,
+    CausalLMOutputWithCrossAttentions,
+    MaskedLMOutput,
+    MultipleChoiceModelOutput,
+    QuestionAnsweringModelOutput,
+    SequenceClassifierOutput,
+    TokenClassifierOutput,
+)
+from ...modeling_utils import PreTrainedModel
+from ...processing_utils import Unpack
+from ...utils import TransformersKwargs, auto_docstring, logging
+from ...utils.generic import can_return_tuple
+from ..roberta.modeling_roberta import (
+    RobertaClassificationHead,
+    RobertaCrossAttention,
+    RobertaEmbeddings,
+    RobertaLayer,
+    RobertaLMHead,
+    RobertaModel,
+    RobertaSelfAttention,
+)
+from .configuration_data2vec_text import Data2VecTextConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+class Data2VecTextEmbeddings(RobertaEmbeddings):
+    pass
+
+
+class Data2VecTextSelfAttention(RobertaSelfAttention):
+    pass
+
+
+class Data2VecTextCrossAttention(RobertaCrossAttention):
+    pass
+
+
+class Data2VecTextLayer(RobertaLayer):
+    pass
+
+
+@auto_docstring
+class Data2VecTextPreTrainedModel(PreTrainedModel):
+    config_class = Data2VecTextConfig
+    base_model_prefix = "data2vec_text"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Data2VecTextForTextEmbeddings", "Data2VecTextLayer"]
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    _supports_flex_attn = True
+    _supports_attention_backend = True
+    _can_record_outputs = {
+        "hidden_states": Data2VecTextLayer,
+        "attentions": Data2VecTextSelfAttention,
+        "cross_attentions": Data2VecTextCrossAttention,
+    }
+
+    def _init_weights(self, module):
+        super()._init_weights(module)
+        if isinstance(module, Data2VecTextEmbeddings):
+            init.copy_(module.position_ids, torch.arange(module.position_ids.shape[-1]).expand((1, -1)))
+            init.zeros_(module.token_type_ids)
+
+
+@auto_docstring
+class Data2VecTextModel(RobertaModel):
+    pass
+
+
+class Data2VecTextLMHead(RobertaLMHead):
+    pass
+
+
+class Data2VecTextClassificationHead(RobertaClassificationHead):
+    pass
+
+
+@auto_docstring(
+    custom_intro="""
+    Data2VecText Model with a `language modeling` head on top for CLM fine-tuning.
+    """
+)
+class Data2VecTextForCausalLM(Data2VecTextPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {
+        "lm_head.decoder.weight": "data2vec_text.embeddings.word_embeddings.weight",
+        "lm_head.decoder.bias": "lm_head.bias",
+    }
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        if not config.is_decoder:
+            logger.warning("If you want to use `Data2VecTextLMHeadModel` as a standalone, add `is_decoder=True.`")
+
+        self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
+        self.lm_head = Data2VecTextLMHead(config)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_output_embeddings(self):
+        return self.lm_head.decoder
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head.decoder = new_embeddings
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        encoder_hidden_states: torch.FloatTensor | None = None,
+        encoder_attention_mask: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        past_key_values: tuple[tuple[torch.FloatTensor]] | None = None,
+        use_cache: bool | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | CausalLMOutputWithCrossAttentions:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
+            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
+            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+
+        Example:
+
+        ```python
+        >>> from transformers import AutoTokenizer, Data2VecTextForCausalLM, Data2VecTextConfig
+        >>> import torch
+
+        >>> tokenizer = AutoTokenizer.from_pretrained("facebook/data2vec-text-base")
+        >>> config = Data2VecTextConfig.from_pretrained("facebook/data2vec-text-base")
+        >>> config.is_decoder = True
+        >>> model = Data2VecTextForCausalLM.from_pretrained("facebook/data2vec-text-base", config=config)
+
+        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
+        >>> outputs = model(**inputs)
+
+        >>> prediction_logits = outputs.logits
+        ```"""
+        if labels is not None:
+            use_cache = False
+
+        outputs: BaseModelOutputWithPoolingAndCrossAttentions = self.data2vec_text(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            return_dict=True,
+            **kwargs,
+        )
+
+        hidden_states = outputs.last_hidden_state
+        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
+
+        return CausalLMOutputWithCrossAttentions(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            cross_attentions=outputs.cross_attentions,
+        )
+
+
+@auto_docstring
+class Data2VecTextForMaskedLM(Data2VecTextPreTrainedModel):
+    _tied_weights_keys = {
+        "lm_head.decoder.weight": "data2vec_text.embeddings.word_embeddings.weight",
+        "lm_head.decoder.bias": "lm_head.bias",
+    }
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        if config.is_decoder:
+            logger.warning(
+                "If you want to use `Data2VecTextForMaskedLM` make sure `config.is_decoder=False` for "
+                "bi-directional self-attention."
+            )
+
+        self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
+        self.lm_head = Data2VecTextLMHead(config)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_output_embeddings(self):
+        return self.lm_head.decoder
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head.decoder = new_embeddings
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        encoder_hidden_states: torch.FloatTensor | None = None,
+        encoder_attention_mask: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | MaskedLMOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+        """
+        outputs = self.data2vec_text(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            return_dict=True,
+            **kwargs,
+        )
+        sequence_output = outputs[0]
+        prediction_scores = self.lm_head(sequence_output)
+
+        masked_lm_loss = None
+        if labels is not None:
+            loss_fct = CrossEntropyLoss()
+
+            labels = labels.to(prediction_scores.device)
+            masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
+
+        return MaskedLMOutput(
+            loss=masked_lm_loss,
+            logits=prediction_scores,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring(
+    custom_intro="""
+    Data2VecText Model transformer with a sequence classification/regression head on top (a linear layer on top of the
+    pooled output) e.g. for GLUE tasks.
+    """
+)
+class Data2VecTextForSequenceClassification(Data2VecTextPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.config = config
+
+        self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
+        self.classifier = Data2VecTextClassificationHead(config)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | SequenceClassifierOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+        outputs = self.data2vec_text(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            return_dict=True,
+            **kwargs,
+        )
+        sequence_output = outputs[0]
+        logits = self.classifier(sequence_output)
+
+        loss = None
+        if labels is not None:
+            labels = labels.to(logits.device)
+
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(logits, labels)
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = BCEWithLogitsLoss()
+                loss = loss_fct(logits, labels)
+
+        return SequenceClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring
+class Data2VecTextForMultipleChoice(Data2VecTextPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.data2vec_text = Data2VecTextModel(config)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.classifier = nn.Linear(config.hidden_size, 1)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | MultipleChoiceModelOutput:
+        r"""
+        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
+            Indices of input sequence tokens in the vocabulary.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
+            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+            1]`:
+
+            - 0 corresponds to a *sentence A* token,
+            - 1 corresponds to a *sentence B* token.
+
+            [What are token type IDs?](../glossary#token-type-ids)
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+            `input_ids` above)
+        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.max_position_embeddings - 1]`.
+
+            [What are position IDs?](../glossary#position-ids)
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+        """
+        num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]
+
+        flat_input_ids = input_ids.view(-1, input_ids.size(-1)) if input_ids is not None else None
+        flat_position_ids = position_ids.view(-1, position_ids.size(-1)) if position_ids is not None else None
+        flat_token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None
+        flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None
+        flat_inputs_embeds = (
+            inputs_embeds.view(-1, inputs_embeds.size(-2), inputs_embeds.size(-1))
+            if inputs_embeds is not None
+            else None
+        )
+
+        outputs = self.data2vec_text(
+            flat_input_ids,
+            position_ids=flat_position_ids,
+            token_type_ids=flat_token_type_ids,
+            attention_mask=flat_attention_mask,
+            inputs_embeds=flat_inputs_embeds,
+            return_dict=True,
+            **kwargs,
+        )
+        pooled_output = outputs[1]
+
+        pooled_output = self.dropout(pooled_output)
+        logits = self.classifier(pooled_output)
+        reshaped_logits = logits.view(-1, num_choices)
+
+        loss = None
+        if labels is not None:
+            loss_fct = CrossEntropyLoss()
+
+            labels = labels.to(reshaped_logits.device)
+            loss = loss_fct(reshaped_logits, labels)
+
+        return MultipleChoiceModelOutput(
+            loss=loss,
+            logits=reshaped_logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring
+class Data2VecTextForTokenClassification(Data2VecTextPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+
+        self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
+        classifier_dropout = (
+            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
+        )
+        self.dropout = nn.Dropout(classifier_dropout)
+        self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | TokenClassifierOutput:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
+        """
+        outputs = self.data2vec_text(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            return_dict=True,
+            **kwargs,
+        )
+
+        sequence_output = outputs[0]
+
+        sequence_output = self.dropout(sequence_output)
+        logits = self.classifier(sequence_output)
+
+        loss = None
+        if labels is not None:
+            loss_fct = CrossEntropyLoss()
+
+            labels = labels.to(logits.device)
+            loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
+
+        return TokenClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+@auto_docstring
+class Data2VecTextForQuestionAnswering(Data2VecTextPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+
+        self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
+        self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.FloatTensor | None = None,
+        token_type_ids: torch.LongTensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        start_positions: torch.LongTensor | None = None,
+        end_positions: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | QuestionAnsweringModelOutput:
+        outputs = self.data2vec_text(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            return_dict=True,
+            **kwargs,
+        )
+
+        sequence_output = outputs[0]
+
+        logits = self.qa_outputs(sequence_output)
+        start_logits, end_logits = logits.split(1, dim=-1)
+        start_logits = start_logits.squeeze(-1).contiguous()
+        end_logits = end_logits.squeeze(-1).contiguous()
+
+        total_loss = None
+        if start_positions is not None and end_positions is not None:
+            # If we are on multi-GPU, split add a dimension
+            if len(start_positions.size()) > 1:
+                start_positions = start_positions.squeeze(-1)
+            if len(end_positions.size()) > 1:
+                end_positions = end_positions.squeeze(-1)
+            # sometimes the start/end positions are outside our model inputs, we ignore these terms
+            ignored_index = start_logits.size(1)
+            start_positions = start_positions.clamp(0, ignored_index)
+            end_positions = end_positions.clamp(0, ignored_index)
+
+            loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
+            start_loss = loss_fct(start_logits, start_positions)
+            end_loss = loss_fct(end_logits, end_positions)
+            total_loss = (start_loss + end_loss) / 2
+
+        return QuestionAnsweringModelOutput(
+            loss=total_loss,
+            start_logits=start_logits,
+            end_logits=end_logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+__all__ = [
+    "Data2VecTextForCausalLM",
+    "Data2VecTextForMaskedLM",
+    "Data2VecTextForMultipleChoice",
+    "Data2VecTextForQuestionAnswering",
+    "Data2VecTextForSequenceClassification",
+    "Data2VecTextForTokenClassification",
+    "Data2VecTextModel",
+    "Data2VecTextPreTrainedModel",
+]
diff --git a/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/sam3/modeling_sam3.py b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/sam3/modeling_sam3.py
new file mode 100644
index 0000000000000000000000000000000000000000..4b88cf846cd3cd03851e62d6ea9c5ef0b3314e63
--- /dev/null
+++ b/LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/sam3/modeling_sam3.py
@@ -0,0 +1,2450 @@
+# Copyright 2025 The Meta AI Authors and The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import Tensor
+
+from ...utils import is_torchvision_available
+
+
+if is_torchvision_available():
+    import torchvision
+
+from transformers import CLIPTextModelWithProjection
+
+from ... import initialization as init
+from ...activations import ACT2FN
+from ...masking_utils import create_bidirectional_mask
+from ...modeling_layers import GradientCheckpointingLayer
+from ...modeling_outputs import (
+    BaseModelOutput,
+    BaseModelOutputWithPooling,
+    ModelOutput,
+)
+from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from ...processing_utils import Unpack
+from ...pytorch_utils import compile_compatible_method_lru_cache
+from ...utils import auto_docstring, can_return_tuple, logging
+from ...utils.generic import (
+    TransformersKwargs,
+    is_flash_attention_requested,
+    merge_with_config_defaults,
+)
+from ...utils.import_utils import requires
+from ...utils.output_capturing import capture_outputs
+from ..auto import AutoModel
+from .configuration_sam3 import (
+    Sam3Config,
+    Sam3DETRDecoderConfig,
+    Sam3DETREncoderConfig,
+    Sam3GeometryEncoderConfig,
+    Sam3MaskDecoderConfig,
+    Sam3VisionConfig,
+    Sam3ViTConfig,
+)
+
+
+logger = logging.get_logger(__name__)
+
+
+@auto_docstring
+@dataclass
+class Sam3VisionEncoderOutput(BaseModelOutputWithPooling):
+    r"""
+    fpn_hidden_states (`tuple[torch.FloatTensor]`):
+        Tuple of multi-level FPN feature maps.
+    fpn_position_encoding (`tuple[torch.FloatTensor]`):
+        Tuple of position encodings for each FPN level.
+    """
+
+    fpn_hidden_states: tuple[torch.FloatTensor, ...] = None
+    fpn_position_encoding: tuple[torch.FloatTensor, ...] = None
+
+
+@auto_docstring
+@dataclass
+class Sam3GeometryEncoderOutput(ModelOutput):
+    r"""
+    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_prompts, hidden_size)`):
+        Encoded geometry prompt features (boxes).
+    attention_mask (`torch.BoolTensor` of shape `(batch_size, num_prompts)`, *optional*):
+        Attention mask for geometry prompts where True indicates valid positions and False indicates padding.
+    """
+
+    last_hidden_state: torch.FloatTensor = None
+    attention_mask: torch.BoolTensor | None = None
+
+
+@auto_docstring
+@dataclass
+class Sam3DETREncoderOutput(ModelOutput):
+    r"""
+    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+        Encoded vision features (flattened from multi-level features).
+    pos_embeds_flattened (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+        Flattened position embeddings for the vision features.
+    text_features (`torch.FloatTensor` of shape `(batch_size, text_seq_len, hidden_size)`, *optional*):
+        Text features (may be pooled after encoder processing).
+    spatial_shapes (`torch.LongTensor` of shape `(num_levels, 2)`, *optional*):
+        Spatial shapes (height, width) for each feature pyramid level.
+    hidden_states (`tuple[torch.FloatTensor]`, *optional*):
+        Tuple of hidden states from all encoder layers.
+    attentions (`tuple[torch.FloatTensor]`, *optional*):
+        Tuple of attention weights from all encoder layers.
+    """
+
+    last_hidden_state: torch.FloatTensor = None
+    pos_embeds_flattened: torch.FloatTensor | None = None
+    text_features: torch.FloatTensor | None = None
+    spatial_shapes: torch.LongTensor | None = None
+    hidden_states: tuple[torch.FloatTensor] | None = None
+    attentions: tuple[torch.FloatTensor] | None = None
+
+
+@auto_docstring
+@dataclass
+class Sam3DETRDecoderOutput(ModelOutput):
+    r"""
+    intermediate_hidden_states (`torch.FloatTensor` of shape `(num_layers, batch_size, num_queries, hidden_size)`):
+        Decoder hidden states from all layers.
+    reference_boxes (`torch.FloatTensor` of shape `(num_layers, batch_size, num_queries, 4)`):
+        Predicted reference boxes from all decoder layers in (cx, cy, w, h) format.
+    presence_logits (`torch.FloatTensor` of shape `(num_layers, batch_size, 1)`):
+        Presence logits from all decoder layers indicating object presence confidence.
+    hidden_states (`tuple[torch.FloatTensor]`, *optional*):
+        Tuple of hidden states from all decoder layers.
+    attentions (`tuple[torch.FloatTensor]`, *optional*):
+        Tuple of attention weights from all decoder layers (self-attention and cross-attention).
+    """
+
+    intermediate_hidden_states: torch.FloatTensor = None
+    reference_boxes: torch.FloatTensor = None
+    presence_logits: torch.FloatTensor = None
+    hidden_states: tuple[torch.FloatTensor] | None = None
+    attentions: tuple[torch.FloatTensor] | None = None
+
+
+@auto_docstring
+@dataclass
+class Sam3MaskDecoderOutput(ModelOutput):
+    r"""
+    pred_masks (`torch.FloatTensor` of shape `(batch_size, num_queries, height, width)`):
+        Predicted segmentation masks for each query.
+    semantic_seg (`torch.FloatTensor` of shape `(batch_size, 1, height, width)`, *optional*):
+        Semantic segmentation output.
+    attentions (`tuple[torch.FloatTensor]`, *optional*):
+        Tuple of attention weights from mask decoder cross-attention layers.
+    """
+
+    pred_masks: torch.FloatTensor = None
+    semantic_seg: torch.FloatTensor | None = None
+    attentions: tuple[torch.FloatTensor] | None = None
+
+
+@auto_docstring
+@dataclass
+class Sam3ImageSegmentationOutput(ModelOutput):
+    r"""
+    pred_masks (`torch.FloatTensor` of shape `(batch_size, num_queries, height, width)`):
+        Predicted segmentation masks for each query.
+    pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`):
+        Predicted bounding boxes in (x1, y1, x2, y2) format.
+    pred_logits (`torch.FloatTensor` of shape `(batch_size, num_queries)`, *optional*):
+        Classification confidence scores for each query, computed via dot product between
+        decoder query features and text features.
+    presence_logits (`torch.FloatTensor` of shape `(batch_size, 1)`, *optional*):
+        Presence logits from the DETR decoder presence token (last layer only). These indicate whether objects
+        are present in the scene. Can be used to compute final scores by multiplying with pred_logits:
+        `final_scores = pred_logits.sigmoid() * presence_logits.sigmoid()`.
+    semantic_seg (`torch.FloatTensor` of shape `(batch_size, 1, height, width)`, *optional*):
+        Semantic segmentation output.
+    decoder_hidden_states (`tuple[torch.FloatTensor]`, *optional*):
+        Tuple of hidden states from all DETR decoder layers. Each tensor has shape `(batch_size, num_queries, hidden_size)`.
+    decoder_reference_boxes (`torch.FloatTensor` of shape `(num_layers, batch_size, num_queries, 4)`, *optional*):
+        Reference boxes from all DETR decoder layers.
+    encoder_hidden_states (`tuple[torch.FloatTensor]`, *optional*):
+        Tuple of hidden states from all DETR encoder layers.
+    vision_hidden_states (`tuple[torch.FloatTensor]`, *optional*):
+        Tuple of hidden states from all vision encoder (ViT) layers.
+    vision_attentions (`tuple[torch.FloatTensor]`, *optional*):
+        Attention weights from vision encoder (ViT) layers.
+    detr_encoder_attentions (`tuple[torch.FloatTensor]`, *optional*):
+        Attention weights from DETR encoder layers.
+    detr_decoder_attentions (`tuple[torch.FloatTensor]`, *optional*):
+        Attention weights from DETR decoder layers (self-attention and cross-attention).
+    mask_decoder_attentions (`tuple[torch.FloatTensor]`, *optional*):
+        Attention weights from mask decoder layers.
+    """
+
+    pred_masks: torch.FloatTensor = None
+    pred_boxes: torch.FloatTensor = None
+    pred_logits: torch.FloatTensor | None = None
+    presence_logits: torch.FloatTensor | None = None
+    semantic_seg: torch.FloatTensor | None = None
+    decoder_hidden_states: tuple[torch.FloatTensor] | None = None
+    decoder_reference_boxes: torch.FloatTensor | None = None
+    encoder_hidden_states: tuple[torch.FloatTensor] | None = None
+    vision_hidden_states: tuple[torch.FloatTensor] | None = None
+    vision_attentions: tuple[torch.FloatTensor] | None = None
+    detr_encoder_attentions: tuple[torch.FloatTensor] | None = None
+    detr_decoder_attentions: tuple[torch.FloatTensor] | None = None
+    mask_decoder_attentions: tuple[torch.FloatTensor] | None = None
+
+
+def inverse_sigmoid(x: torch.Tensor, eps: float = 1e-3) -> torch.Tensor:
+    """The inverse function for sigmoid activation function."""
+    x = x.clamp(min=0, max=1)
+    x1 = x.clamp(min=eps)
+    x2 = (1 - x).clamp(min=eps)
+    return torch.log(x1 / x2)
+
+
+def concat_padded_sequences(seq1, mask1, seq2, mask2, return_index: bool = False):
+    """
+    Concatenates two right-padded sequences, such that the resulting sequence
+    is contiguous and also right-padded.
+
+    Tensors are batch-first, masks are batch-first with True=valid, False=padding.
+
+    Args:
+        seq1: A tensor of shape (batch_size, seq1_length, hidden_size).
+        mask1: A tensor of shape (batch_size, seq1_length) with True=valid, False=padding.
+        seq2: A tensor of shape (batch_size, seq2_length, hidden_size).
+        mask2: A tensor of shape (batch_size, seq2_length) with True=valid, False=padding.
+        return_index: If True, also returns the index of the ids of the element of seq2
+            in the concatenated sequence. This can be used to retrieve the elements of seq2.
+
+    Returns:
+        A tuple (concatenated_sequence, concatenated_mask) if return_index is False,
+        otherwise (concatenated_sequence, concatenated_mask, index).
+        The concatenated_mask uses True=valid, False=padding convention.
+    """
+    batch_size, seq1_length, hidden_size = seq1.shape
+    batch_size2, seq2_length, hidden_size2 = seq2.shape
+
+    assert batch_size == batch_size2 == mask1.size(0) == mask2.size(0)
+    assert hidden_size == hidden_size2
+    assert seq1_length == mask1.size(1)
+    assert seq2_length == mask2.size(1)
+
+    actual_seq1_lengths = mask1.sum(dim=-1)
+    actual_seq2_lengths = mask2.sum(dim=-1)
+
+    final_lengths = actual_seq1_lengths + actual_seq2_lengths
+    max_length = seq1_length + seq2_length
+
+    concatenated_mask = (
+        torch.arange(max_length, device=seq2.device)[None].repeat(batch_size, 1) < final_lengths[:, None]
+    )
+
+    concatenated_sequence = torch.zeros((batch_size, max_length, hidden_size), device=seq2.device, dtype=seq2.dtype)
+    concatenated_sequence[:, :seq1_length, :] = seq1
+
+    # Shift seq2 elements to start at the end of valid seq1
+    index = torch.arange(seq2_length, device=seq2.device)[None].repeat(batch_size, 1)
+    index = index + actual_seq1_lengths[:, None]
+
+    # Scatter seq2 into the right positions
+    concatenated_sequence = concatenated_sequence.scatter(1, index[:, :, None].expand(-1, -1, hidden_size), seq2)
+
+    if return_index:
+        return concatenated_sequence, concatenated_mask, index
+
+    return concatenated_sequence, concatenated_mask
+
+
+def box_cxcywh_to_xyxy(x):
+    """Convert boxes from (cx, cy, w, h) format to (x1, y1, x2, y2) format."""
+    x_c, y_c, w, h = x.unbind(-1)
+    b = [(x_c - 0.5 * w), (y_c - 0.5 * h), (x_c + 0.5 * w), (y_c + 0.5 * h)]
+    return torch.stack(b, dim=-1)
+
+
+class Sam3MLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.activation_fn = ACT2FN[config.hidden_act]
+        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+        self.dropout = nn.Dropout(config.hidden_dropout)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+        hidden_states = self.activation_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        return hidden_states
+
+
+def eager_attention_forward(
+    module: nn.Module,
+    query: torch.Tensor,
+    key: torch.Tensor,
+    value: torch.Tensor,
+    attention_mask: torch.Tensor | None,
+    scaling: float | None = None,
+    dropout: float = 0.0,
+    **kwargs: Unpack[TransformersKwargs],
+):
+    if scaling is None:
+        scaling = query.size(-1) ** -0.5
+
+    # Take the dot product between "query" and "key" to get the raw attention scores.
+    attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling
+
+    if attention_mask is not None:
+        attn_weights = attn_weights + attention_mask
+
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+
+    attn_output = torch.matmul(attn_weights, value)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+class Sam3Attention(nn.Module):
+    """
+    Multi-head attention.
+    Handles standard [batch_size, seq_len, hidden_size] tensors.
+    """
+
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.num_attention_heads = config.num_attention_heads
+        self.head_dim = self.hidden_size // config.num_attention_heads
+        self.scaling = self.head_dim**-0.5
+        self.is_causal = False
+
+        self.q_proj = nn.Linear(self.hidden_size, self.hidden_size)
+        self.k_proj = nn.Linear(self.hidden_size, self.hidden_size)
+        self.v_proj = nn.Linear(self.hidden_size, self.hidden_size)
+        self.o_proj = nn.Linear(self.hidden_size, self.hidden_size)
+
+    def forward(
+        self,
+        query: torch.Tensor,
+        key: torch.Tensor,
+        value: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """
+        Args:
+            query: [batch_size, query_len, hidden_size]
+            key: [batch_size, key_len, hidden_size]
+            value: [batch_size, value_len, hidden_size]
+            attention_mask: [batch_size, num_heads, query_len, key_len] or broadcastable
+
+        Returns:
+            Tuple of (output, attention_weights)
+                output: [batch_size, query_len, hidden_size]
+                attention_weights: [batch_size, num_heads, query_len, key_len]
+        """
+        batch_size = query.shape[0]
+        query_len = query.shape[1]
+        key_len = key.shape[1]
+
+        query = self.q_proj(query).view(batch_size, query_len, self.num_attention_heads, self.head_dim).transpose(1, 2)
+        key = self.k_proj(key).view(batch_size, key_len, self.num_attention_heads, self.head_dim).transpose(1, 2)
+        value = self.v_proj(value).view(batch_size, key_len, self.num_attention_heads, self.head_dim).transpose(1, 2)
+
+        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, eager_attention_forward
+        )
+
+        if (
+            is_flash_attention_requested(self.config)
+            and attention_mask is not None
+            and attention_mask.dtype != torch.bool
+        ):
+            # Relative position bias tensors are represented as float masks and are incompatible with Flash Attention
+            # Fallback to SDPA for this call only so the rest of the model can still benefit from FA
+            attention_interface = ALL_ATTENTION_FUNCTIONS["sdpa"]
+            logger.warning_once(
+                "Sam3Attention: falling back to SDPA for relative-position cross-attention because "
+                "Flash Attention does not support additive bias masks."
+            )
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query,
+            key,
+            value,
+            attention_mask=attention_mask,
+            dropout=0.0,
+            scaling=self.scaling,
+            is_causal=self.is_causal,
+            **kwargs,
+        )
+
+        attn_output = attn_output.reshape(batch_size, query_len, self.num_attention_heads * self.head_dim).contiguous()
+        attn_output = self.o_proj(attn_output)
+
+        return attn_output, attn_weights
+
+
+class Sam3ViTRotaryEmbedding(nn.Module):
+    """
+    Vision Rotary Position Embedding for SAM3, following transformers library standards.
+    Supports 2D (axial) rotary embeddings for spatial dimensions.
+    """
+
+    def __init__(self, config: Sam3ViTConfig, end_x: int, end_y: int, scale: float = 1.0):
+        super().__init__()
+        dim = config.hidden_size // config.num_attention_heads
+        # Ensure even dimension for proper axial splitting
+        if dim % 4 != 0:
+            raise ValueError("Dimension must be divisible by 4 for axial RoPE")
+        self.end_x, self.end_y = end_x, end_y
+        self.dim = dim
+        self.rope_theta = config.rope_theta
+        self.scale = scale
+        freqs = 1.0 / (config.rope_theta ** (torch.arange(0, dim, 4)[: (dim // 4)].float() / dim))
+
+        flattened_indices = torch.arange(end_x * end_y, dtype=torch.long)
+        x_positions = (flattened_indices % end_x) * scale
+        y_positions = torch.div(flattened_indices, end_x, rounding_mode="floor") * scale
+        freqs_x = torch.outer(x_positions, freqs).float()
+        freqs_y = torch.outer(y_positions, freqs).float()
+        inv_freq = torch.cat([freqs_x, freqs_y], dim=-1)
+        inv_freq = inv_freq.repeat_interleave(2, dim=-1)
+        # directly register the cos and sin embeddings as we have a fixed feature shape
+        self.register_buffer("rope_embeddings_cos", inv_freq.cos(), persistent=False)
+        self.register_buffer("rope_embeddings_sin", inv_freq.sin(), persistent=False)
+
+    @torch.no_grad()
+    def forward(self) -> tuple[torch.Tensor, torch.Tensor]:
+        # As the feature map size is fixed for each stage, we can just return the pre-computed embeddings.
+        return self.rope_embeddings_cos, self.rope_embeddings_sin
+
+
+def rotate_pairwise(x):
+    """
+    pairwise rotation of the hidden dims of the input. Differerent from Llama Half-Tensor Rotation.
+
+    This is an optimized version of the following more explicit implementation:
+    ```python
+    x_rotated = torch.zeros_like(x, dtype=x.dtype, device=x.device)
+    x_rotated[..., ::2] = -x[..., 1::2]
+    x_rotated[..., 1::2] = x[..., ::2]
+    return x_rotated
+    ```
+    """
+    x = x.view(*x.shape[:-1], -1, 2)
+    x1, x2 = x.unbind(dim=-1)
+    x = torch.stack((-x2, x1), dim=-1)
+    return x.flatten(start_dim=-2)
+
+
+def apply_rotary_pos_emb_2d(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    cos: torch.Tensor,
+    sin: torch.Tensor,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """
+    Apply rotary position embedding to query and key tensors for self-attention.
+
+    Args:
+        q: Query tensor of shape (batch_size, num_windows, seq_len, num_heads, head_dim)
+        k: Key tensor of shape (batch_size, num_windows, seq_len, num_heads, head_dim)
+        cos: Cosine position embedding of shape (seq_len, head_dim)
+        sin: Sine position embedding of shape (seq_len, head_dim)
+
+    Returns:
+        Rotated (q, k) tensors
+    """
+    q_embed = q.float()
+    q_embed = (q_embed * cos) + (rotate_pairwise(q_embed) * sin)
+
+    k_embed = k.float()
+    k_embed = (k_embed * cos) + (rotate_pairwise(k_embed) * sin)
+
+    return q_embed.type_as(q), k_embed.type_as(k)
+
+
+class Sam3ViTRoPEAttention(nn.Module):
+    """Self-attention with rotary position encoding."""
+
+    def __init__(self, config: Sam3ViTConfig):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.num_attention_heads = config.num_attention_heads
+        self.head_dim = self.hidden_size // config.num_attention_heads
+        self.scaling = self.head_dim**-0.5
+        self.attention_dropout = config.attention_dropout
+        self.is_causal = False
+
+        self.q_proj = nn.Linear(self.hidden_size, self.hidden_size)
+        self.k_proj = nn.Linear(self.hidden_size, self.hidden_size)
+        self.v_proj = nn.Linear(self.hidden_size, self.hidden_size)
+        self.o_proj = nn.Linear(self.hidden_size, self.hidden_size)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: tuple[torch.Tensor, torch.Tensor],
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> Tensor:
+        batch_size, height, width, _ = hidden_states.shape
+        seq_len = height * width
+        new_shape = (batch_size, seq_len, self.num_attention_heads, self.head_dim)
+        query = self.q_proj(hidden_states).view(*new_shape).transpose(1, 2)
+        key = self.k_proj(hidden_states).view(*new_shape).transpose(1, 2)
+        value = self.v_proj(hidden_states).view(*new_shape).transpose(1, 2)
+        cos, sin = position_embeddings
+        query, key = apply_rotary_pos_emb_2d(query, key, cos=cos, sin=sin)
+
+        attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, eager_attention_forward
+        )
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query,
+            key,
+            value,
+            attention_mask=None,
+            dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scaling,
+            is_causal=self.is_causal,
+            **kwargs,
+        )
+        attn_output = attn_output.reshape(batch_size, height, width, -1).contiguous()
+        attn_output = self.o_proj(attn_output)
+        return attn_output, attn_weights
+
+
+class Sam3ViTPatchEmbeddings(nn.Module):
+    """
+    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
+    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
+    Transformer.
+    """
+
+    def __init__(self, config: Sam3ViTConfig):
+        super().__init__()
+        image_size, patch_size = config.pretrain_image_size, config.patch_size
+        num_channels, hidden_size = config.num_channels, config.hidden_size
+
+        image_size = image_size if isinstance(image_size, Iterable) else (image_size, image_size)
+        patch_size = patch_size if isinstance(patch_size, Iterable) else (patch_size, patch_size)
+        num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
+        self.image_size = image_size
+        self.patch_size = patch_size
+        self.num_channels = num_channels
+        self.num_patches = num_patches
+
+        self.projection = nn.Conv2d(num_channels, hidden_size, kernel_size=patch_size, stride=patch_size, bias=False)
+
+    def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
+        embeddings = self.projection(pixel_values.to(self.projection.weight.dtype)).flatten(2).transpose(1, 2)
+        return embeddings
+
+
+class Sam3ViTEmbeddings(nn.Module):
+    """
+    Construct the patch embeddings and position embeddings for SAM3 ViT.
+
+    Position embeddings are tiled (not interpolated) when resizing to match different input sizes.
+    """
+
+    def __init__(self, config: Sam3ViTConfig):
+        super().__init__()
+
+        self.patch_embeddings = Sam3ViTPatchEmbeddings(config)
+        num_patches = self.patch_embeddings.num_patches
+        self.position_embeddings = nn.Parameter(
+            torch.randn(1, num_patches, config.hidden_size)
+        )  # !Remove cls token in convert weights!
+
+        self.dropout = nn.Dropout(config.hidden_dropout)
+        self.patch_size = config.patch_size
+
+    def _tile_position_embeddings(
+        self,
+        position_embeddings: torch.Tensor,
+        height: int,
+        width: int,
+    ) -> torch.Tensor:
+        """
+        Tile position embeddings to match target spatial dimensions.
+        Args:
+            position_embeddings: Shape [1, num_pretrain_patches, hidden_size]
+            height: Target height in patches
+            width: Target width in patches
+
+        Returns:
+            Shape [1, height * width, hidden_size]
+        """
+        pretrain_size = int(position_embeddings.shape[1] ** 0.5)
+
+        # Skip tiling if sizes match (but always tile during tracing for consistent graph)
+        if not torch.jit.is_tracing() and pretrain_size == height and pretrain_size == width:
+            return position_embeddings.reshape(1, height * width, -1)
+
+        # Tile position embeddings to match target spatial dimensions
+        hidden_size = position_embeddings.shape[-1]
+        pos_embed = position_embeddings.reshape(1, pretrain_size, pretrain_size, hidden_size).permute(0, 3, 1, 2)
+        repeat_h = height // pretrain_size + 1
+        repeat_w = width // pretrain_size + 1
+        pos_embed = pos_embed.tile([1, 1, repeat_h, repeat_w])[:, :, :height, :width]
+        return pos_embed.permute(0, 2, 3, 1).reshape(1, height * width, hidden_size)
+
+    def forward(
+        self,
+        pixel_values: torch.Tensor,
+        interpolate_pos_encoding: bool = False,
+    ) -> torch.Tensor:
+        height, width = pixel_values.shape[-2:]
+        embeddings = self.patch_embeddings(pixel_values)
+
+        # Calculate spatial dimensions in patches
+        height_patches = height // self.patch_size
+        width_patches = width // self.patch_size
+
+        position_embeddings = self._tile_position_embeddings(
+            self.position_embeddings,
+            height_patches,
+            width_patches,
+        )
+        embeddings = embeddings + position_embeddings
+        embeddings = self.dropout(embeddings)
+
+        return embeddings
+
+
+def window_partition(hidden_state, window_size):
+    """
+    Partition into non-overlapping windows with padding if needed.
+
+    Args:
+        hidden_state (`torch.Tensor`):
+            Input tokens with [batch_size, height, width, num_channels].
+        window_size (`int`):
+            Window size.
+
+    Returns:
+        `tuple(torch.FloatTensor)` comprising various elements:
+        - windows: windows after partition with [batch_size * num_windows, window_size, window_size, num_channels].
+        - (padded_height, padded_width): padded height and width before partition
+    """
+    batch_size, height, width, num_channels = hidden_state.shape
+    pad_height = (window_size - height % window_size) % window_size
+    pad_width = (window_size - width % window_size) % window_size
+
+    # Noop in case pad_width == 0 and pad_height == 0.
+    hidden_state = nn.functional.pad(hidden_state, (0, 0, 0, pad_width, 0, pad_height))
+
+    padded_height, padded_width = height + pad_height, width + pad_width
+
+    hidden_state = hidden_state.view(
+        batch_size, padded_height // window_size, window_size, padded_width // window_size, window_size, num_channels
+    )
+    windows = hidden_state.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, num_channels)
+    return windows, (padded_height, padded_width)
+
+
+def window_unpartition(windows, window_size, pad_height_width, height_width):
+    """
+    Window unpartition into original sequences and removing padding.
+
+    Args:
+        windows (`torch.Tensor`):
+            Input tokens with [batch_size * num_windows, window_size, window_size, num_channels].
+        window_size (`int`):
+            Window size.
+        pad_height_width (`tuple[int]`):
+            Padded height and width (padded_height, padded_width).
+        height_width (`tuple[int]`):
+            Original height and width before padding.
+
+    Returns:
+        hidden_state: unpartitioned sequences with [batch_size, height, width, num_channels].
+    """
+    padded_height, padded_width = pad_height_width
+    height, width = height_width
+    batch_size = windows.shape[0] // (padded_height * padded_width // window_size // window_size)
+    hidden_state = windows.view(
+        batch_size, padded_height // window_size, padded_width // window_size, window_size, window_size, -1
+    )
+    hidden_state = hidden_state.permute(0, 1, 3, 2, 4, 5).contiguous()
+    hidden_state = hidden_state.view(batch_size, padded_height, padded_width, -1)
+
+    # We always have height <= padded_height and width <= padded_width
+    hidden_state = hidden_state[:, :height, :width, :].contiguous()
+    return hidden_state
+
+
+class Sam3ViTLayerScale(nn.Module):
+    def __init__(self, config) -> None:
+        super().__init__()
+        self.lambda1 = nn.Parameter(config.layer_scale_init_value * torch.ones(config.hidden_size))
+
+    def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
+        return hidden_state * self.lambda1
+
+
+class Sam3ViTLayer(GradientCheckpointingLayer):
+    """Vision Transformer layer with rotary position embeddings and optional windowed attention."""
+
+    def __init__(self, config: Sam3ViTConfig, window_size: int = 0) -> None:
+        super().__init__()
+
+        hidden_size = config.hidden_size
+        image_size = config.image_size
+        image_size = image_size if isinstance(image_size, (list, tuple)) else (image_size, image_size)
+
+        patch_size = config.patch_size
+        patch_size = patch_size if isinstance(patch_size, (list, tuple)) else (patch_size, patch_size)
+
+        input_size = (image_size[0] // patch_size[0], image_size[1] // patch_size[1])
+        self.layer_norm1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_eps)
+        rotary_input_size = input_size if window_size == 0 else (window_size, window_size)
+        rotary_scale = config.window_size / rotary_input_size[0]
+        self.rotary_emb = Sam3ViTRotaryEmbedding(
+            config, end_x=rotary_input_size[0], end_y=rotary_input_size[1], scale=rotary_scale
+        )
+        self.attention = Sam3ViTRoPEAttention(config)
+        self.layer_norm2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_eps)
+        self.mlp = Sam3MLP(config)
+        self.dropout = nn.Dropout(config.hidden_dropout)
+
+        self.window_size = window_size
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> torch.Tensor:
+        residual = hidden_states
+
+        hidden_states = self.layer_norm1(hidden_states)
+
+        if self.window_size > 0:
+            height, width = hidden_states.shape[1], hidden_states.shape[2]
+            # Partition into non-overlapping windows for efficient attention
+            hidden_states, pad_height_width = window_partition(hidden_states, self.window_size)
+
+        position_embeddings = self.rotary_emb()
+        hidden_states, _ = self.attention(hidden_states, position_embeddings, **kwargs)
+
+        if self.window_size > 0:
+            # Reverse window partition to restore original spatial layout
+            hidden_states = window_unpartition(hidden_states, self.window_size, pad_height_width, (height, width))
+
+        hidden_states = residual + hidden_states
+        residual = hidden_states
+        hidden_states = self.layer_norm2(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + self.dropout(hidden_states)
+
+        return hidden_states
+
+
+@auto_docstring
+@requires(backends=("torch", "torchvision"))
+class Sam3PreTrainedModel(PreTrainedModel):
+    config_class = Sam3Config
+    base_model_prefix = "sam3"
+    main_input_name = "pixel_values"
+    input_modalities = ["image", "text"]
+    _supports_sdpa = True
+    _supports_flash_attn = True
+    _supports_flex_attn = True
+    _supports_attention_backend = True
+
+    def _init_weights(self, module):
+        super()._init_weights(module)
+        if isinstance(module, Sam3ViTEmbeddings):
+            init.normal_(module.position_embeddings, mean=0.0, std=self.config.initializer_range)
+        elif isinstance(module, Sam3ViTRotaryEmbedding):
+            end_x, end_y = module.end_x, module.end_y
+            dim = module.dim
+            freqs = 1.0 / (module.rope_theta ** (torch.arange(0, dim, 4)[: (dim // 4)].float() / dim))
+            flattened_indices = torch.arange(end_x * end_y, dtype=torch.long)
+            x_positions = (flattened_indices % end_x) * module.scale
+            y_positions = torch.div(flattened_indices, end_x, rounding_mode="floor") * module.scale
+            freqs_x = torch.outer(x_positions, freqs).float()
+            freqs_y = torch.outer(y_positions, freqs).float()
+            inv_freq = torch.cat([freqs_x, freqs_y], dim=-1)
+            inv_freq = inv_freq.repeat_interleave(2, dim=-1)
+            init.copy_(module.rope_embeddings_cos, inv_freq.cos())
+            init.copy_(module.rope_embeddings_sin, inv_freq.sin())
+
+
+@auto_docstring
+class Sam3ViTModel(Sam3PreTrainedModel):
+    config: Sam3ViTConfig
+    _can_record_outputs = {
+        "hidden_states": Sam3ViTLayer,
+        "attentions": Sam3ViTRoPEAttention,
+    }
+
+    def __init__(self, config: Sam3ViTConfig):
+        super().__init__(config)
+        self.config = config
+        self.embeddings = Sam3ViTEmbeddings(config)
+        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.layers = nn.ModuleList(
+            [
+                Sam3ViTLayer(config, window_size=config.window_size if i not in config.global_attn_indexes else 0)
+                for i in range(config.num_hidden_layers)
+            ]
+        )
+        self.post_init()
+
+    def get_input_embeddings(self) -> Sam3ViTPatchEmbeddings:
+        return self.embeddings.patch_embeddings
+
+    @merge_with_config_defaults
+    @capture_outputs(tie_last_hidden_states=False)
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.Tensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> BaseModelOutput:
+        hidden_states = self.embeddings(pixel_values)  # [batch_size, seq_len, hidden_size]
+
+        batch_size = hidden_states.shape[0]
+        height = pixel_values.shape[-2] // self.config.patch_size
+        width = pixel_values.shape[-1] // self.config.patch_size
+        hidden_size = hidden_states.shape[-1]
+
+        # Reshape to spatial format for windowed attention: [batch_size, height, width, hidden_size]
+        hidden_states = hidden_states.view(batch_size, height, width, hidden_size)
+
+        hidden_states = self.layer_norm(hidden_states)
+        for layer in self.layers:
+            hidden_states = layer(hidden_states, **kwargs)
+
+        # Reshape back to sequence format: [batch_size, height*width, hidden_size]
+        hidden_states = hidden_states.view(batch_size, height * width, hidden_size)
+
+        return BaseModelOutput(last_hidden_state=hidden_states)
+
+
+class Sam3SinePositionEmbedding(nn.Module):
+    """
+    This is a more standard version of the position embedding, very similar to the one used by the Attention is all you
+    need paper, generalized to work on images.
+    """
+
+    def __init__(
+        self,
+        num_position_features: int = 64,
+        temperature: int = 10000,
+        normalize: bool = False,
+        scale: float | None = None,
+    ):
+        super().__init__()
+        if scale is not None and normalize is False:
+            raise ValueError("normalize should be True if scale is passed")
+        self.num_position_features = num_position_features
+        self.temperature = temperature
+        self.normalize = normalize
+        self.scale = 2 * math.pi if scale is None else scale
+
+    def encode_1d_positions(self, x: torch.Tensor, y: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+        """
+        Encode 1D coordinate pairs using sine/cosine positional embeddings.
+
+        Args:
+            x: 1D tensor of x coordinates (flattened)
+            y: 1D tensor of y coordinates (flattened)
+
+        Returns:
+            Tuple of (pos_x, pos_y) positional embeddings
+        """
+        x_embed = x * self.scale
+        y_embed = y * self.scale
+
+        dim_t = torch.arange(self.num_position_features, dtype=torch.int64, device=x.device).to(x.dtype)
+        dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_position_features)
+
+        pos_x = x_embed[:, None] / dim_t
+        pos_y = y_embed[:, None] / dim_t
+        pos_x = torch.stack((pos_x[:, 0::2].sin(), pos_x[:, 1::2].cos()), dim=2).flatten(1)
+        pos_y = torch.stack((pos_y[:, 0::2].sin(), pos_y[:, 1::2].cos()), dim=2).flatten(1)
+        return pos_x, pos_y
+
+    def encode_boxes(self, boxes: torch.Tensor) -> torch.Tensor:
+        """
+        Encode 4D box coordinates (x, y, w, h) for decoder conditioning using sine/cosine embeddings.
+
+        Args:
+            boxes: Box coordinates [batch_size, num_queries, 4] in (x, y, w, h) format
+
+        Returns:
+            Position embeddings [batch_size, num_queries, num_position_features*4]
+        """
+        assert boxes.size(-1) == 4, f"Expected 4D box coordinates (x, y, w, h), got shape {boxes.shape}"
+        dim_t = torch.arange(self.num_position_features, dtype=torch.int64, device=boxes.device).to(boxes.dtype)
+        dim_t = self.temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / self.num_position_features)
+
+        x_embed = boxes[:, :, 0] * self.scale
+        y_embed = boxes[:, :, 1] * self.scale
+        w_embed = boxes[:, :, 2] * self.scale
+        h_embed = boxes[:, :, 3] * self.scale
+
+        pos_x = x_embed[:, :, None] / dim_t
+        pos_y = y_embed[:, :, None] / dim_t
+        pos_w = w_embed[:, :, None] / dim_t
+        pos_h = h_embed[:, :, None] / dim_t
+
+        pos_x = torch.stack((pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos()), dim=3).flatten(2)
+        pos_y = torch.stack((pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos()), dim=3).flatten(2)
+        pos_w = torch.stack((pos_w[:, :, 0::2].sin(), pos_w[:, :, 1::2].cos()), dim=3).flatten(2)
+        pos_h = torch.stack((pos_h[:, :, 0::2].sin(), pos_h[:, :, 1::2].cos()), dim=3).flatten(2)
+
+        pos = torch.cat((pos_y, pos_x, pos_w, pos_h), dim=2)
+
+        return pos
+
+    @staticmethod
+    @compile_compatible_method_lru_cache(maxsize=4)
+    def build_sine_position_embedding(
+        shape: torch.Size,
+        device: torch.device | str,
+        dtype: torch.dtype,
+        num_position_features: int,
+        normalize: bool = False,
+        scale: float | None = None,
+        temperature: int = 10000,
+        mask: torch.Tensor | None = None,
+    ) -> torch.Tensor:
+        if mask is None:
+            mask = torch.ones((shape[0], shape[2], shape[3]), device=device, dtype=torch.bool)
+        y_embed = mask.cumsum(1, dtype=dtype)
+        x_embed = mask.cumsum(2, dtype=dtype)
+        if normalize:
+            eps = 1e-6
+            y_embed = y_embed / (y_embed[:, -1:, :] + eps) * scale
+            x_embed = x_embed / (x_embed[:, :, -1:] + eps) * scale
+
+        dim_t = torch.arange(num_position_features, dtype=torch.int64, device=device).to(dtype)
+        dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / num_position_features)
+
+        pos_x = x_embed[:, :, :, None] / dim_t
+        pos_y = y_embed[:, :, :, None] / dim_t
+        pos_x = torch.stack((pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).flatten(3)
+        pos_y = torch.stack((pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).flatten(3)
+        pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2)
+        return pos
+
+    def forward(
+        self,
+        shape: torch.Size,
+        device: torch.device | str,
+        dtype: torch.dtype,
+        mask: torch.Tensor | None = None,
+    ) -> torch.Tensor:
+        return self.build_sine_position_embedding(
+            shape, device, dtype, self.num_position_features, self.normalize, self.scale, self.temperature, mask
+        )
+
+
+class Sam3FPNLayer(nn.Module):
+    def __init__(self, in_channels: int, fpn_dim: int, scale_factor: float):
+        super().__init__()
+        self.scale_factor = scale_factor
+
+        # Build the upsampling/downsampling layers based on scale factor
+        self.scale_layers = nn.ModuleList()
+
+        if scale_factor == 4.0:
+            self.scale_layers.append(nn.ConvTranspose2d(in_channels, in_channels // 2, kernel_size=2, stride=2))
+            self.scale_layers.append(nn.GELU())
+            self.scale_layers.append(nn.ConvTranspose2d(in_channels // 2, in_channels // 4, kernel_size=2, stride=2))
+            intermediate_channels = in_channels // 4
+        elif scale_factor == 2.0:
+            self.scale_layers.append(nn.ConvTranspose2d(in_channels, in_channels // 2, kernel_size=2, stride=2))
+            intermediate_channels = in_channels // 2
+        elif scale_factor == 1.0:
+            intermediate_channels = in_channels
+        elif scale_factor == 0.5:
+            self.scale_layers.append(nn.MaxPool2d(kernel_size=2, stride=2))
+            intermediate_channels = in_channels
+        else:
+            raise NotImplementedError(f"scale_factor={scale_factor} is not supported yet.")
+
+        self.proj1 = nn.Conv2d(in_channels=intermediate_channels, out_channels=fpn_dim, kernel_size=1)
+        self.proj2 = nn.Conv2d(in_channels=fpn_dim, out_channels=fpn_dim, kernel_size=3, padding=1)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = hidden_states.to(self.proj1.weight.dtype)
+        for layer in self.scale_layers:
+            hidden_states = layer(hidden_states)
+
+        hidden_states = self.proj1(hidden_states)
+        hidden_states = self.proj2(hidden_states)
+
+        return hidden_states
+
+
+class Sam3VisionNeck(nn.Module):
+    def __init__(self, config: Sam3VisionConfig):
+        super().__init__()
+        self.config = config
+
+        self.position_encoding = Sam3SinePositionEmbedding(
+            num_position_features=config.fpn_hidden_size // 2, normalize=True
+        )
+
+        # Create one FPN layer per scale factor
+        self.fpn_layers = nn.ModuleList(
+            [
+                Sam3FPNLayer(
+                    in_channels=config.backbone_config.hidden_size, fpn_dim=config.fpn_hidden_size, scale_factor=scale
+                )
+                for scale in config.scale_factors
+            ]
+        )
+
+    def forward(self, hidden_states: torch.Tensor) -> tuple[tuple[torch.Tensor, ...], tuple[torch.Tensor, ...]]:
+        fpn_hidden_states = ()
+        fpn_position_encoding = ()
+
+        for fpn_layer in self.fpn_layers:
+            fpn_output = fpn_layer(hidden_states)
+            fpn_hidden_states += (fpn_output,)
+            # Generate position encoding for this FPN level
+            pos_enc = self.position_encoding(fpn_output.shape, fpn_output.device, fpn_output.dtype)
+            fpn_position_encoding += (pos_enc,)
+
+        return fpn_hidden_states, fpn_position_encoding
+
+
+@auto_docstring(
+    custom_intro="""
+    The vision model from Sam without any head or projection on top.
+    """
+)
+class Sam3VisionModel(Sam3PreTrainedModel):
+    config_class = Sam3VisionConfig
+    main_input_name = "pixel_values"
+
+    def __init__(self, config: Sam3VisionConfig):
+        super().__init__(config)
+        self.config = config
+        self.backbone = AutoModel.from_config(config.backbone_config)
+        self.neck = Sam3VisionNeck(config)
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.backbone.get_input_embeddings()
+
+    @can_return_tuple
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | Sam3VisionEncoderOutput:
+        if pixel_values is None:
+            raise ValueError("You have to specify pixel_values")
+
+        backbone_output = self.backbone(pixel_values, **kwargs)
+        hidden_states = backbone_output.last_hidden_state  # [batch_size, seq_len, hidden_size]
+
+        # Reshape for FPN neck: [batch_size, seq_len, hidden_size] -> [batch_size, hidden_size, height, width]
+        batch_size = hidden_states.shape[0]
+        height = pixel_values.shape[-2] // self.config.backbone_config.patch_size
+        width = pixel_values.shape[-1] // self.config.backbone_config.patch_size
+        hidden_states_spatial = hidden_states.view(batch_size, height, width, -1).permute(0, 3, 1, 2)
+        fpn_hidden_states, fpn_position_encoding = self.neck(hidden_states_spatial)
+
+        return Sam3VisionEncoderOutput(
+            last_hidden_state=hidden_states,
+            fpn_hidden_states=fpn_hidden_states,
+            fpn_position_encoding=fpn_position_encoding,
+            hidden_states=backbone_output.hidden_states,
+            attentions=backbone_output.attentions,
+        )
+
+
+class Sam3GeometryEncoderLayer(nn.Module):
+    def __init__(self, config: Sam3GeometryEncoderConfig):
+        super().__init__()
+        self.layer_norm1 = nn.LayerNorm(config.hidden_size)
+        self.self_attn = Sam3Attention(config)
+        self.dropout = nn.Dropout(config.dropout)
+
+        self.cross_attn = Sam3Attention(config)
+        self.layer_norm2 = nn.LayerNorm(config.hidden_size)
+
+        self.mlp = Sam3MLP(config)
+        self.layer_norm3 = nn.LayerNorm(config.hidden_size)
+
+    def forward(
+        self,
+        prompt_feats: Tensor,
+        vision_feats: Tensor,
+        vision_pos_encoding: Tensor,
+        prompt_mask: Tensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ):
+        residual = prompt_feats
+        hidden_states = self.layer_norm1(prompt_feats)
+        hidden_states, _ = self.self_attn(
+            query=hidden_states, key=hidden_states, value=hidden_states, attention_mask=prompt_mask, **kwargs
+        )
+        hidden_states = self.dropout(hidden_states) + residual
+        residual = hidden_states
+        hidden_states = self.layer_norm2(hidden_states)
+        key = vision_feats + vision_pos_encoding
+        hidden_states, _ = self.cross_attn(query=hidden_states, key=key, value=vision_feats, **kwargs)
+        hidden_states = self.dropout(hidden_states) + residual
+        residual = hidden_states
+        hidden_states = self.layer_norm3(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = self.dropout(hidden_states) + residual
+
+        return hidden_states
+
+
+class Sam3GeometryEncoder(nn.Module):
+    """
+    Encoder for geometric prompts (boxes).
+
+    Boxes are encoded using three approaches:
+     - Direct projection: linear projection from coordinate space to hidden_size
+     - Pooling: pool features from the backbone at the specified location (ROI align for boxes)
+     - Position encoding: use position encoding of the box center
+
+    These encodings are combined additively and further processed with transformer layers.
+    """
+
+    def __init__(self, config: Sam3GeometryEncoderConfig):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.roi_size = config.roi_size
+
+        self.position_encoding = Sam3SinePositionEmbedding(
+            num_position_features=config.hidden_size // 2, normalize=True
+        )
+        self.label_embed = nn.Embedding(2, self.hidden_size)
+        self.cls_embed = nn.Embedding(1, self.hidden_size)
+
+        # Box encoding layers
+        self.boxes_direct_project = nn.Linear(4, self.hidden_size)
+        self.boxes_pool_project = nn.Conv2d(self.hidden_size, self.hidden_size, self.roi_size)
+        self.boxes_pos_enc_project = nn.Linear(self.hidden_size + 2, self.hidden_size)
+
+        # Image feature normalization
+        self.vision_layer_norm = nn.LayerNorm(self.hidden_size)
+
+        # Prompt projection and normalization
+        self.final_proj = nn.Linear(self.hidden_size, self.hidden_size)
+        self.prompt_layer_norm = nn.LayerNorm(self.hidden_size)
+
+        # Transformer layers
+        self.layers = nn.ModuleList([Sam3GeometryEncoderLayer(config) for _ in range(config.num_layers)])
+        self.output_layer_norm = nn.LayerNorm(self.hidden_size)
+
+    def _encode_box_coordinates(
+        self, center_x: torch.Tensor, center_y: torch.Tensor, width: torch.Tensor, height: torch.Tensor
+    ) -> torch.Tensor:
+        """
+        Encode box coordinates by combining position-encoded centers with raw width/height.
+
+        Args:
+            center_x: 1D tensor of box center x coordinates
+            center_y: 1D tensor of box center y coordinates
+            width: 1D tensor of box widths
+            height: 1D tensor of box heights
+
+        Returns:
+            Encoded box coordinates [N, embedding_dim]
+        """
+        pos_x, pos_y = self.position_encoding.encode_1d_positions(center_x, center_y)
+        pos = torch.cat((pos_y, pos_x, height[:, None], width[:, None]), dim=1)
+        return pos
+
+    def _encode_boxes(self, boxes, boxes_mask, boxes_labels, vision_features):
+        """Encode box prompts. Mask convention: True=valid, False=padding."""
+        batch_size, num_boxes = boxes.shape[:2]
+        height, width = vision_features.shape[-2:]
+        boxes_embed = self.boxes_direct_project(boxes)
+
+        # Pool features using ROI align
+        # Convert boxes from CxCyWH to xyxy format and denormalize
+        boxes_xyxy = box_cxcywh_to_xyxy(boxes)
+        scale = torch.tensor([width, height, width, height], dtype=boxes_xyxy.dtype, device=boxes_xyxy.device)
+        scale = scale.view(1, 1, 4)
+        boxes_xyxy = boxes_xyxy * scale
+        # ROI align expects list of boxes per batch element,
+        # convert from bfloat16 to float16 as roi_align only supports float16 and float32
+        dtype = torch.float16 if vision_features.dtype == torch.bfloat16 else vision_features.dtype
+        sampled_features = torchvision.ops.roi_align(
+            vision_features.to(dtype), boxes_xyxy.to(dtype).unbind(0), self.roi_size
+        ).to(vision_features.dtype)
+
+        pooled_projection = self.boxes_pool_project(sampled_features)
+        pooled_projection = pooled_projection.view(batch_size, num_boxes, self.hidden_size)
+        boxes_embed = boxes_embed + pooled_projection
+
+        # Add position encoding
+        center_x, center_y, box_width, box_height = boxes.unbind(-1)
+        pos_enc = self._encode_box_coordinates(
+            center_x.flatten(), center_y.flatten(), box_width.flatten(), box_height.flatten()
+        )
+        pos_enc = pos_enc.view(batch_size, num_boxes, pos_enc.shape[-1])
+        pos_projection = self.boxes_pos_enc_project(pos_enc)
+        boxes_embed = boxes_embed + pos_projection
+
+        # Add label embeddings (positive/negative)
+        label_embed = self.label_embed(boxes_labels.long())
+        return label_embed + boxes_embed, boxes_mask
+
+    def forward(
+        self,
+        box_embeddings: torch.Tensor,
+        box_mask: torch.Tensor,
+        box_labels: torch.Tensor,
+        img_feats: tuple[torch.Tensor, ...],
+        img_pos_embeds: tuple[torch.Tensor, ...] | None = None,
+    ):
+        """
+        Forward pass for encoding geometric prompts.
+
+        Args:
+            box_embeddings: Box coordinates in CxCyWH format [batch_size, num_boxes, 4]
+            box_mask: Attention mask for boxes [batch_size, num_boxes]
+            box_labels: Labels for boxes (positive/negative) [batch_size, num_boxes]
+            img_feats: Image features from vision encoder
+            img_pos_embeds: Optional position embeddings for image features
+
+        Returns:
+            Sam3GeometryEncoderOutput containing encoded geometry features and attention mask.
+        """
+        batch_size = box_embeddings.shape[0]
+
+        # Prepare vision features for cross-attention: flatten spatial dimensions
+        vision_feats = img_feats[-1]  # [B, C, H, W]
+        vision_pos_embeds = img_pos_embeds[-1] if img_pos_embeds is not None else torch.zeros_like(vision_feats)
+        vision_feats_flat = vision_feats.flatten(2).transpose(1, 2)  # [B, H*W, C]
+        vision_pos_embeds_flat = vision_pos_embeds.flatten(2).transpose(1, 2)  # [B, H*W, C]
+
+        # Normalize image features for pooling operations
+        img_feats_last = img_feats[-1]  # [B, C, H, W]
+        img_feats_last = img_feats_last.permute(0, 2, 3, 1)  # [B, H, W, C]
+        normalized_img_feats = self.vision_layer_norm(img_feats_last)
+        normalized_img_feats = normalized_img_feats.permute(0, 3, 1, 2)  # [B, C, H, W]
+
+        prompt_embeds, prompt_mask = self._encode_boxes(box_embeddings, box_mask, box_labels, normalized_img_feats)
+
+        # Add CLS token (always valid)
+        cls_embed = self.cls_embed.weight.view(1, self.hidden_size).unsqueeze(0).expand(batch_size, -1, -1)
+        cls_mask = torch.ones(batch_size, 1, dtype=prompt_mask.dtype, device=prompt_mask.device)
+        prompt_embeds, prompt_mask = concat_padded_sequences(prompt_embeds, prompt_mask, cls_embed, cls_mask)
+
+        prompt_embeds = self.prompt_layer_norm(self.final_proj(prompt_embeds))
+
+        # Create bidirectional attention mask for transformer layers
+        prompt_attention_mask = None
+        if prompt_mask is not None:
+            prompt_attention_mask = create_bidirectional_mask(
+                config=self.config,
+                inputs_embeds=prompt_embeds,
+                attention_mask=prompt_mask,
+            )
+
+        # Apply transformer layers with cross-attention to vision features
+        for layer in self.layers:
+            prompt_embeds = layer(
+                prompt_feats=prompt_embeds,
+                vision_feats=vision_feats_flat,
+                vision_pos_encoding=vision_pos_embeds_flat,
+                prompt_mask=prompt_attention_mask,
+            )
+
+        # Final output normalization
+        prompt_embeds = self.output_layer_norm(prompt_embeds)
+
+        return Sam3GeometryEncoderOutput(
+            last_hidden_state=prompt_embeds,
+            attention_mask=prompt_mask,
+        )
+
+
+class Sam3DetrEncoderLayer(nn.Module):
+    """DETR encoder layer with self-attention and cross-attention."""
+
+    def __init__(self, config: Sam3DETREncoderConfig):
+        super().__init__()
+        self.config = config
+        self.layer_norm1 = nn.LayerNorm(config.hidden_size)
+        self.self_attn = Sam3Attention(config)
+        self.dropout = nn.Dropout(config.dropout)
+
+        self.cross_attn = Sam3Attention(config)
+        self.layer_norm2 = nn.LayerNorm(config.hidden_size)
+
+        self.mlp = Sam3MLP(config)
+        self.layer_norm3 = nn.LayerNorm(config.hidden_size)
+
+    def forward(
+        self,
+        vision_feats: Tensor,
+        prompt_feats: Tensor,
+        vision_pos_encoding: Tensor,
+        prompt_cross_attn_mask: Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ):
+        """
+        Forward pass for DETR encoder layer.
+
+        Args:
+            vision_feats: Vision features [batch_size, vision_len, hidden_size] (main hidden states)
+            prompt_feats: Text prompt features [batch_size, text_len, hidden_size]
+            vision_pos_encoding: Position encoding for vision [batch_size, vision_len, hidden_size]
+            prompt_cross_attn_mask: Cross-attention mask for prompt features
+
+        Returns:
+            Updated vision features [batch_size, vision_len, hidden_size]
+        """
+        # Self-attention on vision features with position encoding
+        residual = vision_feats
+        hidden_states = self.layer_norm1(vision_feats)
+        hidden_states_with_pos = hidden_states + vision_pos_encoding
+        hidden_states, _ = self.self_attn(
+            query=hidden_states_with_pos,
+            key=hidden_states_with_pos,
+            value=hidden_states,
+            **kwargs,
+        )
+        hidden_states = self.dropout(hidden_states) + residual
+
+        # Cross-attention: vision queries attend to text/prompt features
+        residual = hidden_states
+        hidden_states = self.layer_norm2(hidden_states)
+
+        hidden_states, _ = self.cross_attn(
+            query=hidden_states,
+            key=prompt_feats,
+            value=prompt_feats,
+            attention_mask=prompt_cross_attn_mask,
+            **kwargs,
+        )
+        hidden_states = self.dropout(hidden_states) + residual
+
+        # MLP
+        residual = hidden_states
+        hidden_states = self.layer_norm3(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = self.dropout(hidden_states) + residual
+
+        return hidden_states
+
+
+class Sam3DetrEncoder(Sam3PreTrainedModel):
+    """
+    DETR-style encoder that processes multi-level vision features with text fusion.
+
+    This encoder processes vision features from multiple levels (e.g., FPN features at different
+    resolutions) and fuses them with text prompts through a stack of transformer encoder layers.
+    """
+
+    _can_record_outputs = {
+        "hidden_states": Sam3DetrEncoderLayer,
+        "attentions": Sam3Attention,
+    }
+
+    def __init__(self, config: Sam3DETREncoderConfig):
+        super().__init__(config)
+        self.config = config
+        self.hidden_size = config.hidden_size
+
+        self.layers = nn.ModuleList([Sam3DetrEncoderLayer(config) for _ in range(config.num_layers)])
+
+        self.post_init()
+
+    def _prepare_multilevel_features(
+        self,
+        vision_features: list[torch.Tensor],
+        vision_pos_embeds: list[torch.Tensor],
+    ):
+        """
+        Prepare multi-level vision features by flattening spatial dimensions and adding level embeddings.
+
+        Args:
+            vision_features: List of vision features at different levels [batch_size, channels, height, width]
+            vision_pos_embeds: List of position embeddings for each level [batch_size, channels, height, width]
+
+        Returns:
+            Tuple containing flattened features, position embeddings, and spatial metadata
+        """
+        features_flattened = []
+        pos_embeds_flattened = []
+        spatial_shapes = []
+
+        for features, pos_embed in zip(vision_features, vision_pos_embeds):
+            height, width = features.shape[-2:]
+            spatial_shapes.append((height, width))
+
+            # Flatten spatial dimensions: [batch_size, channels, height, width] -> [batch_size, height*width, channels]
+            features = features.flatten(2).transpose(1, 2)
+            pos_embed = pos_embed.flatten(2).transpose(1, 2)
+
+            features_flattened.append(features)
+            pos_embeds_flattened.append(pos_embed)
+
+        # Concatenate all levels into single sequence
+        features_flattened = torch.cat(features_flattened, dim=1)
+        pos_embeds_flattened = torch.cat(pos_embeds_flattened, dim=1)
+
+        spatial_shapes = torch.tensor(spatial_shapes, dtype=torch.long, device=features_flattened.device)
+
+        return (
+            features_flattened,
+            pos_embeds_flattened,
+            spatial_shapes,
+        )
+
+    @merge_with_config_defaults
+    @capture_outputs
+    def forward(
+        self,
+        vision_features: list[torch.Tensor],
+        text_features: torch.Tensor,
+        vision_pos_embeds: list[torch.Tensor] | None = None,
+        text_mask: torch.Tensor | None = None,
+        spatial_sizes: list[tuple[int, int]] | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | Sam3DETREncoderOutput:
+        """
+        Forward pass for the DETR encoder.
+
+        Args:
+            vision_features: List of vision features at different levels
+            text_features: Text prompt features [batch_size, seq_len, hidden_size]
+            vision_pos_embeds: Optional list of position embeddings for each level
+            text_mask: Optional text padding mask [batch_size, seq_len]
+            spatial_sizes: Optional list of (height, width) tuples for reshaping
+
+        Returns:
+            Sam3DETREncoderOutput containing encoded features and metadata.
+        """
+        batch_size = vision_features[0].shape[0] if vision_features[0].dim() == 4 else vision_features[0].shape[1]
+
+        # TODO: See if we can remove that reshaping and just use the features as is.
+        if spatial_sizes is not None:
+            for i, (height, width) in enumerate(spatial_sizes):
+                # Reshape from [height*width, batch_size, channels] to [batch_size, channels, height, width]
+                vision_features[i] = vision_features[i].reshape(height, width, batch_size, -1).permute(2, 3, 0, 1)
+                vision_pos_embeds[i] = vision_pos_embeds[i].reshape(height, width, batch_size, -1).permute(2, 3, 0, 1)
+
+        # Flatten multi-level features for encoder processing
+        (
+            features_flattened,
+            pos_embeds_flattened,
+            spatial_shapes,
+        ) = self._prepare_multilevel_features(vision_features, vision_pos_embeds)
+
+        prompt_cross_attn_mask = None
+        if text_mask is not None:
+            prompt_cross_attn_mask = create_bidirectional_mask(
+                config=self.config,
+                inputs_embeds=features_flattened,
+                attention_mask=text_mask,
+                encoder_hidden_states=text_features,
+            )
+
+        hidden_states = features_flattened
+        for layer in self.layers:
+            hidden_states = layer(
+                hidden_states,
+                prompt_feats=text_features,
+                vision_pos_encoding=pos_embeds_flattened,
+                prompt_cross_attn_mask=prompt_cross_attn_mask,
+                **kwargs,
+            )
+        return Sam3DETREncoderOutput(
+            last_hidden_state=hidden_states,
+            pos_embeds_flattened=pos_embeds_flattened,
+            text_features=text_features,
+            spatial_shapes=spatial_shapes,
+        )
+
+
+class Sam3DecoderMLP(nn.Module):
+    """Simple 2 or 3-layer MLP for decoder components."""
+
+    def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, num_layers: int = 2):
+        super().__init__()
+        if num_layers == 2:
+            self.layer1 = nn.Linear(input_dim, hidden_dim)
+            self.layer2 = nn.Linear(hidden_dim, output_dim)
+            self.layer3 = None
+        elif num_layers == 3:
+            self.layer1 = nn.Linear(input_dim, hidden_dim)
+            self.layer2 = nn.Linear(hidden_dim, hidden_dim)
+            self.layer3 = nn.Linear(hidden_dim, output_dim)
+        else:
+            raise ValueError(f"Only 2 or 3 layers supported, got {num_layers}")
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = F.relu(self.layer1(x))
+        if self.layer3 is not None:
+            x = F.relu(self.layer2(x))
+            x = self.layer3(x)
+        else:
+            x = self.layer2(x)
+        return x
+
+
+class Sam3DetrDecoderLayer(nn.Module):
+    """DETR decoder layer with self-attention, text cross-attention, and vision cross-attention."""
+
+    def __init__(self, config: Sam3DETRDecoderConfig):
+        super().__init__()
+        self.config = config
+        self.self_attn = Sam3Attention(config)
+        self.self_attn_dropout = nn.Dropout(config.dropout)
+        self.self_attn_layer_norm = nn.LayerNorm(config.hidden_size)
+
+        self.text_cross_attn = Sam3Attention(config)
+        self.text_cross_attn_dropout = nn.Dropout(config.dropout)
+        self.text_cross_attn_layer_norm = nn.LayerNorm(config.hidden_size)
+
+        self.vision_cross_attn = Sam3Attention(config)
+        self.vision_cross_attn_dropout = nn.Dropout(config.dropout)
+        self.vision_cross_attn_layer_norm = nn.LayerNorm(config.hidden_size)
+
+        self.mlp = Sam3MLP(config)
+        self.mlp_layer_norm = nn.LayerNorm(config.hidden_size)
+        self.mlp_dropout = nn.Dropout(config.dropout)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        query_pos: torch.Tensor,
+        text_features: torch.Tensor,
+        vision_features: torch.Tensor,
+        vision_pos_encoding: torch.Tensor,
+        text_cross_attn_mask: torch.Tensor | None = None,
+        vision_cross_attn_mask: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> torch.Tensor:
+        """
+        Forward pass for decoder layer.
+
+        Args:
+            hidden_states: Query features [batch_size, num_queries + 1, hidden_size] (includes presence token at position 0)
+            query_pos: Query position embeddings [batch_size, num_queries, hidden_size]
+            text_features: Text features [batch_size, seq_len, hidden_size]
+            vision_features: Vision features [batch_size, height*width, hidden_size]
+            vision_pos_encoding: Vision position encoding [batch_size, height*width, hidden_size]
+            text_cross_attn_mask: Text cross-attention mask
+            vision_cross_attn_mask: Vision cross-attention mask, already expanded for presence token
+
+        Returns:
+            Updated hidden states (including presence token at position 0)
+        """
+        # Prepend zeros to query_pos for presence token
+        query_pos = F.pad(query_pos, (0, 0, 1, 0), mode="constant", value=0)
+
+        # Self-attention with query position encoding
+        residual = hidden_states
+        query_with_pos = hidden_states + query_pos
+        attn_output, _ = self.self_attn(
+            query=query_with_pos,
+            key=query_with_pos,
+            value=hidden_states,
+            attention_mask=None,
+            **kwargs,
+        )
+        hidden_states = residual + self.self_attn_dropout(attn_output)
+        hidden_states = self.self_attn_layer_norm(hidden_states)
+
+        # Text cross-attention: queries attend to text features
+        residual = hidden_states
+        query_with_pos = hidden_states + query_pos
+
+        attn_output, _ = self.text_cross_attn(
+            query=query_with_pos,
+            key=text_features,
+            value=text_features,
+            attention_mask=text_cross_attn_mask,
+            **kwargs,
+        )
+        hidden_states = residual + self.text_cross_attn_dropout(attn_output)
+        hidden_states = self.text_cross_attn_layer_norm(hidden_states)
+
+        # Vision cross-attention: queries attend to vision features (with RPB)
+        residual = hidden_states
+        query_with_pos = hidden_states + query_pos
+        key_with_pos = vision_features + vision_pos_encoding
+        attn_output, _ = self.vision_cross_attn(
+            query=query_with_pos,
+            key=key_with_pos,
+            value=vision_features,
+            attention_mask=vision_cross_attn_mask,
+            **kwargs,
+        )
+        hidden_states = residual + self.vision_cross_attn_dropout(attn_output)
+        hidden_states = self.vision_cross_attn_layer_norm(hidden_states)
+
+        # MLP
+        residual = hidden_states
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + self.mlp_dropout(hidden_states)
+        hidden_states = self.mlp_layer_norm(hidden_states)
+
+        return hidden_states
+
+
+class Sam3DetrDecoder(Sam3PreTrainedModel):
+    """
+    DETR-style decoder with box refinement and presence token.
+
+    Simplified version that assumes:
+    - Box refinement is always enabled
+    - Intermediate outputs are always returned
+    - BoxRPB (relative position bias) with log-scale encoding
+    - Presence token is used
+    """
+
+    _can_record_outputs = {
+        "hidden_states": Sam3DetrDecoderLayer,
+        "attentions": Sam3Attention,
+    }
+
+    def __init__(
+        self,
+        config: Sam3DETRDecoderConfig,
+    ):
+        super().__init__(config)
+        self.config = config
+        self.hidden_size = config.hidden_size
+
+        self.layers = nn.ModuleList([Sam3DetrDecoderLayer(config) for _ in range(config.num_layers)])
+
+        self.output_layer_norm = nn.LayerNorm(config.hidden_size)
+
+        self.box_head = Sam3DecoderMLP(config.hidden_size, config.hidden_size, 4, 3)
+
+        self.query_embed = nn.Embedding(config.num_queries, config.hidden_size)
+        self.reference_points = nn.Embedding(config.num_queries, 4)
+
+        self.presence_token = nn.Embedding(1, config.hidden_size)
+        self.presence_head = Sam3DecoderMLP(config.hidden_size, config.hidden_size, 1, 3)
+        self.presence_layer_norm = nn.LayerNorm(config.hidden_size)
+        self.clamp_presence_logit_max_val = 10.0
+
+        self.ref_point_head = Sam3DecoderMLP(2 * config.hidden_size, config.hidden_size, config.hidden_size, 2)
+
+        self.box_rpb_embed_x = Sam3DecoderMLP(2, config.hidden_size, config.num_attention_heads, 2)
+        self.box_rpb_embed_y = Sam3DecoderMLP(2, config.hidden_size, config.num_attention_heads, 2)
+
+        self.position_encoding = Sam3SinePositionEmbedding(
+            num_position_features=config.hidden_size // 2, normalize=False
+        )
+
+        self.post_init()
+
+    @compile_compatible_method_lru_cache(maxsize=1)
+    def _get_coords(
+        self, height: torch.Tensor, width: torch.Tensor, dtype: torch.dtype, device: torch.device
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """Generate normalized coordinate grids."""
+        coords_h = torch.arange(0, height, device=device, dtype=dtype) / height
+        coords_w = torch.arange(0, width, device=device, dtype=dtype) / width
+        return coords_h, coords_w
+
+    def _get_rpb_matrix(
+        self, reference_boxes: torch.Tensor, spatial_shape: tuple[torch.Tensor, torch.Tensor]
+    ) -> torch.Tensor:
+        """
+        Compute box relative position bias (RPB) matrix using log-scale encoding.
+        RPB helps the decoder attend to relevant spatial locations based on predicted box positions.
+
+        Args:
+            reference_boxes: Reference boxes [batch_size, num_queries, 4] in sigmoid space
+            spatial_shape: (height, width) of the vision features as tensors
+
+        Returns:
+            RPB matrix [batch_size, num_heads, num_queries, height*width]
+        """
+        height, width = spatial_shape
+        boxes_xyxy = box_cxcywh_to_xyxy(reference_boxes)
+        batch_size, num_queries, _ = boxes_xyxy.shape
+
+        # Generate coordinate grids
+        coords_h, coords_w = self._get_coords(
+            height, width, dtype=reference_boxes.dtype, device=reference_boxes.device
+        )
+
+        # Compute deltas between coordinates and box boundaries
+        deltas_y = coords_h.view(1, -1, 1) - boxes_xyxy.reshape(-1, 1, 4)[:, :, 1:4:2]
+        deltas_y = deltas_y.view(batch_size, num_queries, -1, 2)
+        deltas_x = coords_w.view(1, -1, 1) - boxes_xyxy.reshape(-1, 1, 4)[:, :, 0:3:2]
+        deltas_x = deltas_x.view(batch_size, num_queries, -1, 2)
+
+        # Apply log-scale encoding
+        deltas_x_log = deltas_x * 8
+        deltas_x_log = torch.sign(deltas_x_log) * torch.log2(torch.abs(deltas_x_log) + 1.0) / math.log2(8)
+        deltas_y_log = deltas_y * 8
+        deltas_y_log = torch.sign(deltas_y_log) * torch.log2(torch.abs(deltas_y_log) + 1.0) / math.log2(8)
+
+        # Embed deltas
+        deltas_x = self.box_rpb_embed_x(deltas_x_log)  # [batch_size, num_queries, width, num_heads]
+        deltas_y = self.box_rpb_embed_y(deltas_y_log)  # [batch_size, num_queries, height, num_heads]
+
+        # Combine into 2D bias matrix
+        rpb_matrix = deltas_y.unsqueeze(3) + deltas_x.unsqueeze(
+            2
+        )  # [batch_size, num_queries, height, width, num_heads]
+        rpb_matrix = rpb_matrix.flatten(2, 3)  # [batch_size, num_queries, height*width, num_heads]
+        rpb_matrix = rpb_matrix.permute(0, 3, 1, 2).contiguous()  # [batch_size, num_heads, num_queries, height*width]
+        return rpb_matrix
+
+    @merge_with_config_defaults
+    @capture_outputs
+    def forward(
+        self,
+        vision_features: torch.Tensor,
+        text_features: torch.Tensor,
+        vision_pos_encoding: torch.Tensor,
+        text_mask: torch.Tensor | None = None,
+        spatial_shapes: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | Sam3DETRDecoderOutput:
+        """
+        Forward pass for the DETR decoder.
+
+        Args:
+            vision_features: Vision features [batch_size, height*width, hidden_size]
+            text_features: Text features [batch_size, seq_len, hidden_size]
+            vision_pos_encoding: Vision position encoding [batch_size, height*width, hidden_size]
+            text_mask: Text padding mask [batch_size, seq_len] where True=valid, False=padding
+            spatial_shapes: Spatial shapes [num_levels, 2]
+
+        Returns:
+            Sam3DETRDecoderOutput containing decoder outputs from all layers.
+        """
+        batch_size = vision_features.shape[0]
+
+        query_embeds = self.query_embed.weight.unsqueeze(0).expand(batch_size, -1, -1)
+        reference_boxes = self.reference_points.weight.unsqueeze(0).expand(batch_size, -1, -1)
+        reference_boxes = reference_boxes.sigmoid()
+        presence_token = self.presence_token.weight.unsqueeze(0).expand(batch_size, -1, -1)
+
+        # Concatenate presence token with query embeddings
+        hidden_states = torch.cat([presence_token, query_embeds], dim=1)
+
+        text_cross_attn_mask = None
+        if text_mask is not None:
+            text_cross_attn_mask = create_bidirectional_mask(
+                config=self.config,
+                inputs_embeds=hidden_states,
+                attention_mask=text_mask,
+                encoder_hidden_states=text_features,
+            )
+
+        intermediate_outputs = []
+        intermediate_boxes = [reference_boxes]
+        intermediate_presence_logits = []
+
+        for layer in self.layers:
+            # Generate sine embeddings for conditional queries
+            reference_points_input = reference_boxes.unsqueeze(2)
+            query_sine_embed = self.position_encoding.encode_boxes(reference_points_input[:, :, 0, :])
+            query_pos = self.ref_point_head(query_sine_embed)
+
+            # Compute box relative position bias (RPB) attention mask
+            vision_cross_attn_mask = None
+            if spatial_shapes is not None and spatial_shapes.shape[0] == 1:
+                spatial_shape = (spatial_shapes[0, 0], spatial_shapes[0, 1])
+                rpb_matrix = self._get_rpb_matrix(reference_boxes, spatial_shape)
+                # Prepend zeros row for presence token (it attends to all vision tokens equally)
+                vision_cross_attn_mask = F.pad(rpb_matrix, (0, 0, 1, 0), mode="constant", value=0)
+
+            hidden_states = layer(
+                hidden_states,
+                query_pos=query_pos,
+                text_features=text_features,
+                vision_features=vision_features,
+                vision_pos_encoding=vision_pos_encoding,
+                text_cross_attn_mask=text_cross_attn_mask,
+                vision_cross_attn_mask=vision_cross_attn_mask,
+                **kwargs,
+            )
+
+            # Extract query hidden states (without presence token) for box refinement
+            query_hidden_states = hidden_states[:, 1:]
+
+            # Box refinement: predict delta and update reference boxes
+            reference_boxes_before_sigmoid = inverse_sigmoid(reference_boxes)
+            delta_boxes = self.box_head(self.output_layer_norm(query_hidden_states))
+            new_reference_boxes = (delta_boxes + reference_boxes_before_sigmoid).sigmoid()
+            reference_boxes = new_reference_boxes.detach()
+
+            intermediate_outputs.append(self.output_layer_norm(query_hidden_states))
+            intermediate_boxes.append(new_reference_boxes)
+
+            # Process presence token
+            presence_hidden = hidden_states[:, :1]
+            presence_logits = self.presence_head(self.presence_layer_norm(presence_hidden)).squeeze(-1)
+            presence_logits = presence_logits.clamp(
+                min=-self.clamp_presence_logit_max_val, max=self.clamp_presence_logit_max_val
+            )
+            intermediate_presence_logits.append(presence_logits)
+
+        # Stack outputs from all layers
+        intermediate_outputs = torch.stack(intermediate_outputs)
+        intermediate_boxes = torch.stack(intermediate_boxes[:-1])
+        intermediate_presence_logits = torch.stack(intermediate_presence_logits)
+
+        return Sam3DETRDecoderOutput(
+            intermediate_hidden_states=intermediate_outputs,
+            reference_boxes=intermediate_boxes,
+            presence_logits=intermediate_presence_logits,
+        )
+
+
+class Sam3DotProductScoring(nn.Module):
+    """
+    Computes classification scores by computing dot product between projected decoder queries and pooled text features.
+    This is used to determine confidence/presence scores for each query.
+    """
+
+    def __init__(self, config: Sam3Config):
+        super().__init__()
+        self.config = config
+        hidden_size = config.detr_decoder_config.hidden_size
+        projection_dim = config.detr_decoder_config.hidden_size
+
+        self.text_mlp = Sam3DecoderMLP(
+            input_dim=hidden_size,
+            hidden_dim=config.detr_decoder_config.intermediate_size,
+            output_dim=hidden_size,
+            num_layers=2,
+        )
+        self.text_mlp_dropout = nn.Dropout(config.detr_decoder_config.dropout)
+        self.text_mlp_out_norm = nn.LayerNorm(hidden_size)
+
+        # Projections for text and query features
+        self.text_proj = nn.Linear(hidden_size, projection_dim)
+        self.query_proj = nn.Linear(hidden_size, projection_dim)
+
+        # Scale factor for dot product
+        self.scale = float(1.0 / np.sqrt(projection_dim))
+
+        # Clamping to avoid numerical issues
+        self.clamp_logits = True
+        self.clamp_max_val = 12.0
+
+    def _pool_text_features(self, text_features: torch.Tensor, text_mask: torch.Tensor | None) -> torch.Tensor:
+        """
+        Mean pool text features, accounting for padding.
+
+        Args:
+            text_features: [batch_size, seq_len, hidden_size]
+            text_mask: [batch_size, seq_len] where True indicates valid tokens, False indicates padding
+
+        Returns:
+            pooled_text: [batch_size, hidden_size]
+        """
+        if text_mask is None:
+            # No padding, simple mean
+            return text_features.mean(dim=1)
+
+        is_valid = text_mask.to(text_features.dtype).unsqueeze(-1)  # [batch_size, seq_len, 1]
+
+        # Count valid tokens per batch
+        num_valid = is_valid.sum(dim=1).clamp(min=1.0)  # [batch_size, 1]
+
+        # Mean pool only over valid tokens
+        pooled_text = (text_features * is_valid).sum(dim=1) / num_valid  # [batch_size, hidden_size]
+
+        return pooled_text
+
+    def forward(
+        self,
+        decoder_hidden_states: torch.Tensor,
+        text_features: torch.Tensor,
+        text_mask: torch.Tensor | None = None,
+    ) -> torch.Tensor:
+        """
+        Compute classification scores via dot product.
+
+        Args:
+            decoder_hidden_states: [num_layers, batch_size, num_queries, hidden_size]
+            text_features: [batch_size, seq_len, hidden_size]
+            text_mask: [batch_size, seq_len] where True=valid, False=padding
+
+        Returns:
+            scores: [num_layers, batch_size, num_queries, 1]
+        """
+        orig_text_features = text_features
+        text_features = self.text_mlp(text_features)
+        text_features = self.text_mlp_dropout(text_features)
+        text_features = text_features + orig_text_features
+        text_features = self.text_mlp_out_norm(text_features)
+
+        pooled_text = self._pool_text_features(text_features, text_mask)
+
+        proj_text = self.text_proj(pooled_text)
+        proj_queries = self.query_proj(decoder_hidden_states)
+
+        proj_text = proj_text.unsqueeze(-1)
+        scores = torch.matmul(proj_queries, proj_text.unsqueeze(0))
+        scores = scores * self.scale
+        if self.clamp_logits:
+            scores = scores.clamp(min=-self.clamp_max_val, max=self.clamp_max_val)
+
+        return scores
+
+
+class Sam3MaskEmbedder(nn.Module):
+    """
+    MLP that embeds object queries for mask prediction.
+    Similar to MaskFormer's mask embedder.
+    """
+
+    def __init__(self, config: Sam3MaskDecoderConfig):
+        super().__init__()
+        self.config = config
+        hidden_size = config.hidden_size
+
+        self.layers = nn.ModuleList(
+            [
+                nn.Linear(hidden_size, hidden_size),
+                nn.Linear(hidden_size, hidden_size),
+                nn.Linear(hidden_size, hidden_size),
+            ]
+        )
+        self.activation = nn.ReLU()
+
+    def forward(self, queries: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            queries: Query embeddings [batch_size, num_queries, hidden_size]
+
+        Returns:
+            Mask embeddings [batch_size, num_queries, hidden_size]
+        """
+        hidden_states = queries
+        for i, layer in enumerate(self.layers):
+            hidden_states = layer(hidden_states)
+            if i < len(self.layers) - 1:
+                hidden_states = self.activation(hidden_states)
+        return hidden_states
+
+
+class Sam3PixelDecoder(nn.Module):
+    """
+    Feature Pyramid Network (FPN) decoder that generates pixel-level features.
+    Inspired by MaskFormer's pixel decoder.
+    """
+
+    def __init__(self, config: Sam3MaskDecoderConfig):
+        super().__init__()
+        self.config = config
+        hidden_size = config.hidden_size
+        num_upsampling_stages = config.num_upsampling_stages
+
+        # Create conv layers and norms for FPN
+        self.conv_layers = nn.ModuleList(
+            [
+                nn.Conv2d(hidden_size, hidden_size, kernel_size=3, stride=1, padding=1)
+                for _ in range(num_upsampling_stages)
+            ]
+        )
+        self.norms = nn.ModuleList([nn.GroupNorm(8, hidden_size) for _ in range(num_upsampling_stages)])
+
+        self.out_channels = hidden_size
+
+    def forward(self, backbone_features: list[torch.Tensor]) -> torch.Tensor:
+        """
+        Args:
+            backbone_features: List of backbone features [batch_size, hidden_size, H_i, W_i]
+                              from low to high resolution (assumes already projected to hidden_size)
+
+        Returns:
+            Pixel embeddings [batch_size, hidden_size, H, W] at the finest resolution
+        """
+        # Start from the coarsest feature (last in list)
+        prev_fpn = backbone_features[-1]
+        # Iterate through features from coarse to fine (excluding the last which we started with)
+        for layer_idx, backbone_feat in enumerate(reversed(backbone_features[:-1])):
+            # Upsample previous FPN output to match current backbone feature size
+            prev_fpn = F.interpolate(prev_fpn, size=backbone_feat.shape[-2:], mode="nearest")
+
+            # Add skip connection
+            prev_fpn = prev_fpn + backbone_feat
+
+            # Apply conv and norm
+            prev_fpn = self.conv_layers[layer_idx](prev_fpn)
+            prev_fpn = self.norms[layer_idx](prev_fpn)
+            prev_fpn = F.relu(prev_fpn)
+
+        return prev_fpn
+
+
+class Sam3MaskDecoder(Sam3PreTrainedModel):
+    """
+    Mask decoder that combines object queries with pixel-level features to predict instance masks.
+    Also produces a semantic segmentation output and supports cross-attention to prompts.
+    """
+
+    _can_record_outputs = {
+        "attentions": Sam3Attention,
+    }
+
+    def __init__(self, config: Sam3MaskDecoderConfig):
+        super().__init__(config)
+        self.config = config
+        hidden_size = config.hidden_size
+
+        # Pixel decoder (FPN)
+        self.pixel_decoder = Sam3PixelDecoder(config)
+
+        # Mask embedder (MLP to transform queries)
+        self.mask_embedder = Sam3MaskEmbedder(config)
+
+        # Projection from pixel decoder output to mask embedding space
+        self.instance_projection = nn.Conv2d(self.pixel_decoder.out_channels, hidden_size, kernel_size=1)
+
+        # Semantic segmentation head (always present in UniversalSegmentationHead)
+        self.semantic_projection = nn.Conv2d(self.pixel_decoder.out_channels, 1, kernel_size=1)
+
+        self.prompt_cross_attn = Sam3Attention(config)
+        self.prompt_cross_attn_norm = nn.LayerNorm(hidden_size)
+        self.prompt_cross_attn_dropout = nn.Dropout(config.dropout)
+
+        self.post_init()
+
+    @merge_with_config_defaults
+    @capture_outputs
+    def forward(
+        self,
+        decoder_queries: torch.Tensor,
+        backbone_features: list[torch.Tensor],
+        encoder_hidden_states: torch.Tensor,
+        prompt_features: torch.Tensor | None = None,
+        prompt_mask: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | Sam3MaskDecoderOutput:
+        """
+        Args:
+            decoder_queries: Decoder output queries [batch_size, num_queries, hidden_size]
+            backbone_features: List of backbone features to process through FPN
+            encoder_hidden_states: Encoder outputs [batch_size, seq_len, hidden_size]
+            prompt_features: Prompt features (text + geometry) for cross-attention [batch_size, prompt_len, hidden_size]
+            prompt_mask: Padding mask [batch_size, prompt_len] where True=valid, False=padding
+
+        Returns:
+            Sam3MaskDecoderOutput containing predicted masks and semantic segmentation.
+        """
+        if prompt_features is not None:
+            # Cross-attention: encoder features attend to prompt features
+            residual = encoder_hidden_states
+            normed_hidden_states = self.prompt_cross_attn_norm(encoder_hidden_states)
+
+            cross_attn_mask = None
+            if prompt_mask is not None:
+                cross_attn_mask = create_bidirectional_mask(
+                    config=self.config,
+                    inputs_embeds=normed_hidden_states,
+                    encoder_hidden_states=prompt_features,
+                    attention_mask=prompt_mask,
+                )
+
+            attn_output, _ = self.prompt_cross_attn(
+                query=normed_hidden_states,
+                key=prompt_features,
+                value=prompt_features,
+                attention_mask=cross_attn_mask,
+                **kwargs,
+            )
+            encoder_hidden_states = residual + self.prompt_cross_attn_dropout(attn_output)
+
+        # Process backbone features through FPN to get pixel embeddings
+        pixel_embed = self._embed_pixels(
+            backbone_features=backbone_features,
+            encoder_hidden_states=encoder_hidden_states,
+        )
+
+        # Predict instance masks via dot product between query embeddings and pixel embeddings
+        instance_embeds = self.instance_projection(pixel_embed)
+        mask_embeddings = self.mask_embedder(decoder_queries)
+        pred_masks = torch.einsum("bqc,bchw->bqhw", mask_embeddings, instance_embeds)
+
+        # Generate semantic segmentation
+        semantic_seg = self.semantic_projection(pixel_embed)
+
+        return Sam3MaskDecoderOutput(
+            pred_masks=pred_masks,
+            semantic_seg=semantic_seg,
+        )
+
+    def _embed_pixels(
+        self,
+        backbone_features: list[torch.Tensor],
+        encoder_hidden_states: torch.Tensor,
+    ) -> torch.Tensor:
+        """
+        Embed pixels by combining backbone FPN features with encoder vision features.
+        The encoder vision features replace the finest-resolution backbone feature.
+
+        Args:
+            backbone_features: List of backbone features [batch_size, C, H_i, W_i]
+            encoder_hidden_states: Encoder outputs [batch_size, seq_len, hidden_size]
+
+        Returns:
+            Pixel embeddings [batch_size, hidden_size, H, W]
+        """
+        backbone_visual_feats = [feat.clone() for feat in backbone_features]
+
+        # Extract vision features from encoder output and reshape to spatial format
+        spatial_dim = backbone_features[-1].shape[-2] * backbone_features[-1].shape[-1]
+        encoder_visual_embed = encoder_hidden_states[:, :spatial_dim, :]
+        batch_size, _, hidden_size = encoder_visual_embed.shape
+        height, width = backbone_features[-1].shape[-2:]
+        encoder_visual_embed = encoder_visual_embed.transpose(1, 2).reshape(batch_size, hidden_size, height, width)
+
+        # Replace finest backbone feature with encoder vision features
+        backbone_visual_feats[-1] = encoder_visual_embed
+
+        # Process through FPN decoder
+        pixel_embed = self.pixel_decoder(backbone_visual_feats)
+
+        return pixel_embed
+
+
+class Sam3Model(Sam3PreTrainedModel):
+    input_modalities = ["image", "text"]
+    base_model_prefix = "detector_model"
+    _keys_to_ignore_on_load_unexpected = [
+        r"^tracker_model.",
+        r"^tracker_neck.",
+    ]
+
+    def __init__(self, config: Sam3Config):
+        # loading from a sam3_video config
+        if hasattr(config, "detector_config") and config.detector_config is not None:
+            detector_config = config.detector_config
+            if isinstance(detector_config, dict):
+                detector_config = Sam3Config(**detector_config)
+            config = detector_config
+        super().__init__(config)
+        self.vision_encoder = Sam3VisionModel(config.vision_config)
+        self.text_encoder = CLIPTextModelWithProjection(config.text_config)
+        self.vocab_size = config.text_config.vocab_size
+
+        # Project text features from text encoder hidden size to model hidden size
+        # CLIP text encoder outputs 1024-dim features, but we need 256-dim for DETR
+        self.text_projection = nn.Linear(config.text_config.hidden_size, config.detr_encoder_config.hidden_size)
+
+        # Pass _attn_implementation to subconfigs BEFORE creating modules
+        config.geometry_encoder_config._attn_implementation = config._attn_implementation
+        config.detr_encoder_config._attn_implementation = config._attn_implementation
+        config.detr_decoder_config._attn_implementation = config._attn_implementation
+        config.mask_decoder_config._attn_implementation = config._attn_implementation
+
+        self.geometry_encoder = Sam3GeometryEncoder(config.geometry_encoder_config)
+        self.detr_encoder = Sam3DetrEncoder(config.detr_encoder_config)
+        self.detr_decoder = Sam3DetrDecoder(config.detr_decoder_config)
+        self.mask_decoder = Sam3MaskDecoder(config.mask_decoder_config)
+
+        # Dot product scoring to compute classification scores
+        self.dot_product_scoring = Sam3DotProductScoring(config)
+
+        self.post_init()
+
+    @can_return_tuple
+    @auto_docstring
+    def get_text_features(
+        self,
+        input_ids: torch.LongTensor,
+        attention_mask: torch.Tensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithPooling:
+        r"""
+        Example:
+
+        ```python
+        >>> from transformers import Sam3Model, Sam3Processor
+        >>> from PIL import Image
+        >>> import httpx
+        >>> from io import BytesIO
+
+        >>> model = Sam3Model.from_pretrained("facebook/sam3")
+        >>> processor = Sam3Processor.from_pretrained("facebook/sam3")
+
+        >>> # Pre-compute text embeddings
+        >>> text_inputs = processor(text="cat", return_tensors="pt")
+        >>> text_embeds = model.get_text_features(**text_inputs).pooler_output
+
+        >>> # Reuse text embeddings for multiple images
+        >>> url = "http://images.cocodataset.org/val2017/000000077595.jpg"
+        >>> with httpx.stream("GET", url) as response:
+        ...     image = Image.open(BytesIO(response.read()))
+        >>> img_inputs = processor(images=image, return_tensors="pt")
+        >>> outputs = model(pixel_values=img_inputs.pixel_values, text_embeds=text_embeds)
+        ```
+        """
+        text_outputs = self.text_encoder(
+            input_ids=input_ids, attention_mask=attention_mask, return_dict=True, **kwargs
+        )
+        last_hidden_state = text_outputs.last_hidden_state
+        text_outputs.pooler_output = self.text_projection(last_hidden_state)
+
+        return text_outputs
+
+    @auto_docstring
+    def get_vision_features(
+        self,
+        pixel_values: torch.FloatTensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> Sam3VisionEncoderOutput:
+        r"""
+        Example:
+
+        ```python
+        >>> from transformers import Sam3Model, Sam3Processor
+        >>> from PIL import Image
+        >>> import httpx
+        >>> from io import BytesIO
+
+        >>> model = Sam3Model.from_pretrained("facebook/sam3")
+        >>> processor = Sam3Processor.from_pretrained("facebook/sam3")
+
+        >>> # Pre-compute vision embeddings
+        >>> url = "http://images.cocodataset.org/val2017/000000077595.jpg"
+        >>> with httpx.stream("GET", url) as response:
+        ...     image = Image.open(BytesIO(response.read()))
+        >>> img_inputs = processor(images=image, return_tensors="pt")
+        >>> vision_embeds = model.get_vision_features(pixel_values=img_inputs.pixel_values)
+
+        >>> # Reuse vision embeddings for multiple text prompts
+        >>> text_inputs = processor(text="cat", return_tensors="pt")
+        >>> outputs = model(vision_embeds=vision_embeds, input_ids=text_inputs.input_ids)
+        ```
+        """
+        vision_outputs = self.vision_encoder(pixel_values, **kwargs)
+        return vision_outputs
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor | None = None,
+        vision_embeds: Sam3VisionEncoderOutput | None = None,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        text_embeds: torch.FloatTensor | None = None,
+        input_boxes: torch.FloatTensor | None = None,
+        input_boxes_labels: torch.LongTensor | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> Sam3ImageSegmentationOutput:
+        r"""
+        vision_embeds (`Sam3VisionEncoderOutput`, *optional*):
+            Pre-computed vision embeddings. Can be used to easily reuse vision embeddings. If provided, `pixel_values`
+            should not be passed. Mutually exclusive with `pixel_values`.
+        text_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Pre-computed text embeddings. Can be used to easily reuse text embeddings. If provided, `input_ids`
+            should not be passed. Mutually exclusive with `input_ids`.
+        input_boxes (`torch.FloatTensor` of shape `(batch_size, num_boxes, 4)`, *optional*):
+            Normalized box coordinates in [0, 1] range, in (cx, cy, w, h) format.
+        input_boxes_labels (`torch.LongTensor` of shape `(batch_size, num_boxes)`, *optional*):
+            Labels for boxes: 1 (positive), 0 (negative).
+
+        Example:
+
+        ```python
+        >>> from PIL import Image
+        >>> import httpx
+        >>> from io import BytesIO
+        >>> from transformers import AutoModel, AutoProcessor
+
+        >>> model = AutoModel.from_pretrained("facebook/sam3")
+        >>> processor = AutoProcessor.from_pretrained("facebook/sam3")
+
+        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/sam-car.png"
+        >>> with httpx.stream("GET", url) as response:
+        ...     image = Image.open(BytesIO(response.read())).convert("RGB")
+        >>> text = "car"
+        >>> inputs = processor(images=image, text=text, return_tensors="pt")
+
+        >>> # Get segmentation output
+        >>> outputs = model(**inputs)
+        >>> pred_masks = outputs.pred_masks
+        >>> pred_boxes = outputs.pred_boxes
+        ```
+        """
+        if (pixel_values is None) == (vision_embeds is None):
+            raise ValueError("You must specify exactly one of pixel_values or vision_embeds")
+
+        if (input_ids is None) == (text_embeds is None):
+            raise ValueError("You must specify exactly one of input_ids or text_embeds")
+
+        if pixel_values is not None:
+            batch_size = pixel_values.shape[0]
+            device = pixel_values.device
+        else:
+            batch_size = vision_embeds.fpn_hidden_states[0].shape[0]
+            device = vision_embeds.fpn_hidden_states[0].device
+
+        if vision_embeds is None:
+            vision_outputs = self.vision_encoder(pixel_values, **kwargs)
+        else:
+            vision_outputs = vision_embeds
+
+        fpn_hidden_states = vision_outputs.fpn_hidden_states[:-1]
+        fpn_position_encoding = vision_outputs.fpn_position_encoding[:-1]
+
+        if text_embeds is None:
+            text_embeds = self.get_text_features(input_ids=input_ids, attention_mask=attention_mask, return_dict=True)
+
+        text_features = text_embeds.pooler_output
+        text_mask = attention_mask.bool() if attention_mask is not None else None
+        has_geometry_prompts = input_boxes is not None and input_boxes.numel() > 0
+
+        geometry_prompt_features = None
+        geometry_prompt_mask = None
+
+        if has_geometry_prompts:
+            if input_boxes is not None and input_boxes.numel() > 0:
+                box_embeddings = input_boxes  # [batch_size, num_boxes, 4]
+                box_labels = (
+                    input_boxes_labels
+                    if input_boxes_labels is not None
+                    else torch.ones_like(box_embeddings[..., 0], dtype=torch.long)
+                )
+                box_mask = (
+                    (input_boxes_labels != -10)
+                    if input_boxes_labels is not None
+                    else torch.ones(batch_size, input_boxes.shape[1], dtype=torch.bool, device=device)
+                )
+                box_labels = torch.where(box_labels == -10, 0, box_labels)
+            else:
+                box_embeddings = torch.zeros(batch_size, 0, 4, dtype=text_features.dtype, device=device)
+                box_labels = torch.zeros(batch_size, 0, dtype=torch.long, device=device)
+                box_mask = torch.zeros(batch_size, 0, dtype=torch.bool, device=device)
+
+            geometry_outputs = self.geometry_encoder(
+                box_embeddings=box_embeddings,
+                box_mask=box_mask,
+                box_labels=box_labels,
+                img_feats=fpn_hidden_states,
+                img_pos_embeds=fpn_position_encoding,
+            )
+
+            geometry_prompt_features = geometry_outputs.last_hidden_state
+            geometry_prompt_mask = geometry_outputs.attention_mask
+
+        if geometry_prompt_features is not None:
+            # Repeat text_features for all geometry prompts
+            if text_features.shape[0] == 1 and geometry_prompt_features.shape[0] > 1:
+                text_features = text_features.repeat(geometry_prompt_features.shape[0], 1, 1)
+            combined_prompt_features = torch.cat([text_features, geometry_prompt_features], dim=1)
+            if text_mask is not None and text_mask.shape[0] == 1 and geometry_prompt_mask.shape[0] > 1:
+                text_mask = text_mask.repeat(geometry_prompt_mask.shape[0], 1)
+
+            if text_mask is not None and geometry_prompt_mask is not None:
+                combined_prompt_mask = torch.cat([text_mask, geometry_prompt_mask], dim=1)
+            elif text_mask is not None:
+                geo_valid_mask = torch.ones(
+                    batch_size, geometry_prompt_features.shape[1], dtype=torch.bool, device=device
+                )
+                combined_prompt_mask = torch.cat([text_mask, geo_valid_mask], dim=1)
+            elif geometry_prompt_mask is not None:
+                text_valid_mask = torch.ones(batch_size, text_features.shape[1], dtype=torch.bool, device=device)
+                combined_prompt_mask = torch.cat([text_valid_mask, geometry_prompt_mask], dim=1)
+            else:
+                combined_prompt_mask = None
+        else:
+            combined_prompt_features = text_features
+            combined_prompt_mask = text_mask
+
+        encoder_outputs = self.detr_encoder(
+            vision_features=[fpn_hidden_states[-1]],
+            text_features=combined_prompt_features,
+            vision_pos_embeds=[fpn_position_encoding[-1]],
+            text_mask=combined_prompt_mask,
+            **kwargs,
+        )
+
+        decoder_outputs = self.detr_decoder(
+            vision_features=encoder_outputs.last_hidden_state,
+            text_features=encoder_outputs.text_features,
+            vision_pos_encoding=encoder_outputs.pos_embeds_flattened,
+            text_mask=combined_prompt_mask,
+            spatial_shapes=encoder_outputs.spatial_shapes,
+            **kwargs,
+        )
+
+        # Refine boxes from decoder
+        all_box_offsets = self.detr_decoder.box_head(decoder_outputs.intermediate_hidden_states)
+        reference_boxes_inv_sig = inverse_sigmoid(decoder_outputs.reference_boxes)
+        all_pred_boxes_cxcywh = (reference_boxes_inv_sig + all_box_offsets).sigmoid()
+        all_pred_boxes = box_cxcywh_to_xyxy(all_pred_boxes_cxcywh)
+
+        all_pred_logits = self.dot_product_scoring(
+            decoder_hidden_states=decoder_outputs.intermediate_hidden_states,
+            text_features=encoder_outputs.text_features,
+            text_mask=combined_prompt_mask,
+        ).squeeze(-1)
+
+        pred_logits = all_pred_logits[-1]
+        pred_boxes = all_pred_boxes[-1]
+        decoder_hidden_states = decoder_outputs.intermediate_hidden_states[-1]
+        presence_logits = decoder_outputs.presence_logits[-1]
+
+        mask_outputs = self.mask_decoder(
+            decoder_queries=decoder_hidden_states,
+            backbone_features=list(fpn_hidden_states),
+            encoder_hidden_states=encoder_outputs.last_hidden_state,
+            prompt_features=combined_prompt_features,
+            prompt_mask=combined_prompt_mask,
+            **kwargs,
+        )
+
+        return Sam3ImageSegmentationOutput(
+            pred_masks=mask_outputs.pred_masks,
+            pred_boxes=pred_boxes,
+            pred_logits=pred_logits,
+            presence_logits=presence_logits,
+            semantic_seg=mask_outputs.semantic_seg,
+            decoder_hidden_states=decoder_outputs.hidden_states,
+            decoder_reference_boxes=decoder_outputs.reference_boxes,
+            encoder_hidden_states=encoder_outputs.hidden_states,
+            vision_hidden_states=vision_outputs.hidden_states,
+            vision_attentions=vision_outputs.attentions,
+            detr_encoder_attentions=encoder_outputs.attentions,
+            detr_decoder_attentions=decoder_outputs.attentions,
+            mask_decoder_attentions=mask_outputs.attentions,
+        )
+
+
+__all__ = ["Sam3Model", "Sam3VisionModel", "Sam3ViTModel", "Sam3PreTrainedModel"]