Hugging Face's logo

SSBteam
/
lta

Model card Files
xet
 Community
JinghuiLuAstronaut commited on
Commit
10a07d8
·
verified ·
1 Parent(s): cf00151

Add files using upload-large-folder tool

Browse files
Files changed (20) hide show
  1. LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0018000_logistic_normal_t1p45.log +76 -0
  2. LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0025000_logistic_normal_t1p45.log +76 -0
  3. LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0032000_logistic_normal_t1p45.log +76 -0
  4. LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0039000_logistic_normal_t1p45.log +76 -0
  5. LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0049000_logistic_normal_t1p45.log +76 -0
  6. LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0077000_logistic_normal_t1p45.log +76 -0
  7. LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0103000_logistic_normal_t1p45.log +76 -0
  8. LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0108000_logistic_normal_t1p45.log +76 -0
  9. LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0124000_logistic_normal_t1p45.log +76 -0
  10. LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_bernoulliwrong_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_193815.log +0 -0
  11. LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_bernoulliwrong_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260527_063225.log +167 -0
  12. LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_full_gbs512_8gpu_20260526_155049.log +544 -0
  13. LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_163925.log +597 -0
  14. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/configuration_data2vec_text.py +65 -0
  15. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_audio.py +1324 -0
  16. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_text.py +1208 -0
  17. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_vision.py +1214 -0
  18. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modular_data2vec_audio.py +272 -0
  19. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modular_data2vec_text.py +599 -0
  20. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/sam3/modeling_sam3.py +0 -0
LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0018000_logistic_normal_t1p45.log ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [watch-lognormal-sde] 2026-05-23_00:14:28 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0018000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0018000
2
+ [load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0018000.pt
3
+ [ckpt] step=18000
4
+ [sde] generated 16/256
5
+ [sde] generated 32/256
6
+ [sde] generated 48/256
7
+ [sde] generated 64/256
8
+ [sde] generated 80/256
9
+ [sde] generated 96/256
10
+ [sde] generated 112/256
11
+ [sde] generated 128/256
12
+ [sde] generated 144/256
13
+ [sde] generated 160/256
14
+ [sde] generated 176/256
15
+ [sde] generated 192/256
16
+ [sde] generated 208/256
17
+ [sde] generated 224/256
18
+ [sde] generated 240/256
19
+ [sde] generated 256/256
20
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
21
+ [summary] {
22
+ "type": "summary",
23
+ "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0018000.pt",
24
+ "step": 18000,
25
+ "decode": {
26
+ "decode_rule": "logistic_normal_resample_sde",
27
+ "steps": 128,
28
+ "model_t_mode": "const0.5",
29
+ "mean_mode": "anchor_semantic",
30
+ "endpoint_floor": 0.0,
31
+ "concentration_min": 1.0,
32
+ "concentration_max": 1024.0,
33
+ "endpoint_temp": 1.45,
34
+ "support_power": 1.0,
35
+ "semantic_power": 1.0,
36
+ "noise_init": "logistic_normal",
37
+ "noise_sigma": 3.0,
38
+ "noise_dirichlet_concentration": 1.0,
39
+ "sde_resample": "logistic_normal",
40
+ "logistic_normal_sigma_min": 0.18,
41
+ "logistic_normal_sigma_max": 3.0,
42
+ "logistic_normal_tau_min": 0.65,
43
+ "logistic_normal_tau_max": 1.0,
44
+ "final_from": "blend_0.5",
45
+ "n_samples": 256,
46
+ "seed": 20260522
47
+ },
48
+ "raw_genppl": {
49
+ "ppl": 37.642507957083765,
50
+ "nll_per_token": 3.628133942600674,
51
+ "tokens": 30091,
52
+ "kept_samples": 256,
53
+ "total_samples": 256,
54
+ "empty_rate": 0.0,
55
+ "skipped_samples": 0
56
+ },
57
+ "stripped_genppl": {
58
+ "ppl": 53.23995326170723,
59
+ "nll_per_token": 3.9748091156472474,
60
+ "tokens": 24722,
61
+ "kept_samples": 256,
62
+ "total_samples": 256,
63
+ "empty_rate": 0.0,
64
+ "skipped_samples": 0
65
+ },
66
+ "diversity": {
67
+ "sample_entropy": 3.1327908839169742,
68
+ "unique_tokens": 1552,
69
+ "token_count": 32768,
70
+ "distinct_1": 0.04736328125,
71
+ "distinct_2": 0.24351008858267717,
72
+ "top_token_mass": 0.271881103515625
73
+ }
74
+ }
75
+ [done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0018000/sde_steps128_samples256_scored.jsonl
76
+ [watch-lognormal-sde] 2026-05-23_00:15:56 done step_0018000
LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0025000_logistic_normal_t1p45.log ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [watch-lognormal-sde] 2026-05-23_00:53:13 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0025000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0025000
2
+ [load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0025000.pt
3
+ [ckpt] step=25000
4
+ [sde] generated 16/256
5
+ [sde] generated 32/256
6
+ [sde] generated 48/256
7
+ [sde] generated 64/256
8
+ [sde] generated 80/256
9
+ [sde] generated 96/256
10
+ [sde] generated 112/256
11
+ [sde] generated 128/256
12
+ [sde] generated 144/256
13
+ [sde] generated 160/256
14
+ [sde] generated 176/256
15
+ [sde] generated 192/256
16
+ [sde] generated 208/256
17
+ [sde] generated 224/256
18
+ [sde] generated 240/256
19
+ [sde] generated 256/256
20
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
21
+ [summary] {
22
+ "type": "summary",
23
+ "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0025000.pt",
24
+ "step": 25000,
25
+ "decode": {
26
+ "decode_rule": "logistic_normal_resample_sde",
27
+ "steps": 128,
28
+ "model_t_mode": "const0.5",
29
+ "mean_mode": "anchor_semantic",
30
+ "endpoint_floor": 0.0,
31
+ "concentration_min": 1.0,
32
+ "concentration_max": 1024.0,
33
+ "endpoint_temp": 1.45,
34
+ "support_power": 1.0,
35
+ "semantic_power": 1.0,
36
+ "noise_init": "logistic_normal",
37
+ "noise_sigma": 3.0,
38
+ "noise_dirichlet_concentration": 1.0,
39
+ "sde_resample": "logistic_normal",
40
+ "logistic_normal_sigma_min": 0.18,
41
+ "logistic_normal_sigma_max": 3.0,
42
+ "logistic_normal_tau_min": 0.65,
43
+ "logistic_normal_tau_max": 1.0,
44
+ "final_from": "blend_0.5",
45
+ "n_samples": 256,
46
+ "seed": 20260522
47
+ },
48
+ "raw_genppl": {
49
+ "ppl": 31.231424512837485,
50
+ "nll_per_token": 3.441424783858224,
51
+ "tokens": 37226,
52
+ "kept_samples": 256,
53
+ "total_samples": 256,
54
+ "empty_rate": 0.0,
55
+ "skipped_samples": 0
56
+ },
57
+ "stripped_genppl": {
58
+ "ppl": 43.50436377481887,
59
+ "nll_per_token": 3.772861249725761,
60
+ "tokens": 30767,
61
+ "kept_samples": 256,
62
+ "total_samples": 256,
63
+ "empty_rate": 0.0,
64
+ "skipped_samples": 0
65
+ },
66
+ "diversity": {
67
+ "sample_entropy": 3.79152059307615,
68
+ "unique_tokens": 1705,
69
+ "token_count": 32768,
70
+ "distinct_1": 0.052032470703125,
71
+ "distinct_2": 0.2902005413385827,
72
+ "top_token_mass": 0.07025146484375
73
+ }
74
+ }
75
+ [done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0025000/sde_steps128_samples256_scored.jsonl
76
+ [watch-lognormal-sde] 2026-05-23_00:54:41 done step_0025000
LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0032000_logistic_normal_t1p45.log ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [watch-lognormal-sde] 2026-05-23_01:32:43 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0032000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0032000
2
+ [load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0032000.pt
3
+ [ckpt] step=32000
4
+ [sde] generated 16/256
5
+ [sde] generated 32/256
6
+ [sde] generated 48/256
7
+ [sde] generated 64/256
8
+ [sde] generated 80/256
9
+ [sde] generated 96/256
10
+ [sde] generated 112/256
11
+ [sde] generated 128/256
12
+ [sde] generated 144/256
13
+ [sde] generated 160/256
14
+ [sde] generated 176/256
15
+ [sde] generated 192/256
16
+ [sde] generated 208/256
17
+ [sde] generated 224/256
18
+ [sde] generated 240/256
19
+ [sde] generated 256/256
20
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
21
+ [summary] {
22
+ "type": "summary",
23
+ "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0032000.pt",
24
+ "step": 32000,
25
+ "decode": {
26
+ "decode_rule": "logistic_normal_resample_sde",
27
+ "steps": 128,
28
+ "model_t_mode": "const0.5",
29
+ "mean_mode": "anchor_semantic",
30
+ "endpoint_floor": 0.0,
31
+ "concentration_min": 1.0,
32
+ "concentration_max": 1024.0,
33
+ "endpoint_temp": 1.45,
34
+ "support_power": 1.0,
35
+ "semantic_power": 1.0,
36
+ "noise_init": "logistic_normal",
37
+ "noise_sigma": 3.0,
38
+ "noise_dirichlet_concentration": 1.0,
39
+ "sde_resample": "logistic_normal",
40
+ "logistic_normal_sigma_min": 0.18,
41
+ "logistic_normal_sigma_max": 3.0,
42
+ "logistic_normal_tau_min": 0.65,
43
+ "logistic_normal_tau_max": 1.0,
44
+ "final_from": "blend_0.5",
45
+ "n_samples": 256,
46
+ "seed": 20260522
47
+ },
48
+ "raw_genppl": {
49
+ "ppl": 33.014796153697105,
50
+ "nll_per_token": 3.496955829273304,
51
+ "tokens": 36426,
52
+ "kept_samples": 256,
53
+ "total_samples": 256,
54
+ "empty_rate": 0.0,
55
+ "skipped_samples": 0
56
+ },
57
+ "stripped_genppl": {
58
+ "ppl": 46.797879118519774,
59
+ "nll_per_token": 3.8458378839163636,
60
+ "tokens": 29987,
61
+ "kept_samples": 256,
62
+ "total_samples": 256,
63
+ "empty_rate": 0.0,
64
+ "skipped_samples": 0
65
+ },
66
+ "diversity": {
67
+ "sample_entropy": 3.7235357273924485,
68
+ "unique_tokens": 2030,
69
+ "token_count": 32768,
70
+ "distinct_1": 0.06195068359375,
71
+ "distinct_2": 0.3199741633858268,
72
+ "top_token_mass": 0.106903076171875
73
+ }
74
+ }
75
+ [done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0032000/sde_steps128_samples256_scored.jsonl
76
+ [watch-lognormal-sde] 2026-05-23_01:34:10 done step_0032000
LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0039000_logistic_normal_t1p45.log ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [watch-lognormal-sde] 2026-05-23_02:11:30 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0039000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0039000
2
+ [load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0039000.pt
3
+ [ckpt] step=39000
4
+ [sde] generated 16/256
5
+ [sde] generated 32/256
6
+ [sde] generated 48/256
7
+ [sde] generated 64/256
8
+ [sde] generated 80/256
9
+ [sde] generated 96/256
10
+ [sde] generated 112/256
11
+ [sde] generated 128/256
12
+ [sde] generated 144/256
13
+ [sde] generated 160/256
14
+ [sde] generated 176/256
15
+ [sde] generated 192/256
16
+ [sde] generated 208/256
17
+ [sde] generated 224/256
18
+ [sde] generated 240/256
19
+ [sde] generated 256/256
20
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
21
+ [summary] {
22
+ "type": "summary",
23
+ "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0039000.pt",
24
+ "step": 39000,
25
+ "decode": {
26
+ "decode_rule": "logistic_normal_resample_sde",
27
+ "steps": 128,
28
+ "model_t_mode": "const0.5",
29
+ "mean_mode": "anchor_semantic",
30
+ "endpoint_floor": 0.0,
31
+ "concentration_min": 1.0,
32
+ "concentration_max": 1024.0,
33
+ "endpoint_temp": 1.45,
34
+ "support_power": 1.0,
35
+ "semantic_power": 1.0,
36
+ "noise_init": "logistic_normal",
37
+ "noise_sigma": 3.0,
38
+ "noise_dirichlet_concentration": 1.0,
39
+ "sde_resample": "logistic_normal",
40
+ "logistic_normal_sigma_min": 0.18,
41
+ "logistic_normal_sigma_max": 3.0,
42
+ "logistic_normal_tau_min": 0.65,
43
+ "logistic_normal_tau_max": 1.0,
44
+ "final_from": "blend_0.5",
45
+ "n_samples": 256,
46
+ "seed": 20260522
47
+ },
48
+ "raw_genppl": {
49
+ "ppl": 33.76559505090009,
50
+ "nll_per_token": 3.519442386176717,
51
+ "tokens": 32221,
52
+ "kept_samples": 256,
53
+ "total_samples": 256,
54
+ "empty_rate": 0.0,
55
+ "skipped_samples": 0
56
+ },
57
+ "stripped_genppl": {
58
+ "ppl": 46.10090126017552,
59
+ "nll_per_token": 3.830832499923769,
60
+ "tokens": 26646,
61
+ "kept_samples": 256,
62
+ "total_samples": 256,
63
+ "empty_rate": 0.0,
64
+ "skipped_samples": 0
65
+ },
66
+ "diversity": {
67
+ "sample_entropy": 3.309203124839359,
68
+ "unique_tokens": 1626,
69
+ "token_count": 32768,
70
+ "distinct_1": 0.04962158203125,
71
+ "distinct_2": 0.2572896161417323,
72
+ "top_token_mass": 0.211181640625
73
+ }
74
+ }
75
+ [done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0039000/sde_steps128_samples256_scored.jsonl
76
+ [watch-lognormal-sde] 2026-05-23_02:12:57 done step_0039000
LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0049000_logistic_normal_t1p45.log ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [watch-lognormal-sde] 2026-05-23_03:07:26 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0049000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0049000
2
+ [load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0049000.pt
3
+ [ckpt] step=49000
4
+ [sde] generated 16/256
5
+ [sde] generated 32/256
6
+ [sde] generated 48/256
7
+ [sde] generated 64/256
8
+ [sde] generated 80/256
9
+ [sde] generated 96/256
10
+ [sde] generated 112/256
11
+ [sde] generated 128/256
12
+ [sde] generated 144/256
13
+ [sde] generated 160/256
14
+ [sde] generated 176/256
15
+ [sde] generated 192/256
16
+ [sde] generated 208/256
17
+ [sde] generated 224/256
18
+ [sde] generated 240/256
19
+ [sde] generated 256/256
20
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
21
+ [summary] {
22
+ "type": "summary",
23
+ "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0049000.pt",
24
+ "step": 49000,
25
+ "decode": {
26
+ "decode_rule": "logistic_normal_resample_sde",
27
+ "steps": 128,
28
+ "model_t_mode": "const0.5",
29
+ "mean_mode": "anchor_semantic",
30
+ "endpoint_floor": 0.0,
31
+ "concentration_min": 1.0,
32
+ "concentration_max": 1024.0,
33
+ "endpoint_temp": 1.45,
34
+ "support_power": 1.0,
35
+ "semantic_power": 1.0,
36
+ "noise_init": "logistic_normal",
37
+ "noise_sigma": 3.0,
38
+ "noise_dirichlet_concentration": 1.0,
39
+ "sde_resample": "logistic_normal",
40
+ "logistic_normal_sigma_min": 0.18,
41
+ "logistic_normal_sigma_max": 3.0,
42
+ "logistic_normal_tau_min": 0.65,
43
+ "logistic_normal_tau_max": 1.0,
44
+ "final_from": "blend_0.5",
45
+ "n_samples": 256,
46
+ "seed": 20260522
47
+ },
48
+ "raw_genppl": {
49
+ "ppl": 35.48283825980655,
50
+ "nll_per_token": 3.569049150290528,
51
+ "tokens": 34732,
52
+ "kept_samples": 256,
53
+ "total_samples": 256,
54
+ "empty_rate": 0.0,
55
+ "skipped_samples": 0
56
+ },
57
+ "stripped_genppl": {
58
+ "ppl": 52.17503986958083,
59
+ "nll_per_token": 3.9546042171140185,
60
+ "tokens": 28366,
61
+ "kept_samples": 256,
62
+ "total_samples": 256,
63
+ "empty_rate": 0.0,
64
+ "skipped_samples": 0
65
+ },
66
+ "diversity": {
67
+ "sample_entropy": 3.558572845970964,
68
+ "unique_tokens": 2054,
69
+ "token_count": 32768,
70
+ "distinct_1": 0.06268310546875,
71
+ "distinct_2": 0.30804010826771655,
72
+ "top_token_mass": 0.152191162109375
73
+ }
74
+ }
75
+ [done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0049000/sde_steps128_samples256_scored.jsonl
76
+ [watch-lognormal-sde] 2026-05-23_03:08:54 done step_0049000
LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0077000_logistic_normal_t1p45.log ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [watch-lognormal-sde] 2026-05-23_05:43:25 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0077000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0077000
2
+ [load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0077000.pt
3
+ [ckpt] step=77000
4
+ [sde] generated 16/256
5
+ [sde] generated 32/256
6
+ [sde] generated 48/256
7
+ [sde] generated 64/256
8
+ [sde] generated 80/256
9
+ [sde] generated 96/256
10
+ [sde] generated 112/256
11
+ [sde] generated 128/256
12
+ [sde] generated 144/256
13
+ [sde] generated 160/256
14
+ [sde] generated 176/256
15
+ [sde] generated 192/256
16
+ [sde] generated 208/256
17
+ [sde] generated 224/256
18
+ [sde] generated 240/256
19
+ [sde] generated 256/256
20
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
21
+ [summary] {
22
+ "type": "summary",
23
+ "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0077000.pt",
24
+ "step": 77000,
25
+ "decode": {
26
+ "decode_rule": "logistic_normal_resample_sde",
27
+ "steps": 128,
28
+ "model_t_mode": "const0.5",
29
+ "mean_mode": "anchor_semantic",
30
+ "endpoint_floor": 0.0,
31
+ "concentration_min": 1.0,
32
+ "concentration_max": 1024.0,
33
+ "endpoint_temp": 1.45,
34
+ "support_power": 1.0,
35
+ "semantic_power": 1.0,
36
+ "noise_init": "logistic_normal",
37
+ "noise_sigma": 3.0,
38
+ "noise_dirichlet_concentration": 1.0,
39
+ "sde_resample": "logistic_normal",
40
+ "logistic_normal_sigma_min": 0.18,
41
+ "logistic_normal_sigma_max": 3.0,
42
+ "logistic_normal_tau_min": 0.65,
43
+ "logistic_normal_tau_max": 1.0,
44
+ "final_from": "blend_0.5",
45
+ "n_samples": 256,
46
+ "seed": 20260522
47
+ },
48
+ "raw_genppl": {
49
+ "ppl": 31.90594878776546,
50
+ "nll_per_token": 3.462792474780215,
51
+ "tokens": 37167,
52
+ "kept_samples": 256,
53
+ "total_samples": 256,
54
+ "empty_rate": 0.0,
55
+ "skipped_samples": 0
56
+ },
57
+ "stripped_genppl": {
58
+ "ppl": 46.30221403143558,
59
+ "nll_per_token": 3.8351897792023526,
60
+ "tokens": 30416,
61
+ "kept_samples": 256,
62
+ "total_samples": 256,
63
+ "empty_rate": 0.0,
64
+ "skipped_samples": 0
65
+ },
66
+ "diversity": {
67
+ "sample_entropy": 3.770319501431757,
68
+ "unique_tokens": 2137,
69
+ "token_count": 32768,
70
+ "distinct_1": 0.065216064453125,
71
+ "distinct_2": 0.33744463582677164,
72
+ "top_token_mass": 0.094970703125
73
+ }
74
+ }
75
+ [done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0077000/sde_steps128_samples256_scored.jsonl
76
+ [watch-lognormal-sde] 2026-05-23_05:44:53 done step_0077000
LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0103000_logistic_normal_t1p45.log ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [watch-lognormal-sde] 2026-05-23_08:09:06 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0103000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0103000
2
+ [load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0103000.pt
3
+ [ckpt] step=103000
4
+ [sde] generated 16/256
5
+ [sde] generated 32/256
6
+ [sde] generated 48/256
7
+ [sde] generated 64/256
8
+ [sde] generated 80/256
9
+ [sde] generated 96/256
10
+ [sde] generated 112/256
11
+ [sde] generated 128/256
12
+ [sde] generated 144/256
13
+ [sde] generated 160/256
14
+ [sde] generated 176/256
15
+ [sde] generated 192/256
16
+ [sde] generated 208/256
17
+ [sde] generated 224/256
18
+ [sde] generated 240/256
19
+ [sde] generated 256/256
20
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
21
+ [summary] {
22
+ "type": "summary",
23
+ "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0103000.pt",
24
+ "step": 103000,
25
+ "decode": {
26
+ "decode_rule": "logistic_normal_resample_sde",
27
+ "steps": 128,
28
+ "model_t_mode": "const0.5",
29
+ "mean_mode": "anchor_semantic",
30
+ "endpoint_floor": 0.0,
31
+ "concentration_min": 1.0,
32
+ "concentration_max": 1024.0,
33
+ "endpoint_temp": 1.45,
34
+ "support_power": 1.0,
35
+ "semantic_power": 1.0,
36
+ "noise_init": "logistic_normal",
37
+ "noise_sigma": 3.0,
38
+ "noise_dirichlet_concentration": 1.0,
39
+ "sde_resample": "logistic_normal",
40
+ "logistic_normal_sigma_min": 0.18,
41
+ "logistic_normal_sigma_max": 3.0,
42
+ "logistic_normal_tau_min": 0.65,
43
+ "logistic_normal_tau_max": 1.0,
44
+ "final_from": "blend_0.5",
45
+ "n_samples": 256,
46
+ "seed": 20260522
47
+ },
48
+ "raw_genppl": {
49
+ "ppl": 30.371408722233586,
50
+ "nll_per_token": 3.413501663304038,
51
+ "tokens": 36546,
52
+ "kept_samples": 256,
53
+ "total_samples": 256,
54
+ "empty_rate": 0.0,
55
+ "skipped_samples": 0
56
+ },
57
+ "stripped_genppl": {
58
+ "ppl": 41.969304283345274,
59
+ "nll_per_token": 3.7369385006854787,
60
+ "tokens": 30362,
61
+ "kept_samples": 256,
62
+ "total_samples": 256,
63
+ "empty_rate": 0.0,
64
+ "skipped_samples": 0
65
+ },
66
+ "diversity": {
67
+ "sample_entropy": 3.698089692782413,
68
+ "unique_tokens": 2344,
69
+ "token_count": 32768,
70
+ "distinct_1": 0.071533203125,
71
+ "distinct_2": 0.36540354330708663,
72
+ "top_token_mass": 0.0777587890625
73
+ }
74
+ }
75
+ [done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0103000/sde_steps128_samples256_scored.jsonl
76
+ [watch-lognormal-sde] 2026-05-23_08:10:34 done step_0103000
LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0108000_logistic_normal_t1p45.log ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [watch-lognormal-sde] 2026-05-23_08:36:48 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0108000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0108000
2
+ [load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0108000.pt
3
+ [ckpt] step=108000
4
+ [sde] generated 16/256
5
+ [sde] generated 32/256
6
+ [sde] generated 48/256
7
+ [sde] generated 64/256
8
+ [sde] generated 80/256
9
+ [sde] generated 96/256
10
+ [sde] generated 112/256
11
+ [sde] generated 128/256
12
+ [sde] generated 144/256
13
+ [sde] generated 160/256
14
+ [sde] generated 176/256
15
+ [sde] generated 192/256
16
+ [sde] generated 208/256
17
+ [sde] generated 224/256
18
+ [sde] generated 240/256
19
+ [sde] generated 256/256
20
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
21
+ [summary] {
22
+ "type": "summary",
23
+ "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0108000.pt",
24
+ "step": 108000,
25
+ "decode": {
26
+ "decode_rule": "logistic_normal_resample_sde",
27
+ "steps": 128,
28
+ "model_t_mode": "const0.5",
29
+ "mean_mode": "anchor_semantic",
30
+ "endpoint_floor": 0.0,
31
+ "concentration_min": 1.0,
32
+ "concentration_max": 1024.0,
33
+ "endpoint_temp": 1.45,
34
+ "support_power": 1.0,
35
+ "semantic_power": 1.0,
36
+ "noise_init": "logistic_normal",
37
+ "noise_sigma": 3.0,
38
+ "noise_dirichlet_concentration": 1.0,
39
+ "sde_resample": "logistic_normal",
40
+ "logistic_normal_sigma_min": 0.18,
41
+ "logistic_normal_sigma_max": 3.0,
42
+ "logistic_normal_tau_min": 0.65,
43
+ "logistic_normal_tau_max": 1.0,
44
+ "final_from": "blend_0.5",
45
+ "n_samples": 256,
46
+ "seed": 20260522
47
+ },
48
+ "raw_genppl": {
49
+ "ppl": 34.88124452485366,
50
+ "nll_per_token": 3.551949278589281,
51
+ "tokens": 34859,
52
+ "kept_samples": 256,
53
+ "total_samples": 256,
54
+ "empty_rate": 0.0,
55
+ "skipped_samples": 0
56
+ },
57
+ "stripped_genppl": {
58
+ "ppl": 44.9734082467488,
59
+ "nll_per_token": 3.8060713872536174,
60
+ "tokens": 29578,
61
+ "kept_samples": 256,
62
+ "total_samples": 256,
63
+ "empty_rate": 0.0,
64
+ "skipped_samples": 0
65
+ },
66
+ "diversity": {
67
+ "sample_entropy": 3.532442503084375,
68
+ "unique_tokens": 2417,
69
+ "token_count": 32768,
70
+ "distinct_1": 0.073760986328125,
71
+ "distinct_2": 0.3453494094488189,
72
+ "top_token_mass": 0.121063232421875
73
+ }
74
+ }
75
+ [done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0108000/sde_steps128_samples256_scored.jsonl
76
+ [watch-lognormal-sde] 2026-05-23_08:38:16 done step_0108000
LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0124000_logistic_normal_t1p45.log ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [watch-lognormal-sde] 2026-05-23_10:05:45 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0124000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0124000
2
+ [load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0124000.pt
3
+ [ckpt] step=124000
4
+ [sde] generated 16/256
5
+ [sde] generated 32/256
6
+ [sde] generated 48/256
7
+ [sde] generated 64/256
8
+ [sde] generated 80/256
9
+ [sde] generated 96/256
10
+ [sde] generated 112/256
11
+ [sde] generated 128/256
12
+ [sde] generated 144/256
13
+ [sde] generated 160/256
14
+ [sde] generated 176/256
15
+ [sde] generated 192/256
16
+ [sde] generated 208/256
17
+ [sde] generated 224/256
18
+ [sde] generated 240/256
19
+ [sde] generated 256/256
20
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
21
+ [summary] {
22
+ "type": "summary",
23
+ "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0124000.pt",
24
+ "step": 124000,
25
+ "decode": {
26
+ "decode_rule": "logistic_normal_resample_sde",
27
+ "steps": 128,
28
+ "model_t_mode": "const0.5",
29
+ "mean_mode": "anchor_semantic",
30
+ "endpoint_floor": 0.0,
31
+ "concentration_min": 1.0,
32
+ "concentration_max": 1024.0,
33
+ "endpoint_temp": 1.45,
34
+ "support_power": 1.0,
35
+ "semantic_power": 1.0,
36
+ "noise_init": "logistic_normal",
37
+ "noise_sigma": 3.0,
38
+ "noise_dirichlet_concentration": 1.0,
39
+ "sde_resample": "logistic_normal",
40
+ "logistic_normal_sigma_min": 0.18,
41
+ "logistic_normal_sigma_max": 3.0,
42
+ "logistic_normal_tau_min": 0.65,
43
+ "logistic_normal_tau_max": 1.0,
44
+ "final_from": "blend_0.5",
45
+ "n_samples": 256,
46
+ "seed": 20260522
47
+ },
48
+ "raw_genppl": {
49
+ "ppl": 33.066483898805664,
50
+ "nll_per_token": 3.498520198353581,
51
+ "tokens": 34129,
52
+ "kept_samples": 256,
53
+ "total_samples": 256,
54
+ "empty_rate": 0.0,
55
+ "skipped_samples": 0
56
+ },
57
+ "stripped_genppl": {
58
+ "ppl": 41.22609368102926,
59
+ "nll_per_token": 3.7190713976517773,
60
+ "tokens": 28994,
61
+ "kept_samples": 256,
62
+ "total_samples": 256,
63
+ "empty_rate": 0.0,
64
+ "skipped_samples": 0
65
+ },
66
+ "diversity": {
67
+ "sample_entropy": 3.5810572585206812,
68
+ "unique_tokens": 2244,
69
+ "token_count": 32768,
70
+ "distinct_1": 0.0684814453125,
71
+ "distinct_2": 0.3511011318897638,
72
+ "top_token_mass": 0.14215087890625
73
+ }
74
+ }
75
+ [done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0124000/sde_steps128_samples256_scored.jsonl
76
+ [watch-lognormal-sde] 2026-05-23_10:07:13 done step_0124000
LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_bernoulliwrong_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_193815.log ADDED
The diff for this file is too large to render. See raw diff
 
LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_bernoulliwrong_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260527_063225.log ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ W0527 06:32:27.026000 2692375 torch/distributed/run.py:792]
2
+ W0527 06:32:27.026000 2692375 torch/distributed/run.py:792] *****************************************
3
+ W0527 06:32:27.026000 2692375 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ W0527 06:32:27.026000 2692375 torch/distributed/run.py:792] *****************************************
5
+ [rank6]: Traceback (most recent call last):
6
+ [rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
7
+ [rank6]: main()
8
+ [rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
9
+ [rank6]: rank, world, device = setup_ddp()
10
+ [rank6]: ^^^^^^^^^^^
11
+ [rank6]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
12
+ [rank6]: torch.cuda.set_device(local_rank)
13
+ [rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
14
+ [rank6]: torch._C._cuda_setDevice(device)
15
+ [rank6]: RuntimeError: CUDA error: invalid device ordinal
16
+ [rank6]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
17
+ [rank6]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
18
+ [rank6]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
19
+
20
+ [rank3]: Traceback (most recent call last):
21
+ [rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
22
+ [rank3]: main()
23
+ [rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
24
+ [rank3]: rank, world, device = setup_ddp()
25
+ [rank3]: ^^^^^^^^^^^
26
+ [rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
27
+ [rank3]: torch.cuda.set_device(local_rank)
28
+ [rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
29
+ [rank3]: torch._C._cuda_setDevice(device)
30
+ [rank3]: RuntimeError: CUDA error: invalid device ordinal
31
+ [rank3]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
32
+ [rank3]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
33
+ [rank3]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
34
+
35
+ [rank1]: Traceback (most recent call last):
36
+ [rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
37
+ [rank1]: main()
38
+ [rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
39
+ [rank1]: rank, world, device = setup_ddp()
40
+ [rank1]: ^^^^^^^^^^^
41
+ [rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
42
+ [rank1]: torch.cuda.set_device(local_rank)
43
+ [rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
44
+ [rank1]: torch._C._cuda_setDevice(device)
45
+ [rank1]: RuntimeError: CUDA error: invalid device ordinal
46
+ [rank1]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
47
+ [rank1]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
48
+ [rank1]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
49
+
50
+ [rank7]: Traceback (most recent call last):
51
+ [rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
52
+ [rank7]: main()
53
+ [rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
54
+ [rank7]: rank, world, device = setup_ddp()
55
+ [rank7]: ^^^^^^^^^^^
56
+ [rank7]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
57
+ [rank7]: torch.cuda.set_device(local_rank)
58
+ [rank7]: File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
59
+ [rank7]: torch._C._cuda_setDevice(device)
60
+ [rank7]: RuntimeError: CUDA error: invalid device ordinal
61
+ [rank7]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
62
+ [rank7]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
63
+ [rank7]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
64
+
65
+ [rank4]: Traceback (most recent call last):
66
+ [rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
67
+ [rank4]: main()
68
+ [rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
69
+ [rank4]: rank, world, device = setup_ddp()
70
+ [rank4]: ^^^^^^^^^^^
71
+ [rank4]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
72
+ [rank4]: torch.cuda.set_device(local_rank)
73
+ [rank4]: File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
74
+ [rank4]: torch._C._cuda_setDevice(device)
75
+ [rank4]: RuntimeError: CUDA error: invalid device ordinal
76
+ [rank4]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
77
+ [rank4]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
78
+ [rank4]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
79
+
80
+ [rank2]: Traceback (most recent call last):
81
+ [rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
82
+ [rank2]: main()
83
+ [rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
84
+ [rank2]: rank, world, device = setup_ddp()
85
+ [rank2]: ^^^^^^^^^^^
86
+ [rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
87
+ [rank2]: torch.cuda.set_device(local_rank)
88
+ [rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
89
+ [rank2]: torch._C._cuda_setDevice(device)
90
+ [rank2]: RuntimeError: CUDA error: invalid device ordinal
91
+ [rank2]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
92
+ [rank2]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
93
+ [rank2]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
94
+
95
+ [rank5]: Traceback (most recent call last):
96
+ [rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 232, in <module>
97
+ [rank5]: main()
98
+ [rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 114, in main
99
+ [rank5]: rank, world, device = setup_ddp()
100
+ [rank5]: ^^^^^^^^^^^
101
+ [rank5]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/mini_owt_fit/train.py", line 62, in setup_ddp
102
+ [rank5]: torch.cuda.set_device(local_rank)
103
+ [rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/cuda/__init__.py", line 477, in set_device
104
+ [rank5]: torch._C._cuda_setDevice(device)
105
+ [rank5]: RuntimeError: CUDA error: invalid device ordinal
106
+ [rank5]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
107
+ [rank5]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
108
+ [rank5]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
109
+
110
+ [rank6]:[W527 06:32:28.179191492 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
111
+ [rank3]:[W527 06:32:28.196687749 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
112
+ [rank1]:[W527 06:32:28.200738161 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
113
+ [rank4]:[W527 06:32:28.281200240 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
114
+ [rank7]:[W527 06:32:28.289848591 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
115
+ [rank2]:[W527 06:32:28.328670757 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
116
+ [rank5]:[W527 06:32:28.342647483 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
117
+ W0527 06:32:28.756000 2692375 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 2692488 closing signal SIGTERM
118
+ W0527 06:32:28.757000 2692375 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 2692490 closing signal SIGTERM
119
+ W0527 06:32:28.757000 2692375 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 2692492 closing signal SIGTERM
120
+ W0527 06:32:28.758000 2692375 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 2692493 closing signal SIGTERM
121
+ W0527 06:32:28.759000 2692375 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 2692495 closing signal SIGTERM
122
+ E0527 06:32:30.025000 2692375 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 1 (pid: 2692489) of binary: /usr/bin/python
123
+ Traceback (most recent call last):
124
+ File "/usr/local/bin/torchrun", line 33, in <module>
125
+ sys.exit(load_entry_point('torch==2.7.0a0+ecf3bae40a.nv25.2', 'console_scripts', 'torchrun')())
126
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
127
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
128
+ return f(*args, **kwargs)
129
+ ^^^^^^^^^^^^^^^^^^
130
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main
131
+ run(args)
132
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run
133
+ elastic_launch(
134
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
135
+ return launch_agent(self._config, self._entrypoint, list(args))
136
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
137
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent
138
+ raise ChildFailedError(
139
+ torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
140
+ ============================================================
141
+ train.py FAILED
142
+ ------------------------------------------------------------
143
+ Failures:
144
+ [1]:
145
+ time : 2026-05-27_06:32:28
146
+ host : localhost
147
+ rank : 3 (local_rank: 3)
148
+ exitcode : 1 (pid: 2692491)
149
+ error_file: <N/A>
150
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
151
+ [2]:
152
+ time : 2026-05-27_06:32:28
153
+ host : localhost
154
+ rank : 6 (local_rank: 6)
155
+ exitcode : 1 (pid: 2692494)
156
+ error_file: <N/A>
157
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
158
+ ------------------------------------------------------------
159
+ Root Cause (first observed failure):
160
+ [0]:
161
+ time : 2026-05-27_06:32:28
162
+ host : localhost
163
+ rank : 1 (local_rank: 1)
164
+ exitcode : 1 (pid: 2692489)
165
+ error_file: <N/A>
166
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
167
+ ============================================================
LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_full_gbs512_8gpu_20260526_155049.log ADDED
@@ -0,0 +1,544 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ W0526 15:50:51.060000 10232 torch/distributed/run.py:792]
2
+ W0526 15:50:51.060000 10232 torch/distributed/run.py:792] *****************************************
3
+ W0526 15:50:51.060000 10232 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ W0526 15:50:51.060000 10232 torch/distributed/run.py:792] *****************************************
5
+ [rank7]:[W526 15:50:54.488272530 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
6
+ [rank1]:[W526 15:50:54.606918486 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
7
+ [rank2]:[W526 15:50:54.651901857 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
8
+ [rank5]:[W526 15:50:54.658253820 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
9
+ [rank4]:[W526 15:50:54.674422130 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
10
+ [rank6]:[W526 15:50:54.674769368 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
11
+ [rank3]:[W526 15:50:54.676109529 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
12
+ [data] loaded_cache=cache/owt_t5_payload1022_appendeos1.pt seen=8013769 kept=2860537 dropped=5153232
13
+ [rank0]:[W526 15:51:00.641574510 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
14
+ t-20260526235016-fvc2m-worker-0:10299:10299 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
15
+ t-20260526235016-fvc2m-worker-0:10299:10299 [0] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
16
+ t-20260526235016-fvc2m-worker-0:10299:10299 [0] NCCL INFO cudaDriverVersion 12080
17
+ t-20260526235016-fvc2m-worker-0:10299:10299 [0] NCCL INFO NCCL version 2.25.1+cuda12.8
18
+ t-20260526235016-fvc2m-worker-0:10299:10299 [0] NCCL INFO Comm config Blocking set to 1
19
+ t-20260526235016-fvc2m-worker-0:10300:10300 [1] NCCL INFO cudaDriverVersion 12080
20
+ t-20260526235016-fvc2m-worker-0:10300:10300 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
21
+ t-20260526235016-fvc2m-worker-0:10300:10300 [1] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
22
+ t-20260526235016-fvc2m-worker-0:10300:10300 [1] NCCL INFO NCCL version 2.25.1+cuda12.8
23
+ t-20260526235016-fvc2m-worker-0:10300:10300 [1] NCCL INFO Comm config Blocking set to 1
24
+ t-20260526235016-fvc2m-worker-0:10303:10303 [4] NCCL INFO cudaDriverVersion 12080
25
+ t-20260526235016-fvc2m-worker-0:10303:10303 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
26
+ t-20260526235016-fvc2m-worker-0:10302:10302 [3] NCCL INFO cudaDriverVersion 12080
27
+ t-20260526235016-fvc2m-worker-0:10302:10302 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
28
+ t-20260526235016-fvc2m-worker-0:10303:10303 [4] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
29
+ t-20260526235016-fvc2m-worker-0:10302:10302 [3] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
30
+ t-20260526235016-fvc2m-worker-0:10302:10302 [3] NCCL INFO NCCL version 2.25.1+cuda12.8
31
+ t-20260526235016-fvc2m-worker-0:10303:10303 [4] NCCL INFO NCCL version 2.25.1+cuda12.8
32
+ t-20260526235016-fvc2m-worker-0:10306:10306 [7] NCCL INFO cudaDriverVersion 12080
33
+ t-20260526235016-fvc2m-worker-0:10306:10306 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
34
+ t-20260526235016-fvc2m-worker-0:10306:10306 [7] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
35
+ t-20260526235016-fvc2m-worker-0:10306:10306 [7] NCCL INFO NCCL version 2.25.1+cuda12.8
36
+ t-20260526235016-fvc2m-worker-0:10305:10305 [6] NCCL INFO cudaDriverVersion 12080
37
+ t-20260526235016-fvc2m-worker-0:10305:10305 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
38
+ t-20260526235016-fvc2m-worker-0:10304:10304 [5] NCCL INFO cudaDriverVersion 12080
39
+ t-20260526235016-fvc2m-worker-0:10305:10305 [6] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
40
+ t-20260526235016-fvc2m-worker-0:10305:10305 [6] NCCL INFO NCCL version 2.25.1+cuda12.8
41
+ t-20260526235016-fvc2m-worker-0:10304:10304 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
42
+ t-20260526235016-fvc2m-worker-0:10303:10303 [4] NCCL INFO Comm config Blocking set to 1
43
+ t-20260526235016-fvc2m-worker-0:10302:10302 [3] NCCL INFO Comm config Blocking set to 1
44
+ t-20260526235016-fvc2m-worker-0:10304:10304 [5] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
45
+ t-20260526235016-fvc2m-worker-0:10304:10304 [5] NCCL INFO NCCL version 2.25.1+cuda12.8
46
+ t-20260526235016-fvc2m-worker-0:10301:10301 [2] NCCL INFO cudaDriverVersion 12080
47
+ t-20260526235016-fvc2m-worker-0:10301:10301 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
48
+ t-20260526235016-fvc2m-worker-0:10301:10301 [2] NCCL INFO Bootstrap: Using eth1:10.82.96.104<0>
49
+ t-20260526235016-fvc2m-worker-0:10301:10301 [2] NCCL INFO NCCL version 2.25.1+cuda12.8
50
+ t-20260526235016-fvc2m-worker-0:10306:10306 [7] NCCL INFO Comm config Blocking set to 1
51
+ t-20260526235016-fvc2m-worker-0:10305:10305 [6] NCCL INFO Comm config Blocking set to 1
52
+ t-20260526235016-fvc2m-worker-0:10304:10304 [5] NCCL INFO Comm config Blocking set to 1
53
+ t-20260526235016-fvc2m-worker-0:10301:10301 [2] NCCL INFO Comm config Blocking set to 1
54
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
55
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
56
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
57
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO P2P plugin v9 IBext_v9
58
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
59
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
60
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
61
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
62
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO P2P plugin v9 IBext_v9
63
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
64
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
65
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
66
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
67
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO P2P plugin v9 IBext_v9
68
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
69
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
70
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
71
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
72
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO P2P plugin v9 IBext_v9
73
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
74
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
75
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
76
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
77
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO P2P plugin v9 IBext_v9
78
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
79
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
80
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
81
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
82
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO P2P plugin v9 IBext_v9
83
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
84
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
85
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
86
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
87
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO P2P plugin v9 IBext_v9
88
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
89
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
90
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
91
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
92
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO P2P plugin v9 IBext_v9
93
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
94
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
95
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
96
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
97
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
98
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
99
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
100
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
101
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
102
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
103
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
104
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
105
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO Using network IBext_v9
106
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
107
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO Using network IBext_v9
108
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
109
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Using network IBext_v9
110
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
111
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO Using network IBext_v9
112
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
113
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO Using network IBext_v9
114
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
115
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
116
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
117
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO Using network IBext_v9
118
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
119
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
120
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
121
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO Using network IBext_v9
122
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
123
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.96.104<0>
124
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
125
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO Using network IBext_v9
126
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO ncclCommInitRankConfig comm 0xb143f10 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x153ec126bc8139c1 - Init START
127
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO ncclCommInitRankConfig comm 0xbd4dcd0 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x153ec126bc8139c1 - Init START
128
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO ncclCommInitRankConfig comm 0xa37e4e0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x153ec126bc8139c1 - Init START
129
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO ncclCommInitRankConfig comm 0x9d716c0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x153ec126bc8139c1 - Init START
130
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO ncclCommInitRankConfig comm 0x9b81db0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x153ec126bc8139c1 - Init START
131
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO RAS client listening socket at ::1<28028>
132
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO ncclCommInitRankConfig comm 0x9b30c10 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x153ec126bc8139c1 - Init START
133
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO RAS client listening socket at ::1<28028>
134
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO ncclCommInitRankConfig comm 0x9831810 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x153ec126bc8139c1 - Init START
135
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO RAS client listening socket at ::1<28028>
136
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO RAS client listening socket at ::1<28028>
137
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO RAS client listening socket at ::1<28028>
138
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO ncclCommInitRankConfig comm 0xa99ebf0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x153ec126bc8139c1 - Init START
139
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO RAS client listening socket at ::1<28028>
140
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO RAS client listening socket at ::1<28028>
141
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO RAS client listening socket at ::1<28028>
142
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Bootstrap timings total 0.009389 (create 0.000021, send 0.000076, recv 0.007843, ring 0.001136, delay 0.000001)
143
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO Bootstrap timings total 0.004700 (create 0.000020, send 0.000074, recv 0.000078, ring 0.004231, delay 0.000001)
144
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO Bootstrap timings total 0.006253 (create 0.000025, send 0.000072, recv 0.001602, ring 0.000124, delay 0.000001)
145
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO Bootstrap timings total 0.009861 (create 0.000025, send 0.000073, recv 0.004202, ring 0.001097, delay 0.000001)
146
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO Bootstrap timings total 0.000634 (create 0.000020, send 0.000071, recv 0.000088, ring 0.000198, delay 0.000001)
147
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO Bootstrap timings total 0.001625 (create 0.000020, send 0.000077, recv 0.000101, ring 0.001092, delay 0.000001)
148
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO Bootstrap timings total 0.005219 (create 0.000020, send 0.000072, recv 0.004652, ring 0.000152, delay 0.000001)
149
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO Bootstrap timings total 0.005755 (create 0.000022, send 0.000072, recv 0.000616, ring 0.004733, delay 0.000001)
150
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO MNNVL busId 0x75020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
151
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO MNNVL busId 0x65040 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
152
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO MNNVL busId 0x6f020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
153
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO MNNVL busId 0x73020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
154
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO MNNVL busId 0x71020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
155
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO MNNVL busId 0x69020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
156
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO MNNVL busId 0x67020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
157
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO MNNVL busId 0x6b020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
158
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
159
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
160
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
161
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
162
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
163
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
164
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
165
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
166
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO Setting affinity for GPU 7 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
167
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO NVLS multicast support is available on dev 7
168
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO Setting affinity for GPU 2 to 03ffffff,ffffffff,ffffffff
169
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO NVLS multicast support is available on dev 2
170
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO Setting affinity for GPU 1 to 03ffffff,ffffffff,ffffffff
171
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO NVLS multicast support is available on dev 1
172
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Setting affinity for GPU 0 to 03ffffff,ffffffff,ffffffff
173
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO Setting affinity for GPU 5 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
174
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO NVLS multicast support is available on dev 0
175
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO NVLS multicast support is available on dev 5
176
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO Setting affinity for GPU 6 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
177
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO NVLS multicast support is available on dev 6
178
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO Setting affinity for GPU 4 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
179
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO Setting affinity for GPU 3 to 03ffffff,ffffffff,ffffffff
180
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO NVLS multicast support is available on dev 3
181
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO NVLS multicast support is available on dev 4
182
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO comm 0xa37e4e0 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0
183
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO comm 0x9831810 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0
184
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO comm 0x9d716c0 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0
185
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO comm 0xb143f10 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0
186
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO comm 0x9b81db0 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0
187
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO comm 0xbd4dcd0 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0
188
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO comm 0x9b30c10 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0
189
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO comm 0xa99ebf0 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0
190
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7
191
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7
192
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0
193
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6
194
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2
195
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1
196
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7
197
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO P2P Chunksize set to 524288
198
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO P2P Chunksize set to 524288
199
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4
200
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO P2P Chunksize set to 524288
201
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7
202
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO P2P Chunksize set to 524288
203
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO P2P Chunksize set to 524288
204
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7
205
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5
206
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3
207
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7
208
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7
209
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO P2P Chunksize set to 524288
210
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO P2P Chunksize set to 524288
211
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7
212
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7
213
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7
214
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7
215
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7
216
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7
217
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7
218
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7
219
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7
220
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7
221
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7
222
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7
223
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7
224
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7
225
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7
226
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7
227
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7
228
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1
229
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO P2P Chunksize set to 524288
230
+ t-20260526235016-fvc2m-worker-0:10305:10476 [6] NCCL INFO [Proxy Service UDS] Device 6 CPU core 94
231
+ t-20260526235016-fvc2m-worker-0:10305:10475 [6] NCCL INFO [Proxy Service] Device 6 CPU core 92
232
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Check P2P Type intraNodeP2pSupport 1 directMode 0
233
+ t-20260526235016-fvc2m-worker-0:10299:10477 [0] NCCL INFO [Proxy Service] Device 0 CPU core 2
234
+ t-20260526235016-fvc2m-worker-0:10299:10478 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 4
235
+ t-20260526235016-fvc2m-worker-0:10300:10479 [1] NCCL INFO [Proxy Service] Device 1 CPU core 59
236
+ t-20260526235016-fvc2m-worker-0:10300:10480 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 60
237
+ t-20260526235016-fvc2m-worker-0:10304:10481 [5] NCCL INFO [Proxy Service] Device 5 CPU core 138
238
+ t-20260526235016-fvc2m-worker-0:10304:10482 [5] NCCL INFO [Proxy Service UDS] Device 5 CPU core 141
239
+ t-20260526235016-fvc2m-worker-0:10301:10483 [2] NCCL INFO [Proxy Service] Device 2 CPU core 20
240
+ t-20260526235016-fvc2m-worker-0:10301:10484 [2] NCCL INFO [Proxy Service UDS] Device 2 CPU core 22
241
+ t-20260526235016-fvc2m-worker-0:10302:10485 [3] NCCL INFO [Proxy Service] Device 3 CPU core 82
242
+ t-20260526235016-fvc2m-worker-0:10302:10486 [3] NCCL INFO [Proxy Service UDS] Device 3 CPU core 84
243
+ t-20260526235016-fvc2m-worker-0:10303:10487 [4] NCCL INFO [Proxy Service] Device 4 CPU core 92
244
+ t-20260526235016-fvc2m-worker-0:10303:10488 [4] NCCL INFO [Proxy Service UDS] Device 4 CPU core 94
245
+ t-20260526235016-fvc2m-worker-0:10306:10489 [7] NCCL INFO [Proxy Service] Device 7 CPU core 116
246
+ t-20260526235016-fvc2m-worker-0:10306:10490 [7] NCCL INFO [Proxy Service UDS] Device 7 CPU core 114
247
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
248
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
249
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
250
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
251
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
252
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
253
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
254
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
255
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
256
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
257
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO CC Off, workFifoBytes 1048576
258
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
259
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
260
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
261
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
262
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
263
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
264
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
265
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
266
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
267
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
268
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
269
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
270
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
271
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
272
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
273
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
274
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO ncclCommInitRankConfig comm 0xa99ebf0 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x153ec126bc8139c1 - Init COMPLETE
275
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO ncclCommInitRankConfig comm 0x9b81db0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x153ec126bc8139c1 - Init COMPLETE
276
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO ncclCommInitRankConfig comm 0xbd4dcd0 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x153ec126bc8139c1 - Init COMPLETE
277
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
278
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
279
+ t-20260526235016-fvc2m-worker-0:10304:10401 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 2.20 (kernels 0.21, alloc 1.04, bootstrap 0.00, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.37, rest 0.03)
280
+ t-20260526235016-fvc2m-worker-0:10299:10395 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 2.22 (kernels 0.19, alloc 1.07, bootstrap 0.01, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.38, rest 0.02)
281
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
282
+ t-20260526235016-fvc2m-worker-0:10303:10397 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 2.20 (kernels 0.19, alloc 1.06, bootstrap 0.01, allgathers 0.00, topo 0.53, graphs 0.01, connections 0.37, rest 0.03)
283
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO ncclCommInitRankConfig comm 0x9831810 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x153ec126bc8139c1 - Init COMPLETE
284
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
285
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
286
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO ncclCommInitRankConfig comm 0x9d716c0 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x153ec126bc8139c1 - Init COMPLETE
287
+ t-20260526235016-fvc2m-worker-0:10300:10396 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 2.20 (kernels 0.20, alloc 1.06, bootstrap 0.00, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.37, rest 0.03)
288
+ t-20260526235016-fvc2m-worker-0:10302:10398 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 2.20 (kernels 0.19, alloc 1.06, bootstrap 0.01, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.37, rest 0.03)
289
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
290
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
291
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
292
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO ncclCommInitRankConfig comm 0x9b30c10 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x153ec126bc8139c1 - Init COMPLETE
293
+ t-20260526235016-fvc2m-worker-0:10306:10399 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 2.20 (kernels 0.19, alloc 1.06, bootstrap 0.00, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.37, rest 0.03)
294
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
295
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
296
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
297
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
298
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
299
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
300
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO ncclCommInitRankConfig comm 0xa37e4e0 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x153ec126bc8139c1 - Init COMPLETE
301
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO ncclCommInitRankConfig comm 0xb143f10 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x153ec126bc8139c1 - Init COMPLETE
302
+ t-20260526235016-fvc2m-worker-0:10305:10400 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 2.20 (kernels 0.19, alloc 1.05, bootstrap 0.01, allgathers 0.00, topo 0.53, graphs 0.01, connections 0.38, rest 0.02)
303
+ t-20260526235016-fvc2m-worker-0:10301:10402 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 2.20 (kernels 0.19, alloc 1.05, bootstrap 0.01, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.37, rest 0.03)
304
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM
305
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM
306
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM
307
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM
308
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM
309
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM
310
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM
311
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM
312
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM
313
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM
314
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM
315
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM
316
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM
317
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM
318
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM
319
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM
320
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM
321
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM
322
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM
323
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM
324
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM
325
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM
326
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM
327
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM
328
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM
329
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM
330
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM
331
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM
332
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM
333
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM
334
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM
335
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM
336
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM
337
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM
338
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM
339
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM
340
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM
341
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM
342
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM
343
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM
344
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM
345
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM
346
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM
347
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM
348
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM
349
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM
350
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM
351
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM
352
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM
353
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM
354
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM
355
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM
356
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM
357
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM
358
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM
359
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM
360
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM
361
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM
362
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM
363
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM
364
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM
365
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM
366
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM
367
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM
368
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM
369
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM
370
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM
371
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM
372
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM
373
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM
374
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM
375
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM
376
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM
377
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM
378
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM
379
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM
380
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM
381
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM
382
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM
383
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM
384
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM
385
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM
386
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM
387
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM
388
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM
389
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM
390
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM
391
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM
392
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM
393
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM
394
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM
395
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM
396
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM
397
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM
398
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM
399
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM
400
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM
401
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM
402
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM
403
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM
404
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM
405
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM
406
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM
407
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM
408
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM
409
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM
410
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM
411
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM
412
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM
413
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM
414
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM
415
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM
416
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM
417
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM
418
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM
419
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM
420
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM
421
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM
422
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM
423
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM
424
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM
425
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM
426
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM
427
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM
428
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM
429
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM
430
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM
431
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM
432
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM
433
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM
434
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM
435
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM
436
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM
437
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM
438
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM
439
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM
440
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM
441
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM
442
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM
443
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM
444
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM
445
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM
446
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM
447
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM
448
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM
449
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM
450
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM
451
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM
452
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM
453
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM
454
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM
455
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM
456
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM
457
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM
458
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM
459
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM
460
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM
461
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM
462
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM
463
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM
464
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM
465
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM
466
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM
467
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM
468
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM
469
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM
470
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM
471
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM
472
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM
473
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM
474
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM
475
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM
476
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM
477
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM
478
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM
479
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM
480
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM
481
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM
482
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM
483
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM
484
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM
485
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM
486
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM
487
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM
488
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM
489
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM
490
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM
491
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM
492
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM
493
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM
494
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM
495
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM
496
+ t-20260526235016-fvc2m-worker-0:10301:10497 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
497
+ t-20260526235016-fvc2m-worker-0:10306:10498 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
498
+ t-20260526235016-fvc2m-worker-0:10305:10494 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
499
+ t-20260526235016-fvc2m-worker-0:10304:10493 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
500
+ t-20260526235016-fvc2m-worker-0:10299:10495 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
501
+ t-20260526235016-fvc2m-worker-0:10300:10492 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
502
+ t-20260526235016-fvc2m-worker-0:10303:10491 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
503
+ t-20260526235016-fvc2m-worker-0:10302:10496 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
504
+ {
505
+ "data_path": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext",
506
+ "tokenizer_path": "/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json",
507
+ "out_dir": "runs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_full_gbs512_8gpu_20260526_155049",
508
+ "text_column": "text",
509
+ "subset_size": 0,
510
+ "payload_len": 1022,
511
+ "append_eos": 1,
512
+ "log_skips": 20,
513
+ "cache_path": "cache/owt_t5_payload1022_appendeos1.pt",
514
+ "rebuild_cache": 0,
515
+ "online_data": 0,
516
+ "online_buffer_size": 8192,
517
+ "steps": 20000,
518
+ "batch_size": 32,
519
+ "grad_accum": 2,
520
+ "lr": 0.0003,
521
+ "log_every": 50,
522
+ "save_every": 1000,
523
+ "dim": 768,
524
+ "layers": 12,
525
+ "heads": 12,
526
+ "mlp_dim": 3072,
527
+ "time_tokens": 4,
528
+ "abs_pos": 1,
529
+ "rope": 1,
530
+ "c_min": 1.0,
531
+ "c_max": 1024.0,
532
+ "seed": 1234
533
+ }
534
+ [data] rows=2860537 length=1024 vocab=32100 seen=8013769 dropped=5153232 kept=2860537 bos=1:</s> eos=1:</s>
535
+ [head] ['</s>', '▁Port', '-', 'au', '-', 'Pri', 'nce', ',', '▁Haiti', '▁(', 'C', 'NN', ')', '▁--', '▁Earth', 'qua']
536
+ [tail] ['▁magnitude', '▁earthquake', '▁flat', 't', 'ened', '▁Haiti', "'", 's', '▁capital', '▁city', '▁Tuesday', '▁afternoon', ',', '▁', 'affecting', '</s>']
537
+ t-20260526235016-fvc2m-worker-0:10306:10674 [7] NCCL INFO NVLS comm 0x9b30c10 headRank 7 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
538
+ t-20260526235016-fvc2m-worker-0:10300:10675 [1] NCCL INFO NVLS comm 0x9831810 headRank 1 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
539
+ t-20260526235016-fvc2m-worker-0:10304:10676 [5] NCCL INFO NVLS comm 0xa99ebf0 headRank 5 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
540
+ t-20260526235016-fvc2m-worker-0:10301:10677 [2] NCCL INFO NVLS comm 0xb143f10 headRank 2 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
541
+ t-20260526235016-fvc2m-worker-0:10302:10678 [3] NCCL INFO NVLS comm 0x9d716c0 headRank 3 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
542
+ t-20260526235016-fvc2m-worker-0:10305:10679 [6] NCCL INFO NVLS comm 0xa37e4e0 headRank 6 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
543
+ t-20260526235016-fvc2m-worker-0:10303:10680 [4] NCCL INFO NVLS comm 0x9b81db0 headRank 4 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
544
+ t-20260526235016-fvc2m-worker-0:10299:10681 [0] NCCL INFO NVLS comm 0xbd4dcd0 headRank 0 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
LTA_openwebtext_dualt/mini_owt_fit/logs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_163925.log ADDED
@@ -0,0 +1,597 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ W0526 16:39:26.907000 10232 torch/distributed/run.py:792]
2
+ W0526 16:39:26.907000 10232 torch/distributed/run.py:792] *****************************************
3
+ W0526 16:39:26.907000 10232 torch/distributed/run.py:792] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ W0526 16:39:26.907000 10232 torch/distributed/run.py:792] *****************************************
5
+ [rank3]:[W526 16:39:30.312943735 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
6
+ [rank2]:[W526 16:39:30.393296581 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
7
+ [rank1]:[W526 16:39:30.412360668 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
8
+ [rank5]:[W526 16:39:30.485600324 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
9
+ [rank6]:[W526 16:39:30.506970342 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
10
+ [rank4]:[W526 16:39:30.515612102 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
11
+ [rank7]:[W526 16:39:30.523092350 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
12
+ [data] loaded_cache=cache/owt_t5_payload1022_appendeos1.pt seen=8013769 kept=2860537 dropped=5153232
13
+ [rank0]:[W526 16:39:36.458074028 ProcessGroupNCCL.cpp:4571] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. Specify device_ids in barrier() to force use of a particular device, or call init_process_group() with a device_id.
14
+ t-20260527003833-zv4xx-worker-0:10299:10299 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
15
+ t-20260527003833-zv4xx-worker-0:10299:10299 [0] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
16
+ t-20260527003833-zv4xx-worker-0:10299:10299 [0] NCCL INFO cudaDriverVersion 12080
17
+ t-20260527003833-zv4xx-worker-0:10299:10299 [0] NCCL INFO NCCL version 2.25.1+cuda12.8
18
+ t-20260527003833-zv4xx-worker-0:10299:10299 [0] NCCL INFO Comm config Blocking set to 1
19
+ t-20260527003833-zv4xx-worker-0:10300:10300 [1] NCCL INFO cudaDriverVersion 12080
20
+ t-20260527003833-zv4xx-worker-0:10300:10300 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
21
+ t-20260527003833-zv4xx-worker-0:10300:10300 [1] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
22
+ t-20260527003833-zv4xx-worker-0:10300:10300 [1] NCCL INFO NCCL version 2.25.1+cuda12.8
23
+ t-20260527003833-zv4xx-worker-0:10304:10304 [5] NCCL INFO cudaDriverVersion 12080
24
+ t-20260527003833-zv4xx-worker-0:10304:10304 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
25
+ t-20260527003833-zv4xx-worker-0:10303:10303 [4] NCCL INFO cudaDriverVersion 12080
26
+ t-20260527003833-zv4xx-worker-0:10303:10303 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
27
+ t-20260527003833-zv4xx-worker-0:10300:10300 [1] NCCL INFO Comm config Blocking set to 1
28
+ t-20260527003833-zv4xx-worker-0:10302:10302 [3] NCCL INFO cudaDriverVersion 12080
29
+ t-20260527003833-zv4xx-worker-0:10302:10302 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
30
+ t-20260527003833-zv4xx-worker-0:10304:10304 [5] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
31
+ t-20260527003833-zv4xx-worker-0:10304:10304 [5] NCCL INFO NCCL version 2.25.1+cuda12.8
32
+ t-20260527003833-zv4xx-worker-0:10303:10303 [4] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
33
+ t-20260527003833-zv4xx-worker-0:10303:10303 [4] NCCL INFO NCCL version 2.25.1+cuda12.8
34
+ t-20260527003833-zv4xx-worker-0:10301:10301 [2] NCCL INFO cudaDriverVersion 12080
35
+ t-20260527003833-zv4xx-worker-0:10301:10301 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
36
+ t-20260527003833-zv4xx-worker-0:10306:10306 [7] NCCL INFO cudaDriverVersion 12080
37
+ t-20260527003833-zv4xx-worker-0:10306:10306 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
38
+ t-20260527003833-zv4xx-worker-0:10305:10305 [6] NCCL INFO cudaDriverVersion 12080
39
+ t-20260527003833-zv4xx-worker-0:10305:10305 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
40
+ t-20260527003833-zv4xx-worker-0:10302:10302 [3] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
41
+ t-20260527003833-zv4xx-worker-0:10302:10302 [3] NCCL INFO NCCL version 2.25.1+cuda12.8
42
+ t-20260527003833-zv4xx-worker-0:10301:10301 [2] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
43
+ t-20260527003833-zv4xx-worker-0:10306:10306 [7] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
44
+ t-20260527003833-zv4xx-worker-0:10305:10305 [6] NCCL INFO Bootstrap: Using eth1:10.82.32.105<0>
45
+ t-20260527003833-zv4xx-worker-0:10301:10301 [2] NCCL INFO NCCL version 2.25.1+cuda12.8
46
+ t-20260527003833-zv4xx-worker-0:10306:10306 [7] NCCL INFO NCCL version 2.25.1+cuda12.8
47
+ t-20260527003833-zv4xx-worker-0:10305:10305 [6] NCCL INFO NCCL version 2.25.1+cuda12.8
48
+ t-20260527003833-zv4xx-worker-0:10303:10303 [4] NCCL INFO Comm config Blocking set to 1
49
+ t-20260527003833-zv4xx-worker-0:10304:10304 [5] NCCL INFO Comm config Blocking set to 1
50
+ t-20260527003833-zv4xx-worker-0:10302:10302 [3] NCCL INFO Comm config Blocking set to 1
51
+ t-20260527003833-zv4xx-worker-0:10306:10306 [7] NCCL INFO Comm config Blocking set to 1
52
+ t-20260527003833-zv4xx-worker-0:10305:10305 [6] NCCL INFO Comm config Blocking set to 1
53
+ t-20260527003833-zv4xx-worker-0:10301:10301 [2] NCCL INFO Comm config Blocking set to 1
54
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
55
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
56
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
57
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO P2P plugin v9 IBext_v9
58
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
59
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
60
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
61
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
62
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO P2P plugin v9 IBext_v9
63
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
64
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
65
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
66
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
67
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO P2P plugin v9 IBext_v9
68
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
69
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
70
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
71
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
72
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO P2P plugin v9 IBext_v9
73
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
74
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
75
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
76
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
77
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO P2P plugin v9 IBext_v9
78
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
79
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
80
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
81
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
82
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO P2P plugin v9 IBext_v9
83
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
84
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
85
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
86
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
87
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO P2P plugin v9 IBext_v9
88
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
89
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
90
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
91
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
92
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO P2P plugin v9 IBext_v9
93
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
94
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
95
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
96
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
97
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Using network IBext_v9
98
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
99
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
100
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
101
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
102
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
103
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO Using network IBext_v9
104
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
105
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO Using network IBext_v9
106
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
107
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
108
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
109
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
110
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
111
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
112
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
113
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO Using network IBext_v9
114
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
115
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
116
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO Using network IBext_v9
117
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO Using network IBext_v9
118
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
119
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
120
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
121
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO Using network IBext_v9
122
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
123
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.32.105<0>
124
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
125
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO Using network IBext_v9
126
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO ncclCommInitRankConfig comm 0xabe1e00 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x71c0ab8013683c2b - Init START
127
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO ncclCommInitRankConfig comm 0xaec9340 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x71c0ab8013683c2b - Init START
128
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO ncclCommInitRankConfig comm 0xb193950 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x71c0ab8013683c2b - Init START
129
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO ncclCommInitRankConfig comm 0x97e19d0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x71c0ab8013683c2b - Init START
130
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO ncclCommInitRankConfig comm 0x95b4ea0 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x71c0ab8013683c2b - Init START
131
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO ncclCommInitRankConfig comm 0xa832730 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x71c0ab8013683c2b - Init START
132
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO ncclCommInitRankConfig comm 0xac3b760 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x71c0ab8013683c2b - Init START
133
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO ncclCommInitRankConfig comm 0xafdc9d0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x71c0ab8013683c2b - Init START
134
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO RAS client listening socket at ::1<28028>
135
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO RAS client listening socket at ::1<28028>
136
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO RAS client listening socket at ::1<28028>
137
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO RAS client listening socket at ::1<28028>
138
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO RAS client listening socket at ::1<28028>
139
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO RAS client listening socket at ::1<28028>
140
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO RAS client listening socket at ::1<28028>
141
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO RAS client listening socket at ::1<28028>
142
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO Bootstrap timings total 0.086836 (create 0.000021, send 0.000066, recv 0.011557, ring 0.000081, delay 0.000001)
143
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO Bootstrap timings total 0.074335 (create 0.000019, send 0.000074, recv 0.000066, ring 0.000114, delay 0.000001)
144
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO Bootstrap timings total 0.074834 (create 0.000022, send 0.000067, recv 0.000539, ring 0.000113, delay 0.000001)
145
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO Bootstrap timings total 0.077938 (create 0.000021, send 0.000069, recv 0.003157, ring 0.000124, delay 0.000001)
146
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO Bootstrap timings total 0.075359 (create 0.000020, send 0.000067, recv 0.000093, ring 0.000128, delay 0.000001)
147
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO Bootstrap timings total 0.086289 (create 0.000024, send 0.000069, recv 0.008438, ring 0.000173, delay 0.000001)
148
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO Bootstrap timings total 0.077370 (create 0.000021, send 0.000067, recv 0.000052, ring 0.000165, delay 0.000001)
149
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Bootstrap timings total 0.300148 (create 0.000025, send 0.000079, recv 0.213971, ring 0.000176, delay 0.000001)
150
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO MNNVL busId 0x6f020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
151
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO MNNVL busId 0x71020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
152
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO MNNVL busId 0x6b020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
153
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO MNNVL busId 0x73020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
154
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO MNNVL busId 0x69020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
155
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO MNNVL busId 0x75020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
156
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO MNNVL busId 0x67020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
157
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO MNNVL busId 0x65040 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
158
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
159
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
160
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
161
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
162
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
163
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
164
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
165
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
166
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Setting affinity for GPU 0 to 03ffffff,ffffffff,ffffffff
167
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO NVLS multicast support is available on dev 0
168
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO Setting affinity for GPU 5 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
169
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO NVLS multicast support is available on dev 5
170
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO Setting affinity for GPU 6 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
171
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO NVLS multicast support is available on dev 6
172
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO Setting affinity for GPU 7 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
173
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO NVLS multicast support is available on dev 7
174
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO Setting affinity for GPU 4 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
175
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO NVLS multicast support is available on dev 4
176
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO Setting affinity for GPU 2 to 03ffffff,ffffffff,ffffffff
177
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO NVLS multicast support is available on dev 2
178
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO Setting affinity for GPU 1 to 03ffffff,ffffffff,ffffffff
179
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO NVLS multicast support is available on dev 1
180
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO Setting affinity for GPU 3 to 03ffffff,ffffffff,ffffffff
181
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO NVLS multicast support is available on dev 3
182
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO comm 0xa832730 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0
183
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO comm 0xaec9340 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0
184
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO comm 0xafdc9d0 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0
185
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO comm 0xac3b760 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0
186
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO comm 0x95b4ea0 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0
187
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO comm 0xb193950 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0
188
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO comm 0x97e19d0 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0
189
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO comm 0xabe1e00 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0
190
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7
191
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7
192
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7
193
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7
194
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4
195
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6
196
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7
197
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3
198
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0
199
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO P2P Chunksize set to 524288
200
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO P2P Chunksize set to 524288
201
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5
202
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2
203
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7
204
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1
205
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO P2P Chunksize set to 524288
206
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO P2P Chunksize set to 524288
207
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7
208
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7
209
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO P2P Chunksize set to 524288
210
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO P2P Chunksize set to 524288
211
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO P2P Chunksize set to 524288
212
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7
213
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7
214
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7
215
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7
216
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7
217
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7
218
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7
219
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7
220
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7
221
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7
222
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7
223
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7
224
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7
225
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7
226
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7
227
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7
228
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1
229
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO P2P Chunksize set to 524288
230
+ t-20260527003833-zv4xx-worker-0:10303:10475 [4] NCCL INFO [Proxy Service] Device 4 CPU core 104
231
+ t-20260527003833-zv4xx-worker-0:10303:10476 [4] NCCL INFO [Proxy Service UDS] Device 4 CPU core 107
232
+ t-20260527003833-zv4xx-worker-0:10306:10477 [7] NCCL INFO [Proxy Service] Device 7 CPU core 108
233
+ t-20260527003833-zv4xx-worker-0:10306:10478 [7] NCCL INFO [Proxy Service UDS] Device 7 CPU core 110
234
+ t-20260527003833-zv4xx-worker-0:10305:10479 [6] NCCL INFO [Proxy Service] Device 6 CPU core 94
235
+ t-20260527003833-zv4xx-worker-0:10305:10480 [6] NCCL INFO [Proxy Service UDS] Device 6 CPU core 96
236
+ t-20260527003833-zv4xx-worker-0:10301:10481 [2] NCCL INFO [Proxy Service] Device 2 CPU core 2
237
+ t-20260527003833-zv4xx-worker-0:10301:10482 [2] NCCL INFO [Proxy Service UDS] Device 2 CPU core 4
238
+ t-20260527003833-zv4xx-worker-0:10300:10483 [1] NCCL INFO [Proxy Service] Device 1 CPU core 86
239
+ t-20260527003833-zv4xx-worker-0:10300:10484 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 2
240
+ t-20260527003833-zv4xx-worker-0:10304:10485 [5] NCCL INFO [Proxy Service] Device 5 CPU core 131
241
+ t-20260527003833-zv4xx-worker-0:10304:10486 [5] NCCL INFO [Proxy Service UDS] Device 5 CPU core 132
242
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Check P2P Type intraNodeP2pSupport 1 directMode 0
243
+ t-20260527003833-zv4xx-worker-0:10302:10487 [3] NCCL INFO [Proxy Service] Device 3 CPU core 2
244
+ t-20260527003833-zv4xx-worker-0:10302:10488 [3] NCCL INFO [Proxy Service UDS] Device 3 CPU core 4
245
+ t-20260527003833-zv4xx-worker-0:10299:10489 [0] NCCL INFO [Proxy Service] Device 0 CPU core 77
246
+ t-20260527003833-zv4xx-worker-0:10299:10490 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 79
247
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
248
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
249
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
250
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
251
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
252
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
253
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
254
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
255
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO CC Off, workFifoBytes 1048576
256
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
257
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
258
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
259
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
260
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
261
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
262
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
263
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
264
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
265
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
266
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
267
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
268
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
269
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
270
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
271
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
272
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
273
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
274
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
275
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
276
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
277
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
278
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
279
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
280
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
281
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
282
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
283
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
284
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
285
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
286
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
287
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
288
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO ncclCommInitRankConfig comm 0x95b4ea0 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x71c0ab8013683c2b - Init COMPLETE
289
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO ncclCommInitRankConfig comm 0xabe1e00 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x71c0ab8013683c2b - Init COMPLETE
290
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO ncclCommInitRankConfig comm 0x97e19d0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x71c0ab8013683c2b - Init COMPLETE
291
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO ncclCommInitRankConfig comm 0xa832730 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x71c0ab8013683c2b - Init COMPLETE
292
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO ncclCommInitRankConfig comm 0xac3b760 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x71c0ab8013683c2b - Init COMPLETE
293
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO ncclCommInitRankConfig comm 0xaec9340 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x71c0ab8013683c2b - Init COMPLETE
294
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO ncclCommInitRankConfig comm 0xafdc9d0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x71c0ab8013683c2b - Init COMPLETE
295
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO ncclCommInitRankConfig comm 0xb193950 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x71c0ab8013683c2b - Init COMPLETE
296
+ t-20260527003833-zv4xx-worker-0:10299:10395 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 2.18 (kernels 0.19, alloc 0.85, bootstrap 0.30, allgathers 0.02, topo 0.53, graphs 0.01, connections 0.25, rest 0.04)
297
+ t-20260527003833-zv4xx-worker-0:10306:10400 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 2.16 (kernels 0.20, alloc 1.04, bootstrap 0.08, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.28, rest 0.02)
298
+ t-20260527003833-zv4xx-worker-0:10301:10402 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 2.16 (kernels 0.20, alloc 1.03, bootstrap 0.08, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.27, rest 0.02)
299
+ t-20260527003833-zv4xx-worker-0:10305:10401 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 2.16 (kernels 0.20, alloc 1.04, bootstrap 0.08, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.27, rest 0.02)
300
+ t-20260527003833-zv4xx-worker-0:10304:10398 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 2.17 (kernels 0.20, alloc 1.03, bootstrap 0.09, allgathers 0.01, topo 0.53, graphs 0.01, connections 0.26, rest 0.03)
301
+ t-20260527003833-zv4xx-worker-0:10302:10399 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 2.16 (kernels 0.20, alloc 1.04, bootstrap 0.07, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.25, rest 0.04)
302
+ t-20260527003833-zv4xx-worker-0:10303:10397 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 2.17 (kernels 0.22, alloc 1.02, bootstrap 0.07, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.28, rest 0.01)
303
+ t-20260527003833-zv4xx-worker-0:10300:10396 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 2.17 (kernels 0.20, alloc 1.03, bootstrap 0.09, allgathers 0.01, topo 0.54, graphs 0.01, connections 0.26, rest 0.03)
304
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM
305
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM
306
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM
307
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM
308
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM
309
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM
310
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM
311
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM
312
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM
313
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM
314
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM
315
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM
316
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM
317
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM
318
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM
319
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM
320
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM
321
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM
322
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM
323
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM
324
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM
325
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM
326
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM
327
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM
328
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM
329
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM
330
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM
331
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM
332
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM
333
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM
334
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM
335
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM
336
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM
337
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM
338
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM
339
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM
340
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM
341
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM
342
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM
343
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM
344
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM
345
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM
346
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM
347
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM
348
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM
349
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM
350
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM
351
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM
352
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM
353
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM
354
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM
355
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM
356
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM
357
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM
358
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM
359
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM
360
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM
361
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM
362
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM
363
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM
364
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM
365
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM
366
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM
367
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM
368
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM
369
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM
370
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM
371
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM
372
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM
373
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM
374
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM
375
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM
376
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM
377
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM
378
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM
379
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM
380
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM
381
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM
382
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM
383
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM
384
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM
385
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM
386
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM
387
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM
388
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM
389
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM
390
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM
391
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM
392
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM
393
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM
394
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM
395
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM
396
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM
397
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM
398
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM
399
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM
400
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM
401
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM
402
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM
403
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM
404
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM
405
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM
406
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM
407
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM
408
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM
409
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM
410
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM
411
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM
412
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM
413
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM
414
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM
415
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM
416
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM
417
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM
418
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM
419
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM
420
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM
421
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM
422
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM
423
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM
424
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM
425
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM
426
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM
427
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM
428
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM
429
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM
430
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM
431
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM
432
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM
433
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM
434
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM
435
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM
436
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM
437
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM
438
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM
439
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM
440
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM
441
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM
442
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM
443
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM
444
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM
445
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM
446
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM
447
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM
448
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM
449
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM
450
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM
451
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM
452
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM
453
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM
454
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM
455
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM
456
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM
457
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM
458
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM
459
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM
460
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM
461
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM
462
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM
463
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM
464
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM
465
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM
466
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM
467
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM
468
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM
469
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM
470
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM
471
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM
472
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM
473
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM
474
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM
475
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM
476
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM
477
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM
478
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM
479
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM
480
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM
481
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM
482
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM
483
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM
484
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM
485
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM
486
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM
487
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM
488
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM
489
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM
490
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM
491
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM
492
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM
493
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM
494
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM
495
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM
496
+ t-20260527003833-zv4xx-worker-0:10301:10493 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
497
+ t-20260527003833-zv4xx-worker-0:10305:10492 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
498
+ t-20260527003833-zv4xx-worker-0:10299:10495 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
499
+ t-20260527003833-zv4xx-worker-0:10300:10491 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
500
+ t-20260527003833-zv4xx-worker-0:10302:10498 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
501
+ t-20260527003833-zv4xx-worker-0:10304:10494 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
502
+ t-20260527003833-zv4xx-worker-0:10306:10497 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
503
+ t-20260527003833-zv4xx-worker-0:10303:10496 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
504
+ {
505
+ "data_path": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext",
506
+ "tokenizer_path": "/e2e-data/evad-tech-vla/wanghan58/models/hf/t5-small/tokenizer.json",
507
+ "out_dir": "runs/mini_owt_fit_t5_len1024_bos_eos_C1_to_1024_absrope_time4_d768_l12_h12_native_nofloor_full_gbs512_8gpu_20260526_163925",
508
+ "text_column": "text",
509
+ "subset_size": 0,
510
+ "payload_len": 1022,
511
+ "append_eos": 1,
512
+ "log_skips": 20,
513
+ "cache_path": "cache/owt_t5_payload1022_appendeos1.pt",
514
+ "rebuild_cache": 0,
515
+ "online_data": 0,
516
+ "online_buffer_size": 8192,
517
+ "steps": 1000000,
518
+ "batch_size": 32,
519
+ "grad_accum": 2,
520
+ "lr": 0.0003,
521
+ "log_every": 50,
522
+ "save_every": 1000,
523
+ "dim": 768,
524
+ "layers": 12,
525
+ "heads": 12,
526
+ "mlp_dim": 3072,
527
+ "time_tokens": 4,
528
+ "abs_pos": 1,
529
+ "rope": 1,
530
+ "c_min": 1.0,
531
+ "c_max": 1024.0,
532
+ "seed": 1234
533
+ }
534
+ [data] rows=2860537 length=1024 vocab=32100 seen=8013769 dropped=5153232 kept=2860537 bos=1:</s> eos=1:</s>
535
+ [head] ['</s>', '▁Port', '-', 'au', '-', 'Pri', 'nce', ',', '▁Haiti', '▁(', 'C', 'NN', ')', '▁--', '▁Earth', 'qua']
536
+ [tail] ['▁magnitude', '▁earthquake', '▁flat', 't', 'ened', '▁Haiti', "'", 's', '▁capital', '▁city', '▁Tuesday', '▁afternoon', ',', '▁', 'affecting', '</s>']
537
+ t-20260527003833-zv4xx-worker-0:10304:10610 [5] NCCL INFO NVLS comm 0xaec9340 headRank 5 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
538
+ t-20260527003833-zv4xx-worker-0:10302:10611 [3] NCCL INFO NVLS comm 0xac3b760 headRank 3 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
539
+ t-20260527003833-zv4xx-worker-0:10299:10612 [0] NCCL INFO NVLS comm 0xabe1e00 headRank 0 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
540
+ t-20260527003833-zv4xx-worker-0:10306:10613 [7] NCCL INFO NVLS comm 0x95b4ea0 headRank 7 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
541
+ t-20260527003833-zv4xx-worker-0:10301:10615 [2] NCCL INFO NVLS comm 0x97e19d0 headRank 2 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
542
+ t-20260527003833-zv4xx-worker-0:10300:10614 [1] NCCL INFO NVLS comm 0xb193950 headRank 1 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
543
+ t-20260527003833-zv4xx-worker-0:10303:10616 [4] NCCL INFO NVLS comm 0xafdc9d0 headRank 4 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
544
+ t-20260527003833-zv4xx-worker-0:10305:10617 [6] NCCL INFO NVLS comm 0xa832730 headRank 6 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
545
+ step=50 loss=7.2226 {'pos0_bos_p': 0.8895623087882996, 'pos0_bos_top1': 4, 'last_eos_p': 0.8887543082237244, 'last_eos_top1': 4}
546
+ step=100 loss=7.1043 {'pos0_bos_p': 0.9955054521560669, 'pos0_bos_top1': 4, 'last_eos_p': 0.9959879517555237, 'last_eos_top1': 4}
547
+ step=150 loss=6.0746 {'pos0_bos_p': 0.6761358380317688, 'pos0_bos_top1': 4, 'last_eos_p': 0.7208590507507324, 'last_eos_top1': 4}
548
+ step=200 loss=5.6690 {'pos0_bos_p': 0.9043604731559753, 'pos0_bos_top1': 4, 'last_eos_p': 0.922759473323822, 'last_eos_top1': 4}
549
+ step=250 loss=4.8215 {'pos0_bos_p': 0.933864951133728, 'pos0_bos_top1': 4, 'last_eos_p': 0.953918993473053, 'last_eos_top1': 4}
550
+ step=300 loss=4.5186 {'pos0_bos_p': 0.9616466164588928, 'pos0_bos_top1': 4, 'last_eos_p': 0.9727951288223267, 'last_eos_top1': 4}
551
+ step=350 loss=4.1555 {'pos0_bos_p': 0.965355396270752, 'pos0_bos_top1': 4, 'last_eos_p': 0.9713674783706665, 'last_eos_top1': 4}
552
+ step=400 loss=3.4321 {'pos0_bos_p': 0.982333242893219, 'pos0_bos_top1': 4, 'last_eos_p': 0.984869122505188, 'last_eos_top1': 4}
553
+ step=450 loss=3.5727 {'pos0_bos_p': 0.9869136214256287, 'pos0_bos_top1': 4, 'last_eos_p': 0.9893201589584351, 'last_eos_top1': 4}
554
+ step=500 loss=3.4097 {'pos0_bos_p': 0.9890345335006714, 'pos0_bos_top1': 4, 'last_eos_p': 0.9915169477462769, 'last_eos_top1': 4}
555
+ step=550 loss=2.9839 {'pos0_bos_p': 0.990917980670929, 'pos0_bos_top1': 4, 'last_eos_p': 0.9927908182144165, 'last_eos_top1': 4}
556
+ step=600 loss=2.7384 {'pos0_bos_p': 0.9930053353309631, 'pos0_bos_top1': 4, 'last_eos_p': 0.9942381381988525, 'last_eos_top1': 4}
557
+ step=650 loss=2.4446 {'pos0_bos_p': 0.993517279624939, 'pos0_bos_top1': 4, 'last_eos_p': 0.9944773316383362, 'last_eos_top1': 4}
558
+ step=700 loss=2.3503 {'pos0_bos_p': 0.9943650960922241, 'pos0_bos_top1': 4, 'last_eos_p': 0.9950743317604065, 'last_eos_top1': 4}
559
+ step=750 loss=2.9878 {'pos0_bos_p': 0.9950012564659119, 'pos0_bos_top1': 4, 'last_eos_p': 0.9952785968780518, 'last_eos_top1': 4}
560
+ step=800 loss=2.6886 {'pos0_bos_p': 0.9956516623497009, 'pos0_bos_top1': 4, 'last_eos_p': 0.9956568479537964, 'last_eos_top1': 4}
561
+ step=850 loss=2.6424 {'pos0_bos_p': 0.9948635697364807, 'pos0_bos_top1': 4, 'last_eos_p': 0.9943944215774536, 'last_eos_top1': 4}
562
+ step=900 loss=2.3033 {'pos0_bos_p': 0.9968313574790955, 'pos0_bos_top1': 4, 'last_eos_p': 0.9964890480041504, 'last_eos_top1': 4}
563
+ step=950 loss=2.7804 {'pos0_bos_p': 0.9972208738327026, 'pos0_bos_top1': 4, 'last_eos_p': 0.9968255758285522, 'last_eos_top1': 4}
564
+ step=1000 loss=2.3661 {'pos0_bos_p': 0.9971387386322021, 'pos0_bos_top1': 4, 'last_eos_p': 0.9966385364532471, 'last_eos_top1': 4}
565
+ step=1050 loss=2.2603 {'pos0_bos_p': 0.9974852800369263, 'pos0_bos_top1': 4, 'last_eos_p': 0.9969478249549866, 'last_eos_top1': 4}
566
+ step=1100 loss=2.3556 {'pos0_bos_p': 0.9976263642311096, 'pos0_bos_top1': 4, 'last_eos_p': 0.9970782995223999, 'last_eos_top1': 4}
567
+ step=1150 loss=2.5678 {'pos0_bos_p': 0.9978169202804565, 'pos0_bos_top1': 4, 'last_eos_p': 0.9971928000450134, 'last_eos_top1': 4}
568
+ step=1200 loss=2.8455 {'pos0_bos_p': 0.9978358149528503, 'pos0_bos_top1': 4, 'last_eos_p': 0.9972167015075684, 'last_eos_top1': 4}
569
+ step=1250 loss=1.9140 {'pos0_bos_p': 0.9979123473167419, 'pos0_bos_top1': 4, 'last_eos_p': 0.9973716735839844, 'last_eos_top1': 4}
570
+ step=1300 loss=2.2091 {'pos0_bos_p': 0.9973376393318176, 'pos0_bos_top1': 4, 'last_eos_p': 0.9966146349906921, 'last_eos_top1': 4}
571
+ step=1350 loss=1.5151 {'pos0_bos_p': 0.997620165348053, 'pos0_bos_top1': 4, 'last_eos_p': 0.9968921542167664, 'last_eos_top1': 4}
572
+ step=1400 loss=1.9534 {'pos0_bos_p': 0.9971216320991516, 'pos0_bos_top1': 4, 'last_eos_p': 0.9958294034004211, 'last_eos_top1': 4}
573
+ step=1450 loss=2.0907 {'pos0_bos_p': 0.9967049956321716, 'pos0_bos_top1': 4, 'last_eos_p': 0.995191216468811, 'last_eos_top1': 4}
574
+ step=1500 loss=1.6090 {'pos0_bos_p': 0.9971720576286316, 'pos0_bos_top1': 4, 'last_eos_p': 0.99595707654953, 'last_eos_top1': 4}
575
+ step=1550 loss=1.8623 {'pos0_bos_p': 0.9975792765617371, 'pos0_bos_top1': 4, 'last_eos_p': 0.9965457320213318, 'last_eos_top1': 4}
576
+ step=1600 loss=1.6597 {'pos0_bos_p': 0.9975658655166626, 'pos0_bos_top1': 4, 'last_eos_p': 0.9966031312942505, 'last_eos_top1': 4}
577
+ step=1650 loss=1.7848 {'pos0_bos_p': 0.9976959824562073, 'pos0_bos_top1': 4, 'last_eos_p': 0.9968312382698059, 'last_eos_top1': 4}
578
+ step=1700 loss=1.9018 {'pos0_bos_p': 0.997767448425293, 'pos0_bos_top1': 4, 'last_eos_p': 0.9969239830970764, 'last_eos_top1': 4}
579
+ step=1750 loss=1.6703 {'pos0_bos_p': 0.9975913763046265, 'pos0_bos_top1': 4, 'last_eos_p': 0.9966752529144287, 'last_eos_top1': 4}
580
+ step=1800 loss=1.9970 {'pos0_bos_p': 0.9979997277259827, 'pos0_bos_top1': 4, 'last_eos_p': 0.9973554611206055, 'last_eos_top1': 4}
581
+ step=1850 loss=1.5861 {'pos0_bos_p': 0.9977781176567078, 'pos0_bos_top1': 4, 'last_eos_p': 0.9969488978385925, 'last_eos_top1': 4}
582
+ step=1900 loss=1.9109 {'pos0_bos_p': 0.9981799125671387, 'pos0_bos_top1': 4, 'last_eos_p': 0.9975284934043884, 'last_eos_top1': 4}
583
+ step=1950 loss=2.0138 {'pos0_bos_p': 0.9982655644416809, 'pos0_bos_top1': 4, 'last_eos_p': 0.9977399110794067, 'last_eos_top1': 4}
584
+ step=2000 loss=1.8339 {'pos0_bos_p': 0.9980721473693848, 'pos0_bos_top1': 4, 'last_eos_p': 0.9973933696746826, 'last_eos_top1': 4}
585
+ step=2050 loss=1.6342 {'pos0_bos_p': 0.9986487030982971, 'pos0_bos_top1': 4, 'last_eos_p': 0.9981813430786133, 'last_eos_top1': 4}
586
+ step=2100 loss=1.8772 {'pos0_bos_p': 0.9984073042869568, 'pos0_bos_top1': 4, 'last_eos_p': 0.9978986978530884, 'last_eos_top1': 4}
587
+ step=2150 loss=1.7135 {'pos0_bos_p': 0.9985877275466919, 'pos0_bos_top1': 4, 'last_eos_p': 0.998180627822876, 'last_eos_top1': 4}
588
+ step=2200 loss=1.5222 {'pos0_bos_p': 0.9986546039581299, 'pos0_bos_top1': 4, 'last_eos_p': 0.9982472658157349, 'last_eos_top1': 4}
589
+ step=2250 loss=1.4951 {'pos0_bos_p': 0.9984161853790283, 'pos0_bos_top1': 4, 'last_eos_p': 0.9979872703552246, 'last_eos_top1': 4}
590
+ step=2300 loss=1.3507 {'pos0_bos_p': 0.9987239241600037, 'pos0_bos_top1': 4, 'last_eos_p': 0.9984145164489746, 'last_eos_top1': 4}
591
+ step=2350 loss=1.4153 {'pos0_bos_p': 0.9986024498939514, 'pos0_bos_top1': 4, 'last_eos_p': 0.9983096122741699, 'last_eos_top1': 4}
592
+ step=2400 loss=1.8935 {'pos0_bos_p': 0.9989206790924072, 'pos0_bos_top1': 4, 'last_eos_p': 0.9986801743507385, 'last_eos_top1': 4}
593
+ step=2450 loss=1.8997 {'pos0_bos_p': 0.998845100402832, 'pos0_bos_top1': 4, 'last_eos_p': 0.9985548853874207, 'last_eos_top1': 4}
594
+ step=2500 loss=1.4746 {'pos0_bos_p': 0.9988980293273926, 'pos0_bos_top1': 4, 'last_eos_p': 0.9986351132392883, 'last_eos_top1': 4}
595
+ step=2550 loss=1.6229 {'pos0_bos_p': 0.9988683462142944, 'pos0_bos_top1': 4, 'last_eos_p': 0.998468816280365, 'last_eos_top1': 4}
596
+ step=2600 loss=1.4606 {'pos0_bos_p': 0.99871826171875, 'pos0_bos_top1': 4, 'last_eos_p': 0.9982940554618835, 'last_eos_top1': 4}
597
+ step=2650 loss=1.8987 {'pos0_bos_p': 0.9986856579780579, 'pos0_bos_top1': 4, 'last_eos_p': 0.9983665347099304, 'last_eos_top1': 4}
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/configuration_data2vec_text.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022 The HuggingFace Inc. team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """Data2VecText configuration"""
15
+
16
+ from huggingface_hub.dataclasses import strict
17
+
18
+ from ...configuration_utils import PreTrainedConfig
19
+ from ...utils import auto_docstring
20
+
21
+
22
+ @auto_docstring(checkpoint="facebook/data2vec-text-base")
23
+ @strict
24
+ class Data2VecTextConfig(PreTrainedConfig):
25
+ r"""
26
+ Examples:
27
+
28
+ ```python
29
+ >>> from transformers import Data2VecTextConfig, Data2VecTextModel
30
+
31
+ >>> # Initializing a Data2VecText facebook/data2vec-text-base style configuration
32
+ >>> configuration = Data2VecTextConfig()
33
+
34
+ >>> # Initializing a model (with random weights) from the facebook/data2vec-text-base style configuration
35
+ >>> model = Data2VecTextModel(configuration)
36
+
37
+ >>> # Accessing the model configuration
38
+ >>> configuration = model.config
39
+ ```"""
40
+
41
+ model_type = "data2vec-text"
42
+
43
+ vocab_size: int = 30522
44
+ hidden_size: int = 768
45
+ num_hidden_layers: int = 12
46
+ num_attention_heads: int = 12
47
+ intermediate_size: int = 3072
48
+ hidden_act: str = "gelu"
49
+ hidden_dropout_prob: float | int = 0.1
50
+ attention_probs_dropout_prob: float | int = 0.1
51
+ max_position_embeddings: int = 512
52
+ type_vocab_size: int = 2
53
+ initializer_range: float = 0.02
54
+ layer_norm_eps: float = 1e-12
55
+ pad_token_id: int | None = 1
56
+ bos_token_id: int | None = 0
57
+ eos_token_id: int | list[int] | None = 2
58
+ use_cache: bool = True
59
+ classifier_dropout: float | int | None = None
60
+ is_decoder: bool = False
61
+ add_cross_attention: bool = False
62
+ tie_word_embeddings: bool = True
63
+
64
+
65
+ __all__ = ["Data2VecTextConfig"]
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_audio.py ADDED
@@ -0,0 +1,1324 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
2
+ # This file was automatically generated from src/transformers/models/data2vec/modular_data2vec_audio.py.
3
+ # Do NOT edit this file manually as any edits will be overwritten by the generation of
4
+ # the file from the modular. If any change should be done, please apply the change to the
5
+ # modular_data2vec_audio.py file directly. One of our CI enforces this.
6
+ # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
7
+ # Copyright 2022 The HuggingFace Inc. team.
8
+ #
9
+ # Licensed under the Apache License, Version 2.0 (the "License");
10
+ # you may not use this file except in compliance with the License.
11
+ # You may obtain a copy of the License at
12
+ #
13
+ # http://www.apache.org/licenses/LICENSE-2.0
14
+ #
15
+ # Unless required by applicable law or agreed to in writing, software
16
+ # distributed under the License is distributed on an "AS IS" BASIS,
17
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+ # See the License for the specific language governing permissions and
19
+ # limitations under the License.
20
+
21
+ import math
22
+ import warnings
23
+ from collections.abc import Callable
24
+
25
+ import numpy as np
26
+ import torch
27
+ from torch import nn
28
+ from torch.nn import CrossEntropyLoss
29
+
30
+ from ... import initialization as init
31
+ from ...activations import ACT2FN
32
+ from ...integrations.deepspeed import is_deepspeed_zero3_enabled
33
+ from ...integrations.fsdp import is_fsdp_managed_module
34
+ from ...masking_utils import create_bidirectional_mask
35
+ from ...modeling_flash_attention_utils import FlashAttentionKwargs
36
+ from ...modeling_layers import GradientCheckpointingLayer
37
+ from ...modeling_outputs import (
38
+ BaseModelOutput,
39
+ CausalLMOutput,
40
+ SequenceClassifierOutput,
41
+ TokenClassifierOutput,
42
+ Wav2Vec2BaseModelOutput,
43
+ XVectorOutput,
44
+ )
45
+ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
46
+ from ...processing_utils import Unpack
47
+ from ...utils import TransformersKwargs, auto_docstring, is_peft_available
48
+ from .configuration_data2vec_audio import Data2VecAudioConfig
49
+
50
+
51
+ class Data2VecAudioConvLayer(GradientCheckpointingLayer):
52
+ def __init__(self, config, layer_id=0):
53
+ super().__init__()
54
+ self.in_conv_dim = config.conv_dim[layer_id - 1] if layer_id > 0 else 1
55
+ self.out_conv_dim = config.conv_dim[layer_id]
56
+
57
+ self.conv = nn.Conv1d(
58
+ self.in_conv_dim,
59
+ self.out_conv_dim,
60
+ kernel_size=config.conv_kernel[layer_id],
61
+ stride=config.conv_stride[layer_id],
62
+ bias=config.conv_bias,
63
+ )
64
+ self.layer_norm = nn.LayerNorm(self.out_conv_dim, elementwise_affine=True)
65
+ self.activation = ACT2FN[config.feat_extract_activation]
66
+
67
+ def forward(self, hidden_states):
68
+ hidden_states = self.conv(hidden_states)
69
+
70
+ hidden_states = hidden_states.transpose(-2, -1)
71
+ hidden_states = self.layer_norm(hidden_states)
72
+ hidden_states = hidden_states.transpose(-2, -1)
73
+
74
+ hidden_states = self.activation(hidden_states)
75
+ return hidden_states
76
+
77
+
78
+ class Data2VecAudioPadLayer(nn.Module):
79
+ def __init__(self, num_conv_pos_embeddings):
80
+ super().__init__()
81
+ self.num_pad_remove = 1 if num_conv_pos_embeddings % 2 == 0 else 0
82
+
83
+ def forward(self, hidden_states):
84
+ if self.num_pad_remove > 0:
85
+ hidden_states = hidden_states[:, :, : -self.num_pad_remove]
86
+ return hidden_states
87
+
88
+
89
+ class Data2VecAudioPositionalConvLayer(nn.Module):
90
+ def __init__(self, config):
91
+ super().__init__()
92
+ self.conv = nn.Conv1d(
93
+ config.hidden_size,
94
+ config.hidden_size,
95
+ kernel_size=config.conv_pos_kernel_size,
96
+ padding=config.conv_pos_kernel_size // 2,
97
+ groups=config.num_conv_pos_embedding_groups,
98
+ )
99
+
100
+ self.padding = Data2VecAudioPadLayer(config.conv_pos_kernel_size)
101
+ self.activation = ACT2FN[config.feat_extract_activation]
102
+ # no learnable parameters
103
+ self.layer_norm = nn.LayerNorm(config.hidden_size, elementwise_affine=False)
104
+
105
+ def forward(self, hidden_states):
106
+ hidden_states = self.conv(hidden_states)
107
+ hidden_states = self.padding(hidden_states)
108
+
109
+ hidden_states = hidden_states.transpose(1, 2)
110
+ hidden_states = self.layer_norm(hidden_states)
111
+ hidden_states = hidden_states.transpose(1, 2)
112
+ hidden_states = self.activation(hidden_states)
113
+ return hidden_states
114
+
115
+
116
+ class Data2VecAudioPositionalConvEmbedding(nn.Module):
117
+ def __init__(self, config):
118
+ super().__init__()
119
+ self.layers = nn.ModuleList(
120
+ [Data2VecAudioPositionalConvLayer(config) for _ in range(config.num_conv_pos_embeddings)]
121
+ )
122
+
123
+ def forward(self, hidden_states):
124
+ hidden_states = hidden_states.transpose(1, 2)
125
+ for layer in self.layers:
126
+ hidden_states = layer(hidden_states)
127
+ hidden_states = hidden_states.transpose(1, 2)
128
+ return hidden_states
129
+
130
+
131
+ class Data2VecAudioFeatureEncoder(nn.Module):
132
+ """Construct the features from raw audio waveform"""
133
+
134
+ def __init__(self, config):
135
+ super().__init__()
136
+ self.conv_layers = nn.ModuleList(
137
+ [Data2VecAudioConvLayer(config, layer_id=i) for i in range(config.num_feat_extract_layers)]
138
+ )
139
+ self.gradient_checkpointing = False
140
+ self._requires_grad = True
141
+
142
+ def _freeze_parameters(self):
143
+ for param in self.parameters():
144
+ param.requires_grad = False
145
+ self._requires_grad = False
146
+
147
+ def forward(self, input_values):
148
+ hidden_states = input_values[:, None]
149
+
150
+ # make sure hidden_states require grad for gradient_checkpointing
151
+ if self._requires_grad and self.training:
152
+ hidden_states.requires_grad = True
153
+
154
+ for conv_layer in self.conv_layers:
155
+ hidden_states = conv_layer(hidden_states)
156
+
157
+ return hidden_states
158
+
159
+
160
+ class Data2VecAudioFeatureProjection(nn.Module):
161
+ def __init__(self, config):
162
+ super().__init__()
163
+ self.layer_norm = nn.LayerNorm(config.conv_dim[-1], eps=config.layer_norm_eps)
164
+ self.projection = nn.Linear(config.conv_dim[-1], config.hidden_size)
165
+ self.dropout = nn.Dropout(config.feat_proj_dropout)
166
+
167
+ def forward(self, hidden_states):
168
+ # non-projected hidden states are needed for quantization
169
+ norm_hidden_states = self.layer_norm(hidden_states)
170
+ hidden_states = self.projection(norm_hidden_states)
171
+ hidden_states = self.dropout(hidden_states)
172
+ return hidden_states, norm_hidden_states
173
+
174
+
175
+ def eager_attention_forward(
176
+ module: nn.Module,
177
+ query: torch.Tensor,
178
+ key: torch.Tensor,
179
+ value: torch.Tensor,
180
+ attention_mask: torch.Tensor | None,
181
+ scaling: float | None = None,
182
+ dropout: float = 0.0,
183
+ **kwargs: Unpack[TransformersKwargs],
184
+ ):
185
+ if scaling is None:
186
+ scaling = query.size(-1) ** -0.5
187
+
188
+ # Take the dot product between "query" and "key" to get the raw attention scores.
189
+ attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling
190
+
191
+ if attention_mask is not None:
192
+ attn_weights = attn_weights + attention_mask
193
+
194
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1)
195
+ attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
196
+
197
+ attn_output = torch.matmul(attn_weights, value)
198
+ attn_output = attn_output.transpose(1, 2).contiguous()
199
+
200
+ return attn_output, attn_weights
201
+
202
+
203
+ class Data2VecAudioAttention(nn.Module):
204
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
205
+
206
+ def __init__(
207
+ self,
208
+ embed_dim: int,
209
+ num_heads: int,
210
+ dropout: float = 0.0,
211
+ is_decoder: bool = False,
212
+ bias: bool = True,
213
+ is_causal: bool = False,
214
+ config: Data2VecAudioConfig | None = None,
215
+ ):
216
+ super().__init__()
217
+ self.embed_dim = embed_dim
218
+ self.num_heads = num_heads
219
+ self.dropout = dropout
220
+ self.head_dim = embed_dim // num_heads
221
+ self.config = config
222
+
223
+ if (self.head_dim * num_heads) != self.embed_dim:
224
+ raise ValueError(
225
+ f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
226
+ f" and `num_heads`: {num_heads})."
227
+ )
228
+ self.scaling = self.head_dim**-0.5
229
+ self.is_decoder = is_decoder
230
+ self.is_causal = is_causal
231
+
232
+ self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
233
+ self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
234
+ self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
235
+ self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
236
+
237
+ def forward(
238
+ self,
239
+ hidden_states: torch.Tensor,
240
+ key_value_states: torch.Tensor | None = None,
241
+ attention_mask: torch.Tensor | None = None,
242
+ output_attentions: bool | None = False,
243
+ # TODO: we need a refactor so that the different attention modules can get their specific kwargs
244
+ # ATM, we have mixed things encoder, decoder, and encoder-decoder attn
245
+ **kwargs: Unpack[FlashAttentionKwargs],
246
+ ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]:
247
+ """Input shape: Batch x Time x Channel"""
248
+
249
+ # if key_value_states are provided this layer is used as a cross-attention layer
250
+ # for the decoder
251
+ is_cross_attention = key_value_states is not None
252
+
253
+ # determine input shapes
254
+ input_shape = hidden_states.shape[:-1]
255
+
256
+ hidden_shape = (*input_shape, -1, self.head_dim)
257
+
258
+ # get query proj
259
+ query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
260
+
261
+ current_states = key_value_states if is_cross_attention else hidden_states
262
+ kv_shape = (*current_states.shape[:-1], -1, self.head_dim)
263
+ key_states = self.k_proj(current_states).view(kv_shape).transpose(1, 2)
264
+ value_states = self.v_proj(current_states).view(kv_shape).transpose(1, 2)
265
+
266
+ attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
267
+ self.config._attn_implementation, eager_attention_forward
268
+ )
269
+
270
+ attn_output, attn_weights = attention_interface(
271
+ self,
272
+ query_states,
273
+ key_states,
274
+ value_states,
275
+ attention_mask,
276
+ dropout=0.0 if not self.training else self.dropout,
277
+ scaling=self.scaling,
278
+ output_attentions=output_attentions,
279
+ **kwargs,
280
+ )
281
+
282
+ attn_output = attn_output.reshape(*input_shape, -1).contiguous()
283
+ attn_output = self.out_proj(attn_output)
284
+
285
+ return attn_output, attn_weights, None
286
+
287
+
288
+ class Data2VecAudioFeedForward(nn.Module):
289
+ def __init__(self, config):
290
+ super().__init__()
291
+ self.intermediate_dropout = nn.Dropout(config.activation_dropout)
292
+
293
+ self.intermediate_dense = nn.Linear(config.hidden_size, config.intermediate_size)
294
+ if isinstance(config.hidden_act, str):
295
+ self.intermediate_act_fn = ACT2FN[config.hidden_act]
296
+ else:
297
+ self.intermediate_act_fn = config.hidden_act
298
+
299
+ self.output_dense = nn.Linear(config.intermediate_size, config.hidden_size)
300
+ self.output_dropout = nn.Dropout(config.hidden_dropout)
301
+
302
+ def forward(self, hidden_states):
303
+ hidden_states = self.intermediate_dense(hidden_states)
304
+ hidden_states = self.intermediate_act_fn(hidden_states)
305
+ hidden_states = self.intermediate_dropout(hidden_states)
306
+
307
+ hidden_states = self.output_dense(hidden_states)
308
+ hidden_states = self.output_dropout(hidden_states)
309
+ return hidden_states
310
+
311
+
312
+ class Data2VecAudioEncoderLayer(GradientCheckpointingLayer):
313
+ def __init__(self, config):
314
+ super().__init__()
315
+ self.attention = Data2VecAudioAttention(
316
+ embed_dim=config.hidden_size,
317
+ num_heads=config.num_attention_heads,
318
+ dropout=config.attention_dropout,
319
+ is_decoder=False,
320
+ config=config,
321
+ )
322
+
323
+ self.dropout = nn.Dropout(config.hidden_dropout)
324
+ self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
325
+ self.feed_forward = Data2VecAudioFeedForward(config)
326
+ self.final_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
327
+
328
+ def forward(self, hidden_states, attention_mask=None, output_attentions=False):
329
+ attn_residual = hidden_states
330
+ hidden_states, attn_weights, _ = self.attention(
331
+ hidden_states, attention_mask=attention_mask, output_attentions=output_attentions
332
+ )
333
+ hidden_states = self.dropout(hidden_states)
334
+ hidden_states = attn_residual + hidden_states
335
+
336
+ hidden_states = self.layer_norm(hidden_states)
337
+ hidden_states = hidden_states + self.feed_forward(hidden_states)
338
+ hidden_states = self.final_layer_norm(hidden_states)
339
+
340
+ outputs = (hidden_states,)
341
+
342
+ if output_attentions:
343
+ outputs += (attn_weights,)
344
+
345
+ return outputs
346
+
347
+
348
+ class Data2VecAudioEncoder(nn.Module):
349
+ def __init__(self, config):
350
+ super().__init__()
351
+ self.config = config
352
+ self.pos_conv_embed = Data2VecAudioPositionalConvEmbedding(config)
353
+ self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
354
+ self.dropout = nn.Dropout(config.hidden_dropout)
355
+ self.layers = nn.ModuleList([Data2VecAudioEncoderLayer(config) for _ in range(config.num_hidden_layers)])
356
+ self.gradient_checkpointing = False
357
+
358
+ def forward(
359
+ self,
360
+ hidden_states: torch.tensor,
361
+ attention_mask: torch.Tensor | None = None,
362
+ output_attentions: bool = False,
363
+ output_hidden_states: bool = False,
364
+ return_dict: bool = True,
365
+ ):
366
+ all_hidden_states = () if output_hidden_states else None
367
+ all_self_attentions = () if output_attentions else None
368
+
369
+ if attention_mask is not None:
370
+ # make sure padded tokens output 0
371
+ expand_attention_mask = attention_mask.unsqueeze(-1).repeat(1, 1, hidden_states.shape[2])
372
+ hidden_states[~expand_attention_mask] = 0
373
+
374
+ attention_mask = create_bidirectional_mask(
375
+ config=self.config,
376
+ inputs_embeds=hidden_states,
377
+ attention_mask=attention_mask,
378
+ )
379
+
380
+ position_embeddings = self.pos_conv_embed(hidden_states)
381
+ hidden_states = hidden_states + position_embeddings.to(hidden_states.device)
382
+ hidden_states = self.layer_norm(hidden_states)
383
+ hidden_states = self.dropout(hidden_states)
384
+
385
+ synced_gpus = is_deepspeed_zero3_enabled() or is_fsdp_managed_module(self)
386
+
387
+ for layer in self.layers:
388
+ if output_hidden_states:
389
+ all_hidden_states = all_hidden_states + (hidden_states,)
390
+
391
+ # add LayerDrop (see https://huggingface.co/papers/1909.11556 for description)
392
+ dropout_probability = torch.rand([])
393
+
394
+ skip_the_layer = self.training and dropout_probability < self.config.layerdrop
395
+ if not skip_the_layer or synced_gpus:
396
+ # under fsdp or deepspeed zero3 all gpus must run in sync
397
+ layer_outputs = layer(
398
+ hidden_states, attention_mask=attention_mask, output_attentions=output_attentions
399
+ )
400
+ hidden_states = layer_outputs[0]
401
+
402
+ if skip_the_layer:
403
+ layer_outputs = (None, None)
404
+
405
+ if output_attentions:
406
+ all_self_attentions = all_self_attentions + (layer_outputs[1],)
407
+
408
+ if output_hidden_states:
409
+ all_hidden_states = all_hidden_states + (hidden_states,)
410
+
411
+ if not return_dict:
412
+ return tuple(v for v in [hidden_states, all_hidden_states, all_self_attentions] if v is not None)
413
+ return BaseModelOutput(
414
+ last_hidden_state=hidden_states,
415
+ hidden_states=all_hidden_states,
416
+ attentions=all_self_attentions,
417
+ )
418
+
419
+
420
+ class Data2VecAudioAdapterLayer(nn.Module):
421
+ def __init__(self, config):
422
+ super().__init__()
423
+ self.conv = nn.Conv1d(
424
+ config.output_hidden_size,
425
+ 2 * config.output_hidden_size,
426
+ config.adapter_kernel_size,
427
+ stride=config.adapter_stride,
428
+ padding=1,
429
+ )
430
+
431
+ def forward(self, hidden_states):
432
+ hidden_states = self.conv(hidden_states)
433
+ hidden_states = nn.functional.glu(hidden_states, dim=1)
434
+
435
+ return hidden_states
436
+
437
+
438
+ class Data2VecAudioAdapter(nn.Module):
439
+ def __init__(self, config):
440
+ super().__init__()
441
+
442
+ # feature dim might need to be down-projected
443
+ if config.output_hidden_size != config.hidden_size:
444
+ self.proj = nn.Linear(config.hidden_size, config.output_hidden_size)
445
+ self.proj_layer_norm = nn.LayerNorm(config.output_hidden_size)
446
+ else:
447
+ self.proj = self.proj_layer_norm = None
448
+
449
+ self.layers = nn.ModuleList(Data2VecAudioAdapterLayer(config) for _ in range(config.num_adapter_layers))
450
+ self.layerdrop = config.layerdrop
451
+
452
+ def forward(self, hidden_states):
453
+ # down project hidden_states if necessary
454
+ if self.proj is not None and self.proj_layer_norm is not None:
455
+ hidden_states = self.proj(hidden_states)
456
+ hidden_states = self.proj_layer_norm(hidden_states)
457
+
458
+ hidden_states = hidden_states.transpose(1, 2)
459
+
460
+ for layer in self.layers:
461
+ layerdrop_prob = np.random.random()
462
+ if not self.training or (layerdrop_prob > self.layerdrop):
463
+ hidden_states = layer(hidden_states)
464
+
465
+ hidden_states = hidden_states.transpose(1, 2)
466
+ return hidden_states
467
+
468
+
469
+ @auto_docstring
470
+ class Data2VecAudioPreTrainedModel(PreTrainedModel):
471
+ config: Data2VecAudioConfig
472
+ base_model_prefix = "data2vec_audio"
473
+ main_input_name = "input_values"
474
+ input_modalities = "audio"
475
+ supports_gradient_checkpointing = True
476
+ _supports_flash_attn = True
477
+ _supports_sdpa = True
478
+ _supports_flex_attn = True
479
+
480
+ @torch.no_grad()
481
+ def _init_weights(self, module):
482
+ """Initialize the weights"""
483
+ if isinstance(module, Data2VecAudioFeatureProjection):
484
+ k = math.sqrt(1 / module.projection.in_features)
485
+ init.uniform_(module.projection.weight, a=-k, b=k)
486
+ init.uniform_(module.projection.bias, a=-k, b=k)
487
+ elif isinstance(module, Data2VecAudioPositionalConvLayer):
488
+ init.constant_(module.conv.bias, 0)
489
+ elif isinstance(module, nn.Linear):
490
+ init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
491
+
492
+ if module.bias is not None:
493
+ init.zeros_(module.bias)
494
+ elif isinstance(module, (nn.LayerNorm, nn.GroupNorm)):
495
+ if module.bias is not None:
496
+ init.zeros_(module.bias)
497
+ if module.weight is not None:
498
+ init.ones_(module.weight)
499
+ elif isinstance(module, nn.Conv1d):
500
+ init.kaiming_normal_(module.weight)
501
+
502
+ if module.bias is not None:
503
+ k = math.sqrt(module.groups / (module.in_channels * module.kernel_size[0]))
504
+ init.uniform_(module.bias, a=-k, b=k)
505
+
506
+ def _get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor | int, add_adapter: bool | None = None):
507
+ """
508
+ Computes the output length of the convolutional layers
509
+ """
510
+
511
+ add_adapter = self.config.add_adapter if add_adapter is None else add_adapter
512
+
513
+ def _conv_out_length(input_length, kernel_size, stride):
514
+ # 1D convolutional layer output length formula taken
515
+ # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html
516
+ return torch.div(input_length - kernel_size, stride, rounding_mode="floor") + 1
517
+
518
+ for kernel_size, stride in zip(self.config.conv_kernel, self.config.conv_stride):
519
+ input_lengths = _conv_out_length(input_lengths, kernel_size, stride)
520
+
521
+ if add_adapter:
522
+ for _ in range(self.config.num_adapter_layers):
523
+ input_lengths = _conv_out_length(input_lengths, 1, self.config.adapter_stride)
524
+
525
+ return input_lengths
526
+
527
+ def _get_feature_vector_attention_mask(
528
+ self, feature_vector_length: int, attention_mask: torch.LongTensor, add_adapter=None
529
+ ):
530
+ # Effectively attention_mask.sum(-1), but not inplace to be able to run
531
+ # on inference mode.
532
+ non_padded_lengths = attention_mask.cumsum(dim=-1)[:, -1]
533
+
534
+ output_lengths = self._get_feat_extract_output_lengths(non_padded_lengths, add_adapter=add_adapter)
535
+ output_lengths = output_lengths.to(torch.long)
536
+
537
+ batch_size = attention_mask.shape[0]
538
+
539
+ attention_mask = torch.zeros(
540
+ (batch_size, feature_vector_length), dtype=attention_mask.dtype, device=attention_mask.device
541
+ )
542
+ # these two operations makes sure that all values before the output lengths idxs are attended to
543
+ attention_mask[(torch.arange(attention_mask.shape[0], device=attention_mask.device), output_lengths - 1)] = 1
544
+ attention_mask = attention_mask.flip([-1]).cumsum(-1).flip([-1]).bool()
545
+ return attention_mask
546
+
547
+
548
+ def _compute_mask_indices(
549
+ shape: tuple[int, int],
550
+ mask_prob: float,
551
+ mask_length: int,
552
+ attention_mask: torch.LongTensor | None = None,
553
+ min_masks: int = 0,
554
+ ) -> np.ndarray:
555
+ """
556
+ Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
557
+ ASR](https://huggingface.co/papers/1904.08779). Note that this method is not optimized to run on TPU and should be run on
558
+ CPU as part of the preprocessing during training.
559
+
560
+ Args:
561
+ shape: The shape for which to compute masks. This should be of a tuple of size 2 where
562
+ the first element is the batch size and the second element is the length of the axis to span.
563
+ mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of
564
+ independently generated mask spans of length `mask_length` is computed by
565
+ `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
566
+ actual percentage will be smaller.
567
+ mask_length: size of the mask
568
+ min_masks: minimum number of masked spans
569
+ attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
570
+ each batch dimension.
571
+ """
572
+ batch_size, sequence_length = shape
573
+
574
+ if mask_length < 1:
575
+ raise ValueError("`mask_length` has to be bigger than 0.")
576
+
577
+ if mask_length > sequence_length:
578
+ raise ValueError(
579
+ f"`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: {mask_length}"
580
+ f" and `sequence_length`: {sequence_length}`"
581
+ )
582
+
583
+ # epsilon is used for probabilistic rounding
584
+ epsilon = np.random.rand(1).item()
585
+
586
+ def compute_num_masked_span(input_length):
587
+ """Given input length, compute how many spans should be masked"""
588
+ num_masked_span = int(mask_prob * input_length / mask_length + epsilon)
589
+ num_masked_span = max(num_masked_span, min_masks)
590
+
591
+ # make sure num masked span <= sequence_length
592
+ if num_masked_span * mask_length > sequence_length:
593
+ num_masked_span = sequence_length // mask_length
594
+
595
+ # make sure num_masked span is also <= input_length - (mask_length - 1)
596
+ if input_length - (mask_length - 1) < num_masked_span:
597
+ num_masked_span = max(input_length - (mask_length - 1), 0)
598
+
599
+ return num_masked_span
600
+
601
+ # compute number of masked spans in batch
602
+ input_lengths = (
603
+ attention_mask.detach().sum(-1).tolist()
604
+ if attention_mask is not None
605
+ else [sequence_length for _ in range(batch_size)]
606
+ )
607
+
608
+ # SpecAugment mask to fill
609
+ spec_aug_mask = np.zeros((batch_size, sequence_length), dtype=bool)
610
+ spec_aug_mask_idxs = []
611
+
612
+ max_num_masked_span = compute_num_masked_span(sequence_length)
613
+
614
+ if max_num_masked_span == 0:
615
+ return spec_aug_mask
616
+
617
+ for input_length in input_lengths:
618
+ # compute num of masked spans for this input
619
+ num_masked_span = compute_num_masked_span(input_length)
620
+
621
+ # get random indices to mask
622
+ spec_aug_mask_idx = np.random.choice(
623
+ np.arange(input_length - (mask_length - 1)), num_masked_span, replace=False
624
+ )
625
+
626
+ # pick first sampled index that will serve as a dummy index to pad vector
627
+ # to ensure same dimension for all batches due to probabilistic rounding
628
+ # Picking first sample just pads those vectors twice.
629
+ if len(spec_aug_mask_idx) == 0:
630
+ # this case can only happen if `input_length` is strictly smaller then
631
+ # `sequence_length` in which case the last token has to be a padding
632
+ # token which we can use as a dummy mask id
633
+ dummy_mask_idx = sequence_length - 1
634
+ else:
635
+ dummy_mask_idx = spec_aug_mask_idx[0]
636
+
637
+ spec_aug_mask_idx = np.concatenate(
638
+ [spec_aug_mask_idx, np.ones(max_num_masked_span - num_masked_span, dtype=np.int32) * dummy_mask_idx]
639
+ )
640
+ spec_aug_mask_idxs.append(spec_aug_mask_idx)
641
+
642
+ spec_aug_mask_idxs = np.array(spec_aug_mask_idxs)
643
+
644
+ # expand masked indices to masked spans
645
+ spec_aug_mask_idxs = np.broadcast_to(
646
+ spec_aug_mask_idxs[:, :, None], (batch_size, max_num_masked_span, mask_length)
647
+ )
648
+ spec_aug_mask_idxs = spec_aug_mask_idxs.reshape(batch_size, max_num_masked_span * mask_length)
649
+
650
+ # add offset to the starting indexes so that indexes now create a span
651
+ offsets = np.arange(mask_length)[None, None, :]
652
+ offsets = np.broadcast_to(offsets, (batch_size, max_num_masked_span, mask_length)).reshape(
653
+ batch_size, max_num_masked_span * mask_length
654
+ )
655
+ spec_aug_mask_idxs = spec_aug_mask_idxs + offsets
656
+
657
+ # ensure that we cannot have indices larger than sequence_length
658
+ if spec_aug_mask_idxs.max() > sequence_length - 1:
659
+ spec_aug_mask_idxs[spec_aug_mask_idxs > sequence_length - 1] = sequence_length - 1
660
+
661
+ # scatter indices to mask
662
+ np.put_along_axis(spec_aug_mask, spec_aug_mask_idxs, 1, -1)
663
+
664
+ return spec_aug_mask
665
+
666
+
667
+ Data2VecAudioBaseModelOutput = Wav2Vec2BaseModelOutput
668
+
669
+
670
+ @auto_docstring
671
+ class Data2VecAudioModel(Data2VecAudioPreTrainedModel):
672
+ def __init__(self, config: Data2VecAudioConfig):
673
+ super().__init__(config)
674
+ self.config = config
675
+ self.feature_extractor = Data2VecAudioFeatureEncoder(config)
676
+ self.feature_projection = Data2VecAudioFeatureProjection(config)
677
+
678
+ # model only needs masking vector if mask prob is > 0.0
679
+ if config.mask_time_prob > 0.0 or config.mask_feature_prob > 0.0:
680
+ self.masked_spec_embed = nn.Parameter(torch.Tensor(config.hidden_size).uniform_())
681
+
682
+ self.encoder = Data2VecAudioEncoder(config)
683
+
684
+ self.adapter = Data2VecAudioAdapter(config) if config.add_adapter else None
685
+
686
+ # Initialize weights and apply final processing
687
+ self.post_init()
688
+
689
+ def freeze_feature_encoder(self):
690
+ """
691
+ Calling this function will disable the gradient computation for the feature encoder so that its parameter will
692
+ not be updated during training.
693
+ """
694
+ self.feature_extractor._freeze_parameters()
695
+
696
+ def _mask_hidden_states(
697
+ self,
698
+ hidden_states: torch.FloatTensor,
699
+ mask_time_indices: torch.FloatTensor | None = None,
700
+ attention_mask: torch.LongTensor | None = None,
701
+ ):
702
+ """
703
+ Masks extracted features along time axis and/or along feature axis according to
704
+ [SpecAugment](https://huggingface.co/papers/1904.08779).
705
+ """
706
+
707
+ # `config.apply_spec_augment` can set masking to False
708
+ if not getattr(self.config, "apply_spec_augment", True):
709
+ return hidden_states
710
+
711
+ # generate indices & apply SpecAugment along time axis
712
+ batch_size, sequence_length, hidden_size = hidden_states.size()
713
+
714
+ if mask_time_indices is not None:
715
+ # apply SpecAugment along time axis with given mask_time_indices
716
+ hidden_states[mask_time_indices] = self.masked_spec_embed.to(hidden_states.dtype)
717
+ elif self.config.mask_time_prob > 0 and self.training:
718
+ mask_time_indices = _compute_mask_indices(
719
+ (batch_size, sequence_length),
720
+ mask_prob=self.config.mask_time_prob,
721
+ mask_length=self.config.mask_time_length,
722
+ attention_mask=attention_mask,
723
+ min_masks=self.config.mask_time_min_masks,
724
+ )
725
+ mask_time_indices = torch.tensor(mask_time_indices, device=hidden_states.device, dtype=torch.bool)
726
+ hidden_states[mask_time_indices] = self.masked_spec_embed.to(hidden_states.dtype)
727
+
728
+ if self.config.mask_feature_prob > 0 and self.training:
729
+ # generate indices & apply SpecAugment along feature axis
730
+ mask_feature_indices = _compute_mask_indices(
731
+ (batch_size, hidden_size),
732
+ mask_prob=self.config.mask_feature_prob,
733
+ mask_length=self.config.mask_feature_length,
734
+ min_masks=self.config.mask_feature_min_masks,
735
+ )
736
+ mask_feature_indices = torch.tensor(mask_feature_indices, device=hidden_states.device, dtype=torch.bool)
737
+ mask_feature_indices = mask_feature_indices[:, None].expand(-1, sequence_length, -1)
738
+ hidden_states[mask_feature_indices] = 0
739
+
740
+ return hidden_states
741
+
742
+ @auto_docstring
743
+ def forward(
744
+ self,
745
+ input_values: torch.Tensor | None,
746
+ attention_mask: torch.Tensor | None = None,
747
+ mask_time_indices: torch.FloatTensor | None = None,
748
+ output_attentions: bool | None = None,
749
+ output_hidden_states: bool | None = None,
750
+ return_dict: bool | None = None,
751
+ **kwargs,
752
+ ) -> tuple | Data2VecAudioBaseModelOutput:
753
+ r"""
754
+ mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
755
+ Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
756
+ masked extracted features in *config.proj_codevector_dim* space.
757
+ """
758
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
759
+ output_hidden_states = (
760
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
761
+ )
762
+ return_dict = return_dict if return_dict is not None else self.config.return_dict
763
+
764
+ extract_features = self.feature_extractor(input_values)
765
+ extract_features = extract_features.transpose(1, 2)
766
+
767
+ if attention_mask is not None:
768
+ # compute reduced attention_mask corresponding to feature vectors
769
+ attention_mask = self._get_feature_vector_attention_mask(
770
+ extract_features.shape[1], attention_mask, add_adapter=False
771
+ )
772
+
773
+ hidden_states, extract_features = self.feature_projection(extract_features)
774
+ hidden_states = self._mask_hidden_states(
775
+ hidden_states, mask_time_indices=mask_time_indices, attention_mask=attention_mask
776
+ )
777
+
778
+ encoder_outputs = self.encoder(
779
+ hidden_states,
780
+ attention_mask=attention_mask,
781
+ output_attentions=output_attentions,
782
+ output_hidden_states=output_hidden_states,
783
+ return_dict=return_dict,
784
+ )
785
+
786
+ hidden_states = encoder_outputs[0]
787
+
788
+ if self.adapter is not None:
789
+ hidden_states = self.adapter(hidden_states)
790
+
791
+ if not return_dict:
792
+ return (hidden_states, extract_features) + encoder_outputs[1:]
793
+
794
+ return Data2VecAudioBaseModelOutput(
795
+ last_hidden_state=hidden_states,
796
+ extract_features=extract_features,
797
+ hidden_states=encoder_outputs.hidden_states,
798
+ attentions=encoder_outputs.attentions,
799
+ )
800
+
801
+
802
+ _HIDDEN_STATES_START_POSITION = 2
803
+
804
+
805
+ @auto_docstring(
806
+ custom_intro="""
807
+ Data2VecAudio Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).
808
+ """
809
+ )
810
+ class Data2VecAudioForCTC(Data2VecAudioPreTrainedModel):
811
+ def __init__(self, config):
812
+ r"""
813
+ config ([`Data2VecAudioForCTC`]):
814
+ Model configuration class with all the parameters of the model. Initializing with a config file does not
815
+ load the weights associated with the model, only the configuration. Check out the
816
+ [`~PreTrainedModel.from_pretrained`] method to load the model weights.
817
+ """
818
+ super().__init__(config)
819
+
820
+ self.data2vec_audio = Data2VecAudioModel(config)
821
+ self.dropout = nn.Dropout(config.final_dropout)
822
+
823
+ if config.vocab_size is None:
824
+ raise ValueError(
825
+ f"You are trying to instantiate {self.__class__} with a configuration that "
826
+ "does not define the vocabulary size of the language model head. Please "
827
+ "instantiate the model as follows: `Data2VecAudioForCTC.from_pretrained(..., vocab_size=vocab_size)`. "
828
+ "or define `vocab_size` of your model's configuration."
829
+ )
830
+ output_hidden_size = (
831
+ config.output_hidden_size if hasattr(config, "add_adapter") and config.add_adapter else config.hidden_size
832
+ )
833
+ self.lm_head = nn.Linear(output_hidden_size, config.vocab_size)
834
+
835
+ # Initialize weights and apply final processing
836
+ self.post_init()
837
+
838
+ def freeze_feature_encoder(self):
839
+ """
840
+ Calling this function will disable the gradient computation for the feature encoder so that its parameter will
841
+ not be updated during training.
842
+ """
843
+ self.data2vec_audio.feature_extractor._freeze_parameters()
844
+
845
+ @auto_docstring
846
+ def forward(
847
+ self,
848
+ input_values: torch.Tensor | None,
849
+ attention_mask: torch.Tensor | None = None,
850
+ output_attentions: bool | None = None,
851
+ output_hidden_states: bool | None = None,
852
+ return_dict: bool | None = None,
853
+ labels: torch.Tensor | None = None,
854
+ **kwargs,
855
+ ) -> tuple | CausalLMOutput:
856
+ r"""
857
+ labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
858
+ Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
859
+ the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
860
+ All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
861
+ config.vocab_size - 1]`.
862
+ """
863
+ return_dict = return_dict if return_dict is not None else self.config.return_dict
864
+
865
+ if labels is not None and labels.max() >= self.config.vocab_size:
866
+ raise ValueError(f"Label values must be <= vocab_size: {self.config.vocab_size}")
867
+
868
+ outputs = self.data2vec_audio(
869
+ input_values,
870
+ attention_mask=attention_mask,
871
+ output_attentions=output_attentions,
872
+ output_hidden_states=output_hidden_states,
873
+ return_dict=return_dict,
874
+ )
875
+
876
+ hidden_states = outputs[0]
877
+ hidden_states = self.dropout(hidden_states)
878
+
879
+ logits = self.lm_head(hidden_states)
880
+
881
+ loss = None
882
+ if labels is not None:
883
+ # retrieve loss input_lengths from attention_mask
884
+ attention_mask = (
885
+ attention_mask if attention_mask is not None else torch.ones_like(input_values, dtype=torch.long)
886
+ )
887
+ input_lengths = self._get_feat_extract_output_lengths(attention_mask.sum(-1)).to(torch.long)
888
+
889
+ # assuming that padded tokens are filled with -100
890
+ # when not being attended to
891
+ labels_mask = labels >= 0
892
+ target_lengths = labels_mask.sum(-1)
893
+ flattened_targets = labels.masked_select(labels_mask)
894
+
895
+ # ctc_loss doesn't support fp16
896
+ log_probs = nn.functional.log_softmax(logits, dim=-1, dtype=torch.float32).transpose(0, 1)
897
+
898
+ with torch.backends.cudnn.flags(enabled=False):
899
+ loss = nn.functional.ctc_loss(
900
+ log_probs,
901
+ flattened_targets,
902
+ input_lengths,
903
+ target_lengths,
904
+ blank=self.config.pad_token_id,
905
+ reduction=self.config.ctc_loss_reduction,
906
+ zero_infinity=self.config.ctc_zero_infinity,
907
+ )
908
+
909
+ if not return_dict:
910
+ output = (logits,) + outputs[_HIDDEN_STATES_START_POSITION:]
911
+ return ((loss,) + output) if loss is not None else output
912
+
913
+ return CausalLMOutput(
914
+ loss=loss, logits=logits, hidden_states=outputs.hidden_states, attentions=outputs.attentions
915
+ )
916
+
917
+
918
+ @auto_docstring(
919
+ custom_intro="""
920
+ Data2VecAudio Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
921
+ SUPERB Keyword Spotting.
922
+ """
923
+ )
924
+ class Data2VecAudioForSequenceClassification(Data2VecAudioPreTrainedModel):
925
+ def __init__(self, config):
926
+ super().__init__(config)
927
+
928
+ if hasattr(config, "add_adapter") and config.add_adapter:
929
+ raise ValueError(
930
+ "Sequence classification does not support the use of Data2VecAudio adapters (config.add_adapter=True)"
931
+ )
932
+ self.data2vec_audio = Data2VecAudioModel(config)
933
+ num_layers = config.num_hidden_layers + 1 # transformer layers + input embeddings
934
+ if config.use_weighted_layer_sum:
935
+ self.layer_weights = nn.Parameter(torch.ones(num_layers) / num_layers)
936
+ self.projector = nn.Linear(config.hidden_size, config.classifier_proj_size)
937
+ self.classifier = nn.Linear(config.classifier_proj_size, config.num_labels)
938
+
939
+ # Initialize weights and apply final processing
940
+ self.post_init()
941
+
942
+ def freeze_feature_encoder(self):
943
+ """
944
+ Calling this function will disable the gradient computation for the feature encoder so that its parameter will
945
+ not be updated during training.
946
+ """
947
+ self.data2vec_audio.feature_extractor._freeze_parameters()
948
+
949
+ def freeze_base_model(self):
950
+ """
951
+ Calling this function will disable the gradient computation for the base model so that its parameters will not
952
+ be updated during training. Only the classification head will be updated.
953
+ """
954
+ for param in self.data2vec_audio.parameters():
955
+ param.requires_grad = False
956
+
957
+ @auto_docstring
958
+ def forward(
959
+ self,
960
+ input_values: torch.Tensor | None,
961
+ attention_mask: torch.Tensor | None = None,
962
+ output_attentions: bool | None = None,
963
+ output_hidden_states: bool | None = None,
964
+ return_dict: bool | None = None,
965
+ labels: torch.Tensor | None = None,
966
+ **kwargs,
967
+ ) -> tuple | SequenceClassifierOutput:
968
+ r"""
969
+ input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
970
+ Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
971
+ into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
972
+ (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
973
+ To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
974
+ into a tensor of type `torch.FloatTensor`. See [`Data2VecAudioProcessor.__call__`] for details.
975
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
976
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
977
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
978
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
979
+ """
980
+
981
+ return_dict = return_dict if return_dict is not None else self.config.return_dict
982
+ output_hidden_states = True if self.config.use_weighted_layer_sum else output_hidden_states
983
+
984
+ outputs = self.data2vec_audio(
985
+ input_values,
986
+ attention_mask=attention_mask,
987
+ output_attentions=output_attentions,
988
+ output_hidden_states=output_hidden_states,
989
+ return_dict=return_dict,
990
+ )
991
+
992
+ if self.config.use_weighted_layer_sum:
993
+ hidden_states = outputs[_HIDDEN_STATES_START_POSITION]
994
+ hidden_states = torch.stack(hidden_states, dim=1)
995
+ norm_weights = nn.functional.softmax(self.layer_weights, dim=-1)
996
+ hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(dim=1)
997
+ else:
998
+ hidden_states = outputs[0]
999
+
1000
+ hidden_states = self.projector(hidden_states)
1001
+ if attention_mask is None:
1002
+ pooled_output = hidden_states.mean(dim=1)
1003
+ else:
1004
+ padding_mask = self._get_feature_vector_attention_mask(hidden_states.shape[1], attention_mask)
1005
+ expand_padding_mask = padding_mask.unsqueeze(-1).repeat(1, 1, hidden_states.shape[2])
1006
+ hidden_states[~expand_padding_mask] = 0.0
1007
+ pooled_output = hidden_states.sum(dim=1) / padding_mask.sum(dim=1).view(-1, 1)
1008
+
1009
+ logits = self.classifier(pooled_output)
1010
+
1011
+ loss = None
1012
+ if labels is not None:
1013
+ loss_fct = CrossEntropyLoss()
1014
+ loss = loss_fct(logits.view(-1, self.config.num_labels), labels.view(-1))
1015
+
1016
+ if not return_dict:
1017
+ output = (logits,) + outputs[_HIDDEN_STATES_START_POSITION:]
1018
+ return ((loss,) + output) if loss is not None else output
1019
+
1020
+ return SequenceClassifierOutput(
1021
+ loss=loss,
1022
+ logits=logits,
1023
+ hidden_states=outputs.hidden_states,
1024
+ attentions=outputs.attentions,
1025
+ )
1026
+
1027
+
1028
+ @auto_docstring
1029
+ class Data2VecAudioForAudioFrameClassification(Data2VecAudioPreTrainedModel):
1030
+ def __init__(self, config):
1031
+ super().__init__(config)
1032
+
1033
+ if hasattr(config, "add_adapter") and config.add_adapter:
1034
+ raise ValueError(
1035
+ "Audio frame classification does not support the use of Data2VecAudio adapters (config.add_adapter=True)"
1036
+ )
1037
+ self.data2vec_audio = Data2VecAudioModel(config)
1038
+ num_layers = config.num_hidden_layers + 1 # transformer layers + input embeddings
1039
+ if config.use_weighted_layer_sum:
1040
+ self.layer_weights = nn.Parameter(torch.ones(num_layers) / num_layers)
1041
+ self.classifier = nn.Linear(config.hidden_size, config.num_labels)
1042
+ self.num_labels = config.num_labels
1043
+
1044
+ self.post_init()
1045
+
1046
+ def freeze_feature_encoder(self):
1047
+ """
1048
+ Calling this function will disable the gradient computation for the feature encoder so that its parameter will
1049
+ not be updated during training.
1050
+ """
1051
+ self.data2vec_audio.feature_extractor._freeze_parameters()
1052
+
1053
+ def freeze_base_model(self):
1054
+ """
1055
+ Calling this function will disable the gradient computation for the base model so that its parameters will not
1056
+ be updated during training. Only the classification head will be updated.
1057
+ """
1058
+ for param in self.data2vec_audio.parameters():
1059
+ param.requires_grad = False
1060
+
1061
+ @auto_docstring
1062
+ def forward(
1063
+ self,
1064
+ input_values: torch.Tensor | None,
1065
+ attention_mask: torch.Tensor | None = None,
1066
+ labels: torch.Tensor | None = None,
1067
+ output_attentions: bool | None = None,
1068
+ output_hidden_states: bool | None = None,
1069
+ return_dict: bool | None = None,
1070
+ **kwargs,
1071
+ ) -> tuple | TokenClassifierOutput:
1072
+ r"""
1073
+ input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
1074
+ Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
1075
+ into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
1076
+ (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
1077
+ To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
1078
+ into a tensor of type `torch.FloatTensor`. See [`Data2VecAudioProcessor.__call__`] for details.
1079
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1080
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
1081
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
1082
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
1083
+ """
1084
+
1085
+ return_dict = return_dict if return_dict is not None else self.config.return_dict
1086
+ output_hidden_states = True if self.config.use_weighted_layer_sum else output_hidden_states
1087
+
1088
+ outputs = self.data2vec_audio(
1089
+ input_values,
1090
+ attention_mask=attention_mask,
1091
+ output_attentions=output_attentions,
1092
+ output_hidden_states=output_hidden_states,
1093
+ return_dict=return_dict,
1094
+ )
1095
+
1096
+ if self.config.use_weighted_layer_sum:
1097
+ hidden_states = outputs[_HIDDEN_STATES_START_POSITION]
1098
+ hidden_states = torch.stack(hidden_states, dim=1)
1099
+ norm_weights = nn.functional.softmax(self.layer_weights, dim=-1)
1100
+ hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(dim=1)
1101
+ else:
1102
+ hidden_states = outputs[0]
1103
+
1104
+ logits = self.classifier(hidden_states)
1105
+
1106
+ loss = None
1107
+ if labels is not None:
1108
+ loss_fct = CrossEntropyLoss()
1109
+ loss = loss_fct(logits.view(-1, self.num_labels), torch.argmax(labels.view(-1, self.num_labels), axis=1))
1110
+
1111
+ if not return_dict:
1112
+ output = (logits,) + outputs[_HIDDEN_STATES_START_POSITION:]
1113
+ return output
1114
+
1115
+ return TokenClassifierOutput(
1116
+ loss=loss,
1117
+ logits=logits,
1118
+ hidden_states=outputs.hidden_states,
1119
+ attentions=outputs.attentions,
1120
+ )
1121
+
1122
+
1123
+ class AMSoftmaxLoss(nn.Module):
1124
+ def __init__(self, input_dim, num_labels, scale=30.0, margin=0.4):
1125
+ super().__init__()
1126
+ self.scale = scale
1127
+ self.margin = margin
1128
+ self.num_labels = num_labels
1129
+ self.weight = nn.Parameter(torch.randn(input_dim, num_labels), requires_grad=True)
1130
+ self.loss = nn.CrossEntropyLoss()
1131
+
1132
+ def forward(self, hidden_states, labels):
1133
+ labels = labels.flatten()
1134
+ weight = nn.functional.normalize(self.weight, dim=0)
1135
+ hidden_states = nn.functional.normalize(hidden_states, dim=1)
1136
+ cos_theta = torch.mm(hidden_states, weight)
1137
+ psi = cos_theta - self.margin
1138
+
1139
+ onehot = nn.functional.one_hot(labels, self.num_labels)
1140
+ logits = self.scale * torch.where(onehot.bool(), psi, cos_theta)
1141
+ loss = self.loss(logits, labels)
1142
+
1143
+ return loss
1144
+
1145
+
1146
+ class TDNNLayer(nn.Module):
1147
+ def __init__(self, config, layer_id=0):
1148
+ super().__init__()
1149
+ self.in_conv_dim = config.tdnn_dim[layer_id - 1] if layer_id > 0 else config.tdnn_dim[layer_id]
1150
+ self.out_conv_dim = config.tdnn_dim[layer_id]
1151
+ self.kernel_size = config.tdnn_kernel[layer_id]
1152
+ self.dilation = config.tdnn_dilation[layer_id]
1153
+
1154
+ self.kernel = nn.Linear(self.in_conv_dim * self.kernel_size, self.out_conv_dim)
1155
+ self.activation = nn.ReLU()
1156
+
1157
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
1158
+ if is_peft_available():
1159
+ from peft.tuners.lora import LoraLayer
1160
+
1161
+ if is_peft_available():
1162
+ if isinstance(self.kernel, LoraLayer):
1163
+ warnings.warn(
1164
+ "Detected LoRA on TDNNLayer. LoRA weights won't be applied due to optimization. "
1165
+ "You should exclude TDNNLayer from LoRA's target modules.",
1166
+ )
1167
+
1168
+ # for backward compatibility, we keep nn.Linear but call F.conv1d for speed up
1169
+ hidden_states = hidden_states.transpose(1, 2)
1170
+ weight = self.kernel.weight.view(self.out_conv_dim, self.kernel_size, self.in_conv_dim).transpose(1, 2)
1171
+ hidden_states = nn.functional.conv1d(hidden_states, weight, self.kernel.bias, dilation=self.dilation)
1172
+ hidden_states = hidden_states.transpose(1, 2)
1173
+
1174
+ hidden_states = self.activation(hidden_states)
1175
+ return hidden_states
1176
+
1177
+
1178
+ @auto_docstring(
1179
+ custom_intro="""
1180
+ Data2VecAudio Model with an XVector feature extraction head on top for tasks like Speaker Verification.
1181
+ """
1182
+ )
1183
+ class Data2VecAudioForXVector(Data2VecAudioPreTrainedModel):
1184
+ def __init__(self, config):
1185
+ super().__init__(config)
1186
+
1187
+ self.data2vec_audio = Data2VecAudioModel(config)
1188
+ num_layers = config.num_hidden_layers + 1 # transformer layers + input embeddings
1189
+ if config.use_weighted_layer_sum:
1190
+ self.layer_weights = nn.Parameter(torch.ones(num_layers) / num_layers)
1191
+ self.projector = nn.Linear(config.hidden_size, config.tdnn_dim[0])
1192
+
1193
+ tdnn_layers = [TDNNLayer(config, i) for i in range(len(config.tdnn_dim))]
1194
+ self.tdnn = nn.ModuleList(tdnn_layers)
1195
+
1196
+ self.feature_extractor = nn.Linear(config.tdnn_dim[-1] * 2, config.xvector_output_dim)
1197
+ self.classifier = nn.Linear(config.xvector_output_dim, config.xvector_output_dim)
1198
+
1199
+ self.objective = AMSoftmaxLoss(config.xvector_output_dim, config.num_labels)
1200
+
1201
+ self.post_init()
1202
+
1203
+ def freeze_feature_encoder(self):
1204
+ """
1205
+ Calling this function will disable the gradient computation for the feature encoder so that its parameter will
1206
+ not be updated during training.
1207
+ """
1208
+ self.data2vec_audio.feature_extractor._freeze_parameters()
1209
+
1210
+ def freeze_base_model(self):
1211
+ """
1212
+ Calling this function will disable the gradient computation for the base model so that its parameters will not
1213
+ be updated during training. Only the classification head will be updated.
1214
+ """
1215
+ for param in self.data2vec_audio.parameters():
1216
+ param.requires_grad = False
1217
+
1218
+ def _get_tdnn_output_lengths(self, input_lengths: torch.LongTensor | int):
1219
+ """
1220
+ Computes the output length of the TDNN layers
1221
+ """
1222
+
1223
+ def _conv_out_length(input_length, kernel_size, stride):
1224
+ # 1D convolutional layer output length formula taken
1225
+ # from https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html
1226
+ return (input_length - kernel_size) // stride + 1
1227
+
1228
+ for kernel_size in self.config.tdnn_kernel:
1229
+ input_lengths = _conv_out_length(input_lengths, kernel_size, 1)
1230
+
1231
+ return input_lengths
1232
+
1233
+ @auto_docstring
1234
+ def forward(
1235
+ self,
1236
+ input_values: torch.Tensor | None,
1237
+ attention_mask: torch.Tensor | None = None,
1238
+ output_attentions: bool | None = None,
1239
+ output_hidden_states: bool | None = None,
1240
+ return_dict: bool | None = None,
1241
+ labels: torch.Tensor | None = None,
1242
+ **kwargs,
1243
+ ) -> tuple | XVectorOutput:
1244
+ r"""
1245
+ input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
1246
+ Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
1247
+ into an array of type `list[float]`, a `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library
1248
+ (`pip install torchcodec`) or the soundfile library (`pip install soundfile`).
1249
+ To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion
1250
+ into a tensor of type `torch.FloatTensor`. See [`Data2VecAudioProcessor.__call__`] for details.
1251
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1252
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
1253
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
1254
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
1255
+ """
1256
+
1257
+ return_dict = return_dict if return_dict is not None else self.config.return_dict
1258
+ output_hidden_states = True if self.config.use_weighted_layer_sum else output_hidden_states
1259
+
1260
+ outputs = self.data2vec_audio(
1261
+ input_values,
1262
+ attention_mask=attention_mask,
1263
+ output_attentions=output_attentions,
1264
+ output_hidden_states=output_hidden_states,
1265
+ return_dict=return_dict,
1266
+ )
1267
+
1268
+ if self.config.use_weighted_layer_sum:
1269
+ hidden_states = outputs[_HIDDEN_STATES_START_POSITION]
1270
+ hidden_states = torch.stack(hidden_states, dim=1)
1271
+ norm_weights = nn.functional.softmax(self.layer_weights, dim=-1)
1272
+ hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(dim=1)
1273
+ else:
1274
+ hidden_states = outputs[0]
1275
+
1276
+ hidden_states = self.projector(hidden_states)
1277
+
1278
+ for tdnn_layer in self.tdnn:
1279
+ hidden_states = tdnn_layer(hidden_states)
1280
+
1281
+ # Statistic Pooling
1282
+ if attention_mask is None:
1283
+ mean_features = hidden_states.mean(dim=1)
1284
+ std_features = hidden_states.std(dim=1)
1285
+ else:
1286
+ feat_extract_output_lengths = self._get_feat_extract_output_lengths(attention_mask.sum(dim=1))
1287
+ tdnn_output_lengths = self._get_tdnn_output_lengths(feat_extract_output_lengths)
1288
+ mean_features = []
1289
+ std_features = []
1290
+ for i, length in enumerate(tdnn_output_lengths):
1291
+ mean_features.append(hidden_states[i, :length].mean(dim=0))
1292
+ std_features.append(hidden_states[i, :length].std(dim=0))
1293
+ mean_features = torch.stack(mean_features)
1294
+ std_features = torch.stack(std_features)
1295
+ statistic_pooling = torch.cat([mean_features, std_features], dim=-1)
1296
+
1297
+ output_embeddings = self.feature_extractor(statistic_pooling)
1298
+ logits = self.classifier(output_embeddings)
1299
+
1300
+ loss = None
1301
+ if labels is not None:
1302
+ loss = self.objective(logits, labels)
1303
+
1304
+ if not return_dict:
1305
+ output = (logits, output_embeddings) + outputs[_HIDDEN_STATES_START_POSITION:]
1306
+ return ((loss,) + output) if loss is not None else output
1307
+
1308
+ return XVectorOutput(
1309
+ loss=loss,
1310
+ logits=logits,
1311
+ embeddings=output_embeddings,
1312
+ hidden_states=outputs.hidden_states,
1313
+ attentions=outputs.attentions,
1314
+ )
1315
+
1316
+
1317
+ __all__ = [
1318
+ "Data2VecAudioForAudioFrameClassification",
1319
+ "Data2VecAudioForCTC",
1320
+ "Data2VecAudioForSequenceClassification",
1321
+ "Data2VecAudioForXVector",
1322
+ "Data2VecAudioModel",
1323
+ "Data2VecAudioPreTrainedModel",
1324
+ ]
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_text.py ADDED
@@ -0,0 +1,1208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
2
+ # This file was automatically generated from src/transformers/models/data2vec/modular_data2vec_text.py.
3
+ # Do NOT edit this file manually as any edits will be overwritten by the generation of
4
+ # the file from the modular. If any change should be done, please apply the change to the
5
+ # modular_data2vec_text.py file directly. One of our CI enforces this.
6
+ # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
7
+ # Copyright 2022 The HuggingFace Inc. team.
8
+ #
9
+ # Licensed under the Apache License, Version 2.0 (the "License");
10
+ # you may not use this file except in compliance with the License.
11
+ # You may obtain a copy of the License at
12
+ #
13
+ # http://www.apache.org/licenses/LICENSE-2.0
14
+ #
15
+ # Unless required by applicable law or agreed to in writing, software
16
+ # distributed under the License is distributed on an "AS IS" BASIS,
17
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+ # See the License for the specific language governing permissions and
19
+ # limitations under the License.
20
+
21
+ from collections.abc import Callable
22
+
23
+ import torch
24
+ import torch.nn as nn
25
+ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
26
+
27
+ from ... import initialization as init
28
+ from ...activations import ACT2FN, gelu
29
+ from ...cache_utils import Cache, DynamicCache, EncoderDecoderCache
30
+ from ...generation import GenerationMixin
31
+ from ...masking_utils import create_bidirectional_mask, create_causal_mask
32
+ from ...modeling_layers import GradientCheckpointingLayer
33
+ from ...modeling_outputs import (
34
+ BaseModelOutputWithPastAndCrossAttentions,
35
+ BaseModelOutputWithPoolingAndCrossAttentions,
36
+ CausalLMOutputWithCrossAttentions,
37
+ MaskedLMOutput,
38
+ MultipleChoiceModelOutput,
39
+ QuestionAnsweringModelOutput,
40
+ SequenceClassifierOutput,
41
+ TokenClassifierOutput,
42
+ )
43
+ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
44
+ from ...processing_utils import Unpack
45
+ from ...pytorch_utils import apply_chunking_to_forward
46
+ from ...utils import TransformersKwargs, auto_docstring, logging
47
+ from ...utils.generic import can_return_tuple, merge_with_config_defaults
48
+ from ...utils.output_capturing import capture_outputs
49
+ from .configuration_data2vec_text import Data2VecTextConfig
50
+
51
+
52
+ logger = logging.get_logger(__name__)
53
+
54
+
55
+ class Data2VecTextEmbeddings(nn.Module):
56
+ """Construct the embeddings from word, position and token_type embeddings."""
57
+
58
+ def __init__(self, config):
59
+ super().__init__()
60
+ self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
61
+ self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)
62
+
63
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
64
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
65
+ # position_ids (1, len position emb) is contiguous in memory and exported when serialized
66
+ self.register_buffer(
67
+ "position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)), persistent=False
68
+ )
69
+ self.register_buffer(
70
+ "token_type_ids", torch.zeros(self.position_ids.size(), dtype=torch.long), persistent=False
71
+ )
72
+
73
+ self.padding_idx = config.pad_token_id
74
+ self.position_embeddings = nn.Embedding(
75
+ config.max_position_embeddings, config.hidden_size, padding_idx=self.padding_idx
76
+ )
77
+
78
+ def forward(
79
+ self,
80
+ input_ids: torch.LongTensor | None = None,
81
+ token_type_ids: torch.LongTensor | None = None,
82
+ position_ids: torch.LongTensor | None = None,
83
+ inputs_embeds: torch.FloatTensor | None = None,
84
+ past_key_values_length: int = 0,
85
+ ) -> torch.Tensor:
86
+ if position_ids is None:
87
+ if input_ids is not None:
88
+ # Create the position ids from the input token ids. Any padded tokens remain padded.
89
+ position_ids = self.create_position_ids_from_input_ids(
90
+ input_ids, self.padding_idx, past_key_values_length
91
+ )
92
+ else:
93
+ position_ids = self.create_position_ids_from_inputs_embeds(inputs_embeds, self.padding_idx)
94
+
95
+ if input_ids is not None:
96
+ input_shape = input_ids.size()
97
+ else:
98
+ input_shape = inputs_embeds.size()[:-1]
99
+
100
+ batch_size, seq_length = input_shape
101
+
102
+ # Setting the token_type_ids to the registered buffer in constructor where it is all zeros, which usually occurs
103
+ # when its auto-generated, registered buffer helps users when tracing the model without passing token_type_ids, solves
104
+ # issue #5664
105
+ if token_type_ids is None:
106
+ if hasattr(self, "token_type_ids"):
107
+ # NOTE: We assume either pos ids to have bsz == 1 (broadcastable) or bsz == effective bsz (input_shape[0])
108
+ buffered_token_type_ids = self.token_type_ids.expand(position_ids.shape[0], -1)
109
+ buffered_token_type_ids = torch.gather(buffered_token_type_ids, dim=1, index=position_ids)
110
+ token_type_ids = buffered_token_type_ids.expand(batch_size, seq_length)
111
+ else:
112
+ token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=self.position_ids.device)
113
+
114
+ if inputs_embeds is None:
115
+ inputs_embeds = self.word_embeddings(input_ids)
116
+ token_type_embeddings = self.token_type_embeddings(token_type_ids)
117
+ embeddings = inputs_embeds + token_type_embeddings
118
+
119
+ position_embeddings = self.position_embeddings(position_ids)
120
+ embeddings = embeddings + position_embeddings
121
+
122
+ embeddings = self.LayerNorm(embeddings)
123
+ embeddings = self.dropout(embeddings)
124
+ return embeddings
125
+
126
+ @staticmethod
127
+ def create_position_ids_from_inputs_embeds(inputs_embeds, padding_idx):
128
+ """
129
+ We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.
130
+
131
+ Args:
132
+ inputs_embeds: torch.Tensor
133
+
134
+ Returns: torch.Tensor
135
+ """
136
+ input_shape = inputs_embeds.size()[:-1]
137
+ sequence_length = input_shape[1]
138
+
139
+ position_ids = torch.arange(
140
+ padding_idx + 1, sequence_length + padding_idx + 1, dtype=torch.long, device=inputs_embeds.device
141
+ )
142
+ return position_ids.unsqueeze(0).expand(input_shape)
143
+
144
+ @staticmethod
145
+ def create_position_ids_from_input_ids(input_ids, padding_idx, past_key_values_length=0):
146
+ """
147
+ Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
148
+ are ignored. This is modified from fairseq's `utils.make_positions`.
149
+
150
+ Args:
151
+ x: torch.Tensor x:
152
+
153
+ Returns: torch.Tensor
154
+ """
155
+ # The series of casts and type-conversions here are carefully balanced to both work with ONNX export and XLA.
156
+ mask = input_ids.ne(padding_idx).int()
157
+ incremental_indices = (torch.cumsum(mask, dim=1).type_as(mask) + past_key_values_length) * mask
158
+ return incremental_indices.long() + padding_idx
159
+
160
+
161
+ def eager_attention_forward(
162
+ module: nn.Module,
163
+ query: torch.Tensor,
164
+ key: torch.Tensor,
165
+ value: torch.Tensor,
166
+ attention_mask: torch.Tensor | None,
167
+ scaling: float | None = None,
168
+ dropout: float = 0.0,
169
+ **kwargs: Unpack[TransformersKwargs],
170
+ ):
171
+ if scaling is None:
172
+ scaling = query.size(-1) ** -0.5
173
+
174
+ # Take the dot product between "query" and "key" to get the raw attention scores.
175
+ attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling
176
+
177
+ if attention_mask is not None:
178
+ attn_weights = attn_weights + attention_mask
179
+
180
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1)
181
+ attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
182
+
183
+ attn_output = torch.matmul(attn_weights, value)
184
+ attn_output = attn_output.transpose(1, 2).contiguous()
185
+
186
+ return attn_output, attn_weights
187
+
188
+
189
+ class Data2VecTextSelfAttention(nn.Module):
190
+ def __init__(self, config, is_causal=False, layer_idx=None):
191
+ super().__init__()
192
+ if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
193
+ raise ValueError(
194
+ f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
195
+ f"heads ({config.num_attention_heads})"
196
+ )
197
+ self.config = config
198
+
199
+ self.num_attention_heads = config.num_attention_heads
200
+ self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
201
+ self.all_head_size = self.num_attention_heads * self.attention_head_size
202
+ self.scaling = self.attention_head_size**-0.5
203
+
204
+ self.query = nn.Linear(config.hidden_size, self.all_head_size)
205
+ self.key = nn.Linear(config.hidden_size, self.all_head_size)
206
+ self.value = nn.Linear(config.hidden_size, self.all_head_size)
207
+
208
+ self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
209
+
210
+ self.is_decoder = config.is_decoder
211
+ self.is_causal = is_causal
212
+ self.layer_idx = layer_idx
213
+
214
+ def forward(
215
+ self,
216
+ hidden_states: torch.Tensor,
217
+ attention_mask: torch.FloatTensor | None = None,
218
+ past_key_values: Cache | None = None,
219
+ **kwargs: Unpack[TransformersKwargs],
220
+ ) -> tuple[torch.Tensor]:
221
+ input_shape = hidden_states.shape[:-1]
222
+ hidden_shape = (*input_shape, -1, self.attention_head_size)
223
+
224
+ # get all proj
225
+ query_layer = self.query(hidden_states).view(*hidden_shape).transpose(1, 2)
226
+ key_layer = self.key(hidden_states).view(*hidden_shape).transpose(1, 2)
227
+ value_layer = self.value(hidden_states).view(*hidden_shape).transpose(1, 2)
228
+
229
+ if past_key_values is not None:
230
+ # decoder-only data2vec_text can have a simple dynamic cache for example
231
+ current_past_key_values = past_key_values
232
+ if isinstance(past_key_values, EncoderDecoderCache):
233
+ current_past_key_values = past_key_values.self_attention_cache
234
+
235
+ # save all key/value_layer to cache to be re-used for fast auto-regressive generation
236
+ key_layer, value_layer = current_past_key_values.update(key_layer, value_layer, self.layer_idx)
237
+
238
+ attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
239
+ self.config._attn_implementation, eager_attention_forward
240
+ )
241
+
242
+ attn_output, attn_weights = attention_interface(
243
+ self,
244
+ query_layer,
245
+ key_layer,
246
+ value_layer,
247
+ attention_mask,
248
+ dropout=0.0 if not self.training else self.dropout.p,
249
+ scaling=self.scaling,
250
+ **kwargs,
251
+ )
252
+ attn_output = attn_output.reshape(*input_shape, -1).contiguous()
253
+ return attn_output, attn_weights
254
+
255
+
256
+ class Data2VecTextCrossAttention(nn.Module):
257
+ def __init__(self, config, is_causal=False, layer_idx=None):
258
+ super().__init__()
259
+ if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
260
+ raise ValueError(
261
+ f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
262
+ f"heads ({config.num_attention_heads})"
263
+ )
264
+ self.config = config
265
+
266
+ self.num_attention_heads = config.num_attention_heads
267
+ self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
268
+ self.all_head_size = self.num_attention_heads * self.attention_head_size
269
+ self.scaling = self.attention_head_size**-0.5
270
+
271
+ self.query = nn.Linear(config.hidden_size, self.all_head_size)
272
+ self.key = nn.Linear(config.hidden_size, self.all_head_size)
273
+ self.value = nn.Linear(config.hidden_size, self.all_head_size)
274
+
275
+ self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
276
+
277
+ self.is_causal = is_causal
278
+ self.layer_idx = layer_idx
279
+
280
+ def forward(
281
+ self,
282
+ hidden_states: torch.Tensor,
283
+ encoder_hidden_states: torch.FloatTensor | None = None,
284
+ attention_mask: torch.FloatTensor | None = None,
285
+ past_key_values: EncoderDecoderCache | None = None,
286
+ **kwargs: Unpack[TransformersKwargs],
287
+ ) -> tuple[torch.Tensor]:
288
+ # determine input shapes
289
+ input_shape = hidden_states.shape[:-1]
290
+
291
+ hidden_shape = (*input_shape, -1, self.attention_head_size)
292
+
293
+ # get query proj
294
+ query_layer = self.query(hidden_states).view(hidden_shape).transpose(1, 2)
295
+
296
+ is_updated = past_key_values.is_updated.get(self.layer_idx) if past_key_values is not None else False
297
+ if past_key_values is not None and is_updated:
298
+ # reuse k,v, cross_attentions
299
+ key_layer = past_key_values.cross_attention_cache.layers[self.layer_idx].keys
300
+ value_layer = past_key_values.cross_attention_cache.layers[self.layer_idx].values
301
+ else:
302
+ kv_shape = (*encoder_hidden_states.shape[:-1], -1, self.attention_head_size)
303
+ key_layer = self.key(encoder_hidden_states).view(kv_shape).transpose(1, 2)
304
+ value_layer = self.value(encoder_hidden_states).view(kv_shape).transpose(1, 2)
305
+
306
+ if past_key_values is not None:
307
+ # save all states to the cache
308
+ key_layer, value_layer = past_key_values.cross_attention_cache.update(
309
+ key_layer, value_layer, self.layer_idx
310
+ )
311
+ # set flag that curr layer for cross-attn is already updated so we can re-use in subsequent calls
312
+ past_key_values.is_updated[self.layer_idx] = True
313
+
314
+ attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface(
315
+ self.config._attn_implementation, eager_attention_forward
316
+ )
317
+
318
+ attn_output, attn_weights = attention_interface(
319
+ self,
320
+ query_layer,
321
+ key_layer,
322
+ value_layer,
323
+ attention_mask,
324
+ dropout=0.0 if not self.training else self.dropout.p,
325
+ scaling=self.scaling,
326
+ **kwargs,
327
+ )
328
+ attn_output = attn_output.reshape(*input_shape, -1).contiguous()
329
+ return attn_output, attn_weights
330
+
331
+
332
+ class Data2VecTextSelfOutput(nn.Module):
333
+ def __init__(self, config):
334
+ super().__init__()
335
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
336
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
337
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
338
+
339
+ def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Tensor) -> torch.Tensor:
340
+ hidden_states = self.dense(hidden_states)
341
+ hidden_states = self.dropout(hidden_states)
342
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
343
+ return hidden_states
344
+
345
+
346
+ class Data2VecTextAttention(nn.Module):
347
+ def __init__(self, config, is_causal=False, layer_idx=None, is_cross_attention=False):
348
+ super().__init__()
349
+ self.is_cross_attention = is_cross_attention
350
+ attention_class = Data2VecTextCrossAttention if is_cross_attention else Data2VecTextSelfAttention
351
+ self.self = attention_class(config, is_causal=is_causal, layer_idx=layer_idx)
352
+ self.output = Data2VecTextSelfOutput(config)
353
+
354
+ def forward(
355
+ self,
356
+ hidden_states: torch.Tensor,
357
+ attention_mask: torch.FloatTensor | None = None,
358
+ encoder_hidden_states: torch.FloatTensor | None = None,
359
+ encoder_attention_mask: torch.FloatTensor | None = None,
360
+ past_key_values: Cache | None = None,
361
+ **kwargs: Unpack[TransformersKwargs],
362
+ ) -> tuple[torch.Tensor]:
363
+ attention_mask = attention_mask if not self.is_cross_attention else encoder_attention_mask
364
+ attention_output, attn_weights = self.self(
365
+ hidden_states,
366
+ encoder_hidden_states=encoder_hidden_states,
367
+ attention_mask=attention_mask,
368
+ past_key_values=past_key_values,
369
+ **kwargs,
370
+ )
371
+ attention_output = self.output(attention_output, hidden_states)
372
+ return attention_output, attn_weights
373
+
374
+
375
+ class Data2VecTextIntermediate(nn.Module):
376
+ def __init__(self, config):
377
+ super().__init__()
378
+ self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
379
+ if isinstance(config.hidden_act, str):
380
+ self.intermediate_act_fn = ACT2FN[config.hidden_act]
381
+ else:
382
+ self.intermediate_act_fn = config.hidden_act
383
+
384
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
385
+ hidden_states = self.dense(hidden_states)
386
+ hidden_states = self.intermediate_act_fn(hidden_states)
387
+ return hidden_states
388
+
389
+
390
+ class Data2VecTextOutput(nn.Module):
391
+ def __init__(self, config):
392
+ super().__init__()
393
+ self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
394
+ self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
395
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
396
+
397
+ def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Tensor) -> torch.Tensor:
398
+ hidden_states = self.dense(hidden_states)
399
+ hidden_states = self.dropout(hidden_states)
400
+ hidden_states = self.LayerNorm(hidden_states + input_tensor)
401
+ return hidden_states
402
+
403
+
404
+ class Data2VecTextLayer(GradientCheckpointingLayer):
405
+ def __init__(self, config, layer_idx=None):
406
+ super().__init__()
407
+ self.chunk_size_feed_forward = config.chunk_size_feed_forward
408
+ self.seq_len_dim = 1
409
+ self.attention = Data2VecTextAttention(config, is_causal=config.is_decoder, layer_idx=layer_idx)
410
+ self.is_decoder = config.is_decoder
411
+ self.add_cross_attention = config.add_cross_attention
412
+ if self.add_cross_attention:
413
+ if not self.is_decoder:
414
+ raise ValueError(f"{self} should be used as a decoder model if cross attention is added")
415
+ self.crossattention = Data2VecTextAttention(
416
+ config,
417
+ is_causal=False,
418
+ layer_idx=layer_idx,
419
+ is_cross_attention=True,
420
+ )
421
+ self.intermediate = Data2VecTextIntermediate(config)
422
+ self.output = Data2VecTextOutput(config)
423
+
424
+ def forward(
425
+ self,
426
+ hidden_states: torch.Tensor,
427
+ attention_mask: torch.FloatTensor | None = None,
428
+ encoder_hidden_states: torch.FloatTensor | None = None,
429
+ encoder_attention_mask: torch.FloatTensor | None = None,
430
+ past_key_values: Cache | None = None,
431
+ **kwargs: Unpack[TransformersKwargs],
432
+ ) -> torch.Tensor:
433
+ self_attention_output, _ = self.attention(
434
+ hidden_states,
435
+ attention_mask,
436
+ past_key_values=past_key_values,
437
+ **kwargs,
438
+ )
439
+ attention_output = self_attention_output
440
+
441
+ if self.is_decoder and encoder_hidden_states is not None:
442
+ if not hasattr(self, "crossattention"):
443
+ raise ValueError(
444
+ f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers"
445
+ " by setting `config.add_cross_attention=True`"
446
+ )
447
+
448
+ cross_attention_output, _ = self.crossattention(
449
+ self_attention_output,
450
+ None, # attention_mask
451
+ encoder_hidden_states,
452
+ encoder_attention_mask,
453
+ past_key_values=past_key_values,
454
+ **kwargs,
455
+ )
456
+ attention_output = cross_attention_output
457
+
458
+ layer_output = apply_chunking_to_forward(
459
+ self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
460
+ )
461
+ return layer_output
462
+
463
+ def feed_forward_chunk(self, attention_output):
464
+ intermediate_output = self.intermediate(attention_output)
465
+ layer_output = self.output(intermediate_output, attention_output)
466
+ return layer_output
467
+
468
+
469
+ @auto_docstring
470
+ class Data2VecTextPreTrainedModel(PreTrainedModel):
471
+ config_class = Data2VecTextConfig
472
+ base_model_prefix = "data2vec_text"
473
+ supports_gradient_checkpointing = True
474
+ _no_split_modules = ["Data2VecTextForTextEmbeddings", "Data2VecTextLayer"]
475
+ _supports_flash_attn = True
476
+ _supports_sdpa = True
477
+ _supports_flex_attn = True
478
+ _supports_attention_backend = True
479
+ _can_record_outputs = {
480
+ "hidden_states": Data2VecTextLayer,
481
+ "attentions": Data2VecTextSelfAttention,
482
+ "cross_attentions": Data2VecTextCrossAttention,
483
+ }
484
+
485
+ def _init_weights(self, module):
486
+ super()._init_weights(module)
487
+ if isinstance(module, Data2VecTextEmbeddings):
488
+ init.copy_(module.position_ids, torch.arange(module.position_ids.shape[-1]).expand((1, -1)))
489
+ init.zeros_(module.token_type_ids)
490
+
491
+
492
+ class Data2VecTextEncoder(nn.Module):
493
+ def __init__(self, config):
494
+ super().__init__()
495
+ self.config = config
496
+ self.layer = nn.ModuleList([Data2VecTextLayer(config, layer_idx=i) for i in range(config.num_hidden_layers)])
497
+
498
+ def forward(
499
+ self,
500
+ hidden_states: torch.Tensor,
501
+ attention_mask: torch.FloatTensor | None = None,
502
+ encoder_hidden_states: torch.FloatTensor | None = None,
503
+ encoder_attention_mask: torch.FloatTensor | None = None,
504
+ past_key_values: Cache | None = None,
505
+ use_cache: bool | None = None,
506
+ **kwargs: Unpack[TransformersKwargs],
507
+ ) -> tuple[torch.Tensor] | BaseModelOutputWithPastAndCrossAttentions:
508
+ for i, layer_module in enumerate(self.layer):
509
+ hidden_states = layer_module(
510
+ hidden_states,
511
+ attention_mask,
512
+ encoder_hidden_states, # as a positional argument for gradient checkpointing
513
+ encoder_attention_mask=encoder_attention_mask,
514
+ past_key_values=past_key_values,
515
+ **kwargs,
516
+ )
517
+
518
+ return BaseModelOutputWithPastAndCrossAttentions(
519
+ last_hidden_state=hidden_states,
520
+ past_key_values=past_key_values if use_cache else None,
521
+ )
522
+
523
+
524
+ class Data2VecTextPooler(nn.Module):
525
+ def __init__(self, config):
526
+ super().__init__()
527
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
528
+ self.activation = nn.Tanh()
529
+
530
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
531
+ # We "pool" the model by simply taking the hidden state corresponding
532
+ # to the first token.
533
+ first_token_tensor = hidden_states[:, 0]
534
+ pooled_output = self.dense(first_token_tensor)
535
+ pooled_output = self.activation(pooled_output)
536
+ return pooled_output
537
+
538
+
539
+ @auto_docstring
540
+ class Data2VecTextModel(Data2VecTextPreTrainedModel):
541
+ _no_split_modules = ["Data2VecTextEmbeddings", "Data2VecTextLayer"]
542
+
543
+ def __init__(self, config, add_pooling_layer=True):
544
+ r"""
545
+ add_pooling_layer (bool, *optional*, defaults to `True`):
546
+ Whether to add a pooling layer
547
+ """
548
+ super().__init__(config)
549
+ self.config = config
550
+ self.gradient_checkpointing = False
551
+
552
+ self.embeddings = Data2VecTextEmbeddings(config)
553
+ self.encoder = Data2VecTextEncoder(config)
554
+
555
+ self.pooler = Data2VecTextPooler(config) if add_pooling_layer else None
556
+
557
+ # Initialize weights and apply final processing
558
+ self.post_init()
559
+
560
+ def get_input_embeddings(self):
561
+ return self.embeddings.word_embeddings
562
+
563
+ def set_input_embeddings(self, value):
564
+ self.embeddings.word_embeddings = value
565
+
566
+ @merge_with_config_defaults
567
+ @capture_outputs
568
+ @auto_docstring
569
+ def forward(
570
+ self,
571
+ input_ids: torch.Tensor | None = None,
572
+ attention_mask: torch.Tensor | None = None,
573
+ token_type_ids: torch.Tensor | None = None,
574
+ position_ids: torch.Tensor | None = None,
575
+ inputs_embeds: torch.Tensor | None = None,
576
+ encoder_hidden_states: torch.Tensor | None = None,
577
+ encoder_attention_mask: torch.Tensor | None = None,
578
+ past_key_values: Cache | None = None,
579
+ use_cache: bool | None = None,
580
+ **kwargs: Unpack[TransformersKwargs],
581
+ ) -> tuple[torch.Tensor] | BaseModelOutputWithPoolingAndCrossAttentions:
582
+ if (input_ids is None) ^ (inputs_embeds is not None):
583
+ raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
584
+
585
+ if self.config.is_decoder:
586
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
587
+ else:
588
+ use_cache = False
589
+
590
+ if use_cache and past_key_values is None:
591
+ past_key_values = (
592
+ EncoderDecoderCache(DynamicCache(config=self.config), DynamicCache(config=self.config))
593
+ if encoder_hidden_states is not None or self.config.is_encoder_decoder
594
+ else DynamicCache(config=self.config)
595
+ )
596
+
597
+ past_key_values_length = past_key_values.get_seq_length() if past_key_values is not None else 0
598
+
599
+ embedding_output = self.embeddings(
600
+ input_ids=input_ids,
601
+ position_ids=position_ids,
602
+ token_type_ids=token_type_ids,
603
+ inputs_embeds=inputs_embeds,
604
+ past_key_values_length=past_key_values_length,
605
+ )
606
+
607
+ attention_mask, encoder_attention_mask = self._create_attention_masks(
608
+ attention_mask=attention_mask,
609
+ encoder_attention_mask=encoder_attention_mask,
610
+ embedding_output=embedding_output,
611
+ encoder_hidden_states=encoder_hidden_states,
612
+ past_key_values=past_key_values,
613
+ )
614
+
615
+ encoder_outputs = self.encoder(
616
+ embedding_output,
617
+ attention_mask=attention_mask,
618
+ encoder_hidden_states=encoder_hidden_states,
619
+ encoder_attention_mask=encoder_attention_mask,
620
+ past_key_values=past_key_values,
621
+ use_cache=use_cache,
622
+ position_ids=position_ids,
623
+ **kwargs,
624
+ )
625
+ sequence_output = encoder_outputs.last_hidden_state
626
+ pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
627
+
628
+ return BaseModelOutputWithPoolingAndCrossAttentions(
629
+ last_hidden_state=sequence_output,
630
+ pooler_output=pooled_output,
631
+ past_key_values=encoder_outputs.past_key_values,
632
+ )
633
+
634
+ def _create_attention_masks(
635
+ self,
636
+ attention_mask,
637
+ encoder_attention_mask,
638
+ embedding_output,
639
+ encoder_hidden_states,
640
+ past_key_values,
641
+ ):
642
+ if self.config.is_decoder:
643
+ attention_mask = create_causal_mask(
644
+ config=self.config,
645
+ inputs_embeds=embedding_output,
646
+ attention_mask=attention_mask,
647
+ past_key_values=past_key_values,
648
+ )
649
+ else:
650
+ attention_mask = create_bidirectional_mask(
651
+ config=self.config,
652
+ inputs_embeds=embedding_output,
653
+ attention_mask=attention_mask,
654
+ )
655
+
656
+ if encoder_attention_mask is not None:
657
+ encoder_attention_mask = create_bidirectional_mask(
658
+ config=self.config,
659
+ inputs_embeds=embedding_output,
660
+ attention_mask=encoder_attention_mask,
661
+ encoder_hidden_states=encoder_hidden_states,
662
+ )
663
+
664
+ return attention_mask, encoder_attention_mask
665
+
666
+
667
+ class Data2VecTextLMHead(nn.Module):
668
+ """Data2VecText Head for masked language modeling."""
669
+
670
+ def __init__(self, config):
671
+ super().__init__()
672
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
673
+ self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
674
+
675
+ self.decoder = nn.Linear(config.hidden_size, config.vocab_size)
676
+ self.bias = nn.Parameter(torch.zeros(config.vocab_size))
677
+
678
+ def forward(self, features, **kwargs):
679
+ x = self.dense(features)
680
+ x = gelu(x)
681
+ x = self.layer_norm(x)
682
+
683
+ # project back to size of vocabulary with bias
684
+ x = self.decoder(x)
685
+
686
+ return x
687
+
688
+
689
+ class Data2VecTextClassificationHead(nn.Module):
690
+ """Head for sentence-level classification tasks."""
691
+
692
+ def __init__(self, config):
693
+ super().__init__()
694
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
695
+ classifier_dropout = (
696
+ config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
697
+ )
698
+ self.dropout = nn.Dropout(classifier_dropout)
699
+ self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
700
+
701
+ def forward(self, features, **kwargs):
702
+ x = features[:, 0, :] # take <s> token (equiv. to [CLS])
703
+ x = self.dropout(x)
704
+ x = self.dense(x)
705
+ x = torch.tanh(x)
706
+ x = self.dropout(x)
707
+ x = self.out_proj(x)
708
+ return x
709
+
710
+
711
+ @auto_docstring(
712
+ custom_intro="""
713
+ Data2VecText Model with a `language modeling` head on top for CLM fine-tuning.
714
+ """
715
+ )
716
+ class Data2VecTextForCausalLM(Data2VecTextPreTrainedModel, GenerationMixin):
717
+ _tied_weights_keys = {
718
+ "lm_head.decoder.weight": "data2vec_text.embeddings.word_embeddings.weight",
719
+ "lm_head.decoder.bias": "lm_head.bias",
720
+ }
721
+
722
+ def __init__(self, config):
723
+ super().__init__(config)
724
+
725
+ if not config.is_decoder:
726
+ logger.warning("If you want to use `Data2VecTextLMHeadModel` as a standalone, add `is_decoder=True.`")
727
+
728
+ self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
729
+ self.lm_head = Data2VecTextLMHead(config)
730
+
731
+ # Initialize weights and apply final processing
732
+ self.post_init()
733
+
734
+ def get_output_embeddings(self):
735
+ return self.lm_head.decoder
736
+
737
+ def set_output_embeddings(self, new_embeddings):
738
+ self.lm_head.decoder = new_embeddings
739
+
740
+ @can_return_tuple
741
+ @auto_docstring
742
+ def forward(
743
+ self,
744
+ input_ids: torch.LongTensor | None = None,
745
+ attention_mask: torch.FloatTensor | None = None,
746
+ token_type_ids: torch.LongTensor | None = None,
747
+ position_ids: torch.LongTensor | None = None,
748
+ inputs_embeds: torch.FloatTensor | None = None,
749
+ encoder_hidden_states: torch.FloatTensor | None = None,
750
+ encoder_attention_mask: torch.FloatTensor | None = None,
751
+ labels: torch.LongTensor | None = None,
752
+ past_key_values: tuple[tuple[torch.FloatTensor]] | None = None,
753
+ use_cache: bool | None = None,
754
+ logits_to_keep: int | torch.Tensor = 0,
755
+ **kwargs: Unpack[TransformersKwargs],
756
+ ) -> tuple | CausalLMOutputWithCrossAttentions:
757
+ r"""
758
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
759
+ Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
760
+ `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
761
+ ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
762
+
763
+ Example:
764
+
765
+ ```python
766
+ >>> from transformers import AutoTokenizer, Data2VecTextForCausalLM, Data2VecTextConfig
767
+ >>> import torch
768
+
769
+ >>> tokenizer = AutoTokenizer.from_pretrained("facebook/data2vec-text-base")
770
+ >>> config = Data2VecTextConfig.from_pretrained("facebook/data2vec-text-base")
771
+ >>> config.is_decoder = True
772
+ >>> model = Data2VecTextForCausalLM.from_pretrained("facebook/data2vec-text-base", config=config)
773
+
774
+ >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
775
+ >>> outputs = model(**inputs)
776
+
777
+ >>> prediction_logits = outputs.logits
778
+ ```"""
779
+ if labels is not None:
780
+ use_cache = False
781
+
782
+ outputs: BaseModelOutputWithPoolingAndCrossAttentions = self.data2vec_text(
783
+ input_ids,
784
+ attention_mask=attention_mask,
785
+ token_type_ids=token_type_ids,
786
+ position_ids=position_ids,
787
+ inputs_embeds=inputs_embeds,
788
+ encoder_hidden_states=encoder_hidden_states,
789
+ encoder_attention_mask=encoder_attention_mask,
790
+ past_key_values=past_key_values,
791
+ use_cache=use_cache,
792
+ return_dict=True,
793
+ **kwargs,
794
+ )
795
+
796
+ hidden_states = outputs.last_hidden_state
797
+ # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
798
+ slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
799
+ logits = self.lm_head(hidden_states[:, slice_indices, :])
800
+
801
+ loss = None
802
+ if labels is not None:
803
+ loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
804
+
805
+ return CausalLMOutputWithCrossAttentions(
806
+ loss=loss,
807
+ logits=logits,
808
+ past_key_values=outputs.past_key_values,
809
+ hidden_states=outputs.hidden_states,
810
+ attentions=outputs.attentions,
811
+ cross_attentions=outputs.cross_attentions,
812
+ )
813
+
814
+
815
+ @auto_docstring
816
+ class Data2VecTextForMaskedLM(Data2VecTextPreTrainedModel):
817
+ _tied_weights_keys = {
818
+ "lm_head.decoder.weight": "data2vec_text.embeddings.word_embeddings.weight",
819
+ "lm_head.decoder.bias": "lm_head.bias",
820
+ }
821
+
822
+ def __init__(self, config):
823
+ super().__init__(config)
824
+
825
+ if config.is_decoder:
826
+ logger.warning(
827
+ "If you want to use `Data2VecTextForMaskedLM` make sure `config.is_decoder=False` for "
828
+ "bi-directional self-attention."
829
+ )
830
+
831
+ self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
832
+ self.lm_head = Data2VecTextLMHead(config)
833
+
834
+ # Initialize weights and apply final processing
835
+ self.post_init()
836
+
837
+ def get_output_embeddings(self):
838
+ return self.lm_head.decoder
839
+
840
+ def set_output_embeddings(self, new_embeddings):
841
+ self.lm_head.decoder = new_embeddings
842
+
843
+ @can_return_tuple
844
+ @auto_docstring
845
+ def forward(
846
+ self,
847
+ input_ids: torch.LongTensor | None = None,
848
+ attention_mask: torch.FloatTensor | None = None,
849
+ token_type_ids: torch.LongTensor | None = None,
850
+ position_ids: torch.LongTensor | None = None,
851
+ inputs_embeds: torch.FloatTensor | None = None,
852
+ encoder_hidden_states: torch.FloatTensor | None = None,
853
+ encoder_attention_mask: torch.FloatTensor | None = None,
854
+ labels: torch.LongTensor | None = None,
855
+ **kwargs: Unpack[TransformersKwargs],
856
+ ) -> tuple | MaskedLMOutput:
857
+ r"""
858
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
859
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
860
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
861
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
862
+ """
863
+ outputs = self.data2vec_text(
864
+ input_ids,
865
+ attention_mask=attention_mask,
866
+ token_type_ids=token_type_ids,
867
+ position_ids=position_ids,
868
+ inputs_embeds=inputs_embeds,
869
+ encoder_hidden_states=encoder_hidden_states,
870
+ encoder_attention_mask=encoder_attention_mask,
871
+ return_dict=True,
872
+ **kwargs,
873
+ )
874
+ sequence_output = outputs[0]
875
+ prediction_scores = self.lm_head(sequence_output)
876
+
877
+ masked_lm_loss = None
878
+ if labels is not None:
879
+ loss_fct = CrossEntropyLoss()
880
+
881
+ labels = labels.to(prediction_scores.device)
882
+ masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
883
+
884
+ return MaskedLMOutput(
885
+ loss=masked_lm_loss,
886
+ logits=prediction_scores,
887
+ hidden_states=outputs.hidden_states,
888
+ attentions=outputs.attentions,
889
+ )
890
+
891
+
892
+ @auto_docstring(
893
+ custom_intro="""
894
+ Data2VecText Model transformer with a sequence classification/regression head on top (a linear layer on top of the
895
+ pooled output) e.g. for GLUE tasks.
896
+ """
897
+ )
898
+ class Data2VecTextForSequenceClassification(Data2VecTextPreTrainedModel):
899
+ def __init__(self, config):
900
+ super().__init__(config)
901
+ self.num_labels = config.num_labels
902
+ self.config = config
903
+
904
+ self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
905
+ self.classifier = Data2VecTextClassificationHead(config)
906
+
907
+ # Initialize weights and apply final processing
908
+ self.post_init()
909
+
910
+ @can_return_tuple
911
+ @auto_docstring
912
+ def forward(
913
+ self,
914
+ input_ids: torch.LongTensor | None = None,
915
+ attention_mask: torch.FloatTensor | None = None,
916
+ token_type_ids: torch.LongTensor | None = None,
917
+ position_ids: torch.LongTensor | None = None,
918
+ inputs_embeds: torch.FloatTensor | None = None,
919
+ labels: torch.LongTensor | None = None,
920
+ **kwargs: Unpack[TransformersKwargs],
921
+ ) -> tuple | SequenceClassifierOutput:
922
+ r"""
923
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
924
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
925
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
926
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
927
+ """
928
+ outputs = self.data2vec_text(
929
+ input_ids,
930
+ attention_mask=attention_mask,
931
+ token_type_ids=token_type_ids,
932
+ position_ids=position_ids,
933
+ inputs_embeds=inputs_embeds,
934
+ return_dict=True,
935
+ **kwargs,
936
+ )
937
+ sequence_output = outputs[0]
938
+ logits = self.classifier(sequence_output)
939
+
940
+ loss = None
941
+ if labels is not None:
942
+ labels = labels.to(logits.device)
943
+
944
+ if self.config.problem_type is None:
945
+ if self.num_labels == 1:
946
+ self.config.problem_type = "regression"
947
+ elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
948
+ self.config.problem_type = "single_label_classification"
949
+ else:
950
+ self.config.problem_type = "multi_label_classification"
951
+
952
+ if self.config.problem_type == "regression":
953
+ loss_fct = MSELoss()
954
+ if self.num_labels == 1:
955
+ loss = loss_fct(logits.squeeze(), labels.squeeze())
956
+ else:
957
+ loss = loss_fct(logits, labels)
958
+ elif self.config.problem_type == "single_label_classification":
959
+ loss_fct = CrossEntropyLoss()
960
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
961
+ elif self.config.problem_type == "multi_label_classification":
962
+ loss_fct = BCEWithLogitsLoss()
963
+ loss = loss_fct(logits, labels)
964
+
965
+ return SequenceClassifierOutput(
966
+ loss=loss,
967
+ logits=logits,
968
+ hidden_states=outputs.hidden_states,
969
+ attentions=outputs.attentions,
970
+ )
971
+
972
+
973
+ @auto_docstring
974
+ class Data2VecTextForMultipleChoice(Data2VecTextPreTrainedModel):
975
+ def __init__(self, config):
976
+ super().__init__(config)
977
+
978
+ self.data2vec_text = Data2VecTextModel(config)
979
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
980
+ self.classifier = nn.Linear(config.hidden_size, 1)
981
+
982
+ # Initialize weights and apply final processing
983
+ self.post_init()
984
+
985
+ @can_return_tuple
986
+ @auto_docstring
987
+ def forward(
988
+ self,
989
+ input_ids: torch.LongTensor | None = None,
990
+ token_type_ids: torch.LongTensor | None = None,
991
+ attention_mask: torch.FloatTensor | None = None,
992
+ labels: torch.LongTensor | None = None,
993
+ position_ids: torch.LongTensor | None = None,
994
+ inputs_embeds: torch.FloatTensor | None = None,
995
+ **kwargs: Unpack[TransformersKwargs],
996
+ ) -> tuple | MultipleChoiceModelOutput:
997
+ r"""
998
+ input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
999
+ Indices of input sequence tokens in the vocabulary.
1000
+
1001
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
1002
+ [`PreTrainedTokenizer.__call__`] for details.
1003
+
1004
+ [What are input IDs?](../glossary#input-ids)
1005
+ token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
1006
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
1007
+ 1]`:
1008
+
1009
+ - 0 corresponds to a *sentence A* token,
1010
+ - 1 corresponds to a *sentence B* token.
1011
+
1012
+ [What are token type IDs?](../glossary#token-type-ids)
1013
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
1014
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
1015
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
1016
+ `input_ids` above)
1017
+ position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
1018
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
1019
+ config.max_position_embeddings - 1]`.
1020
+
1021
+ [What are position IDs?](../glossary#position-ids)
1022
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
1023
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
1024
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
1025
+ model's internal embedding lookup matrix.
1026
+ """
1027
+ num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]
1028
+
1029
+ flat_input_ids = input_ids.view(-1, input_ids.size(-1)) if input_ids is not None else None
1030
+ flat_position_ids = position_ids.view(-1, position_ids.size(-1)) if position_ids is not None else None
1031
+ flat_token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None
1032
+ flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None
1033
+ flat_inputs_embeds = (
1034
+ inputs_embeds.view(-1, inputs_embeds.size(-2), inputs_embeds.size(-1))
1035
+ if inputs_embeds is not None
1036
+ else None
1037
+ )
1038
+
1039
+ outputs = self.data2vec_text(
1040
+ flat_input_ids,
1041
+ position_ids=flat_position_ids,
1042
+ token_type_ids=flat_token_type_ids,
1043
+ attention_mask=flat_attention_mask,
1044
+ inputs_embeds=flat_inputs_embeds,
1045
+ return_dict=True,
1046
+ **kwargs,
1047
+ )
1048
+ pooled_output = outputs[1]
1049
+
1050
+ pooled_output = self.dropout(pooled_output)
1051
+ logits = self.classifier(pooled_output)
1052
+ reshaped_logits = logits.view(-1, num_choices)
1053
+
1054
+ loss = None
1055
+ if labels is not None:
1056
+ loss_fct = CrossEntropyLoss()
1057
+
1058
+ labels = labels.to(reshaped_logits.device)
1059
+ loss = loss_fct(reshaped_logits, labels)
1060
+
1061
+ return MultipleChoiceModelOutput(
1062
+ loss=loss,
1063
+ logits=reshaped_logits,
1064
+ hidden_states=outputs.hidden_states,
1065
+ attentions=outputs.attentions,
1066
+ )
1067
+
1068
+
1069
+ @auto_docstring
1070
+ class Data2VecTextForTokenClassification(Data2VecTextPreTrainedModel):
1071
+ def __init__(self, config):
1072
+ super().__init__(config)
1073
+ self.num_labels = config.num_labels
1074
+
1075
+ self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
1076
+ classifier_dropout = (
1077
+ config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
1078
+ )
1079
+ self.dropout = nn.Dropout(classifier_dropout)
1080
+ self.classifier = nn.Linear(config.hidden_size, config.num_labels)
1081
+
1082
+ # Initialize weights and apply final processing
1083
+ self.post_init()
1084
+
1085
+ @can_return_tuple
1086
+ @auto_docstring
1087
+ def forward(
1088
+ self,
1089
+ input_ids: torch.LongTensor | None = None,
1090
+ attention_mask: torch.FloatTensor | None = None,
1091
+ token_type_ids: torch.LongTensor | None = None,
1092
+ position_ids: torch.LongTensor | None = None,
1093
+ inputs_embeds: torch.FloatTensor | None = None,
1094
+ labels: torch.LongTensor | None = None,
1095
+ **kwargs: Unpack[TransformersKwargs],
1096
+ ) -> tuple | TokenClassifierOutput:
1097
+ r"""
1098
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
1099
+ Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
1100
+ """
1101
+ outputs = self.data2vec_text(
1102
+ input_ids,
1103
+ attention_mask=attention_mask,
1104
+ token_type_ids=token_type_ids,
1105
+ position_ids=position_ids,
1106
+ inputs_embeds=inputs_embeds,
1107
+ return_dict=True,
1108
+ **kwargs,
1109
+ )
1110
+
1111
+ sequence_output = outputs[0]
1112
+
1113
+ sequence_output = self.dropout(sequence_output)
1114
+ logits = self.classifier(sequence_output)
1115
+
1116
+ loss = None
1117
+ if labels is not None:
1118
+ loss_fct = CrossEntropyLoss()
1119
+
1120
+ labels = labels.to(logits.device)
1121
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
1122
+
1123
+ return TokenClassifierOutput(
1124
+ loss=loss,
1125
+ logits=logits,
1126
+ hidden_states=outputs.hidden_states,
1127
+ attentions=outputs.attentions,
1128
+ )
1129
+
1130
+
1131
+ @auto_docstring
1132
+ class Data2VecTextForQuestionAnswering(Data2VecTextPreTrainedModel):
1133
+ def __init__(self, config):
1134
+ super().__init__(config)
1135
+ self.num_labels = config.num_labels
1136
+
1137
+ self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
1138
+ self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
1139
+
1140
+ # Initialize weights and apply final processing
1141
+ self.post_init()
1142
+
1143
+ @can_return_tuple
1144
+ @auto_docstring
1145
+ def forward(
1146
+ self,
1147
+ input_ids: torch.LongTensor | None = None,
1148
+ attention_mask: torch.FloatTensor | None = None,
1149
+ token_type_ids: torch.LongTensor | None = None,
1150
+ position_ids: torch.LongTensor | None = None,
1151
+ inputs_embeds: torch.FloatTensor | None = None,
1152
+ start_positions: torch.LongTensor | None = None,
1153
+ end_positions: torch.LongTensor | None = None,
1154
+ **kwargs: Unpack[TransformersKwargs],
1155
+ ) -> tuple | QuestionAnsweringModelOutput:
1156
+ outputs = self.data2vec_text(
1157
+ input_ids,
1158
+ attention_mask=attention_mask,
1159
+ token_type_ids=token_type_ids,
1160
+ position_ids=position_ids,
1161
+ inputs_embeds=inputs_embeds,
1162
+ return_dict=True,
1163
+ **kwargs,
1164
+ )
1165
+
1166
+ sequence_output = outputs[0]
1167
+
1168
+ logits = self.qa_outputs(sequence_output)
1169
+ start_logits, end_logits = logits.split(1, dim=-1)
1170
+ start_logits = start_logits.squeeze(-1).contiguous()
1171
+ end_logits = end_logits.squeeze(-1).contiguous()
1172
+
1173
+ total_loss = None
1174
+ if start_positions is not None and end_positions is not None:
1175
+ # If we are on multi-GPU, split add a dimension
1176
+ if len(start_positions.size()) > 1:
1177
+ start_positions = start_positions.squeeze(-1)
1178
+ if len(end_positions.size()) > 1:
1179
+ end_positions = end_positions.squeeze(-1)
1180
+ # sometimes the start/end positions are outside our model inputs, we ignore these terms
1181
+ ignored_index = start_logits.size(1)
1182
+ start_positions = start_positions.clamp(0, ignored_index)
1183
+ end_positions = end_positions.clamp(0, ignored_index)
1184
+
1185
+ loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
1186
+ start_loss = loss_fct(start_logits, start_positions)
1187
+ end_loss = loss_fct(end_logits, end_positions)
1188
+ total_loss = (start_loss + end_loss) / 2
1189
+
1190
+ return QuestionAnsweringModelOutput(
1191
+ loss=total_loss,
1192
+ start_logits=start_logits,
1193
+ end_logits=end_logits,
1194
+ hidden_states=outputs.hidden_states,
1195
+ attentions=outputs.attentions,
1196
+ )
1197
+
1198
+
1199
+ __all__ = [
1200
+ "Data2VecTextForCausalLM",
1201
+ "Data2VecTextForMaskedLM",
1202
+ "Data2VecTextForMultipleChoice",
1203
+ "Data2VecTextForQuestionAnswering",
1204
+ "Data2VecTextForSequenceClassification",
1205
+ "Data2VecTextForTokenClassification",
1206
+ "Data2VecTextModel",
1207
+ "Data2VecTextPreTrainedModel",
1208
+ ]
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_vision.py ADDED
@@ -0,0 +1,1214 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022 Meta Platforms and The HuggingFace Inc. team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """PyTorch Data2VecVision model."""
15
+
16
+ import collections.abc
17
+ import math
18
+ from dataclasses import dataclass
19
+ from typing import Optional
20
+
21
+ import torch
22
+ from torch import nn
23
+ from torch.nn import CrossEntropyLoss
24
+
25
+ from ... import initialization as init
26
+ from ...activations import ACT2FN
27
+ from ...modeling_layers import GradientCheckpointingLayer
28
+ from ...modeling_outputs import (
29
+ BaseModelOutput,
30
+ BaseModelOutputWithPooling,
31
+ ImageClassifierOutput,
32
+ SemanticSegmenterOutput,
33
+ )
34
+ from ...modeling_utils import PreTrainedModel
35
+ from ...pytorch_utils import compile_compatible_method_lru_cache
36
+ from ...utils import auto_docstring, logging, torch_int
37
+ from .configuration_data2vec_vision import Data2VecVisionConfig
38
+
39
+
40
+ logger = logging.get_logger(__name__)
41
+
42
+
43
+ @auto_docstring(
44
+ custom_intro="""
45
+ Class for outputs of [`Data2VecVisionModel`].
46
+ """
47
+ )
48
+ @dataclass
49
+ # Copied from transformers.models.beit.modeling_beit.BeitModelOutputWithPooling with Beit->Data2VecVision
50
+ class Data2VecVisionModelOutputWithPooling(BaseModelOutputWithPooling):
51
+ r"""
52
+ pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
53
+ Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
54
+ *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
55
+ will be returned.
56
+ """
57
+
58
+
59
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitEmbeddings with Beit->Data2VecVision
60
+ class Data2VecVisionEmbeddings(nn.Module):
61
+ """
62
+ Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
63
+
64
+ """
65
+
66
+ def __init__(self, config: Data2VecVisionConfig) -> None:
67
+ super().__init__()
68
+
69
+ self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
70
+ if config.use_mask_token:
71
+ self.mask_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
72
+ else:
73
+ self.mask_token = None
74
+ self.patch_embeddings = Data2VecVisionPatchEmbeddings(config)
75
+ self.patch_size = config.patch_size
76
+ self.image_size = (
77
+ config.image_size
78
+ if isinstance(config.image_size, collections.abc.Iterable)
79
+ else (config.image_size, config.image_size)
80
+ )
81
+ num_patches = self.patch_embeddings.num_patches
82
+ if config.use_absolute_position_embeddings:
83
+ self.position_embeddings = nn.Parameter(torch.zeros(1, num_patches + 1, config.hidden_size))
84
+ else:
85
+ self.position_embeddings = None
86
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
87
+
88
+ # Copied from transformers.models.vit.modeling_vit.ViTEmbeddings.interpolate_pos_encoding
89
+ def interpolate_pos_encoding(self, embeddings: torch.Tensor, height: int, width: int) -> torch.Tensor:
90
+ """
91
+ This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
92
+ images. This method is also adapted to support torch.jit tracing.
93
+
94
+ Adapted from:
95
+ - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
96
+ - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
97
+ """
98
+
99
+ num_patches = embeddings.shape[1] - 1
100
+ num_positions = self.position_embeddings.shape[1] - 1
101
+
102
+ # always interpolate when tracing to ensure the exported model works for dynamic input shapes
103
+ if not torch.jit.is_tracing() and num_patches == num_positions and height == width:
104
+ return self.position_embeddings
105
+
106
+ class_pos_embed = self.position_embeddings[:, :1]
107
+ patch_pos_embed = self.position_embeddings[:, 1:]
108
+
109
+ dim = embeddings.shape[-1]
110
+
111
+ new_height = height // self.patch_size
112
+ new_width = width // self.patch_size
113
+
114
+ sqrt_num_positions = torch_int(num_positions**0.5)
115
+ patch_pos_embed = patch_pos_embed.reshape(1, sqrt_num_positions, sqrt_num_positions, dim)
116
+ patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2)
117
+
118
+ patch_pos_embed = nn.functional.interpolate(
119
+ patch_pos_embed,
120
+ size=(new_height, new_width),
121
+ mode="bicubic",
122
+ align_corners=False,
123
+ )
124
+
125
+ patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
126
+
127
+ return torch.cat((class_pos_embed, patch_pos_embed), dim=1)
128
+
129
+ def forward(
130
+ self,
131
+ pixel_values: torch.Tensor,
132
+ bool_masked_pos: torch.BoolTensor | None = None,
133
+ ) -> torch.Tensor:
134
+ _, _, height, width = pixel_values.shape
135
+ embeddings, (patch_height, patch_width) = self.patch_embeddings(pixel_values)
136
+ batch_size, seq_len, _ = embeddings.size()
137
+
138
+ if bool_masked_pos is not None:
139
+ mask_tokens = self.mask_token.expand(batch_size, seq_len, -1)
140
+ # replace the masked visual tokens by mask_tokens
141
+ w = bool_masked_pos.unsqueeze(-1).type_as(mask_tokens)
142
+ embeddings = embeddings * (1 - w) + mask_tokens * w
143
+
144
+ cls_tokens = self.cls_token.expand(batch_size, -1, -1)
145
+ embeddings = torch.cat((cls_tokens, embeddings), dim=1)
146
+
147
+ if self.position_embeddings is not None:
148
+ embeddings = embeddings + self.interpolate_pos_encoding(embeddings, height, width)
149
+
150
+ embeddings = self.dropout(embeddings)
151
+
152
+ return embeddings, (patch_height, patch_width)
153
+
154
+
155
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitPatchEmbeddings with Beit->Data2VecVision
156
+ class Data2VecVisionPatchEmbeddings(nn.Module):
157
+ """
158
+ This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
159
+ `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
160
+ Transformer.
161
+ """
162
+
163
+ def __init__(self, config):
164
+ super().__init__()
165
+ image_size, patch_size = config.image_size, config.patch_size
166
+ num_channels, hidden_size = config.num_channels, config.hidden_size
167
+
168
+ image_size = image_size if isinstance(image_size, collections.abc.Iterable) else (image_size, image_size)
169
+ patch_size = patch_size if isinstance(patch_size, collections.abc.Iterable) else (patch_size, patch_size)
170
+ num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
171
+ patch_shape = (image_size[0] // patch_size[0], image_size[1] // patch_size[1])
172
+ self.image_size = image_size
173
+ self.patch_size = patch_size
174
+ self.num_channels = num_channels
175
+ self.num_patches = num_patches
176
+ self.patch_shape = patch_shape
177
+
178
+ self.projection = nn.Conv2d(num_channels, hidden_size, kernel_size=patch_size, stride=patch_size)
179
+
180
+ def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
181
+ batch_size, num_channels, height, width = pixel_values.shape
182
+ if num_channels != self.num_channels:
183
+ raise ValueError(
184
+ "Make sure that the channel dimension of the pixel values match with the one set in the configuration."
185
+ )
186
+
187
+ embeddings = self.projection(pixel_values.to(self.projection.weight.dtype))
188
+ patch_height, patch_width = embeddings.shape[2], embeddings.shape[3]
189
+ embeddings = embeddings.flatten(2).transpose(1, 2)
190
+
191
+ return embeddings, (patch_height, patch_width)
192
+
193
+
194
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitSelfAttention with Beit->Data2VecVision
195
+ class Data2VecVisionSelfAttention(nn.Module):
196
+ def __init__(self, config: Data2VecVisionConfig, window_size: tuple | None = None) -> None:
197
+ super().__init__()
198
+ self.config = config
199
+ if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
200
+ raise ValueError(
201
+ f"The hidden size {config.hidden_size} is not a multiple of the number of attention "
202
+ f"heads {config.num_attention_heads}."
203
+ )
204
+
205
+ self.num_attention_heads = config.num_attention_heads
206
+ self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
207
+ self.all_head_size = self.num_attention_heads * self.attention_head_size
208
+
209
+ self.query = nn.Linear(config.hidden_size, self.all_head_size)
210
+ self.key = nn.Linear(config.hidden_size, self.all_head_size, bias=False)
211
+ self.value = nn.Linear(config.hidden_size, self.all_head_size)
212
+
213
+ self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
214
+
215
+ self.has_relative_position_bias = bool(window_size)
216
+ if self.has_relative_position_bias:
217
+ self.relative_position_bias = Data2VecVisionRelativePositionBias(config, window_size=window_size)
218
+
219
+ def forward(
220
+ self,
221
+ hidden_states: torch.Tensor,
222
+ output_attentions: bool = False,
223
+ relative_position_bias: torch.Tensor | None = None,
224
+ interpolate_pos_encoding: bool = False,
225
+ resolution: tuple[int] | None = None,
226
+ ) -> tuple[torch.Tensor] | tuple[torch.Tensor, torch.Tensor]:
227
+ input_shape = hidden_states.shape[:-1]
228
+ hidden_shape = (*input_shape, -1, self.attention_head_size)
229
+ query_layer = self.query(hidden_states).view(hidden_shape).transpose(1, 2)
230
+ key_layer = self.key(hidden_states).view(hidden_shape).transpose(1, 2)
231
+ value_layer = self.value(hidden_states).view(hidden_shape).transpose(1, 2)
232
+
233
+ # Take the dot product between "query" and "key" to get the raw attention scores.
234
+ attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
235
+
236
+ attention_scores = attention_scores / math.sqrt(self.attention_head_size)
237
+
238
+ # Add relative position bias if present.
239
+ if self.has_relative_position_bias:
240
+ height, width = resolution
241
+ window_size = (height // self.config.patch_size, width // self.config.patch_size)
242
+ attention_scores = attention_scores + self.relative_position_bias(
243
+ window_size, interpolate_pos_encoding, dim_size=hidden_states.shape[1]
244
+ )
245
+
246
+ # Add shared relative position bias if provided.
247
+ if relative_position_bias is not None:
248
+ attention_scores = attention_scores + relative_position_bias
249
+
250
+ # Normalize the attention scores to probabilities.
251
+ attention_probs = nn.functional.softmax(attention_scores, dim=-1)
252
+
253
+ # This is actually dropping out entire tokens to attend to, which might
254
+ # seem a bit unusual, but is taken from the original Transformer paper.
255
+ attention_probs = self.dropout(attention_probs)
256
+
257
+ context_layer = torch.matmul(attention_probs, value_layer)
258
+
259
+ context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
260
+ new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
261
+ context_layer = context_layer.view(*new_context_layer_shape)
262
+
263
+ outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
264
+
265
+ return outputs
266
+
267
+
268
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitSdpaSelfAttention with Beit->Data2VecVision
269
+ class Data2VecVisionSdpaSelfAttention(Data2VecVisionSelfAttention):
270
+ def forward(
271
+ self,
272
+ hidden_states: torch.Tensor,
273
+ output_attentions: bool = False,
274
+ relative_position_bias: torch.Tensor | None = None,
275
+ interpolate_pos_encoding: bool = False,
276
+ resolution: tuple[int] | None = None,
277
+ ) -> tuple[torch.Tensor] | tuple[torch.Tensor, torch.Tensor]:
278
+ if output_attentions:
279
+ logger.warning_once(
280
+ f"{self.__class__.__name__} does not support `output_attentions=True`. The returned attention weights will "
281
+ "be `None`. If you want to get attention weights, please set `attn_implementation='eager'` when loading the model."
282
+ )
283
+ input_shape = hidden_states.shape[:-1]
284
+ hidden_shape = (*input_shape, -1, self.attention_head_size)
285
+ query_layer = self.query(hidden_states).view(hidden_shape).transpose(1, 2)
286
+ key_layer = self.key(hidden_states).view(hidden_shape).transpose(1, 2)
287
+ value_layer = self.value(hidden_states).view(hidden_shape).transpose(1, 2)
288
+
289
+ attn_bias = None
290
+ if self.has_relative_position_bias:
291
+ height, width = resolution
292
+ window_size = (height // self.config.patch_size, width // self.config.patch_size)
293
+ attn_bias = self.relative_position_bias(
294
+ window_size, interpolate_pos_encoding, dim_size=hidden_states.shape[1]
295
+ )
296
+
297
+ # Add shared relative position bias if provided.
298
+ if relative_position_bias is not None:
299
+ if attn_bias is None:
300
+ attn_bias = relative_position_bias
301
+ else:
302
+ attn_bias += relative_position_bias
303
+
304
+ scaling = 1 / math.sqrt(self.attention_head_size)
305
+ context_layer = torch.nn.functional.scaled_dot_product_attention(
306
+ query_layer,
307
+ key_layer,
308
+ value_layer,
309
+ attn_mask=attn_bias,
310
+ dropout_p=self.config.attention_probs_dropout_prob if self.training else 0.0,
311
+ is_causal=False,
312
+ scale=scaling,
313
+ )
314
+ context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
315
+ new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
316
+ context_layer = context_layer.view(*new_context_layer_shape)
317
+ return context_layer, None
318
+
319
+
320
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitSelfOutput with Beit->Data2VecVision
321
+ class Data2VecVisionSelfOutput(nn.Module):
322
+ """
323
+ The residual connection is defined in Data2VecVisionLayer instead of here (as is the case with other models), due to the
324
+ layernorm applied before each block.
325
+ """
326
+
327
+ def __init__(self, config: Data2VecVisionConfig) -> None:
328
+ super().__init__()
329
+ self.dense = nn.Linear(config.hidden_size, config.hidden_size)
330
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
331
+
332
+ def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Tensor, gamma=None) -> torch.Tensor:
333
+ hidden_states = self.dense(hidden_states)
334
+ hidden_states = self.dropout(hidden_states)
335
+
336
+ return hidden_states
337
+
338
+
339
+ DATA2VEC_VISION_SELF_ATTENTION_CLASSES = {
340
+ "eager": Data2VecVisionSelfAttention,
341
+ "sdpa": Data2VecVisionSdpaSelfAttention,
342
+ }
343
+
344
+
345
+ # Copied from tests.models.beit.modeling_beit.BeitAttention with Beit->Data2VecVision, BEIT->DATA2VEC_VISION
346
+ class Data2VecVisionAttention(nn.Module):
347
+ def __init__(self, config: Data2VecVisionConfig, window_size: tuple | None = None) -> None:
348
+ super().__init__()
349
+ self.attention = DATA2VEC_VISION_SELF_ATTENTION_CLASSES[config._attn_implementation](
350
+ config, window_size=window_size
351
+ )
352
+ self.output = Data2VecVisionSelfOutput(config)
353
+
354
+ def forward(
355
+ self,
356
+ hidden_states: torch.Tensor,
357
+ output_attentions: bool = False,
358
+ relative_position_bias: Optional["Data2VecVisionRelativePositionBias"] = None,
359
+ interpolate_pos_encoding: bool = False,
360
+ resolution: tuple[int] | None = None,
361
+ ) -> tuple[torch.Tensor] | tuple[torch.Tensor, torch.Tensor]:
362
+ self_outputs = self.attention(
363
+ hidden_states, output_attentions, relative_position_bias, interpolate_pos_encoding, resolution
364
+ )
365
+
366
+ attention_output = self.output(self_outputs[0], hidden_states)
367
+
368
+ outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them
369
+ return outputs
370
+
371
+
372
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitIntermediate with Beit->Data2VecVision
373
+ class Data2VecVisionIntermediate(nn.Module):
374
+ def __init__(self, config: Data2VecVisionConfig) -> None:
375
+ super().__init__()
376
+ self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
377
+ if isinstance(config.hidden_act, str):
378
+ self.intermediate_act_fn = ACT2FN[config.hidden_act]
379
+ else:
380
+ self.intermediate_act_fn = config.hidden_act
381
+
382
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
383
+ hidden_states = self.dense(hidden_states)
384
+ hidden_states = self.intermediate_act_fn(hidden_states)
385
+
386
+ return hidden_states
387
+
388
+
389
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitOutput with Beit->Data2VecVision
390
+ class Data2VecVisionOutput(nn.Module):
391
+ def __init__(self, config: Data2VecVisionConfig) -> None:
392
+ super().__init__()
393
+ self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
394
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
395
+
396
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
397
+ hidden_states = self.dense(hidden_states)
398
+ hidden_states = self.dropout(hidden_states)
399
+
400
+ return hidden_states
401
+
402
+
403
+ # Copied from transformers.models.swin.modular_swin.SwinDropPath with SwinDropPath->Data2VecVisionDropPath
404
+ class Data2VecVisionDropPath(nn.Module):
405
+ """Stochastic depth (DropPath) per sample, for residual blocks.
406
+
407
+ Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
408
+ <https://arxiv.org/abs/1603.09382>`_.
409
+ """
410
+
411
+ def __init__(self, drop_prob: float = 0.0) -> None:
412
+ super().__init__()
413
+ self.drop_prob = drop_prob
414
+
415
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
416
+ if self.drop_prob == 0.0 or not self.training:
417
+ return hidden_states
418
+ keep_prob = 1 - self.drop_prob
419
+ shape = (hidden_states.shape[0],) + (1,) * (hidden_states.ndim - 1)
420
+ random_tensor = torch.rand(shape, dtype=hidden_states.dtype, device=hidden_states.device)
421
+ random_tensor = torch.floor(random_tensor + keep_prob)
422
+ return hidden_states.div(keep_prob) * random_tensor
423
+
424
+ def extra_repr(self) -> str:
425
+ return f"p={self.drop_prob}"
426
+
427
+
428
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitLayer with Beit->Data2VecVision,BEiT->Data2VecVision
429
+ class Data2VecVisionLayer(GradientCheckpointingLayer):
430
+ """This corresponds to the Block class in the timm implementation."""
431
+
432
+ def __init__(
433
+ self, config: Data2VecVisionConfig, window_size: tuple | None = None, drop_path_rate: float = 0.0
434
+ ) -> None:
435
+ super().__init__()
436
+ self.chunk_size_feed_forward = config.chunk_size_feed_forward
437
+ self.seq_len_dim = 1
438
+ self.attention = Data2VecVisionAttention(config, window_size=window_size)
439
+ self.intermediate = Data2VecVisionIntermediate(config)
440
+ self.output = Data2VecVisionOutput(config)
441
+ self.layernorm_before = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
442
+ self.drop_path = Data2VecVisionDropPath(drop_path_rate) if drop_path_rate > 0.0 else nn.Identity()
443
+ self.layernorm_after = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
444
+
445
+ init_values = config.layer_scale_init_value
446
+ if init_values > 0:
447
+ self.lambda_1 = nn.Parameter(init_values * torch.ones(config.hidden_size), requires_grad=True)
448
+ self.lambda_2 = nn.Parameter(init_values * torch.ones(config.hidden_size), requires_grad=True)
449
+ else:
450
+ self.lambda_1, self.lambda_2 = None, None
451
+
452
+ def forward(
453
+ self,
454
+ hidden_states: torch.Tensor,
455
+ output_attentions: bool = False,
456
+ relative_position_bias: torch.Tensor | None = None,
457
+ interpolate_pos_encoding: bool = False,
458
+ resolution: tuple[int, int] | None = None,
459
+ ) -> tuple[torch.Tensor] | tuple[torch.Tensor, torch.Tensor]:
460
+ self_attention_outputs = self.attention(
461
+ self.layernorm_before(hidden_states), # in Data2VecVision, layernorm is applied before self-attention
462
+ output_attentions=output_attentions,
463
+ relative_position_bias=relative_position_bias,
464
+ interpolate_pos_encoding=interpolate_pos_encoding,
465
+ resolution=resolution,
466
+ )
467
+ attention_output = self_attention_outputs[0]
468
+ outputs = self_attention_outputs[1:] # add self attentions if we output attention weights
469
+
470
+ # apply lambda_1 if present
471
+ if self.lambda_1 is not None:
472
+ attention_output = self.lambda_1 * attention_output
473
+
474
+ # first residual connection
475
+ hidden_states = self.drop_path(attention_output) + hidden_states
476
+
477
+ # in Data2VecVision, layernorm is also applied after self-attention
478
+ layer_output = self.layernorm_after(hidden_states)
479
+
480
+ layer_output = self.intermediate(layer_output)
481
+ layer_output = self.output(layer_output)
482
+
483
+ if self.lambda_2 is not None:
484
+ layer_output = self.lambda_2 * layer_output
485
+
486
+ # second residual connection
487
+ layer_output = self.drop_path(layer_output) + hidden_states
488
+
489
+ outputs = (layer_output,) + outputs
490
+
491
+ return outputs
492
+
493
+
494
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitRelativePositionBias with Beit->Data2VecVision
495
+ class Data2VecVisionRelativePositionBias(nn.Module):
496
+ def __init__(self, config: Data2VecVisionConfig, window_size: tuple) -> None:
497
+ super().__init__()
498
+ self.window_size = window_size
499
+ self.num_relative_distance = (2 * window_size[0] - 1) * (2 * window_size[1] - 1) + 3
500
+ self.relative_position_bias_table = nn.Parameter(
501
+ torch.zeros(self.num_relative_distance, config.num_attention_heads)
502
+ ) # 2*Wh-1 * 2*Ww-1, nH
503
+ # cls to token & token 2 cls & cls to cls
504
+
505
+ @staticmethod
506
+ @compile_compatible_method_lru_cache(maxsize=10)
507
+ def generate_relative_position_index(window_size: tuple[int, int]) -> torch.Tensor:
508
+ """
509
+ This method creates the relative position index, modified to support arbitrary window sizes,
510
+ as introduced in [MiDaS v3.1](https://huggingface.co/papers/2307.14460).
511
+ """
512
+ num_relative_distance = (2 * window_size[0] - 1) * (2 * window_size[1] - 1) + 3
513
+ # cls to token & token 2 cls & cls to cls
514
+ # get pair-wise relative position index for each token inside the window
515
+ window_area = window_size[0] * window_size[1]
516
+ grid = torch.meshgrid(torch.arange(window_size[0]), torch.arange(window_size[1]), indexing="ij")
517
+ coords = torch.stack(grid) # 2, Wh, Ww
518
+ coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww
519
+ relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww
520
+ relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2
521
+ relative_coords[:, :, 0] += window_size[0] - 1 # shift to start from 0
522
+ relative_coords[:, :, 1] += window_size[1] - 1
523
+ relative_coords[:, :, 0] *= 2 * window_size[1] - 1
524
+ relative_position_index = torch.zeros(size=(window_area + 1,) * 2, dtype=relative_coords.dtype)
525
+ relative_position_index[1:, 1:] = relative_coords.sum(-1) # Wh*Ww, Wh*Ww
526
+ relative_position_index[0, 0:] = num_relative_distance - 3
527
+ relative_position_index[0:, 0] = num_relative_distance - 2
528
+ relative_position_index[0, 0] = num_relative_distance - 1
529
+ return relative_position_index
530
+
531
+ def forward(self, window_size, interpolate_pos_encoding: bool = False, dim_size=None) -> torch.Tensor:
532
+ """
533
+ Modification of timm.models.beit.py: Attention._get_rel_pos_bias to support arbitrary window sizes.
534
+ """
535
+ old_height = 2 * self.window_size[0] - 1
536
+ old_width = 2 * self.window_size[1] - 1
537
+
538
+ new_height = 2 * window_size[0] - 1
539
+ new_width = 2 * window_size[1] - 1
540
+
541
+ old_relative_position_bias_table = self.relative_position_bias_table
542
+
543
+ old_num_relative_distance = self.num_relative_distance
544
+ new_num_relative_distance = new_height * new_width + 3
545
+
546
+ old_sub_table = old_relative_position_bias_table[: old_num_relative_distance - 3]
547
+
548
+ old_sub_table = old_sub_table.reshape(1, old_width, old_height, -1).permute(0, 3, 1, 2)
549
+ new_sub_table = nn.functional.interpolate(
550
+ old_sub_table, size=(torch_int(new_height), torch_int(new_width)), mode="bilinear"
551
+ )
552
+ new_sub_table = new_sub_table.permute(0, 2, 3, 1).reshape(new_num_relative_distance - 3, -1)
553
+
554
+ new_relative_position_bias_table = torch.cat(
555
+ [new_sub_table, old_relative_position_bias_table[old_num_relative_distance - 3 :]]
556
+ )
557
+
558
+ relative_position_index = self.generate_relative_position_index(window_size)
559
+ relative_position_bias = new_relative_position_bias_table[relative_position_index.view(-1)]
560
+
561
+ # patch_size*num_patches_height, patch_size*num_patches_width, num_attention_heads
562
+ relative_position_bias = relative_position_bias.view(
563
+ window_size[0] * window_size[1] + 1, window_size[0] * window_size[1] + 1, -1
564
+ )
565
+ # num_attention_heads, patch_size*num_patches_width, patch_size*num_patches_height
566
+ relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous()
567
+
568
+ if interpolate_pos_encoding:
569
+ relative_position_bias = nn.functional.interpolate(
570
+ relative_position_bias.unsqueeze(1),
571
+ size=(dim_size, dim_size),
572
+ mode="bilinear",
573
+ align_corners=False,
574
+ ).squeeze(1)
575
+
576
+ return relative_position_bias.unsqueeze(0)
577
+
578
+
579
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitEncoder with Beit->Data2VecVision
580
+ class Data2VecVisionEncoder(nn.Module):
581
+ def __init__(self, config: Data2VecVisionConfig, window_size: tuple | None = None) -> None:
582
+ super().__init__()
583
+ self.config = config
584
+ self.has_relative_position_bias = config.use_shared_relative_position_bias
585
+ if self.has_relative_position_bias:
586
+ self.relative_position_bias = Data2VecVisionRelativePositionBias(config, window_size=window_size)
587
+
588
+ # stochastic depth decay rule
589
+ dpr = [x.item() for x in torch.linspace(0, config.drop_path_rate, config.num_hidden_layers, device="cpu")]
590
+ self.layer = nn.ModuleList(
591
+ [
592
+ Data2VecVisionLayer(
593
+ config,
594
+ window_size=window_size if config.use_relative_position_bias else None,
595
+ drop_path_rate=dpr[i],
596
+ )
597
+ for i in range(config.num_hidden_layers)
598
+ ]
599
+ )
600
+ self.gradient_checkpointing = False
601
+
602
+ def forward(
603
+ self,
604
+ hidden_states: torch.Tensor,
605
+ output_attentions: bool = False,
606
+ output_hidden_states: bool = False,
607
+ interpolate_pos_encoding: bool = False,
608
+ resolution: tuple[int, int] | None = None,
609
+ return_dict: bool = True,
610
+ ) -> tuple | BaseModelOutput:
611
+ all_hidden_states = () if output_hidden_states else None
612
+ all_self_attentions = () if output_attentions else None
613
+
614
+ for i, layer_module in enumerate(self.layer):
615
+ if output_hidden_states:
616
+ all_hidden_states = all_hidden_states + (hidden_states,)
617
+
618
+ if self.has_relative_position_bias:
619
+ height, width = resolution
620
+ window_size = (height // self.config.patch_size, width // self.config.patch_size)
621
+ relative_position_bias = self.relative_position_bias(
622
+ window_size, interpolate_pos_encoding=interpolate_pos_encoding, dim_size=hidden_states.shape[1]
623
+ )
624
+ else:
625
+ relative_position_bias = None
626
+
627
+ layer_outputs = layer_module(
628
+ hidden_states,
629
+ output_attentions=output_attentions,
630
+ relative_position_bias=relative_position_bias,
631
+ interpolate_pos_encoding=interpolate_pos_encoding,
632
+ resolution=resolution,
633
+ )
634
+
635
+ hidden_states = layer_outputs[0]
636
+
637
+ if output_attentions:
638
+ all_self_attentions = all_self_attentions + (layer_outputs[1],)
639
+
640
+ if output_hidden_states:
641
+ all_hidden_states = all_hidden_states + (hidden_states,)
642
+
643
+ if not return_dict:
644
+ return tuple(v for v in [hidden_states, all_hidden_states, all_self_attentions] if v is not None)
645
+ return BaseModelOutput(
646
+ last_hidden_state=hidden_states,
647
+ hidden_states=all_hidden_states,
648
+ attentions=all_self_attentions,
649
+ )
650
+
651
+
652
+ @auto_docstring
653
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitPreTrainedModel with Beit->Data2VecVision,beit->data2vec_vision
654
+ class Data2VecVisionPreTrainedModel(PreTrainedModel):
655
+ config: Data2VecVisionConfig
656
+ base_model_prefix = "data2vec_vision"
657
+ input_modalities = ("image",)
658
+ main_input_name = "pixel_values"
659
+ supports_gradient_checkpointing = True
660
+ _no_split_modules = ["Data2VecVisionLayer"]
661
+ _keys_to_ignore_on_load_unexpected = [r".*relative_position_index.*"]
662
+ _supports_sdpa = True
663
+
664
+ @torch.no_grad()
665
+ def _init_weights(self, module):
666
+ """Initialize the weights"""
667
+ super()._init_weights(module)
668
+ if isinstance(module, Data2VecVisionEmbeddings):
669
+ init.zeros_(module.cls_token)
670
+ if module.mask_token is not None:
671
+ init.zeros_(module.mask_token)
672
+ if module.position_embeddings is not None:
673
+ init.zeros_(module.position_embeddings)
674
+ elif isinstance(module, Data2VecVisionRelativePositionBias):
675
+ init.zeros_(module.relative_position_bias_table)
676
+ elif isinstance(module, Data2VecVisionLayer):
677
+ if module.lambda_1 is not None:
678
+ init.constant_(module.lambda_1, self.config.layer_scale_init_value)
679
+ init.constant_(module.lambda_2, self.config.layer_scale_init_value)
680
+
681
+
682
+ @auto_docstring
683
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitModel with BEIT->DATA2VEC_VISION,Beit->Data2VecVision,True->False
684
+ class Data2VecVisionModel(Data2VecVisionPreTrainedModel):
685
+ def __init__(self, config: Data2VecVisionConfig, add_pooling_layer: bool = False) -> None:
686
+ r"""
687
+ add_pooling_layer (bool, *optional*, defaults to `False`):
688
+ Whether to add a pooling layer
689
+ """
690
+ super().__init__(config)
691
+ self.config = config
692
+
693
+ self.embeddings = Data2VecVisionEmbeddings(config)
694
+ self.encoder = Data2VecVisionEncoder(config, window_size=self.embeddings.patch_embeddings.patch_shape)
695
+
696
+ self.layernorm = (
697
+ nn.Identity() if config.use_mean_pooling else nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
698
+ )
699
+ self.pooler = Data2VecVisionPooler(config) if add_pooling_layer else None
700
+
701
+ # Initialize weights and apply final processing
702
+ self.post_init()
703
+
704
+ def get_input_embeddings(self):
705
+ return self.embeddings.patch_embeddings
706
+
707
+ @auto_docstring
708
+ def forward(
709
+ self,
710
+ pixel_values: torch.Tensor,
711
+ bool_masked_pos: torch.BoolTensor | None = None,
712
+ output_attentions: bool | None = None,
713
+ output_hidden_states: bool | None = None,
714
+ interpolate_pos_encoding: bool = False,
715
+ return_dict: bool | None = None,
716
+ **kwargs,
717
+ ) -> tuple | Data2VecVisionModelOutputWithPooling:
718
+ r"""
719
+ bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
720
+ Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
721
+ """
722
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
723
+ output_hidden_states = (
724
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
725
+ )
726
+ return_dict = return_dict if return_dict is not None else self.config.return_dict
727
+
728
+ embedding_output, _ = self.embeddings(pixel_values, bool_masked_pos=bool_masked_pos)
729
+ resolution = pixel_values.shape[2:]
730
+
731
+ encoder_outputs = self.encoder(
732
+ embedding_output,
733
+ output_attentions=output_attentions,
734
+ output_hidden_states=output_hidden_states,
735
+ resolution=resolution,
736
+ return_dict=return_dict,
737
+ interpolate_pos_encoding=interpolate_pos_encoding,
738
+ )
739
+ sequence_output = encoder_outputs[0]
740
+ sequence_output = self.layernorm(sequence_output)
741
+ pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
742
+
743
+ if not return_dict:
744
+ head_outputs = (sequence_output, pooled_output) if pooled_output is not None else (sequence_output,)
745
+ return head_outputs + encoder_outputs[1:]
746
+
747
+ return Data2VecVisionModelOutputWithPooling(
748
+ last_hidden_state=sequence_output,
749
+ pooler_output=pooled_output,
750
+ hidden_states=encoder_outputs.hidden_states,
751
+ attentions=encoder_outputs.attentions,
752
+ )
753
+
754
+
755
+ # Copied from transformers.models.beit.modeling_beit.BeitPooler with Beit->Data2VecVision
756
+ class Data2VecVisionPooler(nn.Module):
757
+ def __init__(self, config: Data2VecVisionConfig) -> None:
758
+ super().__init__()
759
+ self.layernorm = (
760
+ nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) if config.use_mean_pooling else None
761
+ )
762
+
763
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
764
+ # Mean pool patch tokens with layernorm, or take the [CLS] token
765
+ return self.layernorm(hidden_states[:, 1:, :].mean(1)) if self.layernorm is not None else hidden_states[:, 0]
766
+
767
+
768
+ @auto_docstring(
769
+ custom_intro="""
770
+ Data2VecVision Model transformer with an image classification head on top (a linear layer on top of the average of
771
+ the final hidden states of the patch tokens) e.g. for ImageNet.
772
+ """
773
+ )
774
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitForImageClassification with BEIT->DATA2VEC_VISION,Beit->Data2VecVision,beit->data2vec_vision
775
+ class Data2VecVisionForImageClassification(Data2VecVisionPreTrainedModel):
776
+ def __init__(self, config: Data2VecVisionConfig) -> None:
777
+ super().__init__(config)
778
+
779
+ self.num_labels = config.num_labels
780
+ self.data2vec_vision = Data2VecVisionModel(config, add_pooling_layer=True)
781
+
782
+ # Classifier head
783
+ self.classifier = nn.Linear(config.hidden_size, config.num_labels) if config.num_labels > 0 else nn.Identity()
784
+
785
+ # Initialize weights and apply final processing
786
+ self.post_init()
787
+
788
+ @auto_docstring
789
+ def forward(
790
+ self,
791
+ pixel_values: torch.Tensor | None = None,
792
+ labels: torch.Tensor | None = None,
793
+ output_attentions: bool | None = None,
794
+ output_hidden_states: bool | None = None,
795
+ interpolate_pos_encoding: bool = False,
796
+ return_dict: bool | None = None,
797
+ **kwargs,
798
+ ) -> tuple | ImageClassifierOutput:
799
+ r"""
800
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
801
+ Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
802
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
803
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
804
+ """
805
+ return_dict = return_dict if return_dict is not None else self.config.return_dict
806
+ outputs = self.data2vec_vision(
807
+ pixel_values,
808
+ output_attentions=output_attentions,
809
+ output_hidden_states=output_hidden_states,
810
+ interpolate_pos_encoding=interpolate_pos_encoding,
811
+ return_dict=return_dict,
812
+ )
813
+
814
+ pooled_output = outputs.pooler_output if return_dict else outputs[1]
815
+
816
+ logits = self.classifier(pooled_output)
817
+
818
+ loss = None
819
+ if labels is not None:
820
+ loss = self.loss_function(labels, logits, self.config)
821
+
822
+ if not return_dict:
823
+ output = (logits,) + outputs[2:]
824
+ return ((loss,) + output) if loss is not None else output
825
+
826
+ return ImageClassifierOutput(
827
+ loss=loss,
828
+ logits=logits,
829
+ hidden_states=outputs.hidden_states,
830
+ attentions=outputs.attentions,
831
+ )
832
+
833
+
834
+ # Copied from transformers.models.beit.modeling_beit.BeitConvLayer with Beit->Data2VecVision
835
+ class Data2VecVisionConvLayer(nn.Module):
836
+ def __init__(
837
+ self,
838
+ in_channels: int,
839
+ out_channels: int,
840
+ kernel_size: int | tuple[int, int] = 3,
841
+ stride: int = 1,
842
+ padding: int | tuple[int, int] | str = 0,
843
+ bias: bool = False,
844
+ dilation: int | tuple[int, int] = 1,
845
+ groups: int = 1,
846
+ activation: str = "relu",
847
+ ):
848
+ super().__init__()
849
+ self.convolution = nn.Conv2d(
850
+ in_channels=in_channels,
851
+ out_channels=out_channels,
852
+ kernel_size=kernel_size,
853
+ stride=stride,
854
+ padding=padding,
855
+ dilation=dilation,
856
+ groups=groups,
857
+ bias=bias,
858
+ )
859
+ self.normalization = nn.BatchNorm2d(out_channels)
860
+ self.activation = ACT2FN[activation] if activation is not None else nn.Identity()
861
+
862
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
863
+ hidden_states = self.convolution(hidden_states)
864
+ hidden_states = self.normalization(hidden_states)
865
+ hidden_states = self.activation(hidden_states)
866
+ return hidden_states
867
+
868
+
869
+ # Copied from transformers.models.beit.modeling_beit.BeitPyramidPoolingBlock with Beit->Data2VecVision
870
+ class Data2VecVisionPyramidPoolingBlock(nn.Module):
871
+ def __init__(self, pool_scale: int, in_channels: int, channels: int) -> None:
872
+ super().__init__()
873
+ self.pooling = nn.AdaptiveAvgPool2d(pool_scale)
874
+ self.conv = Data2VecVisionConvLayer(in_channels, channels, kernel_size=1)
875
+
876
+ def forward(self, input: torch.Tensor, size: tuple[int, int]) -> torch.Tensor:
877
+ hidden_state = self.pooling(input)
878
+ hidden_state = self.conv(hidden_state)
879
+ hidden_state = nn.functional.interpolate(hidden_state, size=size, mode="bilinear", align_corners=False)
880
+ return hidden_state
881
+
882
+
883
+ # Copied from transformers.models.beit.modeling_beit.BeitPyramidPoolingModule with Beit->Data2VecVision
884
+ class Data2VecVisionPyramidPoolingModule(nn.Module):
885
+ """
886
+ Pyramid Pooling Module (PPM) used in PSPNet.
887
+
888
+ Args:
889
+ pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid
890
+ Module.
891
+ in_channels (int): Input channels.
892
+ channels (int): Channels after modules, before conv_seg.
893
+
894
+ Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
895
+ """
896
+
897
+ def __init__(self, pool_scales: tuple[int, ...], in_channels: int, channels: int) -> None:
898
+ super().__init__()
899
+ self.pool_scales = pool_scales
900
+ self.in_channels = in_channels
901
+ self.channels = channels
902
+ self.blocks = nn.ModuleList(
903
+ [
904
+ Data2VecVisionPyramidPoolingBlock(pool_scale=pool_scale, in_channels=in_channels, channels=channels)
905
+ for pool_scale in pool_scales
906
+ ]
907
+ )
908
+
909
+ def forward(self, hidden_states: torch.Tensor) -> list[torch.Tensor]:
910
+ original_size = hidden_states.size()[2:]
911
+ return [block(hidden_states, size=original_size) for block in self.blocks]
912
+
913
+
914
+ # Copied from transformers.models.beit.modeling_beit.BeitUperHead with Beit->Data2VecVision
915
+ class Data2VecVisionUperHead(nn.Module):
916
+ """
917
+ Unified Perceptual Parsing for Scene Understanding. This head is the implementation of
918
+ [UPerNet](https://huggingface.co/papers/1807.10221).
919
+
920
+ Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
921
+ """
922
+
923
+ def __init__(self, config: Data2VecVisionConfig) -> None:
924
+ super().__init__()
925
+
926
+ self.pool_scales = config.pool_scales # e.g. (1, 2, 3, 6)
927
+ self.in_channels = [config.hidden_size] * 4 # e.g. [768, 768, 768, 768]
928
+ self.channels = config.hidden_size
929
+ self.classifier = nn.Conv2d(self.channels, config.num_labels, kernel_size=1)
930
+
931
+ # PSP Module
932
+ self.psp_modules = Data2VecVisionPyramidPoolingModule(
933
+ self.pool_scales,
934
+ self.in_channels[-1],
935
+ self.channels,
936
+ )
937
+ self.psp_bottleneck = Data2VecVisionConvLayer(
938
+ self.in_channels[-1] + len(self.pool_scales) * self.channels,
939
+ self.channels,
940
+ kernel_size=3,
941
+ padding=1,
942
+ )
943
+ # FPN Module
944
+ self.lateral_convs = nn.ModuleList()
945
+ self.fpn_convs = nn.ModuleList()
946
+ for in_channels in self.in_channels[:-1]: # skip the top layer
947
+ self.lateral_convs.append(Data2VecVisionConvLayer(in_channels, self.channels, kernel_size=1))
948
+ self.fpn_convs.append(Data2VecVisionConvLayer(self.channels, self.channels, kernel_size=3, padding=1))
949
+
950
+ self.fpn_bottleneck = Data2VecVisionConvLayer(
951
+ len(self.in_channels) * self.channels,
952
+ self.channels,
953
+ kernel_size=3,
954
+ padding=1,
955
+ )
956
+
957
+ def psp_forward(self, hidden_states: list[torch.Tensor]) -> torch.Tensor:
958
+ hidden_state = hidden_states[-1]
959
+ hidden_state = torch.cat([hidden_state, *self.psp_modules(hidden_state)], dim=1)
960
+ return self.psp_bottleneck(hidden_state)
961
+
962
+ def forward(self, encoder_hidden_states: list[torch.Tensor]) -> torch.Tensor:
963
+ # build laterals
964
+ laterals = []
965
+ for lateral_conv, hidden_state in zip(self.lateral_convs, encoder_hidden_states):
966
+ laterals.append(lateral_conv(hidden_state))
967
+
968
+ laterals.append(self.psp_forward(encoder_hidden_states))
969
+
970
+ # build top-down path
971
+ used_backbone_levels = len(laterals)
972
+ for i in range(used_backbone_levels - 1, 0, -1):
973
+ prev_shape = laterals[i - 1].shape[2:]
974
+ laterals[i - 1] = laterals[i - 1] + nn.functional.interpolate(
975
+ laterals[i], size=prev_shape, mode="bilinear", align_corners=False
976
+ )
977
+
978
+ # build outputs
979
+ fpn_outs = []
980
+ for i in range(used_backbone_levels - 1):
981
+ fpn_outs.append(self.fpn_convs[i](laterals[i]))
982
+ # append psp feature
983
+ fpn_outs.append(laterals[-1])
984
+
985
+ for i in range(used_backbone_levels - 1, 0, -1):
986
+ fpn_outs[i] = nn.functional.interpolate(
987
+ fpn_outs[i], size=fpn_outs[0].shape[2:], mode="bilinear", align_corners=False
988
+ )
989
+ fpn_outs = torch.cat(fpn_outs, dim=1)
990
+ output = self.fpn_bottleneck(fpn_outs)
991
+ output = self.classifier(output)
992
+
993
+ return output
994
+
995
+
996
+ # Copied from transformers.models.beit.modeling_beit.BeitFCNHead with Beit->Data2VecVision
997
+ class Data2VecVisionFCNHead(nn.Module):
998
+ """
999
+ Fully Convolution Networks for Semantic Segmentation. This head is implemented of
1000
+ [FCNNet](https://huggingface.co/papers/1411.4038>).
1001
+
1002
+ Args:
1003
+ config (Data2VecVisionConfig): Configuration.
1004
+ in_channels
1005
+ kernel_size (int): The kernel size for convs in the head. Default: 3.
1006
+ dilation (int): The dilation rate for convs in the head. Default: 1.
1007
+
1008
+
1009
+ Based on OpenMMLab's implementation, found in https://github.com/open-mmlab/mmsegmentation.
1010
+ """
1011
+
1012
+ def __init__(
1013
+ self,
1014
+ config: Data2VecVisionConfig,
1015
+ in_index: int = 2,
1016
+ kernel_size: int = 3,
1017
+ dilation: int | tuple[int, int] = 1,
1018
+ ) -> None:
1019
+ super().__init__()
1020
+ self.in_channels = config.hidden_size
1021
+ self.channels = config.auxiliary_channels
1022
+ self.num_convs = config.auxiliary_num_convs
1023
+ self.concat_input = config.auxiliary_concat_input
1024
+ self.in_index = in_index
1025
+
1026
+ conv_padding = (kernel_size // 2) * dilation
1027
+ self.convs = nn.ModuleList()
1028
+ if self.num_convs > 0:
1029
+ self.convs.append(
1030
+ Data2VecVisionConvLayer(
1031
+ self.in_channels, self.channels, kernel_size=kernel_size, padding=conv_padding, dilation=dilation
1032
+ )
1033
+ )
1034
+ for _ in range(self.num_convs - 1):
1035
+ self.convs.append(
1036
+ Data2VecVisionConvLayer(
1037
+ self.channels,
1038
+ self.channels,
1039
+ kernel_size=kernel_size,
1040
+ padding=conv_padding,
1041
+ dilation=dilation,
1042
+ )
1043
+ )
1044
+ if self.concat_input:
1045
+ self.conv_cat = Data2VecVisionConvLayer(
1046
+ self.in_channels + self.channels, self.channels, kernel_size=kernel_size, padding=kernel_size // 2
1047
+ )
1048
+
1049
+ self.classifier = nn.Conv2d(self.channels, config.num_labels, kernel_size=1)
1050
+
1051
+ def forward(self, encoder_hidden_states: list[torch.Tensor]) -> torch.Tensor:
1052
+ residual = encoder_hidden_states[self.in_index]
1053
+ hidden_states = residual
1054
+ for conv in self.convs:
1055
+ hidden_states = conv(hidden_states)
1056
+ if self.concat_input:
1057
+ hidden_states = self.conv_cat(torch.cat([residual, hidden_states], dim=1))
1058
+ hidden_states = self.classifier(hidden_states)
1059
+ return hidden_states
1060
+
1061
+
1062
+ @auto_docstring
1063
+ # Todo - Refactor as part of vision refactor. Copied from transformers.models.beit.modeling_beit.BeitForSemanticSegmentation with BEIT->DATA2VEC_VISION,Beit->Data2VecVision,microsoft/beit-base-finetuned-ade-640-640->facebook/data2vec-vision-base,beit->data2vec_vision
1064
+ class Data2VecVisionForSemanticSegmentation(Data2VecVisionPreTrainedModel):
1065
+ def __init__(self, config: Data2VecVisionConfig) -> None:
1066
+ super().__init__(config)
1067
+
1068
+ self.num_labels = config.num_labels
1069
+ self.data2vec_vision = Data2VecVisionModel(config, add_pooling_layer=False)
1070
+
1071
+ # FPNs
1072
+ if len(self.config.out_indices) != 4:
1073
+ raise ValueError(
1074
+ "Data2VecVisionForSemanticSegmentation requires config.out_indices to be a list of 4 integers, "
1075
+ "specifying which features to use from the backbone. One can use [3, 5, 7, 11] in case of "
1076
+ "a base-sized architecture."
1077
+ )
1078
+ self.fpn1 = nn.Sequential(
1079
+ nn.ConvTranspose2d(config.hidden_size, config.hidden_size, kernel_size=2, stride=2),
1080
+ nn.BatchNorm2d(config.hidden_size),
1081
+ nn.GELU(),
1082
+ nn.ConvTranspose2d(config.hidden_size, config.hidden_size, kernel_size=2, stride=2),
1083
+ )
1084
+ self.fpn2 = nn.Sequential(
1085
+ nn.ConvTranspose2d(config.hidden_size, config.hidden_size, kernel_size=2, stride=2),
1086
+ )
1087
+ self.fpn3 = nn.Identity()
1088
+ self.fpn4 = nn.MaxPool2d(kernel_size=2, stride=2)
1089
+
1090
+ # Semantic segmentation head(s)
1091
+ self.decode_head = Data2VecVisionUperHead(config)
1092
+ self.auxiliary_head = Data2VecVisionFCNHead(config) if config.use_auxiliary_head else None
1093
+
1094
+ # Initialize weights and apply final processing
1095
+ self.post_init()
1096
+
1097
+ def compute_loss(self, logits, auxiliary_logits, labels):
1098
+ # upsample logits to the images' original size
1099
+ upsampled_logits = nn.functional.interpolate(
1100
+ logits, size=labels.shape[-2:], mode="bilinear", align_corners=False
1101
+ )
1102
+ if auxiliary_logits is not None:
1103
+ upsampled_auxiliary_logits = nn.functional.interpolate(
1104
+ auxiliary_logits, size=labels.shape[-2:], mode="bilinear", align_corners=False
1105
+ )
1106
+ # compute weighted loss
1107
+ loss_fct = CrossEntropyLoss(ignore_index=self.config.semantic_loss_ignore_index)
1108
+ main_loss = loss_fct(upsampled_logits, labels)
1109
+ loss = main_loss
1110
+ if auxiliary_logits is not None:
1111
+ auxiliary_loss = loss_fct(upsampled_auxiliary_logits, labels)
1112
+ loss += self.config.auxiliary_loss_weight * auxiliary_loss
1113
+
1114
+ return loss
1115
+
1116
+ @auto_docstring
1117
+ def forward(
1118
+ self,
1119
+ pixel_values: torch.Tensor | None = None,
1120
+ labels: torch.Tensor | None = None,
1121
+ output_attentions: bool | None = None,
1122
+ output_hidden_states: bool | None = None,
1123
+ interpolate_pos_encoding: bool = False,
1124
+ return_dict: bool | None = None,
1125
+ **kwargs,
1126
+ ) -> tuple | SemanticSegmenterOutput:
1127
+ r"""
1128
+ labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
1129
+ Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
1130
+ config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).
1131
+
1132
+ Examples:
1133
+
1134
+ ```python
1135
+ >>> from transformers import AutoImageProcessor, Data2VecVisionForSemanticSegmentation
1136
+ >>> from PIL import Image
1137
+ >>> import httpx
1138
+ >>> from io import BytesIO
1139
+
1140
+ >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
1141
+ >>> with httpx.stream("GET", url) as response:
1142
+ ... image = Image.open(BytesIO(response.read()))
1143
+
1144
+ >>> image_processor = AutoImageProcessor.from_pretrained("facebook/data2vec-vision-base")
1145
+ >>> model = Data2VecVisionForSemanticSegmentation.from_pretrained("facebook/data2vec-vision-base")
1146
+
1147
+ >>> inputs = image_processor(images=image, return_tensors="pt")
1148
+ >>> outputs = model(**inputs)
1149
+ >>> # logits are of shape (batch_size, num_labels, height, width)
1150
+ >>> logits = outputs.logits
1151
+ ```"""
1152
+ return_dict = return_dict if return_dict is not None else self.config.return_dict
1153
+ output_hidden_states = (
1154
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
1155
+ )
1156
+
1157
+ if labels is not None and self.config.num_labels == 1:
1158
+ raise ValueError("The number of labels should be greater than one")
1159
+
1160
+ outputs = self.data2vec_vision(
1161
+ pixel_values,
1162
+ output_attentions=output_attentions,
1163
+ output_hidden_states=True, # we need the intermediate hidden states
1164
+ interpolate_pos_encoding=interpolate_pos_encoding,
1165
+ return_dict=return_dict,
1166
+ )
1167
+
1168
+ encoder_hidden_states = outputs.hidden_states if return_dict else outputs[1]
1169
+
1170
+ # only keep certain features, and reshape
1171
+ # note that we do +1 as the encoder_hidden_states also includes the initial embeddings
1172
+ features = [feature for idx, feature in enumerate(encoder_hidden_states) if idx + 1 in self.config.out_indices]
1173
+ batch_size = pixel_values.shape[0]
1174
+ patch_resolution = self.config.image_size // self.config.patch_size
1175
+ features = [
1176
+ x[:, 1:, :].permute(0, 2, 1).reshape(batch_size, -1, patch_resolution, patch_resolution) for x in features
1177
+ ]
1178
+
1179
+ # apply FPNs
1180
+ ops = [self.fpn1, self.fpn2, self.fpn3, self.fpn4]
1181
+ for i in range(len(features)):
1182
+ features[i] = ops[i](features[i])
1183
+
1184
+ logits = self.decode_head(features)
1185
+
1186
+ auxiliary_logits = None
1187
+ if self.auxiliary_head is not None:
1188
+ auxiliary_logits = self.auxiliary_head(features)
1189
+
1190
+ loss = None
1191
+ if labels is not None:
1192
+ loss = self.compute_loss(logits, auxiliary_logits, labels)
1193
+
1194
+ if not return_dict:
1195
+ if output_hidden_states:
1196
+ output = (logits,) + outputs[1:]
1197
+ else:
1198
+ output = (logits,) + outputs[2:]
1199
+ return ((loss,) + output) if loss is not None else output
1200
+
1201
+ return SemanticSegmenterOutput(
1202
+ loss=loss,
1203
+ logits=logits,
1204
+ hidden_states=outputs.hidden_states if output_hidden_states else None,
1205
+ attentions=outputs.attentions,
1206
+ )
1207
+
1208
+
1209
+ __all__ = [
1210
+ "Data2VecVisionForImageClassification",
1211
+ "Data2VecVisionForSemanticSegmentation",
1212
+ "Data2VecVisionModel",
1213
+ "Data2VecVisionPreTrainedModel",
1214
+ ]
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modular_data2vec_audio.py ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022 The HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """PyTorch Data2VecText model."""
15
+
16
+ import math
17
+
18
+ import torch
19
+ from torch import nn
20
+
21
+ from ... import initialization as init
22
+ from ...activations import ACT2FN
23
+ from ...modeling_layers import GradientCheckpointingLayer
24
+ from ...modeling_outputs import Wav2Vec2BaseModelOutput
25
+ from ...modeling_utils import PreTrainedModel
26
+ from ..wav2vec2.modeling_wav2vec2 import (
27
+ Wav2Vec2Adapter,
28
+ Wav2Vec2Encoder,
29
+ Wav2Vec2FeatureEncoder,
30
+ Wav2Vec2FeatureProjection,
31
+ Wav2Vec2ForAudioFrameClassification,
32
+ Wav2Vec2ForCTC,
33
+ Wav2Vec2ForSequenceClassification,
34
+ Wav2Vec2ForXVector,
35
+ Wav2Vec2Model,
36
+ Wav2Vec2PreTrainedModel,
37
+ Wav2Vec2SamePadLayer,
38
+ )
39
+ from .configuration_data2vec_audio import Data2VecAudioConfig
40
+
41
+
42
+ class Data2VecAudioConvLayer(GradientCheckpointingLayer):
43
+ def __init__(self, config, layer_id=0):
44
+ super().__init__()
45
+ self.in_conv_dim = config.conv_dim[layer_id - 1] if layer_id > 0 else 1
46
+ self.out_conv_dim = config.conv_dim[layer_id]
47
+
48
+ self.conv = nn.Conv1d(
49
+ self.in_conv_dim,
50
+ self.out_conv_dim,
51
+ kernel_size=config.conv_kernel[layer_id],
52
+ stride=config.conv_stride[layer_id],
53
+ bias=config.conv_bias,
54
+ )
55
+ self.layer_norm = nn.LayerNorm(self.out_conv_dim, elementwise_affine=True)
56
+ self.activation = ACT2FN[config.feat_extract_activation]
57
+
58
+ def forward(self, hidden_states):
59
+ hidden_states = self.conv(hidden_states)
60
+
61
+ hidden_states = hidden_states.transpose(-2, -1)
62
+ hidden_states = self.layer_norm(hidden_states)
63
+ hidden_states = hidden_states.transpose(-2, -1)
64
+
65
+ hidden_states = self.activation(hidden_states)
66
+ return hidden_states
67
+
68
+
69
+ class Data2VecAudioPadLayer(Wav2Vec2SamePadLayer):
70
+ pass
71
+
72
+
73
+ class Data2VecAudioPositionalConvLayer(nn.Module):
74
+ def __init__(self, config):
75
+ super().__init__()
76
+ self.conv = nn.Conv1d(
77
+ config.hidden_size,
78
+ config.hidden_size,
79
+ kernel_size=config.conv_pos_kernel_size,
80
+ padding=config.conv_pos_kernel_size // 2,
81
+ groups=config.num_conv_pos_embedding_groups,
82
+ )
83
+
84
+ self.padding = Data2VecAudioPadLayer(config.conv_pos_kernel_size)
85
+ self.activation = ACT2FN[config.feat_extract_activation]
86
+ # no learnable parameters
87
+ self.layer_norm = nn.LayerNorm(config.hidden_size, elementwise_affine=False)
88
+
89
+ def forward(self, hidden_states):
90
+ hidden_states = self.conv(hidden_states)
91
+ hidden_states = self.padding(hidden_states)
92
+
93
+ hidden_states = hidden_states.transpose(1, 2)
94
+ hidden_states = self.layer_norm(hidden_states)
95
+ hidden_states = hidden_states.transpose(1, 2)
96
+ hidden_states = self.activation(hidden_states)
97
+ return hidden_states
98
+
99
+
100
+ class Data2VecAudioPositionalConvEmbedding(nn.Module):
101
+ def __init__(self, config):
102
+ super().__init__()
103
+ self.layers = nn.ModuleList(
104
+ [Data2VecAudioPositionalConvLayer(config) for _ in range(config.num_conv_pos_embeddings)]
105
+ )
106
+
107
+ def forward(self, hidden_states):
108
+ hidden_states = hidden_states.transpose(1, 2)
109
+ for layer in self.layers:
110
+ hidden_states = layer(hidden_states)
111
+ hidden_states = hidden_states.transpose(1, 2)
112
+ return hidden_states
113
+
114
+
115
+ class Data2VecAudioFeatureEncoder(Wav2Vec2FeatureEncoder):
116
+ def __init__(self, config):
117
+ nn.Module.__init__(self)
118
+ self.conv_layers = nn.ModuleList(
119
+ [Data2VecAudioConvLayer(config, layer_id=i) for i in range(config.num_feat_extract_layers)]
120
+ )
121
+ self.gradient_checkpointing = False
122
+ self._requires_grad = True
123
+
124
+
125
+ class Data2VecAudioFeatureProjection(Wav2Vec2FeatureProjection):
126
+ pass
127
+
128
+
129
+ class Data2VecAudioEncoder(Wav2Vec2Encoder):
130
+ pass
131
+
132
+
133
+ class Data2VecAudioAdapter(Wav2Vec2Adapter):
134
+ pass
135
+
136
+
137
+ class Data2VecAudioPreTrainedModel(PreTrainedModel, Wav2Vec2PreTrainedModel):
138
+ config: Data2VecAudioConfig
139
+ base_model_prefix = "data2vec_audio"
140
+ main_input_name = "input_values"
141
+ input_modalities = "audio"
142
+ supports_gradient_checkpointing = True
143
+ _supports_flash_attn = True
144
+ _supports_sdpa = True
145
+ _supports_flex_attn = True
146
+
147
+ @torch.no_grad()
148
+ def _init_weights(self, module):
149
+ """Initialize the weights"""
150
+ if isinstance(module, Data2VecAudioFeatureProjection):
151
+ k = math.sqrt(1 / module.projection.in_features)
152
+ init.uniform_(module.projection.weight, a=-k, b=k)
153
+ init.uniform_(module.projection.bias, a=-k, b=k)
154
+ elif isinstance(module, Data2VecAudioPositionalConvLayer):
155
+ init.constant_(module.conv.bias, 0)
156
+ elif isinstance(module, nn.Linear):
157
+ init.normal_(module.weight, mean=0.0, std=self.config.initializer_range)
158
+
159
+ if module.bias is not None:
160
+ init.zeros_(module.bias)
161
+ elif isinstance(module, (nn.LayerNorm, nn.GroupNorm)):
162
+ if module.bias is not None:
163
+ init.zeros_(module.bias)
164
+ if module.weight is not None:
165
+ init.ones_(module.weight)
166
+ elif isinstance(module, nn.Conv1d):
167
+ init.kaiming_normal_(module.weight)
168
+
169
+ if module.bias is not None:
170
+ k = math.sqrt(module.groups / (module.in_channels * module.kernel_size[0]))
171
+ init.uniform_(module.bias, a=-k, b=k)
172
+
173
+ def _get_adapters(self):
174
+ raise AttributeError("Not needed for Data2VecAudio")
175
+
176
+ def init_adapter_layers(self):
177
+ raise AttributeError("Not needed for Data2VecAudio")
178
+
179
+ def load_adapter(self):
180
+ raise AttributeError("Not needed for Data2VecAudio")
181
+
182
+
183
+ Data2VecAudioBaseModelOutput = Wav2Vec2BaseModelOutput
184
+
185
+
186
+ class Data2VecAudioModel(Data2VecAudioPreTrainedModel, Wav2Vec2Model):
187
+ def __init__(self, config: Data2VecAudioConfig):
188
+ Data2VecAudioPreTrainedModel.__init__(self, config)
189
+ self.config = config
190
+ self.feature_extractor = Data2VecAudioFeatureEncoder(config)
191
+ self.feature_projection = Data2VecAudioFeatureProjection(config)
192
+
193
+ # model only needs masking vector if mask prob is > 0.0
194
+ if config.mask_time_prob > 0.0 or config.mask_feature_prob > 0.0:
195
+ self.masked_spec_embed = nn.Parameter(torch.Tensor(config.hidden_size).uniform_())
196
+
197
+ self.encoder = Data2VecAudioEncoder(config)
198
+
199
+ self.adapter = Data2VecAudioAdapter(config) if config.add_adapter else None
200
+
201
+ # Initialize weights and apply final processing
202
+ self.post_init()
203
+
204
+ def freeze_feature_encoder(self):
205
+ """
206
+ Calling this function will disable the gradient computation for the feature encoder so that its parameter will
207
+ not be updated during training.
208
+ """
209
+ self.feature_extractor._freeze_parameters()
210
+
211
+ def forward(self, **super_kwargs):
212
+ return super().forward(**super_kwargs)
213
+
214
+
215
+ class Data2VecAudioForCTC(Data2VecAudioPreTrainedModel, Wav2Vec2ForCTC):
216
+ def __init__(self, config):
217
+ r"""
218
+ config ([`Data2VecAudioForCTC`]):
219
+ Model configuration class with all the parameters of the model. Initializing with a config file does not
220
+ load the weights associated with the model, only the configuration. Check out the
221
+ [`~PreTrainedModel.from_pretrained`] method to load the model weights.
222
+ """
223
+ Data2VecAudioPreTrainedModel.__init__(self, config)
224
+
225
+ self.data2vec_audio = Data2VecAudioModel(config)
226
+ self.dropout = nn.Dropout(config.final_dropout)
227
+
228
+ if config.vocab_size is None:
229
+ raise ValueError(
230
+ f"You are trying to instantiate {self.__class__} with a configuration that "
231
+ "does not define the vocabulary size of the language model head. Please "
232
+ "instantiate the model as follows: `Data2VecAudioForCTC.from_pretrained(..., vocab_size=vocab_size)`. "
233
+ "or define `vocab_size` of your model's configuration."
234
+ )
235
+ output_hidden_size = (
236
+ config.output_hidden_size if hasattr(config, "add_adapter") and config.add_adapter else config.hidden_size
237
+ )
238
+ self.lm_head = nn.Linear(output_hidden_size, config.vocab_size)
239
+
240
+ # Initialize weights and apply final processing
241
+ self.post_init()
242
+
243
+ def freeze_base_model(self):
244
+ raise AttributeError("Not needed for Data2VecAudio")
245
+
246
+ def tie_weights(self):
247
+ raise AttributeError("Not needed for Data2VecAudio")
248
+
249
+ def forward(self, **super_kwargs):
250
+ return super().forward(**super_kwargs)
251
+
252
+
253
+ class Data2VecAudioForSequenceClassification(Wav2Vec2ForSequenceClassification):
254
+ pass
255
+
256
+
257
+ class Data2VecAudioForAudioFrameClassification(Wav2Vec2ForAudioFrameClassification):
258
+ pass
259
+
260
+
261
+ class Data2VecAudioForXVector(Wav2Vec2ForXVector):
262
+ pass
263
+
264
+
265
+ __all__ = [
266
+ "Data2VecAudioForAudioFrameClassification",
267
+ "Data2VecAudioForCTC",
268
+ "Data2VecAudioForSequenceClassification",
269
+ "Data2VecAudioForXVector",
270
+ "Data2VecAudioModel",
271
+ "Data2VecAudioPreTrainedModel",
272
+ ]
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/data2vec/modular_data2vec_text.py ADDED
@@ -0,0 +1,599 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2022 The HuggingFace Inc. team.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """PyTorch Data2VecText model."""
15
+
16
+ import torch
17
+ import torch.nn as nn
18
+ from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
19
+
20
+ from ... import initialization as init
21
+ from ...generation import GenerationMixin
22
+ from ...modeling_outputs import (
23
+ BaseModelOutputWithPoolingAndCrossAttentions,
24
+ CausalLMOutputWithCrossAttentions,
25
+ MaskedLMOutput,
26
+ MultipleChoiceModelOutput,
27
+ QuestionAnsweringModelOutput,
28
+ SequenceClassifierOutput,
29
+ TokenClassifierOutput,
30
+ )
31
+ from ...modeling_utils import PreTrainedModel
32
+ from ...processing_utils import Unpack
33
+ from ...utils import TransformersKwargs, auto_docstring, logging
34
+ from ...utils.generic import can_return_tuple
35
+ from ..roberta.modeling_roberta import (
36
+ RobertaClassificationHead,
37
+ RobertaCrossAttention,
38
+ RobertaEmbeddings,
39
+ RobertaLayer,
40
+ RobertaLMHead,
41
+ RobertaModel,
42
+ RobertaSelfAttention,
43
+ )
44
+ from .configuration_data2vec_text import Data2VecTextConfig
45
+
46
+
47
+ logger = logging.get_logger(__name__)
48
+
49
+
50
+ class Data2VecTextEmbeddings(RobertaEmbeddings):
51
+ pass
52
+
53
+
54
+ class Data2VecTextSelfAttention(RobertaSelfAttention):
55
+ pass
56
+
57
+
58
+ class Data2VecTextCrossAttention(RobertaCrossAttention):
59
+ pass
60
+
61
+
62
+ class Data2VecTextLayer(RobertaLayer):
63
+ pass
64
+
65
+
66
+ @auto_docstring
67
+ class Data2VecTextPreTrainedModel(PreTrainedModel):
68
+ config_class = Data2VecTextConfig
69
+ base_model_prefix = "data2vec_text"
70
+ supports_gradient_checkpointing = True
71
+ _no_split_modules = ["Data2VecTextForTextEmbeddings", "Data2VecTextLayer"]
72
+ _supports_flash_attn = True
73
+ _supports_sdpa = True
74
+ _supports_flex_attn = True
75
+ _supports_attention_backend = True
76
+ _can_record_outputs = {
77
+ "hidden_states": Data2VecTextLayer,
78
+ "attentions": Data2VecTextSelfAttention,
79
+ "cross_attentions": Data2VecTextCrossAttention,
80
+ }
81
+
82
+ def _init_weights(self, module):
83
+ super()._init_weights(module)
84
+ if isinstance(module, Data2VecTextEmbeddings):
85
+ init.copy_(module.position_ids, torch.arange(module.position_ids.shape[-1]).expand((1, -1)))
86
+ init.zeros_(module.token_type_ids)
87
+
88
+
89
+ @auto_docstring
90
+ class Data2VecTextModel(RobertaModel):
91
+ pass
92
+
93
+
94
+ class Data2VecTextLMHead(RobertaLMHead):
95
+ pass
96
+
97
+
98
+ class Data2VecTextClassificationHead(RobertaClassificationHead):
99
+ pass
100
+
101
+
102
+ @auto_docstring(
103
+ custom_intro="""
104
+ Data2VecText Model with a `language modeling` head on top for CLM fine-tuning.
105
+ """
106
+ )
107
+ class Data2VecTextForCausalLM(Data2VecTextPreTrainedModel, GenerationMixin):
108
+ _tied_weights_keys = {
109
+ "lm_head.decoder.weight": "data2vec_text.embeddings.word_embeddings.weight",
110
+ "lm_head.decoder.bias": "lm_head.bias",
111
+ }
112
+
113
+ def __init__(self, config):
114
+ super().__init__(config)
115
+
116
+ if not config.is_decoder:
117
+ logger.warning("If you want to use `Data2VecTextLMHeadModel` as a standalone, add `is_decoder=True.`")
118
+
119
+ self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
120
+ self.lm_head = Data2VecTextLMHead(config)
121
+
122
+ # Initialize weights and apply final processing
123
+ self.post_init()
124
+
125
+ def get_output_embeddings(self):
126
+ return self.lm_head.decoder
127
+
128
+ def set_output_embeddings(self, new_embeddings):
129
+ self.lm_head.decoder = new_embeddings
130
+
131
+ @can_return_tuple
132
+ @auto_docstring
133
+ def forward(
134
+ self,
135
+ input_ids: torch.LongTensor | None = None,
136
+ attention_mask: torch.FloatTensor | None = None,
137
+ token_type_ids: torch.LongTensor | None = None,
138
+ position_ids: torch.LongTensor | None = None,
139
+ inputs_embeds: torch.FloatTensor | None = None,
140
+ encoder_hidden_states: torch.FloatTensor | None = None,
141
+ encoder_attention_mask: torch.FloatTensor | None = None,
142
+ labels: torch.LongTensor | None = None,
143
+ past_key_values: tuple[tuple[torch.FloatTensor]] | None = None,
144
+ use_cache: bool | None = None,
145
+ logits_to_keep: int | torch.Tensor = 0,
146
+ **kwargs: Unpack[TransformersKwargs],
147
+ ) -> tuple | CausalLMOutputWithCrossAttentions:
148
+ r"""
149
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
150
+ Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
151
+ `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
152
+ ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
153
+
154
+ Example:
155
+
156
+ ```python
157
+ >>> from transformers import AutoTokenizer, Data2VecTextForCausalLM, Data2VecTextConfig
158
+ >>> import torch
159
+
160
+ >>> tokenizer = AutoTokenizer.from_pretrained("facebook/data2vec-text-base")
161
+ >>> config = Data2VecTextConfig.from_pretrained("facebook/data2vec-text-base")
162
+ >>> config.is_decoder = True
163
+ >>> model = Data2VecTextForCausalLM.from_pretrained("facebook/data2vec-text-base", config=config)
164
+
165
+ >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
166
+ >>> outputs = model(**inputs)
167
+
168
+ >>> prediction_logits = outputs.logits
169
+ ```"""
170
+ if labels is not None:
171
+ use_cache = False
172
+
173
+ outputs: BaseModelOutputWithPoolingAndCrossAttentions = self.data2vec_text(
174
+ input_ids,
175
+ attention_mask=attention_mask,
176
+ token_type_ids=token_type_ids,
177
+ position_ids=position_ids,
178
+ inputs_embeds=inputs_embeds,
179
+ encoder_hidden_states=encoder_hidden_states,
180
+ encoder_attention_mask=encoder_attention_mask,
181
+ past_key_values=past_key_values,
182
+ use_cache=use_cache,
183
+ return_dict=True,
184
+ **kwargs,
185
+ )
186
+
187
+ hidden_states = outputs.last_hidden_state
188
+ # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
189
+ slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
190
+ logits = self.lm_head(hidden_states[:, slice_indices, :])
191
+
192
+ loss = None
193
+ if labels is not None:
194
+ loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
195
+
196
+ return CausalLMOutputWithCrossAttentions(
197
+ loss=loss,
198
+ logits=logits,
199
+ past_key_values=outputs.past_key_values,
200
+ hidden_states=outputs.hidden_states,
201
+ attentions=outputs.attentions,
202
+ cross_attentions=outputs.cross_attentions,
203
+ )
204
+
205
+
206
+ @auto_docstring
207
+ class Data2VecTextForMaskedLM(Data2VecTextPreTrainedModel):
208
+ _tied_weights_keys = {
209
+ "lm_head.decoder.weight": "data2vec_text.embeddings.word_embeddings.weight",
210
+ "lm_head.decoder.bias": "lm_head.bias",
211
+ }
212
+
213
+ def __init__(self, config):
214
+ super().__init__(config)
215
+
216
+ if config.is_decoder:
217
+ logger.warning(
218
+ "If you want to use `Data2VecTextForMaskedLM` make sure `config.is_decoder=False` for "
219
+ "bi-directional self-attention."
220
+ )
221
+
222
+ self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
223
+ self.lm_head = Data2VecTextLMHead(config)
224
+
225
+ # Initialize weights and apply final processing
226
+ self.post_init()
227
+
228
+ def get_output_embeddings(self):
229
+ return self.lm_head.decoder
230
+
231
+ def set_output_embeddings(self, new_embeddings):
232
+ self.lm_head.decoder = new_embeddings
233
+
234
+ @can_return_tuple
235
+ @auto_docstring
236
+ def forward(
237
+ self,
238
+ input_ids: torch.LongTensor | None = None,
239
+ attention_mask: torch.FloatTensor | None = None,
240
+ token_type_ids: torch.LongTensor | None = None,
241
+ position_ids: torch.LongTensor | None = None,
242
+ inputs_embeds: torch.FloatTensor | None = None,
243
+ encoder_hidden_states: torch.FloatTensor | None = None,
244
+ encoder_attention_mask: torch.FloatTensor | None = None,
245
+ labels: torch.LongTensor | None = None,
246
+ **kwargs: Unpack[TransformersKwargs],
247
+ ) -> tuple | MaskedLMOutput:
248
+ r"""
249
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
250
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
251
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
252
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
253
+ """
254
+ outputs = self.data2vec_text(
255
+ input_ids,
256
+ attention_mask=attention_mask,
257
+ token_type_ids=token_type_ids,
258
+ position_ids=position_ids,
259
+ inputs_embeds=inputs_embeds,
260
+ encoder_hidden_states=encoder_hidden_states,
261
+ encoder_attention_mask=encoder_attention_mask,
262
+ return_dict=True,
263
+ **kwargs,
264
+ )
265
+ sequence_output = outputs[0]
266
+ prediction_scores = self.lm_head(sequence_output)
267
+
268
+ masked_lm_loss = None
269
+ if labels is not None:
270
+ loss_fct = CrossEntropyLoss()
271
+
272
+ labels = labels.to(prediction_scores.device)
273
+ masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1))
274
+
275
+ return MaskedLMOutput(
276
+ loss=masked_lm_loss,
277
+ logits=prediction_scores,
278
+ hidden_states=outputs.hidden_states,
279
+ attentions=outputs.attentions,
280
+ )
281
+
282
+
283
+ @auto_docstring(
284
+ custom_intro="""
285
+ Data2VecText Model transformer with a sequence classification/regression head on top (a linear layer on top of the
286
+ pooled output) e.g. for GLUE tasks.
287
+ """
288
+ )
289
+ class Data2VecTextForSequenceClassification(Data2VecTextPreTrainedModel):
290
+ def __init__(self, config):
291
+ super().__init__(config)
292
+ self.num_labels = config.num_labels
293
+ self.config = config
294
+
295
+ self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
296
+ self.classifier = Data2VecTextClassificationHead(config)
297
+
298
+ # Initialize weights and apply final processing
299
+ self.post_init()
300
+
301
+ @can_return_tuple
302
+ @auto_docstring
303
+ def forward(
304
+ self,
305
+ input_ids: torch.LongTensor | None = None,
306
+ attention_mask: torch.FloatTensor | None = None,
307
+ token_type_ids: torch.LongTensor | None = None,
308
+ position_ids: torch.LongTensor | None = None,
309
+ inputs_embeds: torch.FloatTensor | None = None,
310
+ labels: torch.LongTensor | None = None,
311
+ **kwargs: Unpack[TransformersKwargs],
312
+ ) -> tuple | SequenceClassifierOutput:
313
+ r"""
314
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
315
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
316
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
317
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
318
+ """
319
+ outputs = self.data2vec_text(
320
+ input_ids,
321
+ attention_mask=attention_mask,
322
+ token_type_ids=token_type_ids,
323
+ position_ids=position_ids,
324
+ inputs_embeds=inputs_embeds,
325
+ return_dict=True,
326
+ **kwargs,
327
+ )
328
+ sequence_output = outputs[0]
329
+ logits = self.classifier(sequence_output)
330
+
331
+ loss = None
332
+ if labels is not None:
333
+ labels = labels.to(logits.device)
334
+
335
+ if self.config.problem_type is None:
336
+ if self.num_labels == 1:
337
+ self.config.problem_type = "regression"
338
+ elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
339
+ self.config.problem_type = "single_label_classification"
340
+ else:
341
+ self.config.problem_type = "multi_label_classification"
342
+
343
+ if self.config.problem_type == "regression":
344
+ loss_fct = MSELoss()
345
+ if self.num_labels == 1:
346
+ loss = loss_fct(logits.squeeze(), labels.squeeze())
347
+ else:
348
+ loss = loss_fct(logits, labels)
349
+ elif self.config.problem_type == "single_label_classification":
350
+ loss_fct = CrossEntropyLoss()
351
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
352
+ elif self.config.problem_type == "multi_label_classification":
353
+ loss_fct = BCEWithLogitsLoss()
354
+ loss = loss_fct(logits, labels)
355
+
356
+ return SequenceClassifierOutput(
357
+ loss=loss,
358
+ logits=logits,
359
+ hidden_states=outputs.hidden_states,
360
+ attentions=outputs.attentions,
361
+ )
362
+
363
+
364
+ @auto_docstring
365
+ class Data2VecTextForMultipleChoice(Data2VecTextPreTrainedModel):
366
+ def __init__(self, config):
367
+ super().__init__(config)
368
+
369
+ self.data2vec_text = Data2VecTextModel(config)
370
+ self.dropout = nn.Dropout(config.hidden_dropout_prob)
371
+ self.classifier = nn.Linear(config.hidden_size, 1)
372
+
373
+ # Initialize weights and apply final processing
374
+ self.post_init()
375
+
376
+ @can_return_tuple
377
+ @auto_docstring
378
+ def forward(
379
+ self,
380
+ input_ids: torch.LongTensor | None = None,
381
+ token_type_ids: torch.LongTensor | None = None,
382
+ attention_mask: torch.FloatTensor | None = None,
383
+ labels: torch.LongTensor | None = None,
384
+ position_ids: torch.LongTensor | None = None,
385
+ inputs_embeds: torch.FloatTensor | None = None,
386
+ **kwargs: Unpack[TransformersKwargs],
387
+ ) -> tuple | MultipleChoiceModelOutput:
388
+ r"""
389
+ input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
390
+ Indices of input sequence tokens in the vocabulary.
391
+
392
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
393
+ [`PreTrainedTokenizer.__call__`] for details.
394
+
395
+ [What are input IDs?](../glossary#input-ids)
396
+ token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
397
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
398
+ 1]`:
399
+
400
+ - 0 corresponds to a *sentence A* token,
401
+ - 1 corresponds to a *sentence B* token.
402
+
403
+ [What are token type IDs?](../glossary#token-type-ids)
404
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
405
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
406
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
407
+ `input_ids` above)
408
+ position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
409
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
410
+ config.max_position_embeddings - 1]`.
411
+
412
+ [What are position IDs?](../glossary#position-ids)
413
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
414
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
415
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
416
+ model's internal embedding lookup matrix.
417
+ """
418
+ num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]
419
+
420
+ flat_input_ids = input_ids.view(-1, input_ids.size(-1)) if input_ids is not None else None
421
+ flat_position_ids = position_ids.view(-1, position_ids.size(-1)) if position_ids is not None else None
422
+ flat_token_type_ids = token_type_ids.view(-1, token_type_ids.size(-1)) if token_type_ids is not None else None
423
+ flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None
424
+ flat_inputs_embeds = (
425
+ inputs_embeds.view(-1, inputs_embeds.size(-2), inputs_embeds.size(-1))
426
+ if inputs_embeds is not None
427
+ else None
428
+ )
429
+
430
+ outputs = self.data2vec_text(
431
+ flat_input_ids,
432
+ position_ids=flat_position_ids,
433
+ token_type_ids=flat_token_type_ids,
434
+ attention_mask=flat_attention_mask,
435
+ inputs_embeds=flat_inputs_embeds,
436
+ return_dict=True,
437
+ **kwargs,
438
+ )
439
+ pooled_output = outputs[1]
440
+
441
+ pooled_output = self.dropout(pooled_output)
442
+ logits = self.classifier(pooled_output)
443
+ reshaped_logits = logits.view(-1, num_choices)
444
+
445
+ loss = None
446
+ if labels is not None:
447
+ loss_fct = CrossEntropyLoss()
448
+
449
+ labels = labels.to(reshaped_logits.device)
450
+ loss = loss_fct(reshaped_logits, labels)
451
+
452
+ return MultipleChoiceModelOutput(
453
+ loss=loss,
454
+ logits=reshaped_logits,
455
+ hidden_states=outputs.hidden_states,
456
+ attentions=outputs.attentions,
457
+ )
458
+
459
+
460
+ @auto_docstring
461
+ class Data2VecTextForTokenClassification(Data2VecTextPreTrainedModel):
462
+ def __init__(self, config):
463
+ super().__init__(config)
464
+ self.num_labels = config.num_labels
465
+
466
+ self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
467
+ classifier_dropout = (
468
+ config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob
469
+ )
470
+ self.dropout = nn.Dropout(classifier_dropout)
471
+ self.classifier = nn.Linear(config.hidden_size, config.num_labels)
472
+
473
+ # Initialize weights and apply final processing
474
+ self.post_init()
475
+
476
+ @can_return_tuple
477
+ @auto_docstring
478
+ def forward(
479
+ self,
480
+ input_ids: torch.LongTensor | None = None,
481
+ attention_mask: torch.FloatTensor | None = None,
482
+ token_type_ids: torch.LongTensor | None = None,
483
+ position_ids: torch.LongTensor | None = None,
484
+ inputs_embeds: torch.FloatTensor | None = None,
485
+ labels: torch.LongTensor | None = None,
486
+ **kwargs: Unpack[TransformersKwargs],
487
+ ) -> tuple | TokenClassifierOutput:
488
+ r"""
489
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
490
+ Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
491
+ """
492
+ outputs = self.data2vec_text(
493
+ input_ids,
494
+ attention_mask=attention_mask,
495
+ token_type_ids=token_type_ids,
496
+ position_ids=position_ids,
497
+ inputs_embeds=inputs_embeds,
498
+ return_dict=True,
499
+ **kwargs,
500
+ )
501
+
502
+ sequence_output = outputs[0]
503
+
504
+ sequence_output = self.dropout(sequence_output)
505
+ logits = self.classifier(sequence_output)
506
+
507
+ loss = None
508
+ if labels is not None:
509
+ loss_fct = CrossEntropyLoss()
510
+
511
+ labels = labels.to(logits.device)
512
+ loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
513
+
514
+ return TokenClassifierOutput(
515
+ loss=loss,
516
+ logits=logits,
517
+ hidden_states=outputs.hidden_states,
518
+ attentions=outputs.attentions,
519
+ )
520
+
521
+
522
+ @auto_docstring
523
+ class Data2VecTextForQuestionAnswering(Data2VecTextPreTrainedModel):
524
+ def __init__(self, config):
525
+ super().__init__(config)
526
+ self.num_labels = config.num_labels
527
+
528
+ self.data2vec_text = Data2VecTextModel(config, add_pooling_layer=False)
529
+ self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
530
+
531
+ # Initialize weights and apply final processing
532
+ self.post_init()
533
+
534
+ @can_return_tuple
535
+ @auto_docstring
536
+ def forward(
537
+ self,
538
+ input_ids: torch.LongTensor | None = None,
539
+ attention_mask: torch.FloatTensor | None = None,
540
+ token_type_ids: torch.LongTensor | None = None,
541
+ position_ids: torch.LongTensor | None = None,
542
+ inputs_embeds: torch.FloatTensor | None = None,
543
+ start_positions: torch.LongTensor | None = None,
544
+ end_positions: torch.LongTensor | None = None,
545
+ **kwargs: Unpack[TransformersKwargs],
546
+ ) -> tuple | QuestionAnsweringModelOutput:
547
+ outputs = self.data2vec_text(
548
+ input_ids,
549
+ attention_mask=attention_mask,
550
+ token_type_ids=token_type_ids,
551
+ position_ids=position_ids,
552
+ inputs_embeds=inputs_embeds,
553
+ return_dict=True,
554
+ **kwargs,
555
+ )
556
+
557
+ sequence_output = outputs[0]
558
+
559
+ logits = self.qa_outputs(sequence_output)
560
+ start_logits, end_logits = logits.split(1, dim=-1)
561
+ start_logits = start_logits.squeeze(-1).contiguous()
562
+ end_logits = end_logits.squeeze(-1).contiguous()
563
+
564
+ total_loss = None
565
+ if start_positions is not None and end_positions is not None:
566
+ # If we are on multi-GPU, split add a dimension
567
+ if len(start_positions.size()) > 1:
568
+ start_positions = start_positions.squeeze(-1)
569
+ if len(end_positions.size()) > 1:
570
+ end_positions = end_positions.squeeze(-1)
571
+ # sometimes the start/end positions are outside our model inputs, we ignore these terms
572
+ ignored_index = start_logits.size(1)
573
+ start_positions = start_positions.clamp(0, ignored_index)
574
+ end_positions = end_positions.clamp(0, ignored_index)
575
+
576
+ loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
577
+ start_loss = loss_fct(start_logits, start_positions)
578
+ end_loss = loss_fct(end_logits, end_positions)
579
+ total_loss = (start_loss + end_loss) / 2
580
+
581
+ return QuestionAnsweringModelOutput(
582
+ loss=total_loss,
583
+ start_logits=start_logits,
584
+ end_logits=end_logits,
585
+ hidden_states=outputs.hidden_states,
586
+ attentions=outputs.attentions,
587
+ )
588
+
589
+
590
+ __all__ = [
591
+ "Data2VecTextForCausalLM",
592
+ "Data2VecTextForMaskedLM",
593
+ "Data2VecTextForMultipleChoice",
594
+ "Data2VecTextForQuestionAnswering",
595
+ "Data2VecTextForSequenceClassification",
596
+ "Data2VecTextForTokenClassification",
597
+ "Data2VecTextModel",
598
+ "Data2VecTextPreTrainedModel",
599
+ ]
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/sam3/modeling_sam3.py ADDED
The diff for this file is too large to render. See raw diff