Hugging Face's logo

SSBteam
/
lta

Model card Files
xet
 Community
JinghuiLuAstronaut commited on
Commit
4979f07
·
verified ·
1 Parent(s): c0802e1

Add files using upload-large-folder tool

Browse files
Files changed (20) hide show
  1. LTA_openwebtext_dualt/logs/ar_lm1b_flmpack_bert_small_len128_gbs512_1gpu_1m.log +21 -0
  2. LTA_openwebtext_dualt/logs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525_watcher.log +1854 -0
  3. LTA_openwebtext_dualt/logs/lta_owt_c1024_len1024_t0to1_lowk64plus_cleanbridge_droplowt0p2_buf1000_gbs128_4gpu_2k.log +81 -0
  4. LTA_openwebtext_dualt/logs/lta_owt_dirichlet_categorical_fullvocab_c1024_fullycoupled_gpt2cached_len1024_rollin_p0p5_s4_i32_outwd-1_gbs512_8gpu_1m_20260517_201804.log +658 -0
  5. LTA_openwebtext_dualt/logs/lta_owt_gpt2cached_len1024_rollout1_p05_smoke4gpu_20260513_143811.log +231 -0
  6. LTA_openwebtext_dualt/logs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525_watcher.sh +87 -0
  7. LTA_openwebtext_dualt/logs/watch_lm1b_len128_Cv2v_sde_20260523.pid +1 -0
  8. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/_cl_builtins.py +231 -0
  9. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/blueprint.py +173 -0
  10. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/cddl.py +172 -0
  11. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/codeql.py +80 -0
  12. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/data.py +763 -0
  13. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/graph.py +108 -0
  14. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/julia.py +294 -0
  15. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/maple.py +291 -0
  16. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/ptx.py +119 -0
  17. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/scripting.py +1638 -0
  18. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/tact.py +303 -0
  19. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/thingsdb.py +143 -0
  20. LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/yara.py +69 -0
LTA_openwebtext_dualt/logs/ar_lm1b_flmpack_bert_small_len128_gbs512_1gpu_1m.log ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "task": "ar_lm",
3
+ "device": "cuda:0",
4
+ "rank": 0,
5
+ "world_size": 1,
6
+ "samples": "wrapped_streaming",
7
+ "vocab_size": 30522,
8
+ "bos_id": 101,
9
+ "eos_id": 102,
10
+ "save_dir": "runs/ar_lm1b_flmpack_bert_small_len128_gbs512_1gpu_1m",
11
+ "params": 108440832,
12
+ "batch_size": 64,
13
+ "grad_accum": 8,
14
+ "effective_batch_size": 512,
15
+ "global_batch_size": 512,
16
+ "max_len": 128,
17
+ "wrap": true,
18
+ "openwebtext_split": "all",
19
+ "torch_compile": false
20
+ }
21
+ step=100 micro_steps=800 elapsed=153.3s lr=1.212000e-05 loss=9.3046 ppl=14004.0317 acc=0.0481 tokens=8128.0000
LTA_openwebtext_dualt/logs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525_watcher.log ADDED
@@ -0,0 +1,1854 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [watch-gumbel] run_dir=runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525
2
+ [watch-gumbel] out_base=docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525
3
+ [watch-gumbel] interval=1000 max_len=1024 steps=128 c=30522->61044 temp=1.45 top_p=0.95 tau=1.0->0.2 n=128
4
+ [watch-gumbel] 2026-05-25_18:56:19 no ckpt yet
5
+ [watch-gumbel] 2026-05-25_18:56:49 no ckpt yet
6
+ [watch-gumbel] 2026-05-25_18:57:19 no ckpt yet
7
+ [watch-gumbel] 2026-05-25_18:57:49 no ckpt yet
8
+ [watch-gumbel] 2026-05-25_18:58:19 no ckpt yet
9
+ [watch-gumbel] 2026-05-25_18:58:49 no ckpt yet
10
+ [watch-gumbel] 2026-05-25_18:59:19 no ckpt yet
11
+ [watch-gumbel] 2026-05-25_18:59:49 no ckpt yet
12
+ [watch-gumbel] 2026-05-25_19:00:19 no ckpt yet
13
+ [watch-gumbel] 2026-05-25_19:00:49 no ckpt yet
14
+ [watch-gumbel] 2026-05-25_19:01:19 no ckpt yet
15
+ [watch-gumbel] 2026-05-25_19:01:49 no ckpt yet
16
+ [watch-gumbel] 2026-05-25_19:02:19 no ckpt yet
17
+ [watch-gumbel] 2026-05-25_19:02:49 no ckpt yet
18
+ [watch-gumbel] 2026-05-25_19:03:19 no ckpt yet
19
+ [watch-gumbel] 2026-05-25_19:03:49 no ckpt yet
20
+ [watch-gumbel] 2026-05-25_19:04:19 no ckpt yet
21
+ [watch-gumbel] 2026-05-25_19:04:49 no ckpt yet
22
+ [watch-gumbel] 2026-05-25_19:05:19 no ckpt yet
23
+ [watch-gumbel] 2026-05-25_19:05:49 no ckpt yet
24
+ [watch-gumbel] 2026-05-25_19:06:19 no ckpt yet
25
+ [watch-gumbel] 2026-05-25_19:06:49 no ckpt yet
26
+ [watch-gumbel] 2026-05-25_19:07:19 no ckpt yet
27
+ [watch-gumbel] 2026-05-25_19:07:49 no ckpt yet
28
+ [watch-gumbel] 2026-05-25_19:08:19 no ckpt yet
29
+ [watch-gumbel] 2026-05-25_19:08:49 no ckpt yet
30
+ [watch-gumbel] 2026-05-25_19:09:19 no ckpt yet
31
+ [watch-gumbel] 2026-05-25_19:09:49 no ckpt yet
32
+ [watch-gumbel] 2026-05-25_19:10:19 no ckpt yet
33
+ [watch-gumbel] 2026-05-25_19:10:49 no ckpt yet
34
+ [watch-gumbel] 2026-05-25_19:11:19 no ckpt yet
35
+ [watch-gumbel] 2026-05-25_19:11:49 no ckpt yet
36
+ [watch-gumbel] 2026-05-25_19:12:19 no ckpt yet
37
+ [watch-gumbel] 2026-05-25_19:12:49 no ckpt yet
38
+ [watch-gumbel] 2026-05-25_19:13:19 no ckpt yet
39
+ [watch-gumbel] 2026-05-25_19:13:49 no ckpt yet
40
+ [watch-gumbel] 2026-05-25_19:14:19 no ckpt yet
41
+ [watch-gumbel] 2026-05-25_19:14:49 no ckpt yet
42
+ [watch-gumbel] 2026-05-25_19:15:19 no ckpt yet
43
+ [watch-gumbel] 2026-05-25_19:15:49 no ckpt yet
44
+ [watch-gumbel] 2026-05-25_19:16:19 no ckpt yet
45
+ [watch-gumbel] 2026-05-25_19:16:49 no ckpt yet
46
+ [watch-gumbel] 2026-05-25_19:17:19 no ckpt yet
47
+ [watch-gumbel] 2026-05-25_19:17:49 no ckpt yet
48
+ [watch-gumbel] 2026-05-25_19:18:19 no ckpt yet
49
+ [watch-gumbel] 2026-05-25_19:18:49 no ckpt yet
50
+ [watch-gumbel] 2026-05-25_19:19:19 no ckpt yet
51
+ [watch-gumbel] 2026-05-25_19:19:49 no ckpt yet
52
+ [watch-gumbel] 2026-05-25_19:20:19 no ckpt yet
53
+ [watch-gumbel] 2026-05-25_19:20:49 no ckpt yet
54
+ [watch-gumbel] 2026-05-25_19:21:19 no ckpt yet
55
+ [watch-gumbel] 2026-05-25_19:21:49 no ckpt yet
56
+ [watch-gumbel] 2026-05-25_19:22:19 no ckpt yet
57
+ [watch-gumbel] 2026-05-25_19:22:49 no ckpt yet
58
+ [watch-gumbel] 2026-05-25_19:23:19 no ckpt yet
59
+ [watch-gumbel] 2026-05-25_19:23:49 no ckpt yet
60
+ [watch-gumbel] 2026-05-25_19:24:19 no ckpt yet
61
+ [watch-gumbel] 2026-05-25_19:24:49 no ckpt yet
62
+ [watch-gumbel] 2026-05-25_19:25:19 no ckpt yet
63
+ [watch-gumbel] 2026-05-25_19:25:49 no ckpt yet
64
+ [watch-gumbel] 2026-05-25_19:26:19 no ckpt yet
65
+ [watch-gumbel] 2026-05-25_19:26:49 no ckpt yet
66
+ [watch-gumbel] 2026-05-25_19:27:19 no ckpt yet
67
+ [watch-gumbel] 2026-05-25_19:27:49 no ckpt yet
68
+ [watch-gumbel] 2026-05-25_19:28:19 no ckpt yet
69
+ [watch-gumbel] 2026-05-25_19:28:49 no ckpt yet
70
+ [watch-gumbel] 2026-05-25_19:29:19 no ckpt yet
71
+ [watch-gumbel] 2026-05-25_19:29:49 no ckpt yet
72
+ [watch-gumbel] 2026-05-25_19:30:19 no ckpt yet
73
+ [watch-gumbel] 2026-05-25_19:30:49 no ckpt yet
74
+ [watch-gumbel] 2026-05-25_19:31:19 no ckpt yet
75
+ [watch-gumbel] 2026-05-25_19:31:49 no ckpt yet
76
+ [watch-gumbel] 2026-05-25_19:32:19 no ckpt yet
77
+ [watch-gumbel] 2026-05-25_19:32:49 no ckpt yet
78
+ [watch-gumbel] 2026-05-25_19:33:19 no ckpt yet
79
+ [watch-gumbel] 2026-05-25_19:33:49 no ckpt yet
80
+ [watch-gumbel] 2026-05-25_19:34:19 no ckpt yet
81
+ [watch-gumbel] 2026-05-25_19:34:49 no ckpt yet
82
+ [watch-gumbel] 2026-05-25_19:35:19 no ckpt yet
83
+ [watch-gumbel] 2026-05-25_19:35:49 no ckpt yet
84
+ [watch-gumbel] 2026-05-25_19:36:19 no ckpt yet
85
+ [watch-gumbel] 2026-05-25_19:36:49 no ckpt yet
86
+ [watch-gumbel] 2026-05-25_19:37:19 no ckpt yet
87
+ [watch-gumbel] 2026-05-25_19:37:49 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0001000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0001000
88
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0001000.pt
89
+ [ckpt] step=1000
90
+ [sde] generated 2/128
91
+ [sde] generated 4/128
92
+ [sde] generated 6/128
93
+ [sde] generated 8/128
94
+ [sde] generated 10/128
95
+ [sde] generated 12/128
96
+ [sde] generated 14/128
97
+ [sde] generated 16/128
98
+ [sde] generated 18/128
99
+ [sde] generated 20/128
100
+ [sde] generated 22/128
101
+ [sde] generated 24/128
102
+ [sde] generated 26/128
103
+ [sde] generated 28/128
104
+ [sde] generated 30/128
105
+ [sde] generated 32/128
106
+ [sde] generated 34/128
107
+ [sde] generated 36/128
108
+ [sde] generated 38/128
109
+ [sde] generated 40/128
110
+ [sde] generated 42/128
111
+ [sde] generated 44/128
112
+ [sde] generated 46/128
113
+ [sde] generated 48/128
114
+ [sde] generated 50/128
115
+ [sde] generated 52/128
116
+ [sde] generated 54/128
117
+ [sde] generated 56/128
118
+ [sde] generated 58/128
119
+ [sde] generated 60/128
120
+ [sde] generated 62/128
121
+ [sde] generated 64/128
122
+ [sde] generated 66/128
123
+ [sde] generated 68/128
124
+ [sde] generated 70/128
125
+ [sde] generated 72/128
126
+ [sde] generated 74/128
127
+ [sde] generated 76/128
128
+ [sde] generated 78/128
129
+ [sde] generated 80/128
130
+ [sde] generated 82/128
131
+ [sde] generated 84/128
132
+ [sde] generated 86/128
133
+ [sde] generated 88/128
134
+ [sde] generated 90/128
135
+ [sde] generated 92/128
136
+ [sde] generated 94/128
137
+ [sde] generated 96/128
138
+ [sde] generated 98/128
139
+ [sde] generated 100/128
140
+ [sde] generated 102/128
141
+ [sde] generated 104/128
142
+ [sde] generated 106/128
143
+ [sde] generated 108/128
144
+ [sde] generated 110/128
145
+ [sde] generated 112/128
146
+ [sde] generated 114/128
147
+ [sde] generated 116/128
148
+ [sde] generated 118/128
149
+ [sde] generated 120/128
150
+ [sde] generated 122/128
151
+ [sde] generated 124/128
152
+ [sde] generated 126/128
153
+ [sde] generated 128/128
154
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
155
+ [summary] {
156
+ "type": "summary",
157
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0001000.pt",
158
+ "step": 1000,
159
+ "decode": {
160
+ "decode_rule": "dirichlet_resample_sde",
161
+ "steps": 128,
162
+ "model_t_mode": "support_t",
163
+ "mean_mode": "endpoint_only",
164
+ "anchor_gamma": 1.0,
165
+ "endpoint_floor": 0.0,
166
+ "concentration_min": 30522.0,
167
+ "concentration_max": 61044.0,
168
+ "endpoint_temp": 1.45,
169
+ "endpoint_temp_start": null,
170
+ "endpoint_temp_end": null,
171
+ "endpoint_projection": "gumbel_softmax",
172
+ "endpoint_top_k": 0,
173
+ "endpoint_top_p": 0.95,
174
+ "gumbel_tau_start": 1.0,
175
+ "gumbel_tau_end": 0.2,
176
+ "gumbel_noise_scale_start": 1.0,
177
+ "gumbel_noise_scale_end": 1.0,
178
+ "ban_special_tokens": false,
179
+ "banned_endpoint_ids": [],
180
+ "support_power": 1.0,
181
+ "semantic_power": 1.0,
182
+ "noise_init": "dirichlet",
183
+ "noise_sigma": -1.0,
184
+ "noise_dirichlet_concentration": 30522.0,
185
+ "sde_resample": "dirichlet",
186
+ "logistic_normal_sigma_min": 0.18,
187
+ "logistic_normal_sigma_max": 3.0,
188
+ "logistic_normal_tau_min": 0.65,
189
+ "logistic_normal_tau_max": 1.0,
190
+ "final_from": "blend_0.5",
191
+ "n_samples": 128,
192
+ "seed": 20260524
193
+ },
194
+ "raw_genppl": {
195
+ "ppl": 22.3410697800894,
196
+ "nll_per_token": 3.10642667880114,
197
+ "tokens": 126527,
198
+ "kept_samples": 128,
199
+ "total_samples": 128,
200
+ "empty_rate": 0.0,
201
+ "skipped_samples": 0
202
+ },
203
+ "stripped_genppl": {
204
+ "ppl": 22.040057312418394,
205
+ "nll_per_token": 3.0928615846632925,
206
+ "tokens": 125697,
207
+ "kept_samples": 128,
208
+ "total_samples": 128,
209
+ "empty_rate": 0.0,
210
+ "skipped_samples": 0
211
+ },
212
+ "diversity": {
213
+ "sample_entropy": 2.407557866090484,
214
+ "unique_tokens": 3579,
215
+ "token_count": 131072,
216
+ "distinct_1": 0.02730560302734375,
217
+ "distinct_2": 0.09034396383186706,
218
+ "top_token_mass": 0.2610321044921875
219
+ }
220
+ }
221
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0001000/sde_steps128_samples128_scored.jsonl
222
+ [watch-gumbel] 2026-05-25_19:49:09 done step_0001000
223
+ [watch-gumbel] 2026-05-25_20:24:39 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0002000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0002000
224
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0002000.pt
225
+ [ckpt] step=2000
226
+ [sde] generated 2/128
227
+ [sde] generated 4/128
228
+ [sde] generated 6/128
229
+ [sde] generated 8/128
230
+ [sde] generated 10/128
231
+ [sde] generated 12/128
232
+ [sde] generated 14/128
233
+ [sde] generated 16/128
234
+ [sde] generated 18/128
235
+ [sde] generated 20/128
236
+ [sde] generated 22/128
237
+ [sde] generated 24/128
238
+ [sde] generated 26/128
239
+ [sde] generated 28/128
240
+ [sde] generated 30/128
241
+ [sde] generated 32/128
242
+ [sde] generated 34/128
243
+ [sde] generated 36/128
244
+ [sde] generated 38/128
245
+ [sde] generated 40/128
246
+ [sde] generated 42/128
247
+ [sde] generated 44/128
248
+ [sde] generated 46/128
249
+ [sde] generated 48/128
250
+ [sde] generated 50/128
251
+ [sde] generated 52/128
252
+ [sde] generated 54/128
253
+ [sde] generated 56/128
254
+ [sde] generated 58/128
255
+ [sde] generated 60/128
256
+ [sde] generated 62/128
257
+ [sde] generated 64/128
258
+ [sde] generated 66/128
259
+ [sde] generated 68/128
260
+ [sde] generated 70/128
261
+ [sde] generated 72/128
262
+ [sde] generated 74/128
263
+ [sde] generated 76/128
264
+ [sde] generated 78/128
265
+ [sde] generated 80/128
266
+ [sde] generated 82/128
267
+ [sde] generated 84/128
268
+ [sde] generated 86/128
269
+ [sde] generated 88/128
270
+ [sde] generated 90/128
271
+ [sde] generated 92/128
272
+ [sde] generated 94/128
273
+ [sde] generated 96/128
274
+ [sde] generated 98/128
275
+ [sde] generated 100/128
276
+ [sde] generated 102/128
277
+ [sde] generated 104/128
278
+ [sde] generated 106/128
279
+ [sde] generated 108/128
280
+ [sde] generated 110/128
281
+ [sde] generated 112/128
282
+ [sde] generated 114/128
283
+ [sde] generated 116/128
284
+ [sde] generated 118/128
285
+ [sde] generated 120/128
286
+ [sde] generated 122/128
287
+ [sde] generated 124/128
288
+ [sde] generated 126/128
289
+ [sde] generated 128/128
290
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
291
+ [summary] {
292
+ "type": "summary",
293
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0002000.pt",
294
+ "step": 2000,
295
+ "decode": {
296
+ "decode_rule": "dirichlet_resample_sde",
297
+ "steps": 128,
298
+ "model_t_mode": "support_t",
299
+ "mean_mode": "endpoint_only",
300
+ "anchor_gamma": 1.0,
301
+ "endpoint_floor": 0.0,
302
+ "concentration_min": 30522.0,
303
+ "concentration_max": 61044.0,
304
+ "endpoint_temp": 1.45,
305
+ "endpoint_temp_start": null,
306
+ "endpoint_temp_end": null,
307
+ "endpoint_projection": "gumbel_softmax",
308
+ "endpoint_top_k": 0,
309
+ "endpoint_top_p": 0.95,
310
+ "gumbel_tau_start": 1.0,
311
+ "gumbel_tau_end": 0.2,
312
+ "gumbel_noise_scale_start": 1.0,
313
+ "gumbel_noise_scale_end": 1.0,
314
+ "ban_special_tokens": false,
315
+ "banned_endpoint_ids": [],
316
+ "support_power": 1.0,
317
+ "semantic_power": 1.0,
318
+ "noise_init": "dirichlet",
319
+ "noise_sigma": -1.0,
320
+ "noise_dirichlet_concentration": 30522.0,
321
+ "sde_resample": "dirichlet",
322
+ "logistic_normal_sigma_min": 0.18,
323
+ "logistic_normal_sigma_max": 3.0,
324
+ "logistic_normal_tau_min": 0.65,
325
+ "logistic_normal_tau_max": 1.0,
326
+ "final_from": "blend_0.5",
327
+ "n_samples": 128,
328
+ "seed": 20260524
329
+ },
330
+ "raw_genppl": {
331
+ "ppl": 1.0299565730229217,
332
+ "nll_per_token": 0.029516639239029304,
333
+ "tokens": 16258,
334
+ "kept_samples": 128,
335
+ "total_samples": 128,
336
+ "empty_rate": 0.0,
337
+ "skipped_samples": 0
338
+ },
339
+ "stripped_genppl": {
340
+ "ppl": 1.0299565730229217,
341
+ "nll_per_token": 0.029516639239029304,
342
+ "tokens": 16258,
343
+ "kept_samples": 128,
344
+ "total_samples": 128,
345
+ "empty_rate": 0.0,
346
+ "skipped_samples": 0
347
+ },
348
+ "diversity": {
349
+ "sample_entropy": 6.05086011148535e-05,
350
+ "unique_tokens": 2,
351
+ "token_count": 131072,
352
+ "distinct_1": 1.52587890625e-05,
353
+ "distinct_2": 2.2910557184750733e-05,
354
+ "top_token_mass": 0.9999923706054688
355
+ }
356
+ }
357
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0002000/sde_steps128_samples128_scored.jsonl
358
+ [watch-gumbel] 2026-05-25_20:35:40 done step_0002000
359
+ [watch-gumbel] 2026-05-25_21:11:10 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0003000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0003000
360
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0003000.pt
361
+ [ckpt] step=3000
362
+ [sde] generated 2/128
363
+ [sde] generated 4/128
364
+ [sde] generated 6/128
365
+ [sde] generated 8/128
366
+ [sde] generated 10/128
367
+ [sde] generated 12/128
368
+ [sde] generated 14/128
369
+ [sde] generated 16/128
370
+ [sde] generated 18/128
371
+ [sde] generated 20/128
372
+ [sde] generated 22/128
373
+ [sde] generated 24/128
374
+ [sde] generated 26/128
375
+ [sde] generated 28/128
376
+ [sde] generated 30/128
377
+ [sde] generated 32/128
378
+ [sde] generated 34/128
379
+ [sde] generated 36/128
380
+ [sde] generated 38/128
381
+ [sde] generated 40/128
382
+ [sde] generated 42/128
383
+ [sde] generated 44/128
384
+ [sde] generated 46/128
385
+ [sde] generated 48/128
386
+ [sde] generated 50/128
387
+ [sde] generated 52/128
388
+ [sde] generated 54/128
389
+ [sde] generated 56/128
390
+ [sde] generated 58/128
391
+ [sde] generated 60/128
392
+ [sde] generated 62/128
393
+ [sde] generated 64/128
394
+ [sde] generated 66/128
395
+ [sde] generated 68/128
396
+ [sde] generated 70/128
397
+ [sde] generated 72/128
398
+ [sde] generated 74/128
399
+ [sde] generated 76/128
400
+ [sde] generated 78/128
401
+ [sde] generated 80/128
402
+ [sde] generated 82/128
403
+ [sde] generated 84/128
404
+ [sde] generated 86/128
405
+ [sde] generated 88/128
406
+ [sde] generated 90/128
407
+ [sde] generated 92/128
408
+ [sde] generated 94/128
409
+ [sde] generated 96/128
410
+ [sde] generated 98/128
411
+ [sde] generated 100/128
412
+ [sde] generated 102/128
413
+ [sde] generated 104/128
414
+ [sde] generated 106/128
415
+ [sde] generated 108/128
416
+ [sde] generated 110/128
417
+ [sde] generated 112/128
418
+ [sde] generated 114/128
419
+ [sde] generated 116/128
420
+ [sde] generated 118/128
421
+ [sde] generated 120/128
422
+ [sde] generated 122/128
423
+ [sde] generated 124/128
424
+ [sde] generated 126/128
425
+ [sde] generated 128/128
426
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
427
+ [summary] {
428
+ "type": "summary",
429
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0003000.pt",
430
+ "step": 3000,
431
+ "decode": {
432
+ "decode_rule": "dirichlet_resample_sde",
433
+ "steps": 128,
434
+ "model_t_mode": "support_t",
435
+ "mean_mode": "endpoint_only",
436
+ "anchor_gamma": 1.0,
437
+ "endpoint_floor": 0.0,
438
+ "concentration_min": 30522.0,
439
+ "concentration_max": 61044.0,
440
+ "endpoint_temp": 1.45,
441
+ "endpoint_temp_start": null,
442
+ "endpoint_temp_end": null,
443
+ "endpoint_projection": "gumbel_softmax",
444
+ "endpoint_top_k": 0,
445
+ "endpoint_top_p": 0.95,
446
+ "gumbel_tau_start": 1.0,
447
+ "gumbel_tau_end": 0.2,
448
+ "gumbel_noise_scale_start": 1.0,
449
+ "gumbel_noise_scale_end": 1.0,
450
+ "ban_special_tokens": false,
451
+ "banned_endpoint_ids": [],
452
+ "support_power": 1.0,
453
+ "semantic_power": 1.0,
454
+ "noise_init": "dirichlet",
455
+ "noise_sigma": -1.0,
456
+ "noise_dirichlet_concentration": 30522.0,
457
+ "sde_resample": "dirichlet",
458
+ "logistic_normal_sigma_min": 0.18,
459
+ "logistic_normal_sigma_max": 3.0,
460
+ "logistic_normal_tau_min": 0.65,
461
+ "logistic_normal_tau_max": 1.0,
462
+ "final_from": "blend_0.5",
463
+ "n_samples": 128,
464
+ "seed": 20260524
465
+ },
466
+ "raw_genppl": {
467
+ "ppl": 3.8147677280337264,
468
+ "nll_per_token": 1.3388797790123395,
469
+ "tokens": 125410,
470
+ "kept_samples": 128,
471
+ "total_samples": 128,
472
+ "empty_rate": 0.0,
473
+ "skipped_samples": 0
474
+ },
475
+ "stripped_genppl": {
476
+ "ppl": 3.7623815682257042,
477
+ "nll_per_token": 1.3250521526960528,
478
+ "tokens": 124984,
479
+ "kept_samples": 128,
480
+ "total_samples": 128,
481
+ "empty_rate": 0.0,
482
+ "skipped_samples": 0
483
+ },
484
+ "diversity": {
485
+ "sample_entropy": 1.1669265594308051,
486
+ "unique_tokens": 557,
487
+ "token_count": 131072,
488
+ "distinct_1": 0.00424957275390625,
489
+ "distinct_2": 0.02731702101661779,
490
+ "top_token_mass": 0.5301742553710938
491
+ }
492
+ }
493
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0003000/sde_steps128_samples128_scored.jsonl
494
+ [watch-gumbel] 2026-05-25_21:22:30 done step_0003000
495
+ [watch-gumbel] 2026-05-25_21:57:31 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0004000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0004000
496
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0004000.pt
497
+ [ckpt] step=4000
498
+ [sde] generated 2/128
499
+ [sde] generated 4/128
500
+ [sde] generated 6/128
501
+ [sde] generated 8/128
502
+ [sde] generated 10/128
503
+ [sde] generated 12/128
504
+ [sde] generated 14/128
505
+ [sde] generated 16/128
506
+ [sde] generated 18/128
507
+ [sde] generated 20/128
508
+ [sde] generated 22/128
509
+ [sde] generated 24/128
510
+ [sde] generated 26/128
511
+ [sde] generated 28/128
512
+ [sde] generated 30/128
513
+ [sde] generated 32/128
514
+ [sde] generated 34/128
515
+ [sde] generated 36/128
516
+ [sde] generated 38/128
517
+ [sde] generated 40/128
518
+ [sde] generated 42/128
519
+ [sde] generated 44/128
520
+ [sde] generated 46/128
521
+ [sde] generated 48/128
522
+ [sde] generated 50/128
523
+ [sde] generated 52/128
524
+ [sde] generated 54/128
525
+ [sde] generated 56/128
526
+ [sde] generated 58/128
527
+ [sde] generated 60/128
528
+ [sde] generated 62/128
529
+ [sde] generated 64/128
530
+ [sde] generated 66/128
531
+ [sde] generated 68/128
532
+ [sde] generated 70/128
533
+ [sde] generated 72/128
534
+ [sde] generated 74/128
535
+ [sde] generated 76/128
536
+ [sde] generated 78/128
537
+ [sde] generated 80/128
538
+ [sde] generated 82/128
539
+ [sde] generated 84/128
540
+ [sde] generated 86/128
541
+ [sde] generated 88/128
542
+ [sde] generated 90/128
543
+ [sde] generated 92/128
544
+ [sde] generated 94/128
545
+ [sde] generated 96/128
546
+ [sde] generated 98/128
547
+ [sde] generated 100/128
548
+ [sde] generated 102/128
549
+ [sde] generated 104/128
550
+ [sde] generated 106/128
551
+ [sde] generated 108/128
552
+ [sde] generated 110/128
553
+ [sde] generated 112/128
554
+ [sde] generated 114/128
555
+ [sde] generated 116/128
556
+ [sde] generated 118/128
557
+ [sde] generated 120/128
558
+ [sde] generated 122/128
559
+ [sde] generated 124/128
560
+ [sde] generated 126/128
561
+ [sde] generated 128/128
562
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
563
+ [summary] {
564
+ "type": "summary",
565
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0004000.pt",
566
+ "step": 4000,
567
+ "decode": {
568
+ "decode_rule": "dirichlet_resample_sde",
569
+ "steps": 128,
570
+ "model_t_mode": "support_t",
571
+ "mean_mode": "endpoint_only",
572
+ "anchor_gamma": 1.0,
573
+ "endpoint_floor": 0.0,
574
+ "concentration_min": 30522.0,
575
+ "concentration_max": 61044.0,
576
+ "endpoint_temp": 1.45,
577
+ "endpoint_temp_start": null,
578
+ "endpoint_temp_end": null,
579
+ "endpoint_projection": "gumbel_softmax",
580
+ "endpoint_top_k": 0,
581
+ "endpoint_top_p": 0.95,
582
+ "gumbel_tau_start": 1.0,
583
+ "gumbel_tau_end": 0.2,
584
+ "gumbel_noise_scale_start": 1.0,
585
+ "gumbel_noise_scale_end": 1.0,
586
+ "ban_special_tokens": false,
587
+ "banned_endpoint_ids": [],
588
+ "support_power": 1.0,
589
+ "semantic_power": 1.0,
590
+ "noise_init": "dirichlet",
591
+ "noise_sigma": -1.0,
592
+ "noise_dirichlet_concentration": 30522.0,
593
+ "sde_resample": "dirichlet",
594
+ "logistic_normal_sigma_min": 0.18,
595
+ "logistic_normal_sigma_max": 3.0,
596
+ "logistic_normal_tau_min": 0.65,
597
+ "logistic_normal_tau_max": 1.0,
598
+ "final_from": "blend_0.5",
599
+ "n_samples": 128,
600
+ "seed": 20260524
601
+ },
602
+ "raw_genppl": {
603
+ "ppl": 4.962894031005841,
604
+ "nll_per_token": 1.601989044575183,
605
+ "tokens": 130463,
606
+ "kept_samples": 128,
607
+ "total_samples": 128,
608
+ "empty_rate": 0.0,
609
+ "skipped_samples": 0
610
+ },
611
+ "stripped_genppl": {
612
+ "ppl": 4.902226614272038,
613
+ "nll_per_token": 1.5896895129790605,
614
+ "tokens": 130277,
615
+ "kept_samples": 128,
616
+ "total_samples": 128,
617
+ "empty_rate": 0.0,
618
+ "skipped_samples": 0
619
+ },
620
+ "diversity": {
621
+ "sample_entropy": 1.7944221879574278,
622
+ "unique_tokens": 973,
623
+ "token_count": 131072,
624
+ "distinct_1": 0.00742340087890625,
625
+ "distinct_2": 0.05484023704789834,
626
+ "top_token_mass": 0.2004547119140625
627
+ }
628
+ }
629
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0004000/sde_steps128_samples128_scored.jsonl
630
+ [watch-gumbel] 2026-05-25_22:08:52 done step_0004000
631
+ [watch-gumbel] 2026-05-25_23:21:24 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0005000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0005000
632
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0005000.pt
633
+ [ckpt] step=5000
634
+ [sde] generated 2/128
635
+ [sde] generated 4/128
636
+ [sde] generated 6/128
637
+ [sde] generated 8/128
638
+ [sde] generated 10/128
639
+ [sde] generated 12/128
640
+ [sde] generated 14/128
641
+ [sde] generated 16/128
642
+ [sde] generated 18/128
643
+ [sde] generated 20/128
644
+ [sde] generated 22/128
645
+ [sde] generated 24/128
646
+ [sde] generated 26/128
647
+ [sde] generated 28/128
648
+ [sde] generated 30/128
649
+ [sde] generated 32/128
650
+ [sde] generated 34/128
651
+ [sde] generated 36/128
652
+ [sde] generated 38/128
653
+ [sde] generated 40/128
654
+ [sde] generated 42/128
655
+ [sde] generated 44/128
656
+ [sde] generated 46/128
657
+ [sde] generated 48/128
658
+ [sde] generated 50/128
659
+ [sde] generated 52/128
660
+ [sde] generated 54/128
661
+ [sde] generated 56/128
662
+ [sde] generated 58/128
663
+ [sde] generated 60/128
664
+ [sde] generated 62/128
665
+ [sde] generated 64/128
666
+ [sde] generated 66/128
667
+ [sde] generated 68/128
668
+ [sde] generated 70/128
669
+ [sde] generated 72/128
670
+ [sde] generated 74/128
671
+ [sde] generated 76/128
672
+ [sde] generated 78/128
673
+ [sde] generated 80/128
674
+ [sde] generated 82/128
675
+ [sde] generated 84/128
676
+ [sde] generated 86/128
677
+ [sde] generated 88/128
678
+ [sde] generated 90/128
679
+ [sde] generated 92/128
680
+ [sde] generated 94/128
681
+ [sde] generated 96/128
682
+ [sde] generated 98/128
683
+ [sde] generated 100/128
684
+ [sde] generated 102/128
685
+ [sde] generated 104/128
686
+ [sde] generated 106/128
687
+ [sde] generated 108/128
688
+ [sde] generated 110/128
689
+ [sde] generated 112/128
690
+ [sde] generated 114/128
691
+ [sde] generated 116/128
692
+ [sde] generated 118/128
693
+ [sde] generated 120/128
694
+ [sde] generated 122/128
695
+ [sde] generated 124/128
696
+ [sde] generated 126/128
697
+ [sde] generated 128/128
698
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
699
+ [summary] {
700
+ "type": "summary",
701
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0005000.pt",
702
+ "step": 5000,
703
+ "decode": {
704
+ "decode_rule": "dirichlet_resample_sde",
705
+ "steps": 128,
706
+ "model_t_mode": "support_t",
707
+ "mean_mode": "endpoint_only",
708
+ "anchor_gamma": 1.0,
709
+ "endpoint_floor": 0.0,
710
+ "concentration_min": 30522.0,
711
+ "concentration_max": 61044.0,
712
+ "endpoint_temp": 1.45,
713
+ "endpoint_temp_start": null,
714
+ "endpoint_temp_end": null,
715
+ "endpoint_projection": "gumbel_softmax",
716
+ "endpoint_top_k": 0,
717
+ "endpoint_top_p": 0.95,
718
+ "gumbel_tau_start": 1.0,
719
+ "gumbel_tau_end": 0.2,
720
+ "gumbel_noise_scale_start": 1.0,
721
+ "gumbel_noise_scale_end": 1.0,
722
+ "ban_special_tokens": false,
723
+ "banned_endpoint_ids": [],
724
+ "support_power": 1.0,
725
+ "semantic_power": 1.0,
726
+ "noise_init": "dirichlet",
727
+ "noise_sigma": -1.0,
728
+ "noise_dirichlet_concentration": 30522.0,
729
+ "sde_resample": "dirichlet",
730
+ "logistic_normal_sigma_min": 0.18,
731
+ "logistic_normal_sigma_max": 3.0,
732
+ "logistic_normal_tau_min": 0.65,
733
+ "logistic_normal_tau_max": 1.0,
734
+ "final_from": "blend_0.5",
735
+ "n_samples": 128,
736
+ "seed": 20260524
737
+ },
738
+ "raw_genppl": {
739
+ "ppl": 5.962372484300887,
740
+ "nll_per_token": 1.785468469704996,
741
+ "tokens": 126709,
742
+ "kept_samples": 128,
743
+ "total_samples": 128,
744
+ "empty_rate": 0.0,
745
+ "skipped_samples": 0
746
+ },
747
+ "stripped_genppl": {
748
+ "ppl": 5.833573527331135,
749
+ "nll_per_token": 1.7636297675275545,
750
+ "tokens": 126134,
751
+ "kept_samples": 128,
752
+ "total_samples": 128,
753
+ "empty_rate": 0.0,
754
+ "skipped_samples": 0
755
+ },
756
+ "diversity": {
757
+ "sample_entropy": 1.9941871274167644,
758
+ "unique_tokens": 982,
759
+ "token_count": 131072,
760
+ "distinct_1": 0.0074920654296875,
761
+ "distinct_2": 0.051121089931573806,
762
+ "top_token_mass": 0.2200775146484375
763
+ }
764
+ }
765
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0005000/sde_steps128_samples128_scored.jsonl
766
+ [watch-gumbel] 2026-05-25_23:40:12 done step_0005000
767
+ [watch-gumbel] 2026-05-26_01:14:44 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0006000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0006000
768
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0006000.pt
769
+ [ckpt] step=6000
770
+ [sde] generated 2/128
771
+ [sde] generated 4/128
772
+ [sde] generated 6/128
773
+ [sde] generated 8/128
774
+ [sde] generated 10/128
775
+ [sde] generated 12/128
776
+ [sde] generated 14/128
777
+ [sde] generated 16/128
778
+ [sde] generated 18/128
779
+ [sde] generated 20/128
780
+ [sde] generated 22/128
781
+ [sde] generated 24/128
782
+ [sde] generated 26/128
783
+ [sde] generated 28/128
784
+ [sde] generated 30/128
785
+ [sde] generated 32/128
786
+ [sde] generated 34/128
787
+ [sde] generated 36/128
788
+ [sde] generated 38/128
789
+ [sde] generated 40/128
790
+ [sde] generated 42/128
791
+ [sde] generated 44/128
792
+ [sde] generated 46/128
793
+ [sde] generated 48/128
794
+ [sde] generated 50/128
795
+ [sde] generated 52/128
796
+ [sde] generated 54/128
797
+ [sde] generated 56/128
798
+ [sde] generated 58/128
799
+ [sde] generated 60/128
800
+ [sde] generated 62/128
801
+ [sde] generated 64/128
802
+ [sde] generated 66/128
803
+ [sde] generated 68/128
804
+ [sde] generated 70/128
805
+ [sde] generated 72/128
806
+ [sde] generated 74/128
807
+ [sde] generated 76/128
808
+ [sde] generated 78/128
809
+ [sde] generated 80/128
810
+ [sde] generated 82/128
811
+ [sde] generated 84/128
812
+ [sde] generated 86/128
813
+ [sde] generated 88/128
814
+ [sde] generated 90/128
815
+ [sde] generated 92/128
816
+ [sde] generated 94/128
817
+ [sde] generated 96/128
818
+ [sde] generated 98/128
819
+ [sde] generated 100/128
820
+ [sde] generated 102/128
821
+ [sde] generated 104/128
822
+ [sde] generated 106/128
823
+ [sde] generated 108/128
824
+ [sde] generated 110/128
825
+ [sde] generated 112/128
826
+ [sde] generated 114/128
827
+ [sde] generated 116/128
828
+ [sde] generated 118/128
829
+ [sde] generated 120/128
830
+ [sde] generated 122/128
831
+ [sde] generated 124/128
832
+ [sde] generated 126/128
833
+ [sde] generated 128/128
834
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
835
+ [summary] {
836
+ "type": "summary",
837
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0006000.pt",
838
+ "step": 6000,
839
+ "decode": {
840
+ "decode_rule": "dirichlet_resample_sde",
841
+ "steps": 128,
842
+ "model_t_mode": "support_t",
843
+ "mean_mode": "endpoint_only",
844
+ "anchor_gamma": 1.0,
845
+ "endpoint_floor": 0.0,
846
+ "concentration_min": 30522.0,
847
+ "concentration_max": 61044.0,
848
+ "endpoint_temp": 1.45,
849
+ "endpoint_temp_start": null,
850
+ "endpoint_temp_end": null,
851
+ "endpoint_projection": "gumbel_softmax",
852
+ "endpoint_top_k": 0,
853
+ "endpoint_top_p": 0.95,
854
+ "gumbel_tau_start": 1.0,
855
+ "gumbel_tau_end": 0.2,
856
+ "gumbel_noise_scale_start": 1.0,
857
+ "gumbel_noise_scale_end": 1.0,
858
+ "ban_special_tokens": false,
859
+ "banned_endpoint_ids": [],
860
+ "support_power": 1.0,
861
+ "semantic_power": 1.0,
862
+ "noise_init": "dirichlet",
863
+ "noise_sigma": -1.0,
864
+ "noise_dirichlet_concentration": 30522.0,
865
+ "sde_resample": "dirichlet",
866
+ "logistic_normal_sigma_min": 0.18,
867
+ "logistic_normal_sigma_max": 3.0,
868
+ "logistic_normal_tau_min": 0.65,
869
+ "logistic_normal_tau_max": 1.0,
870
+ "final_from": "blend_0.5",
871
+ "n_samples": 128,
872
+ "seed": 20260524
873
+ },
874
+ "raw_genppl": {
875
+ "ppl": 2.0555040043182493,
876
+ "nll_per_token": 0.7205210754532657,
877
+ "tokens": 128145,
878
+ "kept_samples": 128,
879
+ "total_samples": 128,
880
+ "empty_rate": 0.0,
881
+ "skipped_samples": 0
882
+ },
883
+ "stripped_genppl": {
884
+ "ppl": 2.0238469957582215,
885
+ "nll_per_token": 0.7050001535866586,
886
+ "tokens": 127950,
887
+ "kept_samples": 128,
888
+ "total_samples": 128,
889
+ "empty_rate": 0.0,
890
+ "skipped_samples": 0
891
+ },
892
+ "diversity": {
893
+ "sample_entropy": 1.2342076949480105,
894
+ "unique_tokens": 923,
895
+ "token_count": 131072,
896
+ "distinct_1": 0.00704193115234375,
897
+ "distinct_2": 0.037894061583577714,
898
+ "top_token_mass": 0.26293182373046875
899
+ }
900
+ }
901
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0006000/sde_steps128_samples128_scored.jsonl
902
+ [watch-gumbel] 2026-05-26_01:33:28 done step_0006000
903
+ [watch-gumbel] 2026-05-26_03:08:30 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0007000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0007000
904
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0007000.pt
905
+ [ckpt] step=7000
906
+ [sde] generated 2/128
907
+ [sde] generated 4/128
908
+ [sde] generated 6/128
909
+ [sde] generated 8/128
910
+ [sde] generated 10/128
911
+ [sde] generated 12/128
912
+ [sde] generated 14/128
913
+ [sde] generated 16/128
914
+ [sde] generated 18/128
915
+ [sde] generated 20/128
916
+ [sde] generated 22/128
917
+ [sde] generated 24/128
918
+ [sde] generated 26/128
919
+ [sde] generated 28/128
920
+ [sde] generated 30/128
921
+ [sde] generated 32/128
922
+ [sde] generated 34/128
923
+ [sde] generated 36/128
924
+ [sde] generated 38/128
925
+ [sde] generated 40/128
926
+ [sde] generated 42/128
927
+ [sde] generated 44/128
928
+ [sde] generated 46/128
929
+ [sde] generated 48/128
930
+ [sde] generated 50/128
931
+ [sde] generated 52/128
932
+ [sde] generated 54/128
933
+ [sde] generated 56/128
934
+ [sde] generated 58/128
935
+ [sde] generated 60/128
936
+ [sde] generated 62/128
937
+ [sde] generated 64/128
938
+ [sde] generated 66/128
939
+ [sde] generated 68/128
940
+ [sde] generated 70/128
941
+ [sde] generated 72/128
942
+ [sde] generated 74/128
943
+ [sde] generated 76/128
944
+ [sde] generated 78/128
945
+ [sde] generated 80/128
946
+ [sde] generated 82/128
947
+ [sde] generated 84/128
948
+ [sde] generated 86/128
949
+ [sde] generated 88/128
950
+ [sde] generated 90/128
951
+ [sde] generated 92/128
952
+ [sde] generated 94/128
953
+ [sde] generated 96/128
954
+ [sde] generated 98/128
955
+ [sde] generated 100/128
956
+ [sde] generated 102/128
957
+ [sde] generated 104/128
958
+ [sde] generated 106/128
959
+ [sde] generated 108/128
960
+ [sde] generated 110/128
961
+ [sde] generated 112/128
962
+ [sde] generated 114/128
963
+ [sde] generated 116/128
964
+ [sde] generated 118/128
965
+ [sde] generated 120/128
966
+ [sde] generated 122/128
967
+ [sde] generated 124/128
968
+ [sde] generated 126/128
969
+ [sde] generated 128/128
970
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
971
+ [summary] {
972
+ "type": "summary",
973
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0007000.pt",
974
+ "step": 7000,
975
+ "decode": {
976
+ "decode_rule": "dirichlet_resample_sde",
977
+ "steps": 128,
978
+ "model_t_mode": "support_t",
979
+ "mean_mode": "endpoint_only",
980
+ "anchor_gamma": 1.0,
981
+ "endpoint_floor": 0.0,
982
+ "concentration_min": 30522.0,
983
+ "concentration_max": 61044.0,
984
+ "endpoint_temp": 1.45,
985
+ "endpoint_temp_start": null,
986
+ "endpoint_temp_end": null,
987
+ "endpoint_projection": "gumbel_softmax",
988
+ "endpoint_top_k": 0,
989
+ "endpoint_top_p": 0.95,
990
+ "gumbel_tau_start": 1.0,
991
+ "gumbel_tau_end": 0.2,
992
+ "gumbel_noise_scale_start": 1.0,
993
+ "gumbel_noise_scale_end": 1.0,
994
+ "ban_special_tokens": false,
995
+ "banned_endpoint_ids": [],
996
+ "support_power": 1.0,
997
+ "semantic_power": 1.0,
998
+ "noise_init": "dirichlet",
999
+ "noise_sigma": -1.0,
1000
+ "noise_dirichlet_concentration": 30522.0,
1001
+ "sde_resample": "dirichlet",
1002
+ "logistic_normal_sigma_min": 0.18,
1003
+ "logistic_normal_sigma_max": 3.0,
1004
+ "logistic_normal_tau_min": 0.65,
1005
+ "logistic_normal_tau_max": 1.0,
1006
+ "final_from": "blend_0.5",
1007
+ "n_samples": 128,
1008
+ "seed": 20260524
1009
+ },
1010
+ "raw_genppl": {
1011
+ "ppl": 5.688809754755273,
1012
+ "nll_per_token": 1.738501044315569,
1013
+ "tokens": 130803,
1014
+ "kept_samples": 128,
1015
+ "total_samples": 128,
1016
+ "empty_rate": 0.0,
1017
+ "skipped_samples": 0
1018
+ },
1019
+ "stripped_genppl": {
1020
+ "ppl": 5.361685487735275,
1021
+ "nll_per_token": 1.6792783823186042,
1022
+ "tokens": 130626,
1023
+ "kept_samples": 128,
1024
+ "total_samples": 128,
1025
+ "empty_rate": 0.0,
1026
+ "skipped_samples": 0
1027
+ },
1028
+ "diversity": {
1029
+ "sample_entropy": 1.5534247280182494,
1030
+ "unique_tokens": 548,
1031
+ "token_count": 131072,
1032
+ "distinct_1": 0.004180908203125,
1033
+ "distinct_2": 0.029951735092864126,
1034
+ "top_token_mass": 0.396453857421875
1035
+ }
1036
+ }
1037
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0007000/sde_steps128_samples128_scored.jsonl
1038
+ [watch-gumbel] 2026-05-26_03:27:41 done step_0007000
1039
+ [watch-gumbel] 2026-05-26_05:01:44 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0008000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0008000
1040
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0008000.pt
1041
+ [ckpt] step=8000
1042
+ [sde] generated 2/128
1043
+ [sde] generated 4/128
1044
+ [sde] generated 6/128
1045
+ [sde] generated 8/128
1046
+ [sde] generated 10/128
1047
+ [sde] generated 12/128
1048
+ [sde] generated 14/128
1049
+ [sde] generated 16/128
1050
+ [sde] generated 18/128
1051
+ [sde] generated 20/128
1052
+ [sde] generated 22/128
1053
+ [sde] generated 24/128
1054
+ [sde] generated 26/128
1055
+ [sde] generated 28/128
1056
+ [sde] generated 30/128
1057
+ [sde] generated 32/128
1058
+ [sde] generated 34/128
1059
+ [sde] generated 36/128
1060
+ [sde] generated 38/128
1061
+ [sde] generated 40/128
1062
+ [sde] generated 42/128
1063
+ [sde] generated 44/128
1064
+ [sde] generated 46/128
1065
+ [sde] generated 48/128
1066
+ [sde] generated 50/128
1067
+ [sde] generated 52/128
1068
+ [sde] generated 54/128
1069
+ [sde] generated 56/128
1070
+ [sde] generated 58/128
1071
+ [sde] generated 60/128
1072
+ [sde] generated 62/128
1073
+ [sde] generated 64/128
1074
+ [sde] generated 66/128
1075
+ [sde] generated 68/128
1076
+ [sde] generated 70/128
1077
+ [sde] generated 72/128
1078
+ [sde] generated 74/128
1079
+ [sde] generated 76/128
1080
+ [sde] generated 78/128
1081
+ [sde] generated 80/128
1082
+ [sde] generated 82/128
1083
+ [sde] generated 84/128
1084
+ [sde] generated 86/128
1085
+ [sde] generated 88/128
1086
+ [sde] generated 90/128
1087
+ [sde] generated 92/128
1088
+ [sde] generated 94/128
1089
+ [sde] generated 96/128
1090
+ [sde] generated 98/128
1091
+ [sde] generated 100/128
1092
+ [sde] generated 102/128
1093
+ [sde] generated 104/128
1094
+ [sde] generated 106/128
1095
+ [sde] generated 108/128
1096
+ [sde] generated 110/128
1097
+ [sde] generated 112/128
1098
+ [sde] generated 114/128
1099
+ [sde] generated 116/128
1100
+ [sde] generated 118/128
1101
+ [sde] generated 120/128
1102
+ [sde] generated 122/128
1103
+ [sde] generated 124/128
1104
+ [sde] generated 126/128
1105
+ [sde] generated 128/128
1106
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
1107
+ [summary] {
1108
+ "type": "summary",
1109
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0008000.pt",
1110
+ "step": 8000,
1111
+ "decode": {
1112
+ "decode_rule": "dirichlet_resample_sde",
1113
+ "steps": 128,
1114
+ "model_t_mode": "support_t",
1115
+ "mean_mode": "endpoint_only",
1116
+ "anchor_gamma": 1.0,
1117
+ "endpoint_floor": 0.0,
1118
+ "concentration_min": 30522.0,
1119
+ "concentration_max": 61044.0,
1120
+ "endpoint_temp": 1.45,
1121
+ "endpoint_temp_start": null,
1122
+ "endpoint_temp_end": null,
1123
+ "endpoint_projection": "gumbel_softmax",
1124
+ "endpoint_top_k": 0,
1125
+ "endpoint_top_p": 0.95,
1126
+ "gumbel_tau_start": 1.0,
1127
+ "gumbel_tau_end": 0.2,
1128
+ "gumbel_noise_scale_start": 1.0,
1129
+ "gumbel_noise_scale_end": 1.0,
1130
+ "ban_special_tokens": false,
1131
+ "banned_endpoint_ids": [],
1132
+ "support_power": 1.0,
1133
+ "semantic_power": 1.0,
1134
+ "noise_init": "dirichlet",
1135
+ "noise_sigma": -1.0,
1136
+ "noise_dirichlet_concentration": 30522.0,
1137
+ "sde_resample": "dirichlet",
1138
+ "logistic_normal_sigma_min": 0.18,
1139
+ "logistic_normal_sigma_max": 3.0,
1140
+ "logistic_normal_tau_min": 0.65,
1141
+ "logistic_normal_tau_max": 1.0,
1142
+ "final_from": "blend_0.5",
1143
+ "n_samples": 128,
1144
+ "seed": 20260524
1145
+ },
1146
+ "raw_genppl": {
1147
+ "ppl": 1.9376819357617616,
1148
+ "nll_per_token": 0.6614923801654685,
1149
+ "tokens": 123676,
1150
+ "kept_samples": 128,
1151
+ "total_samples": 128,
1152
+ "empty_rate": 0.0,
1153
+ "skipped_samples": 0
1154
+ },
1155
+ "stripped_genppl": {
1156
+ "ppl": 1.9049121999831735,
1157
+ "nll_per_token": 0.6444359182688443,
1158
+ "tokens": 123415,
1159
+ "kept_samples": 128,
1160
+ "total_samples": 128,
1161
+ "empty_rate": 0.0,
1162
+ "skipped_samples": 0
1163
+ },
1164
+ "diversity": {
1165
+ "sample_entropy": 0.865856147139675,
1166
+ "unique_tokens": 613,
1167
+ "token_count": 131072,
1168
+ "distinct_1": 0.00467681884765625,
1169
+ "distinct_2": 0.027240652492668622,
1170
+ "top_token_mass": 0.568145751953125
1171
+ }
1172
+ }
1173
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0008000/sde_steps128_samples128_scored.jsonl
1174
+ [watch-gumbel] 2026-05-26_05:20:27 done step_0008000
1175
+ [watch-gumbel] 2026-05-26_06:55:00 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0009000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0009000
1176
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0009000.pt
1177
+ [ckpt] step=9000
1178
+ [sde] generated 2/128
1179
+ [sde] generated 4/128
1180
+ [sde] generated 6/128
1181
+ [sde] generated 8/128
1182
+ [sde] generated 10/128
1183
+ [sde] generated 12/128
1184
+ [sde] generated 14/128
1185
+ [sde] generated 16/128
1186
+ [sde] generated 18/128
1187
+ [sde] generated 20/128
1188
+ [sde] generated 22/128
1189
+ [sde] generated 24/128
1190
+ [sde] generated 26/128
1191
+ [sde] generated 28/128
1192
+ [sde] generated 30/128
1193
+ [sde] generated 32/128
1194
+ [sde] generated 34/128
1195
+ [sde] generated 36/128
1196
+ [sde] generated 38/128
1197
+ [sde] generated 40/128
1198
+ [sde] generated 42/128
1199
+ [sde] generated 44/128
1200
+ [sde] generated 46/128
1201
+ [sde] generated 48/128
1202
+ [sde] generated 50/128
1203
+ [sde] generated 52/128
1204
+ [sde] generated 54/128
1205
+ [sde] generated 56/128
1206
+ [sde] generated 58/128
1207
+ [sde] generated 60/128
1208
+ [sde] generated 62/128
1209
+ [sde] generated 64/128
1210
+ [sde] generated 66/128
1211
+ [sde] generated 68/128
1212
+ [sde] generated 70/128
1213
+ [sde] generated 72/128
1214
+ [sde] generated 74/128
1215
+ [sde] generated 76/128
1216
+ [sde] generated 78/128
1217
+ [sde] generated 80/128
1218
+ [sde] generated 82/128
1219
+ [sde] generated 84/128
1220
+ [sde] generated 86/128
1221
+ [sde] generated 88/128
1222
+ [sde] generated 90/128
1223
+ [sde] generated 92/128
1224
+ [sde] generated 94/128
1225
+ [sde] generated 96/128
1226
+ [sde] generated 98/128
1227
+ [sde] generated 100/128
1228
+ [sde] generated 102/128
1229
+ [sde] generated 104/128
1230
+ [sde] generated 106/128
1231
+ [sde] generated 108/128
1232
+ [sde] generated 110/128
1233
+ [sde] generated 112/128
1234
+ [sde] generated 114/128
1235
+ [sde] generated 116/128
1236
+ [sde] generated 118/128
1237
+ [sde] generated 120/128
1238
+ [sde] generated 122/128
1239
+ [sde] generated 124/128
1240
+ [sde] generated 126/128
1241
+ [sde] generated 128/128
1242
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
1243
+ [summary] {
1244
+ "type": "summary",
1245
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0009000.pt",
1246
+ "step": 9000,
1247
+ "decode": {
1248
+ "decode_rule": "dirichlet_resample_sde",
1249
+ "steps": 128,
1250
+ "model_t_mode": "support_t",
1251
+ "mean_mode": "endpoint_only",
1252
+ "anchor_gamma": 1.0,
1253
+ "endpoint_floor": 0.0,
1254
+ "concentration_min": 30522.0,
1255
+ "concentration_max": 61044.0,
1256
+ "endpoint_temp": 1.45,
1257
+ "endpoint_temp_start": null,
1258
+ "endpoint_temp_end": null,
1259
+ "endpoint_projection": "gumbel_softmax",
1260
+ "endpoint_top_k": 0,
1261
+ "endpoint_top_p": 0.95,
1262
+ "gumbel_tau_start": 1.0,
1263
+ "gumbel_tau_end": 0.2,
1264
+ "gumbel_noise_scale_start": 1.0,
1265
+ "gumbel_noise_scale_end": 1.0,
1266
+ "ban_special_tokens": false,
1267
+ "banned_endpoint_ids": [],
1268
+ "support_power": 1.0,
1269
+ "semantic_power": 1.0,
1270
+ "noise_init": "dirichlet",
1271
+ "noise_sigma": -1.0,
1272
+ "noise_dirichlet_concentration": 30522.0,
1273
+ "sde_resample": "dirichlet",
1274
+ "logistic_normal_sigma_min": 0.18,
1275
+ "logistic_normal_sigma_max": 3.0,
1276
+ "logistic_normal_tau_min": 0.65,
1277
+ "logistic_normal_tau_max": 1.0,
1278
+ "final_from": "blend_0.5",
1279
+ "n_samples": 128,
1280
+ "seed": 20260524
1281
+ },
1282
+ "raw_genppl": {
1283
+ "ppl": 5.088105610903924,
1284
+ "nll_per_token": 1.6269055826824483,
1285
+ "tokens": 129820,
1286
+ "kept_samples": 128,
1287
+ "total_samples": 128,
1288
+ "empty_rate": 0.0,
1289
+ "skipped_samples": 0
1290
+ },
1291
+ "stripped_genppl": {
1292
+ "ppl": 5.014868702762704,
1293
+ "nll_per_token": 1.612407240166428,
1294
+ "tokens": 129623,
1295
+ "kept_samples": 128,
1296
+ "total_samples": 128,
1297
+ "empty_rate": 0.0,
1298
+ "skipped_samples": 0
1299
+ },
1300
+ "diversity": {
1301
+ "sample_entropy": 1.3782953161976421,
1302
+ "unique_tokens": 1229,
1303
+ "token_count": 131072,
1304
+ "distinct_1": 0.00937652587890625,
1305
+ "distinct_2": 0.0396963587487781,
1306
+ "top_token_mass": 0.366668701171875
1307
+ }
1308
+ }
1309
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0009000/sde_steps128_samples128_scored.jsonl
1310
+ [watch-gumbel] 2026-05-26_07:13:03 done step_0009000
1311
+ [watch-gumbel] 2026-05-26_08:47:07 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0010000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0010000
1312
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0010000.pt
1313
+ [ckpt] step=10000
1314
+ [sde] generated 2/128
1315
+ [sde] generated 4/128
1316
+ [sde] generated 6/128
1317
+ [sde] generated 8/128
1318
+ [sde] generated 10/128
1319
+ [sde] generated 12/128
1320
+ [sde] generated 14/128
1321
+ [sde] generated 16/128
1322
+ [sde] generated 18/128
1323
+ [sde] generated 20/128
1324
+ [sde] generated 22/128
1325
+ [sde] generated 24/128
1326
+ [sde] generated 26/128
1327
+ [sde] generated 28/128
1328
+ [sde] generated 30/128
1329
+ [sde] generated 32/128
1330
+ [sde] generated 34/128
1331
+ [sde] generated 36/128
1332
+ [sde] generated 38/128
1333
+ [sde] generated 40/128
1334
+ [sde] generated 42/128
1335
+ [sde] generated 44/128
1336
+ [sde] generated 46/128
1337
+ [sde] generated 48/128
1338
+ [sde] generated 50/128
1339
+ [sde] generated 52/128
1340
+ [sde] generated 54/128
1341
+ [sde] generated 56/128
1342
+ [sde] generated 58/128
1343
+ [sde] generated 60/128
1344
+ [sde] generated 62/128
1345
+ [sde] generated 64/128
1346
+ [sde] generated 66/128
1347
+ [sde] generated 68/128
1348
+ [sde] generated 70/128
1349
+ [sde] generated 72/128
1350
+ [sde] generated 74/128
1351
+ [sde] generated 76/128
1352
+ [sde] generated 78/128
1353
+ [sde] generated 80/128
1354
+ [sde] generated 82/128
1355
+ [sde] generated 84/128
1356
+ [sde] generated 86/128
1357
+ [sde] generated 88/128
1358
+ [sde] generated 90/128
1359
+ [sde] generated 92/128
1360
+ [sde] generated 94/128
1361
+ [sde] generated 96/128
1362
+ [sde] generated 98/128
1363
+ [sde] generated 100/128
1364
+ [sde] generated 102/128
1365
+ [sde] generated 104/128
1366
+ [sde] generated 106/128
1367
+ [sde] generated 108/128
1368
+ [sde] generated 110/128
1369
+ [sde] generated 112/128
1370
+ [sde] generated 114/128
1371
+ [sde] generated 116/128
1372
+ [sde] generated 118/128
1373
+ [sde] generated 120/128
1374
+ [sde] generated 122/128
1375
+ [sde] generated 124/128
1376
+ [sde] generated 126/128
1377
+ [sde] generated 128/128
1378
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
1379
+ [summary] {
1380
+ "type": "summary",
1381
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0010000.pt",
1382
+ "step": 10000,
1383
+ "decode": {
1384
+ "decode_rule": "dirichlet_resample_sde",
1385
+ "steps": 128,
1386
+ "model_t_mode": "support_t",
1387
+ "mean_mode": "endpoint_only",
1388
+ "anchor_gamma": 1.0,
1389
+ "endpoint_floor": 0.0,
1390
+ "concentration_min": 30522.0,
1391
+ "concentration_max": 61044.0,
1392
+ "endpoint_temp": 1.45,
1393
+ "endpoint_temp_start": null,
1394
+ "endpoint_temp_end": null,
1395
+ "endpoint_projection": "gumbel_softmax",
1396
+ "endpoint_top_k": 0,
1397
+ "endpoint_top_p": 0.95,
1398
+ "gumbel_tau_start": 1.0,
1399
+ "gumbel_tau_end": 0.2,
1400
+ "gumbel_noise_scale_start": 1.0,
1401
+ "gumbel_noise_scale_end": 1.0,
1402
+ "ban_special_tokens": false,
1403
+ "banned_endpoint_ids": [],
1404
+ "support_power": 1.0,
1405
+ "semantic_power": 1.0,
1406
+ "noise_init": "dirichlet",
1407
+ "noise_sigma": -1.0,
1408
+ "noise_dirichlet_concentration": 30522.0,
1409
+ "sde_resample": "dirichlet",
1410
+ "logistic_normal_sigma_min": 0.18,
1411
+ "logistic_normal_sigma_max": 3.0,
1412
+ "logistic_normal_tau_min": 0.65,
1413
+ "logistic_normal_tau_max": 1.0,
1414
+ "final_from": "blend_0.5",
1415
+ "n_samples": 128,
1416
+ "seed": 20260524
1417
+ },
1418
+ "raw_genppl": {
1419
+ "ppl": 7.382004975613158,
1420
+ "nll_per_token": 1.9990452786272435,
1421
+ "tokens": 108846,
1422
+ "kept_samples": 128,
1423
+ "total_samples": 128,
1424
+ "empty_rate": 0.0,
1425
+ "skipped_samples": 0
1426
+ },
1427
+ "stripped_genppl": {
1428
+ "ppl": 7.193271401079397,
1429
+ "nll_per_token": 1.9731460614509597,
1430
+ "tokens": 108120,
1431
+ "kept_samples": 128,
1432
+ "total_samples": 128,
1433
+ "empty_rate": 0.0,
1434
+ "skipped_samples": 0
1435
+ },
1436
+ "diversity": {
1437
+ "sample_entropy": 1.9640364761127493,
1438
+ "unique_tokens": 1041,
1439
+ "token_count": 131072,
1440
+ "distinct_1": 0.00794219970703125,
1441
+ "distinct_2": 0.05942234848484849,
1442
+ "top_token_mass": 0.31719970703125
1443
+ }
1444
+ }
1445
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0010000/sde_steps128_samples128_scored.jsonl
1446
+ [watch-gumbel] 2026-05-26_09:06:14 done step_0010000
1447
+ [watch-gumbel] 2026-05-26_10:41:18 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0011000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0011000
1448
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0011000.pt
1449
+ [ckpt] step=11000
1450
+ [sde] generated 2/128
1451
+ [sde] generated 4/128
1452
+ [sde] generated 6/128
1453
+ [sde] generated 8/128
1454
+ [sde] generated 10/128
1455
+ [sde] generated 12/128
1456
+ [sde] generated 14/128
1457
+ [sde] generated 16/128
1458
+ [sde] generated 18/128
1459
+ [sde] generated 20/128
1460
+ [sde] generated 22/128
1461
+ [sde] generated 24/128
1462
+ [sde] generated 26/128
1463
+ [sde] generated 28/128
1464
+ [sde] generated 30/128
1465
+ [sde] generated 32/128
1466
+ [sde] generated 34/128
1467
+ [sde] generated 36/128
1468
+ [sde] generated 38/128
1469
+ [sde] generated 40/128
1470
+ [sde] generated 42/128
1471
+ [sde] generated 44/128
1472
+ [sde] generated 46/128
1473
+ [sde] generated 48/128
1474
+ [sde] generated 50/128
1475
+ [sde] generated 52/128
1476
+ [sde] generated 54/128
1477
+ [sde] generated 56/128
1478
+ [sde] generated 58/128
1479
+ [sde] generated 60/128
1480
+ [sde] generated 62/128
1481
+ [sde] generated 64/128
1482
+ [sde] generated 66/128
1483
+ [sde] generated 68/128
1484
+ [sde] generated 70/128
1485
+ [sde] generated 72/128
1486
+ [sde] generated 74/128
1487
+ [sde] generated 76/128
1488
+ [sde] generated 78/128
1489
+ [sde] generated 80/128
1490
+ [sde] generated 82/128
1491
+ [sde] generated 84/128
1492
+ [sde] generated 86/128
1493
+ [sde] generated 88/128
1494
+ [sde] generated 90/128
1495
+ [sde] generated 92/128
1496
+ [sde] generated 94/128
1497
+ [sde] generated 96/128
1498
+ [sde] generated 98/128
1499
+ [sde] generated 100/128
1500
+ [sde] generated 102/128
1501
+ [sde] generated 104/128
1502
+ [sde] generated 106/128
1503
+ [sde] generated 108/128
1504
+ [sde] generated 110/128
1505
+ [sde] generated 112/128
1506
+ [sde] generated 114/128
1507
+ [sde] generated 116/128
1508
+ [sde] generated 118/128
1509
+ [sde] generated 120/128
1510
+ [sde] generated 122/128
1511
+ [sde] generated 124/128
1512
+ [sde] generated 126/128
1513
+ [sde] generated 128/128
1514
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
1515
+ [summary] {
1516
+ "type": "summary",
1517
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0011000.pt",
1518
+ "step": 11000,
1519
+ "decode": {
1520
+ "decode_rule": "dirichlet_resample_sde",
1521
+ "steps": 128,
1522
+ "model_t_mode": "support_t",
1523
+ "mean_mode": "endpoint_only",
1524
+ "anchor_gamma": 1.0,
1525
+ "endpoint_floor": 0.0,
1526
+ "concentration_min": 30522.0,
1527
+ "concentration_max": 61044.0,
1528
+ "endpoint_temp": 1.45,
1529
+ "endpoint_temp_start": null,
1530
+ "endpoint_temp_end": null,
1531
+ "endpoint_projection": "gumbel_softmax",
1532
+ "endpoint_top_k": 0,
1533
+ "endpoint_top_p": 0.95,
1534
+ "gumbel_tau_start": 1.0,
1535
+ "gumbel_tau_end": 0.2,
1536
+ "gumbel_noise_scale_start": 1.0,
1537
+ "gumbel_noise_scale_end": 1.0,
1538
+ "ban_special_tokens": false,
1539
+ "banned_endpoint_ids": [],
1540
+ "support_power": 1.0,
1541
+ "semantic_power": 1.0,
1542
+ "noise_init": "dirichlet",
1543
+ "noise_sigma": -1.0,
1544
+ "noise_dirichlet_concentration": 30522.0,
1545
+ "sde_resample": "dirichlet",
1546
+ "logistic_normal_sigma_min": 0.18,
1547
+ "logistic_normal_sigma_max": 3.0,
1548
+ "logistic_normal_tau_min": 0.65,
1549
+ "logistic_normal_tau_max": 1.0,
1550
+ "final_from": "blend_0.5",
1551
+ "n_samples": 128,
1552
+ "seed": 20260524
1553
+ },
1554
+ "raw_genppl": {
1555
+ "ppl": 2.1435701575313577,
1556
+ "nll_per_token": 0.7624727368824596,
1557
+ "tokens": 129410,
1558
+ "kept_samples": 128,
1559
+ "total_samples": 128,
1560
+ "empty_rate": 0.0,
1561
+ "skipped_samples": 0
1562
+ },
1563
+ "stripped_genppl": {
1564
+ "ppl": 2.1047372288167483,
1565
+ "nll_per_token": 0.7441906274261585,
1566
+ "tokens": 129173,
1567
+ "kept_samples": 128,
1568
+ "total_samples": 128,
1569
+ "empty_rate": 0.0,
1570
+ "skipped_samples": 0
1571
+ },
1572
+ "diversity": {
1573
+ "sample_entropy": 1.0596923862199497,
1574
+ "unique_tokens": 1295,
1575
+ "token_count": 131072,
1576
+ "distinct_1": 0.00988006591796875,
1577
+ "distinct_2": 0.05237353372434018,
1578
+ "top_token_mass": 0.6339111328125
1579
+ }
1580
+ }
1581
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0011000/sde_steps128_samples128_scored.jsonl
1582
+ [watch-gumbel] 2026-05-26_11:01:12 done step_0011000
1583
+ [watch-gumbel] 2026-05-26_12:52:47 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0012000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0012000
1584
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0012000.pt
1585
+ [ckpt] step=12000
1586
+ [sde] generated 2/128
1587
+ [sde] generated 4/128
1588
+ [sde] generated 6/128
1589
+ [sde] generated 8/128
1590
+ [sde] generated 10/128
1591
+ [sde] generated 12/128
1592
+ [sde] generated 14/128
1593
+ [sde] generated 16/128
1594
+ [sde] generated 18/128
1595
+ [sde] generated 20/128
1596
+ [sde] generated 22/128
1597
+ [sde] generated 24/128
1598
+ [sde] generated 26/128
1599
+ [sde] generated 28/128
1600
+ [sde] generated 30/128
1601
+ [sde] generated 32/128
1602
+ [sde] generated 34/128
1603
+ [sde] generated 36/128
1604
+ [sde] generated 38/128
1605
+ [sde] generated 40/128
1606
+ [sde] generated 42/128
1607
+ [sde] generated 44/128
1608
+ [sde] generated 46/128
1609
+ [sde] generated 48/128
1610
+ [sde] generated 50/128
1611
+ [sde] generated 52/128
1612
+ [sde] generated 54/128
1613
+ [sde] generated 56/128
1614
+ [sde] generated 58/128
1615
+ [sde] generated 60/128
1616
+ [sde] generated 62/128
1617
+ [sde] generated 64/128
1618
+ [sde] generated 66/128
1619
+ [sde] generated 68/128
1620
+ [sde] generated 70/128
1621
+ [sde] generated 72/128
1622
+ [sde] generated 74/128
1623
+ [sde] generated 76/128
1624
+ [sde] generated 78/128
1625
+ [sde] generated 80/128
1626
+ [sde] generated 82/128
1627
+ [sde] generated 84/128
1628
+ [sde] generated 86/128
1629
+ [sde] generated 88/128
1630
+ [sde] generated 90/128
1631
+ [sde] generated 92/128
1632
+ [sde] generated 94/128
1633
+ [sde] generated 96/128
1634
+ [sde] generated 98/128
1635
+ [sde] generated 100/128
1636
+ [sde] generated 102/128
1637
+ [sde] generated 104/128
1638
+ [sde] generated 106/128
1639
+ [sde] generated 108/128
1640
+ [sde] generated 110/128
1641
+ [sde] generated 112/128
1642
+ [sde] generated 114/128
1643
+ [sde] generated 116/128
1644
+ [sde] generated 118/128
1645
+ [sde] generated 120/128
1646
+ [sde] generated 122/128
1647
+ [sde] generated 124/128
1648
+ [sde] generated 126/128
1649
+ [sde] generated 128/128
1650
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
1651
+ [summary] {
1652
+ "type": "summary",
1653
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0012000.pt",
1654
+ "step": 12000,
1655
+ "decode": {
1656
+ "decode_rule": "dirichlet_resample_sde",
1657
+ "steps": 128,
1658
+ "model_t_mode": "support_t",
1659
+ "mean_mode": "endpoint_only",
1660
+ "anchor_gamma": 1.0,
1661
+ "endpoint_floor": 0.0,
1662
+ "concentration_min": 30522.0,
1663
+ "concentration_max": 61044.0,
1664
+ "endpoint_temp": 1.45,
1665
+ "endpoint_temp_start": null,
1666
+ "endpoint_temp_end": null,
1667
+ "endpoint_projection": "gumbel_softmax",
1668
+ "endpoint_top_k": 0,
1669
+ "endpoint_top_p": 0.95,
1670
+ "gumbel_tau_start": 1.0,
1671
+ "gumbel_tau_end": 0.2,
1672
+ "gumbel_noise_scale_start": 1.0,
1673
+ "gumbel_noise_scale_end": 1.0,
1674
+ "ban_special_tokens": false,
1675
+ "banned_endpoint_ids": [],
1676
+ "support_power": 1.0,
1677
+ "semantic_power": 1.0,
1678
+ "noise_init": "dirichlet",
1679
+ "noise_sigma": -1.0,
1680
+ "noise_dirichlet_concentration": 30522.0,
1681
+ "sde_resample": "dirichlet",
1682
+ "logistic_normal_sigma_min": 0.18,
1683
+ "logistic_normal_sigma_max": 3.0,
1684
+ "logistic_normal_tau_min": 0.65,
1685
+ "logistic_normal_tau_max": 1.0,
1686
+ "final_from": "blend_0.5",
1687
+ "n_samples": 128,
1688
+ "seed": 20260524
1689
+ },
1690
+ "raw_genppl": {
1691
+ "ppl": 9.787557925258849,
1692
+ "nll_per_token": 2.281111979588946,
1693
+ "tokens": 121050,
1694
+ "kept_samples": 128,
1695
+ "total_samples": 128,
1696
+ "empty_rate": 0.0,
1697
+ "skipped_samples": 0
1698
+ },
1699
+ "stripped_genppl": {
1700
+ "ppl": 9.575427309198085,
1701
+ "nll_per_token": 2.2592001616634176,
1702
+ "tokens": 120277,
1703
+ "kept_samples": 128,
1704
+ "total_samples": 128,
1705
+ "empty_rate": 0.0,
1706
+ "skipped_samples": 0
1707
+ },
1708
+ "diversity": {
1709
+ "sample_entropy": 2.3105578319413413,
1710
+ "unique_tokens": 1540,
1711
+ "token_count": 131072,
1712
+ "distinct_1": 0.011749267578125,
1713
+ "distinct_2": 0.08389082355816227,
1714
+ "top_token_mass": 0.17154693603515625
1715
+ }
1716
+ }
1717
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0012000/sde_steps128_samples128_scored.jsonl
1718
+ [watch-gumbel] 2026-05-26_13:16:51 done step_0012000
1719
+ [watch-gumbel] 2026-05-26_15:32:28 infer runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0013000.pt -> docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0013000
1720
+ [load] runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0013000.pt
1721
+ [ckpt] step=13000
1722
+ [sde] generated 2/128
1723
+ [sde] generated 4/128
1724
+ [sde] generated 6/128
1725
+ [sde] generated 8/128
1726
+ [sde] generated 10/128
1727
+ [sde] generated 12/128
1728
+ [sde] generated 14/128
1729
+ [sde] generated 16/128
1730
+ [sde] generated 18/128
1731
+ [sde] generated 20/128
1732
+ [sde] generated 22/128
1733
+ [sde] generated 24/128
1734
+ [sde] generated 26/128
1735
+ [sde] generated 28/128
1736
+ [sde] generated 30/128
1737
+ [sde] generated 32/128
1738
+ [sde] generated 34/128
1739
+ [sde] generated 36/128
1740
+ [sde] generated 38/128
1741
+ [sde] generated 40/128
1742
+ [sde] generated 42/128
1743
+ [sde] generated 44/128
1744
+ [sde] generated 46/128
1745
+ [sde] generated 48/128
1746
+ [sde] generated 50/128
1747
+ [sde] generated 52/128
1748
+ [sde] generated 54/128
1749
+ [sde] generated 56/128
1750
+ [sde] generated 58/128
1751
+ [sde] generated 60/128
1752
+ [sde] generated 62/128
1753
+ [sde] generated 64/128
1754
+ [sde] generated 66/128
1755
+ [sde] generated 68/128
1756
+ [sde] generated 70/128
1757
+ [sde] generated 72/128
1758
+ [sde] generated 74/128
1759
+ [sde] generated 76/128
1760
+ [sde] generated 78/128
1761
+ [sde] generated 80/128
1762
+ [sde] generated 82/128
1763
+ [sde] generated 84/128
1764
+ [sde] generated 86/128
1765
+ [sde] generated 88/128
1766
+ [sde] generated 90/128
1767
+ [sde] generated 92/128
1768
+ [sde] generated 94/128
1769
+ [sde] generated 96/128
1770
+ [sde] generated 98/128
1771
+ [sde] generated 100/128
1772
+ [sde] generated 102/128
1773
+ [sde] generated 104/128
1774
+ [sde] generated 106/128
1775
+ [sde] generated 108/128
1776
+ [sde] generated 110/128
1777
+ [sde] generated 112/128
1778
+ [sde] generated 114/128
1779
+ [sde] generated 116/128
1780
+ [sde] generated 118/128
1781
+ [sde] generated 120/128
1782
+ [sde] generated 122/128
1783
+ [sde] generated 124/128
1784
+ [sde] generated 126/128
1785
+ [sde] generated 128/128
1786
+ [score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
1787
+ [summary] {
1788
+ "type": "summary",
1789
+ "checkpoint": "runs/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0013000.pt",
1790
+ "step": 13000,
1791
+ "decode": {
1792
+ "decode_rule": "dirichlet_resample_sde",
1793
+ "steps": 128,
1794
+ "model_t_mode": "support_t",
1795
+ "mean_mode": "endpoint_only",
1796
+ "anchor_gamma": 1.0,
1797
+ "endpoint_floor": 0.0,
1798
+ "concentration_min": 30522.0,
1799
+ "concentration_max": 61044.0,
1800
+ "endpoint_temp": 1.45,
1801
+ "endpoint_temp_start": null,
1802
+ "endpoint_temp_end": null,
1803
+ "endpoint_projection": "gumbel_softmax",
1804
+ "endpoint_top_k": 0,
1805
+ "endpoint_top_p": 0.95,
1806
+ "gumbel_tau_start": 1.0,
1807
+ "gumbel_tau_end": 0.2,
1808
+ "gumbel_noise_scale_start": 1.0,
1809
+ "gumbel_noise_scale_end": 1.0,
1810
+ "ban_special_tokens": false,
1811
+ "banned_endpoint_ids": [],
1812
+ "support_power": 1.0,
1813
+ "semantic_power": 1.0,
1814
+ "noise_init": "dirichlet",
1815
+ "noise_sigma": -1.0,
1816
+ "noise_dirichlet_concentration": 30522.0,
1817
+ "sde_resample": "dirichlet",
1818
+ "logistic_normal_sigma_min": 0.18,
1819
+ "logistic_normal_sigma_max": 3.0,
1820
+ "logistic_normal_tau_min": 0.65,
1821
+ "logistic_normal_tau_max": 1.0,
1822
+ "final_from": "blend_0.5",
1823
+ "n_samples": 128,
1824
+ "seed": 20260524
1825
+ },
1826
+ "raw_genppl": {
1827
+ "ppl": 11.019391485874875,
1828
+ "nll_per_token": 2.399656583132524,
1829
+ "tokens": 130289,
1830
+ "kept_samples": 128,
1831
+ "total_samples": 128,
1832
+ "empty_rate": 0.0,
1833
+ "skipped_samples": 0
1834
+ },
1835
+ "stripped_genppl": {
1836
+ "ppl": 10.84729878838137,
1837
+ "nll_per_token": 2.3839160893946967,
1838
+ "tokens": 129965,
1839
+ "kept_samples": 128,
1840
+ "total_samples": 128,
1841
+ "empty_rate": 0.0,
1842
+ "skipped_samples": 0
1843
+ },
1844
+ "diversity": {
1845
+ "sample_entropy": 2.991297718897202,
1846
+ "unique_tokens": 1773,
1847
+ "token_count": 131072,
1848
+ "distinct_1": 0.01352691650390625,
1849
+ "distinct_2": 0.11194098240469208,
1850
+ "top_token_mass": 0.14432525634765625
1851
+ }
1852
+ }
1853
+ [done] docs/lta_samples/metrics_20260525/owt_bert_absrope_adaln_Cv_to_2v_mask1_sameT_every1k_sde_gumbel_topp0p95_tau1p0_to_0p2_blend_c30522_61044_n128/lta_owt_bert_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask1_sameT_gbs512_b4x4_1m_save1k_watch_20260525/step_0013000/sde_steps128_samples128_scored.jsonl
1854
+ [watch-gumbel] 2026-05-26_15:50:49 done step_0013000
LTA_openwebtext_dualt/logs/lta_owt_c1024_len1024_t0to1_lowk64plus_cleanbridge_droplowt0p2_buf1000_gbs128_4gpu_2k.log ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ *****************************************
3
+ Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ *****************************************
5
+ NCCL version 2.25.1+cuda12.8
6
+ {
7
+ "device": "cuda:0",
8
+ "rank": 0,
9
+ "world_size": 4,
10
+ "samples": "wrapped_stream_online_shuffle:1000",
11
+ "vocab_size": 50257,
12
+ "save_dir": "runs/lta_owt_c1024_len1024_t0to1_lowk64plus_cleanbridge_droplowt0p2_buf1000_gbs128_4gpu_2k",
13
+ "batch_size": 16,
14
+ "grad_accum": 2,
15
+ "effective_batch_size": 128,
16
+ "global_batch_size": 128,
17
+ "lr_schedule": "constant_warmup",
18
+ "warmup_steps": 100,
19
+ "adam_beta1": 0.9,
20
+ "adam_beta2": 0.999,
21
+ "adam_eps": 1e-08,
22
+ "model_type": "ddit",
23
+ "dual_t": true,
24
+ "corrupt_t_mode": "same",
25
+ "corrupt_min_t": 0.0,
26
+ "corrupt_max_t": 1.0,
27
+ "dirichlet_endpoint_mode": "categorical_dual_t",
28
+ "dirichlet_semantic_t_mode": "same",
29
+ "dirichlet_semantic_t_value": 0.0,
30
+ "categorical_wrong_from_full_vocab": true,
31
+ "simplex_bridge_sampler": "dirichlet",
32
+ "logistic_normal_sigma_min": 0.18,
33
+ "logistic_normal_sigma_max": 2.2,
34
+ "logistic_normal_tau_min": 0.65,
35
+ "logistic_normal_tau_max": 1.15,
36
+ "torch_compile": false,
37
+ "compile_mode": "max-autotune",
38
+ "state_format": "prob",
39
+ "target_loss": "hard_ce",
40
+ "meanflow_weight": 0.0,
41
+ "bridge_noise_init": "logistic_normal",
42
+ "noise_sigma": -1.0,
43
+ "wrap": true,
44
+ "wrap_mode": "stream",
45
+ "wrap_record_buffer_size": 200,
46
+ "owt_cached_chunks": false,
47
+ "owt_chunk_cache_dir": "",
48
+ "owt_chunk_cache_rebuild": false,
49
+ "owt_chunk_cache_write_batch": 4096,
50
+ "online_chunk_shuffle": true,
51
+ "online_chunk_shuffle_buffer": 1000,
52
+ "openwebtext_split": "train_minus_100k",
53
+ "detokenizer": "auto",
54
+ "resolved_detokenizer": null,
55
+ "num_workers": 0,
56
+ "latest_every": 500,
57
+ "resume_path": ""
58
+ }
59
+ step=25 micro_steps=50 elapsed=29.3s lr=7.800000e-05 loss_all=10.6501 acc_all=0.3912 loss_corrupt=10.6544 acc_corrupt=0.3448 corrupt_frac=0.7759 loss=10.6475 loss_recon=10.6475 loss_meanflow=0.0000 mean_model_t=0.5108 mean_corrupt_t=0.5108 mean_loss_t_weight=0.8363 prior_center_loss_beta=0.0000 wrong_frac=0.4916 init_acc_corrupt=0.4742 init_gold_top10=0.5033 init_gold_top100=0.5307
60
+ step=50 micro_steps=100 elapsed=25.2s lr=1.530000e-04 loss_all=9.1700 acc_all=0.1655 loss_corrupt=9.1781 acc_corrupt=0.1521 corrupt_frac=0.7866 loss=9.1563 loss_recon=9.1563 loss_meanflow=0.0000 mean_model_t=0.5058 mean_corrupt_t=0.5058 mean_loss_t_weight=0.7987 prior_center_loss_beta=0.0000 wrong_frac=0.4977 init_acc_corrupt=0.4698 init_gold_top10=0.4967 init_gold_top100=0.5274
61
+ step=75 micro_steps=150 elapsed=25.1s lr=2.280000e-04 loss_all=7.2596 acc_all=0.1964 loss_corrupt=7.3334 acc_corrupt=0.1765 corrupt_frac=0.7806 loss=7.2117 loss_recon=7.2117 loss_meanflow=0.0000 mean_model_t=0.5048 mean_corrupt_t=0.5048 mean_loss_t_weight=0.7937 prior_center_loss_beta=0.0000 wrong_frac=0.4932 init_acc_corrupt=0.4726 init_gold_top10=0.5009 init_gold_top100=0.5315
62
+ step=100 micro_steps=200 elapsed=25.3s lr=3.000000e-04 loss_all=5.3227 acc_all=0.3612 loss_corrupt=5.6663 acc_corrupt=0.3162 corrupt_frac=0.7777 loss=5.1685 loss_recon=5.1685 loss_meanflow=0.0000 mean_model_t=0.4906 mean_corrupt_t=0.4906 mean_loss_t_weight=0.7987 prior_center_loss_beta=0.0000 wrong_frac=0.5081 init_acc_corrupt=0.4579 init_gold_top10=0.4861 init_gold_top100=0.5164
63
+ step=125 micro_steps=250 elapsed=25.2s lr=3.000000e-04 loss_all=4.2821 acc_all=0.4956 loss_corrupt=4.7957 acc_corrupt=0.4340 corrupt_frac=0.7931 loss=4.0574 loss_recon=4.0574 loss_meanflow=0.0000 mean_model_t=0.4954 mean_corrupt_t=0.4954 mean_loss_t_weight=0.8075 prior_center_loss_beta=0.0000 wrong_frac=0.5049 init_acc_corrupt=0.4569 init_gold_top10=0.4893 init_gold_top100=0.5199
64
+ step=150 micro_steps=300 elapsed=25.2s lr=3.000000e-04 loss_all=3.9885 acc_all=0.5171 loss_corrupt=4.5149 acc_corrupt=0.4525 corrupt_frac=0.7826 loss=3.7021 loss_recon=3.7021 loss_meanflow=0.0000 mean_model_t=0.4859 mean_corrupt_t=0.4859 mean_loss_t_weight=0.7913 prior_center_loss_beta=0.0000 wrong_frac=0.5146 init_acc_corrupt=0.4506 init_gold_top10=0.4797 init_gold_top100=0.5106
65
+ step=175 micro_steps=350 elapsed=25.2s lr=3.000000e-04 loss_all=3.7626 acc_all=0.5451 loss_corrupt=4.2718 acc_corrupt=0.4824 corrupt_frac=0.7915 loss=3.4175 loss_recon=3.4175 loss_meanflow=0.0000 mean_model_t=0.5100 mean_corrupt_t=0.5100 mean_loss_t_weight=0.8000 prior_center_loss_beta=0.0000 wrong_frac=0.4906 init_acc_corrupt=0.4760 init_gold_top10=0.5040 init_gold_top100=0.5332
66
+ step=200 micro_steps=400 elapsed=25.1s lr=3.000000e-04 loss_all=3.7931 acc_all=0.5390 loss_corrupt=4.3632 acc_corrupt=0.4687 corrupt_frac=0.7807 loss=3.4321 loss_recon=3.4321 loss_meanflow=0.0000 mean_model_t=0.4984 mean_corrupt_t=0.4984 mean_loss_t_weight=0.7850 prior_center_loss_beta=0.0000 wrong_frac=0.5058 init_acc_corrupt=0.4583 init_gold_top10=0.4880 init_gold_top100=0.5201
67
+ step=225 micro_steps=450 elapsed=25.2s lr=3.000000e-04 loss_all=3.5848 acc_all=0.5637 loss_corrupt=4.0758 acc_corrupt=0.5027 corrupt_frac=0.7883 loss=3.2648 loss_recon=3.2648 loss_meanflow=0.0000 mean_model_t=0.5213 mean_corrupt_t=0.5213 mean_loss_t_weight=0.8100 prior_center_loss_beta=0.0000 wrong_frac=0.4770 init_acc_corrupt=0.4930 init_gold_top10=0.5177 init_gold_top100=0.5450
68
+ step=250 micro_steps=500 elapsed=25.3s lr=3.000000e-04 loss_all=3.7362 acc_all=0.5461 loss_corrupt=4.2148 acc_corrupt=0.4879 corrupt_frac=0.7874 loss=3.3531 loss_recon=3.3531 loss_meanflow=0.0000 mean_model_t=0.5055 mean_corrupt_t=0.5055 mean_loss_t_weight=0.7863 prior_center_loss_beta=0.0000 wrong_frac=0.4915 init_acc_corrupt=0.4751 init_gold_top10=0.5027 init_gold_top100=0.5318
69
+ step=275 micro_steps=550 elapsed=25.3s lr=3.000000e-04 loss_all=3.7222 acc_all=0.5429 loss_corrupt=4.2287 acc_corrupt=0.4809 corrupt_frac=0.7828 loss=3.3629 loss_recon=3.3629 loss_meanflow=0.0000 mean_model_t=0.4952 mean_corrupt_t=0.4952 mean_loss_t_weight=0.7875 prior_center_loss_beta=0.0000 wrong_frac=0.5007 init_acc_corrupt=0.4653 init_gold_top10=0.4934 init_gold_top100=0.5242
70
+ step=300 micro_steps=600 elapsed=25.3s lr=3.000000e-04 loss_all=3.6758 acc_all=0.5442 loss_corrupt=4.2142 acc_corrupt=0.4776 corrupt_frac=0.7859 loss=3.4857 loss_recon=3.4857 loss_meanflow=0.0000 mean_model_t=0.4962 mean_corrupt_t=0.4962 mean_loss_t_weight=0.8175 prior_center_loss_beta=0.0000 wrong_frac=0.5066 init_acc_corrupt=0.4563 init_gold_top10=0.4879 init_gold_top100=0.5165
71
+ step=325 micro_steps=650 elapsed=25.2s lr=3.000000e-04 loss_all=3.5373 acc_all=0.5624 loss_corrupt=4.0185 acc_corrupt=0.5028 corrupt_frac=0.7840 loss=3.1492 loss_recon=3.1492 loss_meanflow=0.0000 mean_model_t=0.5095 mean_corrupt_t=0.5095 mean_loss_t_weight=0.8025 prior_center_loss_beta=0.0000 wrong_frac=0.4868 init_acc_corrupt=0.4819 init_gold_top10=0.5075 init_gold_top100=0.5383
72
+ step=350 micro_steps=700 elapsed=25.2s lr=3.000000e-04 loss_all=3.6158 acc_all=0.5486 loss_corrupt=4.1173 acc_corrupt=0.4865 corrupt_frac=0.7842 loss=3.3154 loss_recon=3.3154 loss_meanflow=0.0000 mean_model_t=0.4933 mean_corrupt_t=0.4933 mean_loss_t_weight=0.8025 prior_center_loss_beta=0.0000 wrong_frac=0.5044 init_acc_corrupt=0.4587 init_gold_top10=0.4899 init_gold_top100=0.5200
73
+ step=375 micro_steps=750 elapsed=25.2s lr=3.000000e-04 loss_all=3.4370 acc_all=0.5712 loss_corrupt=3.9058 acc_corrupt=0.5125 corrupt_frac=0.7920 loss=3.0150 loss_recon=3.0150 loss_meanflow=0.0000 mean_model_t=0.5195 mean_corrupt_t=0.5195 mean_loss_t_weight=0.8050 prior_center_loss_beta=0.0000 wrong_frac=0.4825 init_acc_corrupt=0.4866 init_gold_top10=0.5119 init_gold_top100=0.5400
74
+ step=400 micro_steps=800 elapsed=25.2s lr=3.000000e-04 loss_all=3.3831 acc_all=0.5751 loss_corrupt=3.9147 acc_corrupt=0.5085 corrupt_frac=0.7748 loss=3.1365 loss_recon=3.1365 loss_meanflow=0.0000 mean_model_t=0.5116 mean_corrupt_t=0.5116 mean_loss_t_weight=0.8200 prior_center_loss_beta=0.0000 wrong_frac=0.4892 init_acc_corrupt=0.4786 init_gold_top10=0.5057 init_gold_top100=0.5320
75
+ step=425 micro_steps=850 elapsed=25.2s lr=3.000000e-04 loss_all=3.5427 acc_all=0.5542 loss_corrupt=4.0855 acc_corrupt=0.4860 corrupt_frac=0.7837 loss=3.0889 loss_recon=3.0889 loss_meanflow=0.0000 mean_model_t=0.4945 mean_corrupt_t=0.4945 mean_loss_t_weight=0.7812 prior_center_loss_beta=0.0000 wrong_frac=0.5101 init_acc_corrupt=0.4534 init_gold_top10=0.4837 init_gold_top100=0.5162
76
+ step=450 micro_steps=900 elapsed=25.3s lr=3.000000e-04 loss_all=3.5278 acc_all=0.5548 loss_corrupt=4.0352 acc_corrupt=0.4907 corrupt_frac=0.7848 loss=3.0848 loss_recon=3.0848 loss_meanflow=0.0000 mean_model_t=0.4917 mean_corrupt_t=0.4917 mean_loss_t_weight=0.7950 prior_center_loss_beta=0.0000 wrong_frac=0.5098 init_acc_corrupt=0.4567 init_gold_top10=0.4848 init_gold_top100=0.5149
77
+ step=475 micro_steps=950 elapsed=25.2s lr=3.000000e-04 loss_all=3.3691 acc_all=0.5724 loss_corrupt=3.8616 acc_corrupt=0.5102 corrupt_frac=0.7849 loss=3.0133 loss_recon=3.0133 loss_meanflow=0.0000 mean_model_t=0.5069 mean_corrupt_t=0.5069 mean_loss_t_weight=0.8137 prior_center_loss_beta=0.0000 wrong_frac=0.4903 init_acc_corrupt=0.4767 init_gold_top10=0.5044 init_gold_top100=0.5338
78
+ step=500 micro_steps=1000 elapsed=25.2s lr=3.000000e-04 loss_all=3.4379 acc_all=0.5629 loss_corrupt=3.9421 acc_corrupt=0.4984 corrupt_frac=0.7745 loss=3.1358 loss_recon=3.1358 loss_meanflow=0.0000 mean_model_t=0.4968 mean_corrupt_t=0.4968 mean_loss_t_weight=0.8150 prior_center_loss_beta=0.0000 wrong_frac=0.5031 init_acc_corrupt=0.4606 init_gold_top10=0.4916 init_gold_top100=0.5204
79
+ step=525 micro_steps=1050 elapsed=28.9s lr=3.000000e-04 loss_all=3.2989 acc_all=0.5792 loss_corrupt=3.7982 acc_corrupt=0.5150 corrupt_frac=0.7826 loss=3.0613 loss_recon=3.0613 loss_meanflow=0.0000 mean_model_t=0.5094 mean_corrupt_t=0.5094 mean_loss_t_weight=0.8275 prior_center_loss_beta=0.0000 wrong_frac=0.4894 init_acc_corrupt=0.4759 init_gold_top10=0.5055 init_gold_top100=0.5316
80
+ step=550 micro_steps=1100 elapsed=25.2s lr=3.000000e-04 loss_all=3.4631 acc_all=0.5589 loss_corrupt=3.9581 acc_corrupt=0.4955 corrupt_frac=0.7784 loss=3.0085 loss_recon=3.0085 loss_meanflow=0.0000 mean_model_t=0.4917 mean_corrupt_t=0.4917 mean_loss_t_weight=0.7837 prior_center_loss_beta=0.0000 wrong_frac=0.5064 init_acc_corrupt=0.4554 init_gold_top10=0.4873 init_gold_top100=0.5196
81
+ step=575 micro_steps=1150 elapsed=25.3s lr=3.000000e-04 loss_all=3.4780 acc_all=0.5578 loss_corrupt=4.0191 acc_corrupt=0.4889 corrupt_frac=0.7895 loss=3.0881 loss_recon=3.0881 loss_meanflow=0.0000 mean_model_t=0.4925 mean_corrupt_t=0.4925 mean_loss_t_weight=0.8013 prior_center_loss_beta=0.0000 wrong_frac=0.5146 init_acc_corrupt=0.4480 init_gold_top10=0.4793 init_gold_top100=0.5110
LTA_openwebtext_dualt/logs/lta_owt_dirichlet_categorical_fullvocab_c1024_fullycoupled_gpt2cached_len1024_rollin_p0p5_s4_i32_outwd-1_gbs512_8gpu_1m_20260517_201804.log ADDED
@@ -0,0 +1,658 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ *****************************************
3
+ Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
4
+ *****************************************
5
+ t-20260518041721-mttss-worker-0:10318:10318 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
6
+ t-20260518041721-mttss-worker-0:10318:10318 [0] NCCL INFO Bootstrap: Using eth1:10.82.48.5<0>
7
+ t-20260518041721-mttss-worker-0:10318:10318 [0] NCCL INFO cudaDriverVersion 12080
8
+ t-20260518041721-mttss-worker-0:10318:10318 [0] NCCL INFO NCCL version 2.25.1+cuda12.8
9
+ t-20260518041721-mttss-worker-0:10318:10318 [0] NCCL INFO Comm config Blocking set to 1
10
+ t-20260518041721-mttss-worker-0:10320:10320 [2] NCCL INFO cudaDriverVersion 12080
11
+ t-20260518041721-mttss-worker-0:10320:10320 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
12
+ t-20260518041721-mttss-worker-0:10320:10320 [2] NCCL INFO Bootstrap: Using eth1:10.82.48.5<0>
13
+ t-20260518041721-mttss-worker-0:10320:10320 [2] NCCL INFO NCCL version 2.25.1+cuda12.8
14
+ t-20260518041721-mttss-worker-0:10320:10320 [2] NCCL INFO Comm config Blocking set to 1
15
+ t-20260518041721-mttss-worker-0:10323:10323 [5] NCCL INFO cudaDriverVersion 12080
16
+ t-20260518041721-mttss-worker-0:10323:10323 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
17
+ t-20260518041721-mttss-worker-0:10323:10323 [5] NCCL INFO Bootstrap: Using eth1:10.82.48.5<0>
18
+ t-20260518041721-mttss-worker-0:10323:10323 [5] NCCL INFO NCCL version 2.25.1+cuda12.8
19
+ t-20260518041721-mttss-worker-0:10323:10323 [5] NCCL INFO Comm config Blocking set to 1
20
+ t-20260518041721-mttss-worker-0:10322:10322 [4] NCCL INFO cudaDriverVersion 12080
21
+ t-20260518041721-mttss-worker-0:10322:10322 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
22
+ t-20260518041721-mttss-worker-0:10322:10322 [4] NCCL INFO Bootstrap: Using eth1:10.82.48.5<0>
23
+ t-20260518041721-mttss-worker-0:10322:10322 [4] NCCL INFO NCCL version 2.25.1+cuda12.8
24
+ t-20260518041721-mttss-worker-0:10322:10322 [4] NCCL INFO Comm config Blocking set to 1
25
+ t-20260518041721-mttss-worker-0:10319:10319 [1] NCCL INFO cudaDriverVersion 12080
26
+ t-20260518041721-mttss-worker-0:10319:10319 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
27
+ t-20260518041721-mttss-worker-0:10319:10319 [1] NCCL INFO Bootstrap: Using eth1:10.82.48.5<0>
28
+ t-20260518041721-mttss-worker-0:10319:10319 [1] NCCL INFO NCCL version 2.25.1+cuda12.8
29
+ t-20260518041721-mttss-worker-0:10319:10319 [1] NCCL INFO Comm config Blocking set to 1
30
+ t-20260518041721-mttss-worker-0:10324:10324 [6] NCCL INFO cudaDriverVersion 12080
31
+ t-20260518041721-mttss-worker-0:10324:10324 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
32
+ t-20260518041721-mttss-worker-0:10324:10324 [6] NCCL INFO Bootstrap: Using eth1:10.82.48.5<0>
33
+ t-20260518041721-mttss-worker-0:10324:10324 [6] NCCL INFO NCCL version 2.25.1+cuda12.8
34
+ t-20260518041721-mttss-worker-0:10324:10324 [6] NCCL INFO Comm config Blocking set to 1
35
+ t-20260518041721-mttss-worker-0:10321:10321 [3] NCCL INFO cudaDriverVersion 12080
36
+ t-20260518041721-mttss-worker-0:10321:10321 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
37
+ t-20260518041721-mttss-worker-0:10321:10321 [3] NCCL INFO Bootstrap: Using eth1:10.82.48.5<0>
38
+ t-20260518041721-mttss-worker-0:10321:10321 [3] NCCL INFO NCCL version 2.25.1+cuda12.8
39
+ t-20260518041721-mttss-worker-0:10321:10321 [3] NCCL INFO Comm config Blocking set to 1
40
+ t-20260518041721-mttss-worker-0:10325:10325 [7] NCCL INFO cudaDriverVersion 12080
41
+ t-20260518041721-mttss-worker-0:10325:10325 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
42
+ t-20260518041721-mttss-worker-0:10325:10325 [7] NCCL INFO Bootstrap: Using eth1:10.82.48.5<0>
43
+ t-20260518041721-mttss-worker-0:10325:10325 [7] NCCL INFO NCCL version 2.25.1+cuda12.8
44
+ t-20260518041721-mttss-worker-0:10325:10325 [7] NCCL INFO Comm config Blocking set to 1
45
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
46
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
47
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
48
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO P2P plugin v9 IBext_v9
49
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
50
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
51
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
52
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
53
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO P2P plugin v9 IBext_v9
54
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
55
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
56
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.48.5<0>
57
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
58
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Using network IBext_v9
59
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
60
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.48.5<0>
61
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
62
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO Using network IBext_v9
63
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
64
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
65
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
66
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO P2P plugin v9 IBext_v9
67
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
68
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
69
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.48.5<0>
70
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
71
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO Using network IBext_v9
72
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
73
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
74
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
75
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO P2P plugin v9 IBext_v9
76
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
77
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
78
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
79
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
80
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO P2P plugin v9 IBext_v9
81
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
82
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
83
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.48.5<0>
84
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
85
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO Using network IBext_v9
86
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
87
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.48.5<0>
88
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
89
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO Using network IBext_v9
90
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
91
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
92
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
93
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO P2P plugin v9 IBext_v9
94
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
95
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO ncclCommInitRankConfig comm 0xbc85d50 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x2f3876d210e12b32 - Init START
96
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
97
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.48.5<0>
98
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
99
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO Using network IBext_v9
100
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
101
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
102
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
103
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO P2P plugin v9 IBext_v9
104
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
105
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO NET/Plugin: Loaded net plugin NCCL RDMA Plugin v9 (v9)
106
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO NET/Plugin: Loaded collnet plugin SHARP (v9)
107
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO Plugin Path : /opt/hpcx/nccl_rdma_sharp_plugin/lib/libnccl-net.so
108
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO P2P plugin v9 IBext_v9
109
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth1
110
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
111
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.48.5<0>
112
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO NCCL_IB_PCI_RELAXED_ORDERING set by environment to 1.
113
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO NET/IB : Using [0]mlx5_1:1/RoCE [1]mlx5_4:1/RoCE [2]mlx5_5:1/RoCE [3]mlx5_6:1/RoCE [4]mlx5_7:1/RoCE [5]mlx5_8:1/RoCE [6]mlx5_9:1/RoCE [7]mlx5_10:1/RoCE [RO]; OOB eth1:10.82.48.5<0>
114
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
115
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO Using network IBext_v9
116
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO PROFILER/Plugin: Could not find: libnccl-profiler.so.
117
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO Using network IBext_v9
118
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO ncclCommInitRankConfig comm 0xa2915b0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x2f3876d210e12b32 - Init START
119
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO ncclCommInitRankConfig comm 0xb62de00 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x2f3876d210e12b32 - Init START
120
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO ncclCommInitRankConfig comm 0xa0849c0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x2f3876d210e12b32 - Init START
121
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO ncclCommInitRankConfig comm 0xb06fb20 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x2f3876d210e12b32 - Init START
122
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO RAS client listening socket at ::1<28028>
123
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO ncclCommInitRankConfig comm 0xb38f670 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x2f3876d210e12b32 - Init START
124
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO RAS client listening socket at ::1<28028>
125
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO ncclCommInitRankConfig comm 0xb85e190 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x2f3876d210e12b32 - Init START
126
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO RAS client listening socket at ::1<28028>
127
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO RAS client listening socket at ::1<28028>
128
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO RAS client listening socket at ::1<28028>
129
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO ncclCommInitRankConfig comm 0xa149c50 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x2f3876d210e12b32 - Init START
130
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO RAS client listening socket at ::1<28028>
131
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO RAS client listening socket at ::1<28028>
132
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO RAS client listening socket at ::1<28028>
133
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Bootstrap timings total 0.726898 (create 0.000036, send 0.000066, recv 0.570777, ring 0.006418, delay 0.000001)
134
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO Bootstrap timings total 0.006962 (create 0.000024, send 0.000075, recv 0.000132, ring 0.006459, delay 0.000001)
135
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO Bootstrap timings total 0.021801 (create 0.000023, send 0.000063, recv 0.014952, ring 0.006465, delay 0.000001)
136
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO Bootstrap timings total 0.191055 (create 0.000024, send 0.000079, recv 0.000117, ring 0.000124, delay 0.000001)
137
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO Bootstrap timings total 0.000735 (create 0.000021, send 0.000062, recv 0.000126, ring 0.000233, delay 0.000001)
138
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO Bootstrap timings total 0.546182 (create 0.000024, send 0.000069, recv 0.545578, ring 0.000116, delay 0.000001)
139
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO Bootstrap timings total 0.156220 (create 0.000020, send 0.000062, recv 0.000122, ring 0.155700, delay 0.000001)
140
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO Bootstrap timings total 0.305584 (create 0.000023, send 0.000067, recv 0.283891, ring 0.021271, delay 0.000001)
141
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO MNNVL busId 0x73020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
142
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO MNNVL busId 0x6b020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
143
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO MNNVL busId 0x75020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
144
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO MNNVL busId 0x65040 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
145
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO MNNVL busId 0x69020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
146
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO MNNVL busId 0x67020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
147
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO MNNVL busId 0x6f020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
148
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO MNNVL busId 0x71020 fabric UUID 0.0 cliqueId 0x0 state 3 healthMask 0x0
149
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
150
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
151
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
152
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
153
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
154
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
155
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
156
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO NCCL_TOPO_FILE set by environment to /var/run/nvidia-topologyd/virtualTopology.xml
157
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO Setting affinity for GPU 6 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
158
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO NVLS multicast support is available on dev 6
159
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO Setting affinity for GPU 5 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
160
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO Setting affinity for GPU 4 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
161
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO Setting affinity for GPU 2 to 03ffffff,ffffffff,ffffffff
162
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO NVLS multicast support is available on dev 5
163
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO NVLS multicast support is available on dev 2
164
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Setting affinity for GPU 0 to 03ffffff,ffffffff,ffffffff
165
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO Setting affinity for GPU 7 to 0fffff,ffffffff,ffffffff,fc000000,00000000,00000000
166
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO Setting affinity for GPU 1 to 03ffffff,ffffffff,ffffffff
167
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO NVLS multicast support is available on dev 7
168
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO Setting affinity for GPU 3 to 03ffffff,ffffffff,ffffffff
169
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO NVLS multicast support is available on dev 0
170
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO NVLS multicast support is available on dev 4
171
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO NVLS multicast support is available on dev 1
172
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO NVLS multicast support is available on dev 3
173
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO comm 0xa2915b0 rank 2 nRanks 8 nNodes 1 localRanks 8 localRank 2 MNNVL 0
174
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO comm 0xa149c50 rank 3 nRanks 8 nNodes 1 localRanks 8 localRank 3 MNNVL 0
175
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO comm 0xb06fb20 rank 1 nRanks 8 nNodes 1 localRanks 8 localRank 1 MNNVL 0
176
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO comm 0xb85e190 rank 7 nRanks 8 nNodes 1 localRanks 8 localRank 7 MNNVL 0
177
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO comm 0xb38f670 rank 6 nRanks 8 nNodes 1 localRanks 8 localRank 6 MNNVL 0
178
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO comm 0xbc85d50 rank 0 nRanks 8 nNodes 1 localRanks 8 localRank 0 MNNVL 0
179
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO comm 0xa0849c0 rank 4 nRanks 8 nNodes 1 localRanks 8 localRank 4 MNNVL 0
180
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO comm 0xb62de00 rank 5 nRanks 8 nNodes 1 localRanks 8 localRank 5 MNNVL 0
181
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 00/24 : 0 1 2 3 4 5 6 7
182
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 [2] 3/-1/-1->2->1 [3] 3/-1/-1->2->1 [4] 3/-1/-1->2->1 [5] 3/-1/-1->2->1 [6] 3/-1/-1->2->1 [7] 3/-1/-1->2->1 [8] 3/-1/-1->2->1 [9] 3/-1/-1->2->1 [10] 3/-1/-1->2->1 [11] 3/-1/-1->2->1 [12] 3/-1/-1->2->1 [13] 3/-1/-1->2->1 [14] 3/-1/-1->2->1 [15] 3/-1/-1->2->1 [16] 3/-1/-1->2->1 [17] 3/-1/-1->2->1 [18] 3/-1/-1->2->1 [19] 3/-1/-1->2->1 [20] 3/-1/-1->2->1 [21] 3/-1/-1->2->1 [22] 3/-1/-1->2->1 [23] 3/-1/-1->2->1
183
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO Trees [0] 4/-1/-1->3->2 [1] 4/-1/-1->3->2 [2] 4/-1/-1->3->2 [3] 4/-1/-1->3->2 [4] 4/-1/-1->3->2 [5] 4/-1/-1->3->2 [6] 4/-1/-1->3->2 [7] 4/-1/-1->3->2 [8] 4/-1/-1->3->2 [9] 4/-1/-1->3->2 [10] 4/-1/-1->3->2 [11] 4/-1/-1->3->2 [12] 4/-1/-1->3->2 [13] 4/-1/-1->3->2 [14] 4/-1/-1->3->2 [15] 4/-1/-1->3->2 [16] 4/-1/-1->3->2 [17] 4/-1/-1->3->2 [18] 4/-1/-1->3->2 [19] 4/-1/-1->3->2 [20] 4/-1/-1->3->2 [21] 4/-1/-1->3->2 [22] 4/-1/-1->3->2 [23] 4/-1/-1->3->2
184
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 [2] 7/-1/-1->6->5 [3] 7/-1/-1->6->5 [4] 7/-1/-1->6->5 [5] 7/-1/-1->6->5 [6] 7/-1/-1->6->5 [7] 7/-1/-1->6->5 [8] 7/-1/-1->6->5 [9] 7/-1/-1->6->5 [10] 7/-1/-1->6->5 [11] 7/-1/-1->6->5 [12] 7/-1/-1->6->5 [13] 7/-1/-1->6->5 [14] 7/-1/-1->6->5 [15] 7/-1/-1->6->5 [16] 7/-1/-1->6->5 [17] 7/-1/-1->6->5 [18] 7/-1/-1->6->5 [19] 7/-1/-1->6->5 [20] 7/-1/-1->6->5 [21] 7/-1/-1->6->5 [22] 7/-1/-1->6->5 [23] 7/-1/-1->6->5
185
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO P2P Chunksize set to 524288
186
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 [2] -1/-1/-1->7->6 [3] -1/-1/-1->7->6 [4] -1/-1/-1->7->6 [5] -1/-1/-1->7->6 [6] -1/-1/-1->7->6 [7] -1/-1/-1->7->6 [8] -1/-1/-1->7->6 [9] -1/-1/-1->7->6 [10] -1/-1/-1->7->6 [11] -1/-1/-1->7->6 [12] -1/-1/-1->7->6 [13] -1/-1/-1->7->6 [14] -1/-1/-1->7->6 [15] -1/-1/-1->7->6 [16] -1/-1/-1->7->6 [17] -1/-1/-1->7->6 [18] -1/-1/-1->7->6 [19] -1/-1/-1->7->6 [20] -1/-1/-1->7->6 [21] -1/-1/-1->7->6 [22] -1/-1/-1->7->6 [23] -1/-1/-1->7->6
187
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO P2P Chunksize set to 524288
188
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 01/24 : 0 1 2 3 4 5 6 7
189
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO P2P Chunksize set to 524288
190
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 [2] 2/-1/-1->1->0 [3] 2/-1/-1->1->0 [4] 2/-1/-1->1->0 [5] 2/-1/-1->1->0 [6] 2/-1/-1->1->0 [7] 2/-1/-1->1->0 [8] 2/-1/-1->1->0 [9] 2/-1/-1->1->0 [10] 2/-1/-1->1->0 [11] 2/-1/-1->1->0 [12] 2/-1/-1->1->0 [13] 2/-1/-1->1->0 [14] 2/-1/-1->1->0 [15] 2/-1/-1->1->0 [16] 2/-1/-1->1->0 [17] 2/-1/-1->1->0 [18] 2/-1/-1->1->0 [19] 2/-1/-1->1->0 [20] 2/-1/-1->1->0 [21] 2/-1/-1->1->0 [22] 2/-1/-1->1->0 [23] 2/-1/-1->1->0
191
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO P2P Chunksize set to 524288
192
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 02/24 : 0 1 2 3 4 5 6 7
193
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO Trees [0] 5/-1/-1->4->3 [1] 5/-1/-1->4->3 [2] 5/-1/-1->4->3 [3] 5/-1/-1->4->3 [4] 5/-1/-1->4->3 [5] 5/-1/-1->4->3 [6] 5/-1/-1->4->3 [7] 5/-1/-1->4->3 [8] 5/-1/-1->4->3 [9] 5/-1/-1->4->3 [10] 5/-1/-1->4->3 [11] 5/-1/-1->4->3 [12] 5/-1/-1->4->3 [13] 5/-1/-1->4->3 [14] 5/-1/-1->4->3 [15] 5/-1/-1->4->3 [16] 5/-1/-1->4->3 [17] 5/-1/-1->4->3 [18] 5/-1/-1->4->3 [19] 5/-1/-1->4->3 [20] 5/-1/-1->4->3 [21] 5/-1/-1->4->3 [22] 5/-1/-1->4->3 [23] 5/-1/-1->4->3
194
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/-1/-1->5->4 [2] 6/-1/-1->5->4 [3] 6/-1/-1->5->4 [4] 6/-1/-1->5->4 [5] 6/-1/-1->5->4 [6] 6/-1/-1->5->4 [7] 6/-1/-1->5->4 [8] 6/-1/-1->5->4 [9] 6/-1/-1->5->4 [10] 6/-1/-1->5->4 [11] 6/-1/-1->5->4 [12] 6/-1/-1->5->4 [13] 6/-1/-1->5->4 [14] 6/-1/-1->5->4 [15] 6/-1/-1->5->4 [16] 6/-1/-1->5->4 [17] 6/-1/-1->5->4 [18] 6/-1/-1->5->4 [19] 6/-1/-1->5->4 [20] 6/-1/-1->5->4 [21] 6/-1/-1->5->4 [22] 6/-1/-1->5->4 [23] 6/-1/-1->5->4
195
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO P2P Chunksize set to 524288
196
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 03/24 : 0 1 2 3 4 5 6 7
197
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO P2P Chunksize set to 524288
198
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 04/24 : 0 1 2 3 4 5 6 7
199
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO P2P Chunksize set to 524288
200
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 05/24 : 0 1 2 3 4 5 6 7
201
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 06/24 : 0 1 2 3 4 5 6 7
202
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 07/24 : 0 1 2 3 4 5 6 7
203
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 08/24 : 0 1 2 3 4 5 6 7
204
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 09/24 : 0 1 2 3 4 5 6 7
205
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 10/24 : 0 1 2 3 4 5 6 7
206
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 11/24 : 0 1 2 3 4 5 6 7
207
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 12/24 : 0 1 2 3 4 5 6 7
208
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 13/24 : 0 1 2 3 4 5 6 7
209
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 14/24 : 0 1 2 3 4 5 6 7
210
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 15/24 : 0 1 2 3 4 5 6 7
211
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 16/24 : 0 1 2 3 4 5 6 7
212
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 17/24 : 0 1 2 3 4 5 6 7
213
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 18/24 : 0 1 2 3 4 5 6 7
214
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 19/24 : 0 1 2 3 4 5 6 7
215
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 20/24 : 0 1 2 3 4 5 6 7
216
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 21/24 : 0 1 2 3 4 5 6 7
217
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 22/24 : 0 1 2 3 4 5 6 7
218
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Channel 23/24 : 0 1 2 3 4 5 6 7
219
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 [2] 1/-1/-1->0->-1 [3] 1/-1/-1->0->-1 [4] 1/-1/-1->0->-1 [5] 1/-1/-1->0->-1 [6] 1/-1/-1->0->-1 [7] 1/-1/-1->0->-1 [8] 1/-1/-1->0->-1 [9] 1/-1/-1->0->-1 [10] 1/-1/-1->0->-1 [11] 1/-1/-1->0->-1 [12] 1/-1/-1->0->-1 [13] 1/-1/-1->0->-1 [14] 1/-1/-1->0->-1 [15] 1/-1/-1->0->-1 [16] 1/-1/-1->0->-1 [17] 1/-1/-1->0->-1 [18] 1/-1/-1->0->-1 [19] 1/-1/-1->0->-1 [20] 1/-1/-1->0->-1 [21] 1/-1/-1->0->-1 [22] 1/-1/-1->0->-1 [23] 1/-1/-1->0->-1
220
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO P2P Chunksize set to 524288
221
+ t-20260518041721-mttss-worker-0:10325:10470 [7] NCCL INFO [Proxy Service] Device 7 CPU core 144
222
+ t-20260518041721-mttss-worker-0:10325:10471 [7] NCCL INFO [Proxy Service UDS] Device 7 CPU core 148
223
+ t-20260518041721-mttss-worker-0:10323:10472 [5] NCCL INFO [Proxy Service] Device 5 CPU core 93
224
+ t-20260518041721-mttss-worker-0:10323:10473 [5] NCCL INFO [Proxy Service UDS] Device 5 CPU core 94
225
+ t-20260518041721-mttss-worker-0:10322:10474 [4] NCCL INFO [Proxy Service] Device 4 CPU core 93
226
+ t-20260518041721-mttss-worker-0:10322:10475 [4] NCCL INFO [Proxy Service UDS] Device 4 CPU core 94
227
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Check P2P Type intraNodeP2pSupport 1 directMode 0
228
+ t-20260518041721-mttss-worker-0:10318:10476 [0] NCCL INFO [Proxy Service] Device 0 CPU core 4
229
+ t-20260518041721-mttss-worker-0:10318:10477 [0] NCCL INFO [Proxy Service UDS] Device 0 CPU core 10
230
+ t-20260518041721-mttss-worker-0:10319:10478 [1] NCCL INFO [Proxy Service] Device 1 CPU core 4
231
+ t-20260518041721-mttss-worker-0:10319:10479 [1] NCCL INFO [Proxy Service UDS] Device 1 CPU core 8
232
+ t-20260518041721-mttss-worker-0:10324:10480 [6] NCCL INFO [Proxy Service] Device 6 CPU core 124
233
+ t-20260518041721-mttss-worker-0:10324:10481 [6] NCCL INFO [Proxy Service UDS] Device 6 CPU core 128
234
+ t-20260518041721-mttss-worker-0:10321:10482 [3] NCCL INFO [Proxy Service] Device 3 CPU core 14
235
+ t-20260518041721-mttss-worker-0:10321:10483 [3] NCCL INFO [Proxy Service UDS] Device 3 CPU core 16
236
+ t-20260518041721-mttss-worker-0:10320:10484 [2] NCCL INFO [Proxy Service] Device 2 CPU core 42
237
+ t-20260518041721-mttss-worker-0:10320:10485 [2] NCCL INFO [Proxy Service UDS] Device 2 CPU core 44
238
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
239
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
240
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
241
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
242
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
243
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
244
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
245
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
246
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO CC Off, workFifoBytes 1048576
247
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
248
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
249
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
250
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
251
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
252
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
253
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO threadThresholds 8/8/64 | 64/8/64 | 512 | 512
254
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO 24 coll channels, 24 collnet channels, 16 nvls channels, 32 p2p channels, 32 p2p channels per peer
255
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
256
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
257
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
258
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
259
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
260
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
261
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
262
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
263
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
264
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
265
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
266
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
267
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
268
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
269
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v4 symbol.
270
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
271
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
272
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
273
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO ncclCommInitRankConfig comm 0xb85e190 rank 7 nranks 8 cudaDev 7 nvmlDev 7 busId 75020 commId 0x2f3876d210e12b32 - Init COMPLETE
274
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
275
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO ncclCommInitRankConfig comm 0xa0849c0 rank 4 nranks 8 cudaDev 4 nvmlDev 4 busId 6f020 commId 0x2f3876d210e12b32 - Init COMPLETE
276
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
277
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO ncclCommInitRankConfig comm 0xb06fb20 rank 1 nranks 8 cudaDev 1 nvmlDev 1 busId 67020 commId 0x2f3876d210e12b32 - Init COMPLETE
278
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
279
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
280
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v3 symbol.
281
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO ncclCommInitRankConfig comm 0xb38f670 rank 6 nranks 8 cudaDev 6 nvmlDev 6 busId 73020 commId 0x2f3876d210e12b32 - Init COMPLETE
282
+ t-20260518041721-mttss-worker-0:10325:10397 [7] NCCL INFO Init timings - ncclCommInitRankConfig: rank 7 nranks 8 total 2.26 (kernels 0.51, alloc 0.70, bootstrap 0.01, allgathers 0.02, topo 0.59, graphs 0.01, connections 0.38, rest 0.05)
283
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO ncclCommInitRankConfig comm 0xb62de00 rank 5 nranks 8 cudaDev 5 nvmlDev 5 busId 71020 commId 0x2f3876d210e12b32 - Init COMPLETE
284
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO ncclCommInitRankConfig comm 0xa2915b0 rank 2 nranks 8 cudaDev 2 nvmlDev 2 busId 69020 commId 0x2f3876d210e12b32 - Init COMPLETE
285
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO ncclCommInitRankConfig comm 0xa149c50 rank 3 nranks 8 cudaDev 3 nvmlDev 3 busId 6b020 commId 0x2f3876d210e12b32 - Init COMPLETE
286
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO TUNER/Plugin: Failed to find ncclTunerPlugin_v2 symbol, using internal tuner instead.
287
+ t-20260518041721-mttss-worker-0:10322:10393 [4] NCCL INFO Init timings - ncclCommInitRankConfig: rank 4 nranks 8 total 2.31 (kernels 0.27, alloc 0.79, bootstrap 0.19, allgathers 0.01, topo 0.59, graphs 0.01, connections 0.38, rest 0.06)
288
+ t-20260518041721-mttss-worker-0:10319:10394 [1] NCCL INFO Init timings - ncclCommInitRankConfig: rank 1 nranks 8 total 2.29 (kernels 0.30, alloc 0.79, bootstrap 0.16, allgathers 0.01, topo 0.59, graphs 0.01, connections 0.37, rest 0.06)
289
+ t-20260518041721-mttss-worker-0:10324:10395 [6] NCCL INFO Init timings - ncclCommInitRankConfig: rank 6 nranks 8 total 2.27 (kernels 0.45, alloc 0.75, bootstrap 0.02, allgathers 0.02, topo 0.59, graphs 0.01, connections 0.36, rest 0.06)
290
+ t-20260518041721-mttss-worker-0:10323:10392 [5] NCCL INFO Init timings - ncclCommInitRankConfig: rank 5 nranks 8 total 2.32 (kernels 0.23, alloc 0.73, bootstrap 0.31, allgathers 0.02, topo 0.59, graphs 0.01, connections 0.38, rest 0.05)
291
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO ncclCommInitRankConfig comm 0xbc85d50 rank 0 nranks 8 cudaDev 0 nvmlDev 0 busId 65040 commId 0x2f3876d210e12b32 - Init COMPLETE
292
+ t-20260518041721-mttss-worker-0:10321:10396 [3] NCCL INFO Init timings - ncclCommInitRankConfig: rank 3 nranks 8 total 2.26 (kernels 0.51, alloc 0.71, bootstrap 0.00, allgathers 0.02, topo 0.59, graphs 0.01, connections 0.35, rest 0.08)
293
+ t-20260518041721-mttss-worker-0:10320:10385 [2] NCCL INFO Init timings - ncclCommInitRankConfig: rank 2 nranks 8 total 2.37 (kernels 0.19, alloc 0.59, bootstrap 0.55, allgathers 0.02, topo 0.59, graphs 0.01, connections 0.35, rest 0.08)
294
+ t-20260518041721-mttss-worker-0:10318:10378 [0] NCCL INFO Init timings - ncclCommInitRankConfig: rank 0 nranks 8 total 2.41 (kernels 0.19, alloc 0.45, bootstrap 0.73, allgathers 0.02, topo 0.59, graphs 0.01, connections 0.37, rest 0.06)
295
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 00/0 : 4[4] -> 5[5] via P2P/CUMEM
296
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 01/0 : 4[4] -> 5[5] via P2P/CUMEM
297
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 02/0 : 4[4] -> 5[5] via P2P/CUMEM
298
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 03/0 : 4[4] -> 5[5] via P2P/CUMEM
299
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 04/0 : 4[4] -> 5[5] via P2P/CUMEM
300
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 05/0 : 4[4] -> 5[5] via P2P/CUMEM
301
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 06/0 : 4[4] -> 5[5] via P2P/CUMEM
302
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 07/0 : 4[4] -> 5[5] via P2P/CUMEM
303
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 08/0 : 4[4] -> 5[5] via P2P/CUMEM
304
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 09/0 : 4[4] -> 5[5] via P2P/CUMEM
305
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 10/0 : 4[4] -> 5[5] via P2P/CUMEM
306
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 00/0 : 0[0] -> 1[1] via P2P/CUMEM
307
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 01/0 : 0[0] -> 1[1] via P2P/CUMEM
308
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 02/0 : 0[0] -> 1[1] via P2P/CUMEM
309
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 03/0 : 0[0] -> 1[1] via P2P/CUMEM
310
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 11/0 : 4[4] -> 5[5] via P2P/CUMEM
311
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 04/0 : 0[0] -> 1[1] via P2P/CUMEM
312
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 12/0 : 4[4] -> 5[5] via P2P/CUMEM
313
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 05/0 : 0[0] -> 1[1] via P2P/CUMEM
314
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 13/0 : 4[4] -> 5[5] via P2P/CUMEM
315
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 06/0 : 0[0] -> 1[1] via P2P/CUMEM
316
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 14/0 : 4[4] -> 5[5] via P2P/CUMEM
317
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 07/0 : 0[0] -> 1[1] via P2P/CUMEM
318
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 15/0 : 4[4] -> 5[5] via P2P/CUMEM
319
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 08/0 : 0[0] -> 1[1] via P2P/CUMEM
320
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 16/0 : 4[4] -> 5[5] via P2P/CUMEM
321
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 09/0 : 0[0] -> 1[1] via P2P/CUMEM
322
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 00/0 : 3[3] -> 4[4] via P2P/CUMEM
323
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 17/0 : 4[4] -> 5[5] via P2P/CUMEM
324
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 18/0 : 4[4] -> 5[5] via P2P/CUMEM
325
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 10/0 : 0[0] -> 1[1] via P2P/CUMEM
326
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 19/0 : 4[4] -> 5[5] via P2P/CUMEM
327
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 11/0 : 0[0] -> 1[1] via P2P/CUMEM
328
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 01/0 : 3[3] -> 4[4] via P2P/CUMEM
329
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 20/0 : 4[4] -> 5[5] via P2P/CUMEM
330
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 12/0 : 0[0] -> 1[1] via P2P/CUMEM
331
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 02/0 : 3[3] -> 4[4] via P2P/CUMEM
332
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 21/0 : 4[4] -> 5[5] via P2P/CUMEM
333
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 13/0 : 0[0] -> 1[1] via P2P/CUMEM
334
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 03/0 : 3[3] -> 4[4] via P2P/CUMEM
335
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 04/0 : 3[3] -> 4[4] via P2P/CUMEM
336
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 05/0 : 3[3] -> 4[4] via P2P/CUMEM
337
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 22/0 : 4[4] -> 5[5] via P2P/CUMEM
338
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 00/0 : 5[5] -> 6[6] via P2P/CUMEM
339
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 06/0 : 3[3] -> 4[4] via P2P/CUMEM
340
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 14/0 : 0[0] -> 1[1] via P2P/CUMEM
341
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 01/0 : 5[5] -> 6[6] via P2P/CUMEM
342
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Channel 23/0 : 4[4] -> 5[5] via P2P/CUMEM
343
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 07/0 : 3[3] -> 4[4] via P2P/CUMEM
344
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 15/0 : 0[0] -> 1[1] via P2P/CUMEM
345
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 08/0 : 3[3] -> 4[4] via P2P/CUMEM
346
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 16/0 : 0[0] -> 1[1] via P2P/CUMEM
347
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 09/0 : 3[3] -> 4[4] via P2P/CUMEM
348
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 00/0 : 2[2] -> 3[3] via P2P/CUMEM
349
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 17/0 : 0[0] -> 1[1] via P2P/CUMEM
350
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 10/0 : 3[3] -> 4[4] via P2P/CUMEM
351
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 01/0 : 2[2] -> 3[3] via P2P/CUMEM
352
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 18/0 : 0[0] -> 1[1] via P2P/CUMEM
353
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 11/0 : 3[3] -> 4[4] via P2P/CUMEM
354
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 02/0 : 2[2] -> 3[3] via P2P/CUMEM
355
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 19/0 : 0[0] -> 1[1] via P2P/CUMEM
356
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 12/0 : 3[3] -> 4[4] via P2P/CUMEM
357
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 03/0 : 2[2] -> 3[3] via P2P/CUMEM
358
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 20/0 : 0[0] -> 1[1] via P2P/CUMEM
359
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 13/0 : 3[3] -> 4[4] via P2P/CUMEM
360
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 04/0 : 2[2] -> 3[3] via P2P/CUMEM
361
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 00/0 : 1[1] -> 2[2] via P2P/CUMEM
362
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 02/0 : 5[5] -> 6[6] via P2P/CUMEM
363
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 21/0 : 0[0] -> 1[1] via P2P/CUMEM
364
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 14/0 : 3[3] -> 4[4] via P2P/CUMEM
365
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 05/0 : 2[2] -> 3[3] via P2P/CUMEM
366
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 01/0 : 1[1] -> 2[2] via P2P/CUMEM
367
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 03/0 : 5[5] -> 6[6] via P2P/CUMEM
368
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 22/0 : 0[0] -> 1[1] via P2P/CUMEM
369
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 15/0 : 3[3] -> 4[4] via P2P/CUMEM
370
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 00/0 : 6[6] -> 7[7] via P2P/CUMEM
371
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 06/0 : 2[2] -> 3[3] via P2P/CUMEM
372
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 02/0 : 1[1] -> 2[2] via P2P/CUMEM
373
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 04/0 : 5[5] -> 6[6] via P2P/CUMEM
374
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Channel 23/0 : 0[0] -> 1[1] via P2P/CUMEM
375
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 16/0 : 3[3] -> 4[4] via P2P/CUMEM
376
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 01/0 : 6[6] -> 7[7] via P2P/CUMEM
377
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 07/0 : 2[2] -> 3[3] via P2P/CUMEM
378
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 03/0 : 1[1] -> 2[2] via P2P/CUMEM
379
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 05/0 : 5[5] -> 6[6] via P2P/CUMEM
380
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 17/0 : 3[3] -> 4[4] via P2P/CUMEM
381
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 02/0 : 6[6] -> 7[7] via P2P/CUMEM
382
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 08/0 : 2[2] -> 3[3] via P2P/CUMEM
383
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 04/0 : 1[1] -> 2[2] via P2P/CUMEM
384
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 06/0 : 5[5] -> 6[6] via P2P/CUMEM
385
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 18/0 : 3[3] -> 4[4] via P2P/CUMEM
386
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 03/0 : 6[6] -> 7[7] via P2P/CUMEM
387
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 09/0 : 2[2] -> 3[3] via P2P/CUMEM
388
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 05/0 : 1[1] -> 2[2] via P2P/CUMEM
389
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 07/0 : 5[5] -> 6[6] via P2P/CUMEM
390
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 19/0 : 3[3] -> 4[4] via P2P/CUMEM
391
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 04/0 : 6[6] -> 7[7] via P2P/CUMEM
392
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 10/0 : 2[2] -> 3[3] via P2P/CUMEM
393
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 06/0 : 1[1] -> 2[2] via P2P/CUMEM
394
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 08/0 : 5[5] -> 6[6] via P2P/CUMEM
395
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 20/0 : 3[3] -> 4[4] via P2P/CUMEM
396
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 11/0 : 2[2] -> 3[3] via P2P/CUMEM
397
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 05/0 : 6[6] -> 7[7] via P2P/CUMEM
398
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 07/0 : 1[1] -> 2[2] via P2P/CUMEM
399
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 09/0 : 5[5] -> 6[6] via P2P/CUMEM
400
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 21/0 : 3[3] -> 4[4] via P2P/CUMEM
401
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 12/0 : 2[2] -> 3[3] via P2P/CUMEM
402
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 06/0 : 6[6] -> 7[7] via P2P/CUMEM
403
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 08/0 : 1[1] -> 2[2] via P2P/CUMEM
404
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 10/0 : 5[5] -> 6[6] via P2P/CUMEM
405
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 22/0 : 3[3] -> 4[4] via P2P/CUMEM
406
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 13/0 : 2[2] -> 3[3] via P2P/CUMEM
407
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 07/0 : 6[6] -> 7[7] via P2P/CUMEM
408
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 09/0 : 1[1] -> 2[2] via P2P/CUMEM
409
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 11/0 : 5[5] -> 6[6] via P2P/CUMEM
410
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 00/0 : 7[7] -> 0[0] via P2P/CUMEM
411
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 14/0 : 2[2] -> 3[3] via P2P/CUMEM
412
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Channel 23/0 : 3[3] -> 4[4] via P2P/CUMEM
413
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 08/0 : 6[6] -> 7[7] via P2P/CUMEM
414
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 10/0 : 1[1] -> 2[2] via P2P/CUMEM
415
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 12/0 : 5[5] -> 6[6] via P2P/CUMEM
416
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 01/0 : 7[7] -> 0[0] via P2P/CUMEM
417
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 15/0 : 2[2] -> 3[3] via P2P/CUMEM
418
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 09/0 : 6[6] -> 7[7] via P2P/CUMEM
419
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 11/0 : 1[1] -> 2[2] via P2P/CUMEM
420
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 13/0 : 5[5] -> 6[6] via P2P/CUMEM
421
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 02/0 : 7[7] -> 0[0] via P2P/CUMEM
422
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 16/0 : 2[2] -> 3[3] via P2P/CUMEM
423
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 10/0 : 6[6] -> 7[7] via P2P/CUMEM
424
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 12/0 : 1[1] -> 2[2] via P2P/CUMEM
425
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 14/0 : 5[5] -> 6[6] via P2P/CUMEM
426
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 03/0 : 7[7] -> 0[0] via P2P/CUMEM
427
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 17/0 : 2[2] -> 3[3] via P2P/CUMEM
428
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 11/0 : 6[6] -> 7[7] via P2P/CUMEM
429
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 13/0 : 1[1] -> 2[2] via P2P/CUMEM
430
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 15/0 : 5[5] -> 6[6] via P2P/CUMEM
431
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 04/0 : 7[7] -> 0[0] via P2P/CUMEM
432
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 18/0 : 2[2] -> 3[3] via P2P/CUMEM
433
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 12/0 : 6[6] -> 7[7] via P2P/CUMEM
434
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 14/0 : 1[1] -> 2[2] via P2P/CUMEM
435
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 16/0 : 5[5] -> 6[6] via P2P/CUMEM
436
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 05/0 : 7[7] -> 0[0] via P2P/CUMEM
437
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 19/0 : 2[2] -> 3[3] via P2P/CUMEM
438
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 13/0 : 6[6] -> 7[7] via P2P/CUMEM
439
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 15/0 : 1[1] -> 2[2] via P2P/CUMEM
440
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 17/0 : 5[5] -> 6[6] via P2P/CUMEM
441
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 06/0 : 7[7] -> 0[0] via P2P/CUMEM
442
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 20/0 : 2[2] -> 3[3] via P2P/CUMEM
443
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 14/0 : 6[6] -> 7[7] via P2P/CUMEM
444
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 16/0 : 1[1] -> 2[2] via P2P/CUMEM
445
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 18/0 : 5[5] -> 6[6] via P2P/CUMEM
446
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 07/0 : 7[7] -> 0[0] via P2P/CUMEM
447
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 21/0 : 2[2] -> 3[3] via P2P/CUMEM
448
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 15/0 : 6[6] -> 7[7] via P2P/CUMEM
449
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 17/0 : 1[1] -> 2[2] via P2P/CUMEM
450
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 19/0 : 5[5] -> 6[6] via P2P/CUMEM
451
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 08/0 : 7[7] -> 0[0] via P2P/CUMEM
452
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 22/0 : 2[2] -> 3[3] via P2P/CUMEM
453
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 16/0 : 6[6] -> 7[7] via P2P/CUMEM
454
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 18/0 : 1[1] -> 2[2] via P2P/CUMEM
455
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 20/0 : 5[5] -> 6[6] via P2P/CUMEM
456
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 09/0 : 7[7] -> 0[0] via P2P/CUMEM
457
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Channel 23/0 : 2[2] -> 3[3] via P2P/CUMEM
458
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 17/0 : 6[6] -> 7[7] via P2P/CUMEM
459
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 19/0 : 1[1] -> 2[2] via P2P/CUMEM
460
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 21/0 : 5[5] -> 6[6] via P2P/CUMEM
461
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 10/0 : 7[7] -> 0[0] via P2P/CUMEM
462
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 18/0 : 6[6] -> 7[7] via P2P/CUMEM
463
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 20/0 : 1[1] -> 2[2] via P2P/CUMEM
464
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 22/0 : 5[5] -> 6[6] via P2P/CUMEM
465
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 11/0 : 7[7] -> 0[0] via P2P/CUMEM
466
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Channel 23/0 : 5[5] -> 6[6] via P2P/CUMEM
467
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 12/0 : 7[7] -> 0[0] via P2P/CUMEM
468
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 21/0 : 1[1] -> 2[2] via P2P/CUMEM
469
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 19/0 : 6[6] -> 7[7] via P2P/CUMEM
470
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 22/0 : 1[1] -> 2[2] via P2P/CUMEM
471
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 13/0 : 7[7] -> 0[0] via P2P/CUMEM
472
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 20/0 : 6[6] -> 7[7] via P2P/CUMEM
473
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Channel 23/0 : 1[1] -> 2[2] via P2P/CUMEM
474
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 14/0 : 7[7] -> 0[0] via P2P/CUMEM
475
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 21/0 : 6[6] -> 7[7] via P2P/CUMEM
476
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 15/0 : 7[7] -> 0[0] via P2P/CUMEM
477
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 22/0 : 6[6] -> 7[7] via P2P/CUMEM
478
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 16/0 : 7[7] -> 0[0] via P2P/CUMEM
479
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Channel 23/0 : 6[6] -> 7[7] via P2P/CUMEM
480
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 17/0 : 7[7] -> 0[0] via P2P/CUMEM
481
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 18/0 : 7[7] -> 0[0] via P2P/CUMEM
482
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 19/0 : 7[7] -> 0[0] via P2P/CUMEM
483
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 20/0 : 7[7] -> 0[0] via P2P/CUMEM
484
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 21/0 : 7[7] -> 0[0] via P2P/CUMEM
485
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 22/0 : 7[7] -> 0[0] via P2P/CUMEM
486
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Channel 23/0 : 7[7] -> 0[0] via P2P/CUMEM
487
+ t-20260518041721-mttss-worker-0:10324:10488 [6] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
488
+ t-20260518041721-mttss-worker-0:10323:10489 [5] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
489
+ t-20260518041721-mttss-worker-0:10325:10493 [7] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
490
+ t-20260518041721-mttss-worker-0:10321:10490 [3] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
491
+ t-20260518041721-mttss-worker-0:10322:10487 [4] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
492
+ t-20260518041721-mttss-worker-0:10318:10492 [0] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
493
+ t-20260518041721-mttss-worker-0:10319:10486 [1] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
494
+ t-20260518041721-mttss-worker-0:10320:10491 [2] NCCL INFO Connected all rings, use ring PXN 0 GDR 1
495
+ {
496
+ "device": "cuda:0",
497
+ "rank": 0,
498
+ "world_size": 8,
499
+ "samples": "owt_cached_chunks:8734897",
500
+ "vocab_size": 50257,
501
+ "tokenizer_vocab_size": 50257,
502
+ "save_dir": "runs/lta_owt_dirichlet_categorical_fullvocab_c1024_fullycoupled_gpt2cached_len1024_rollin_p0p5_s4_i32_outwd-1_gbs512_8gpu_1m_20260517_201804",
503
+ "batch_size": 32,
504
+ "grad_accum": 2,
505
+ "effective_batch_size": 512,
506
+ "global_batch_size": 512,
507
+ "lr_schedule": "cosine",
508
+ "optimizer": "adamw",
509
+ "epochs": 0.0,
510
+ "steps_per_epoch": 17061,
511
+ "total_steps": 1000000,
512
+ "warmup_steps": 2000,
513
+ "warmup_epochs": -1.0,
514
+ "min_lr": 6e-05,
515
+ "weight_decay": 0.1,
516
+ "output_weight_decay": -1.0,
517
+ "adamw_param_groups": "nanogpt",
518
+ "adam_beta1": 0.9,
519
+ "adam_beta2": 0.95,
520
+ "adam_eps": 1e-08,
521
+ "muon_impl": "legacy",
522
+ "muon_momentum": 0.95,
523
+ "muon_ns_steps": 5,
524
+ "muon_update_scale": 1.0,
525
+ "muon_nesterov": false,
526
+ "muon_width_scale": false,
527
+ "muon_grouping": "",
528
+ "muon_param_count": 0,
529
+ "muon_adam_param_count": 0,
530
+ "muon_param_names": [],
531
+ "muon_adam_param_names": [],
532
+ "muon_effective_nesterov": false,
533
+ "muon_effective_width_scale": false,
534
+ "muon_effective_weight_decay": 0.1,
535
+ "muon_adam_fallback_nesterov": false,
536
+ "muon_adam_fallback_weight_decay": 0.1,
537
+ "ema_decay": 0.0,
538
+ "ema_start_step": 0,
539
+ "model_type": "ddit",
540
+ "ddit_mlp_type": "gelu",
541
+ "elf_num_time_tokens": 4,
542
+ "elf_num_model_mode_tokens": 0,
543
+ "qk_norm": true,
544
+ "output_bias": false,
545
+ "output_init_std": -1.0,
546
+ "norm_type": "rmsnorm",
547
+ "target_loss": "hard_ce",
548
+ "linear_soft_target_power": 1.0,
549
+ "linear_soft_target_min_conf": 0.0,
550
+ "linear_soft_target_max_conf": 1.0,
551
+ "t_sampling_mode": "uniform",
552
+ "t_sampling_power": 1.0,
553
+ "t_sampling_eps": 0.0001,
554
+ "t_sampling_logit_mean": -1.5,
555
+ "t_sampling_logit_std": 0.8,
556
+ "dual_t": true,
557
+ "corrupt_t_mode": "same",
558
+ "corrupt_min_t": 0.0,
559
+ "corrupt_max_t": 1.0,
560
+ "prefix_block_prob": 0.0,
561
+ "prefix_block_len": 128,
562
+ "mask_ratio_floor_schedule": "none",
563
+ "dirichlet_endpoint_mode": "categorical_dual_t",
564
+ "dirichlet_semantic_t_mode": "same",
565
+ "dirichlet_semantic_t_value": 0.0,
566
+ "dirichlet_semantic_t_curve": "linear",
567
+ "dirichlet_semantic_t_power": 1.0,
568
+ "endpoint_sequence_random_prob_alpha": 0.0,
569
+ "categorical_wrong_from_full_vocab": true,
570
+ "categorical_wrong_from_batch_valid_tokens": false,
571
+ "categorical_wrong_basin_token_ids": "",
572
+ "categorical_wrong_basin_prob": 0.0,
573
+ "categorical_wrong_unigram_prob": 0.0,
574
+ "categorical_wrong_uniform_prob": 0.0,
575
+ "categorical_wrong_prob_floor": 0.0,
576
+ "categorical_wrong_corpus_unigram_path": "",
577
+ "categorical_wrong_corpus_unigram_alpha": 1.0,
578
+ "categorical_wrong_basin_shared_prob": 0.0,
579
+ "categorical_wrong_unigram_shared_prob": 0.0,
580
+ "mask_mixture_original_prob": 0.0,
581
+ "mask_mixture_lowk_prob": 0.0,
582
+ "mask_mixture_lowcorrupt_prob": 0.0,
583
+ "mask_mixture_block_prob": 0.0,
584
+ "mask_mixture_all_prob": 0.0,
585
+ "mask_mixture_lowk_clean_tokens": "1,2,4,8,16,32,64",
586
+ "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
587
+ "mask_mixture_block_tokens": "64,128",
588
+ "simplex_bridge_sampler": "dirichlet",
589
+ "logistic_normal_sigma_min": 0.18,
590
+ "logistic_normal_sigma_max": 2.2,
591
+ "logistic_normal_tau_min": 0.65,
592
+ "logistic_normal_tau_max": 1.15,
593
+ "torch_compile": false,
594
+ "compile_mode": "max-autotune",
595
+ "state_format": "prob",
596
+ "meanflow_weight": 0.0,
597
+ "rollout_train_prob": 0.5,
598
+ "rollout_train_steps": 4,
599
+ "rollout_train_infer_steps": 32,
600
+ "rollout_train_temp": 1.45,
601
+ "rollout_train_max_gamma": 1.0,
602
+ "rollout_train_corrupt_only": true,
603
+ "rollout_train_samplewise": true,
604
+ "rollout_train_compute_always": false,
605
+ "rollout_train_sync_t": false,
606
+ "bridge_noise_init": "logistic_normal",
607
+ "noise_sigma": -1.0,
608
+ "allow_tf32": true,
609
+ "activation_checkpointing": false,
610
+ "activation_checkpoint_interval": 1,
611
+ "activation_checkpoint_scope": "block",
612
+ "ddp_static_graph": false,
613
+ "ddp_gradient_as_bucket_view": true,
614
+ "blocking_data_transfer": false,
615
+ "dataloader_prefetch_factor": 2,
616
+ "full_train_stats": false,
617
+ "tokenized_hf": false,
618
+ "tokenized_pad_token": "pad",
619
+ "elf_conditional_hf": false,
620
+ "record_pad_truncate": false,
621
+ "record_add_eos": false,
622
+ "record_add_special_tokens": false,
623
+ "record_pad_token": "pad",
624
+ "record_shuffle_buffer": 10000,
625
+ "wrap": true,
626
+ "wrap_mode": "stream",
627
+ "wrap_record_buffer_size": 200,
628
+ "owt_cached_chunks": true,
629
+ "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train_minus_100k",
630
+ "owt_chunk_cache_rebuild": false,
631
+ "owt_chunk_cache_write_batch": 4096,
632
+ "owt_exact_repeat_per_chunk": 0,
633
+ "online_chunk_shuffle": false,
634
+ "online_chunk_shuffle_buffer": 10000,
635
+ "openwebtext_split": "train_minus_100k",
636
+ "detokenizer": "auto",
637
+ "resolved_detokenizer": null,
638
+ "num_workers": 4,
639
+ "latest_every": 1000,
640
+ "resume_path": ""
641
+ }
642
+ t-20260518041721-mttss-worker-0:10323:10718 [5] NCCL INFO NVLS comm 0xb62de00 headRank 5 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
643
+ t-20260518041721-mttss-worker-0:10322:10719 [4] NCCL INFO NVLS comm 0xa0849c0 headRank 4 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
644
+ t-20260518041721-mttss-worker-0:10324:10720 [6] NCCL INFO NVLS comm 0xb38f670 headRank 6 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
645
+ t-20260518041721-mttss-worker-0:10325:10721 [7] NCCL INFO NVLS comm 0xb85e190 headRank 7 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
646
+ t-20260518041721-mttss-worker-0:10319:10722 [1] NCCL INFO NVLS comm 0xb06fb20 headRank 1 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
647
+ t-20260518041721-mttss-worker-0:10321:10723 [3] NCCL INFO NVLS comm 0xa149c50 headRank 3 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
648
+ t-20260518041721-mttss-worker-0:10320:10724 [2] NCCL INFO NVLS comm 0xa2915b0 headRank 2 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
649
+ t-20260518041721-mttss-worker-0:10318:10725 [0] NCCL INFO NVLS comm 0xbc85d50 headRank 0 nHeads 8 buffSize 1048576 nvlsPerRankSize 33554432 nvlsTotalSize 268435456
650
+ step=100 epoch=1/59 epoch_step=100/17061 micro_steps=200 elapsed=198.7s lr=3.030000e-05 loss=10.4221 loss_recon=10.4221 loss_meanflow=0.0000 mean_model_t=0.4957 mean_corrupt_t=0.4957 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5038 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3987 corrupt_frac=0.5429 acc_corrupt=0.2200 loss_corrupt=10.4221 wrong_frac=0.5015 init_acc_corrupt=0.4061 acc_corrupt_t_0p0_0p2=0.0371 corrupt_frac_t_0p0_0p2=0.2023 acc_corrupt_t_0p2_0p4=0.1318 corrupt_frac_t_0p2_0p4=0.1948 acc_corrupt_t_0p4_0p6=0.2538 corrupt_frac_t_0p4_0p6=0.2114 acc_corrupt_t_0p6_0p8=0.3604 corrupt_frac_t_0p6_0p8=0.1961 acc_corrupt_t_0p8_1p0=0.3188 corrupt_frac_t_0p8_1p0=0.1963 out_w_norm=2.4721 out_g_norm=1.3885 loss_all=9.4921 init_gold_top10=0.3802 init_gold_top100=0.4676 rollout_applied_pos_frac=0.3793 init_acc_rollout_applied=0.3813 init_acc_rollout_kept=0.3246 logit_acc_rollout_applied=0.0355 logit_acc_rollout_kept=0.0500
651
+ step=200 epoch=1/59 epoch_step=200/17061 micro_steps=400 elapsed=198.1s lr=6.030000e-05 loss=8.2416 loss_recon=8.2416 loss_meanflow=0.0000 mean_model_t=0.5036 mean_corrupt_t=0.5036 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5072 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.0816 corrupt_frac=0.5536 acc_corrupt=0.0496 loss_corrupt=8.2416 wrong_frac=0.4951 init_acc_corrupt=0.4116 acc_corrupt_t_0p0_0p2=0.0349 corrupt_frac_t_0p0_0p2=0.1945 acc_corrupt_t_0p2_0p4=0.0383 corrupt_frac_t_0p2_0p4=0.1988 acc_corrupt_t_0p4_0p6=0.0460 corrupt_frac_t_0p4_0p6=0.1999 acc_corrupt_t_0p6_0p8=0.0565 corrupt_frac_t_0p6_0p8=0.2062 acc_corrupt_t_0p8_1p0=0.0714 corrupt_frac_t_0p8_1p0=0.2006 out_w_norm=16.2238 out_g_norm=1.6261 loss_all=7.0861 init_gold_top10=0.4720 init_gold_top100=0.6049 rollout_applied_pos_frac=0.5360 init_acc_rollout_applied=0.3968 init_acc_rollout_kept=0.4669 logit_acc_rollout_applied=0.0647 logit_acc_rollout_kept=0.0823
652
+ step=300 epoch=1/59 epoch_step=300/17061 micro_steps=600 elapsed=198.2s lr=9.030000e-05 loss=6.1058 loss_recon=6.1058 loss_meanflow=0.0000 mean_model_t=0.4968 mean_corrupt_t=0.4968 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4988 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.3637 corrupt_frac=0.5526 acc_corrupt=0.2361 loss_corrupt=6.1058 wrong_frac=0.5036 init_acc_corrupt=0.4184 acc_corrupt_t_0p0_0p2=0.0488 corrupt_frac_t_0p0_0p2=0.2036 acc_corrupt_t_0p2_0p4=0.1369 corrupt_frac_t_0p2_0p4=0.2011 acc_corrupt_t_0p4_0p6=0.2582 corrupt_frac_t_0p4_0p6=0.2002 acc_corrupt_t_0p6_0p8=0.3703 corrupt_frac_t_0p6_0p8=0.2040 acc_corrupt_t_0p8_1p0=0.3732 corrupt_frac_t_0p8_1p0=0.1933 out_w_norm=29.6194 out_g_norm=0.5237 loss_all=2.9747 init_gold_top10=0.5026 init_gold_top100=0.6129 rollout_applied_pos_frac=0.4836 init_acc_rollout_applied=0.4245 init_acc_rollout_kept=0.5271 logit_acc_rollout_applied=0.3860 logit_acc_rollout_kept=0.4452
653
+ step=400 epoch=1/59 epoch_step=400/17061 micro_steps=800 elapsed=198.2s lr=1.203000e-04 loss=4.5389 loss_recon=4.5389 loss_meanflow=0.0000 mean_model_t=0.5047 mean_corrupt_t=0.5047 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5083 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6617 corrupt_frac=0.5494 acc_corrupt=0.4456 loss_corrupt=4.5389 wrong_frac=0.4962 init_acc_corrupt=0.4585 acc_corrupt_t_0p0_0p2=0.0571 corrupt_frac_t_0p0_0p2=0.1906 acc_corrupt_t_0p2_0p4=0.2447 corrupt_frac_t_0p2_0p4=0.2033 acc_corrupt_t_0p4_0p6=0.4699 corrupt_frac_t_0p4_0p6=0.2045 acc_corrupt_t_0p6_0p8=0.6551 corrupt_frac_t_0p6_0p8=0.1963 acc_corrupt_t_0p8_1p0=0.7801 corrupt_frac_t_0p8_1p0=0.2062 out_w_norm=38.4290 out_g_norm=0.3547 loss_all=2.5956 init_gold_top10=0.4494 init_gold_top100=0.5732 rollout_applied_pos_frac=0.5930 init_acc_rollout_applied=0.4734 init_acc_rollout_kept=0.3075 logit_acc_rollout_applied=0.4816 logit_acc_rollout_kept=0.3247
654
+ step=500 epoch=1/59 epoch_step=500/17061 micro_steps=1000 elapsed=198.1s lr=1.503000e-04 loss=4.3322 loss_recon=4.3322 loss_meanflow=0.0000 mean_model_t=0.5021 mean_corrupt_t=0.5021 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4898 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6976 corrupt_frac=0.5476 acc_corrupt=0.4685 loss_corrupt=4.3322 wrong_frac=0.5001 init_acc_corrupt=0.4607 acc_corrupt_t_0p0_0p2=0.0579 corrupt_frac_t_0p0_0p2=0.1936 acc_corrupt_t_0p2_0p4=0.2444 corrupt_frac_t_0p2_0p4=0.2080 acc_corrupt_t_0p4_0p6=0.4943 corrupt_frac_t_0p4_0p6=0.2043 acc_corrupt_t_0p6_0p8=0.6959 corrupt_frac_t_0p6_0p8=0.1959 acc_corrupt_t_0p8_1p0=0.8533 corrupt_frac_t_0p8_1p0=0.1982 out_w_norm=43.1801 out_g_norm=0.4719 loss_all=2.2372 init_gold_top10=0.4701 init_gold_top100=0.5680 rollout_applied_pos_frac=0.5047 init_acc_rollout_applied=0.5660 init_acc_rollout_kept=0.2967 logit_acc_rollout_applied=0.5707 logit_acc_rollout_kept=0.3183
655
+ step=600 epoch=1/59 epoch_step=600/17061 micro_steps=1200 elapsed=198.2s lr=1.803000e-04 loss=4.2636 loss_recon=4.2636 loss_meanflow=0.0000 mean_model_t=0.5009 mean_corrupt_t=0.5009 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5016 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.6979 corrupt_frac=0.5442 acc_corrupt=0.4748 loss_corrupt=4.2636 wrong_frac=0.5027 init_acc_corrupt=0.4597 acc_corrupt_t_0p0_0p2=0.0641 corrupt_frac_t_0p0_0p2=0.2000 acc_corrupt_t_0p2_0p4=0.2544 corrupt_frac_t_0p2_0p4=0.2097 acc_corrupt_t_0p4_0p6=0.4976 corrupt_frac_t_0p4_0p6=0.1962 acc_corrupt_t_0p6_0p8=0.6978 corrupt_frac_t_0p6_0p8=0.1925 acc_corrupt_t_0p8_1p0=0.8763 corrupt_frac_t_0p8_1p0=0.2015 out_w_norm=46.3865 out_g_norm=0.4602 loss_all=2.6210 init_gold_top10=0.4605 init_gold_top100=0.5593 rollout_applied_pos_frac=0.4724 init_acc_rollout_applied=0.5274 init_acc_rollout_kept=0.2993 logit_acc_rollout_applied=0.5498 logit_acc_rollout_kept=0.3387
656
+ step=700 epoch=1/59 epoch_step=700/17061 micro_steps=1400 elapsed=198.2s lr=2.103000e-04 loss=4.0183 loss_recon=4.0183 loss_meanflow=0.0000 mean_model_t=0.4960 mean_corrupt_t=0.4960 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.5052 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7082 corrupt_frac=0.5415 acc_corrupt=0.4965 loss_corrupt=4.0183 wrong_frac=0.5023 init_acc_corrupt=0.4637 acc_corrupt_t_0p0_0p2=0.0885 corrupt_frac_t_0p0_0p2=0.2035 acc_corrupt_t_0p2_0p4=0.2857 corrupt_frac_t_0p2_0p4=0.1984 acc_corrupt_t_0p4_0p6=0.5180 corrupt_frac_t_0p4_0p6=0.2022 acc_corrupt_t_0p6_0p8=0.7111 corrupt_frac_t_0p6_0p8=0.1992 acc_corrupt_t_0p8_1p0=0.8915 corrupt_frac_t_0p8_1p0=0.1968 out_w_norm=49.4214 out_g_norm=0.4775 loss_all=2.4216 init_gold_top10=0.5060 init_gold_top100=0.6303 rollout_applied_pos_frac=0.5217 init_acc_rollout_applied=0.4598 init_acc_rollout_kept=0.4608 logit_acc_rollout_applied=0.4877 logit_acc_rollout_kept=0.5123
657
+ step=800 epoch=1/59 epoch_step=800/17061 micro_steps=1600 elapsed=198.2s lr=2.403000e-04 loss=3.7500 loss_recon=3.7500 loss_meanflow=0.0000 mean_model_t=0.4982 mean_corrupt_t=0.4982 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4995 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7184 corrupt_frac=0.5435 acc_corrupt=0.5141 loss_corrupt=3.7500 wrong_frac=0.5008 init_acc_corrupt=0.4665 acc_corrupt_t_0p0_0p2=0.1102 corrupt_frac_t_0p0_0p2=0.2006 acc_corrupt_t_0p2_0p4=0.3047 corrupt_frac_t_0p2_0p4=0.2002 acc_corrupt_t_0p4_0p6=0.5365 corrupt_frac_t_0p4_0p6=0.2053 acc_corrupt_t_0p6_0p8=0.7199 corrupt_frac_t_0p6_0p8=0.1959 acc_corrupt_t_0p8_1p0=0.9011 corrupt_frac_t_0p8_1p0=0.2011 out_w_norm=52.8468 out_g_norm=0.4661 loss_all=2.3611 init_gold_top10=0.4487 init_gold_top100=0.5721 rollout_applied_pos_frac=0.4284 init_acc_rollout_applied=0.4275 init_acc_rollout_kept=0.3689 logit_acc_rollout_applied=0.4749 logit_acc_rollout_kept=0.4374
658
+ step=900 epoch=1/59 epoch_step=900/17061 micro_steps=1800 elapsed=198.2s lr=2.703000e-04 loss=3.4955 loss_recon=3.4955 loss_meanflow=0.0000 mean_model_t=0.5009 mean_corrupt_t=0.5009 mean_loss_t_weight=1.0000 linear_soft_target_mean_conf=0.0000 prior_center_loss_beta=0.0000 rollout_train_applied=0.4984 grad_enabled_before_rollout=1.0000 grad_enabled_after_rollout=1.0000 logits_requires_grad=1.0000 raw_loss_requires_grad=1.0000 acc_all=0.7286 corrupt_frac=0.5526 acc_corrupt=0.5344 loss_corrupt=3.4955 wrong_frac=0.5007 init_acc_corrupt=0.4667 acc_corrupt_t_0p0_0p2=0.1349 corrupt_frac_t_0p0_0p2=0.1925 acc_corrupt_t_0p2_0p4=0.3312 corrupt_frac_t_0p2_0p4=0.2070 acc_corrupt_t_0p4_0p6=0.5646 corrupt_frac_t_0p4_0p6=0.2098 acc_corrupt_t_0p6_0p8=0.7408 corrupt_frac_t_0p6_0p8=0.1950 acc_corrupt_t_0p8_1p0=0.9043 corrupt_frac_t_0p8_1p0=0.1957 out_w_norm=56.3063 out_g_norm=0.4883 loss_all=2.3322 init_gold_top10=0.4710 init_gold_top100=0.6259 rollout_applied_pos_frac=0.5629 init_acc_rollout_applied=0.4131 init_acc_rollout_kept=0.3983 logit_acc_rollout_applied=0.4993 logit_acc_rollout_kept=0.4578
LTA_openwebtext_dualt/logs/lta_owt_gpt2cached_len1024_rollout1_p05_smoke4gpu_20260513_143811.log ADDED
@@ -0,0 +1,231 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ NCCL version 2.25.1+cuda12.8
2
+ {
3
+ "device": "cuda:0",
4
+ "rank": 0,
5
+ "world_size": 4,
6
+ "samples": "owt_cached_chunks:8734897",
7
+ "vocab_size": 50257,
8
+ "tokenizer_vocab_size": 50257,
9
+ "save_dir": "runs/lta_owt_gpt2cached_len1024_rollout1_p05_smoke4gpu_20260513_143811",
10
+ "batch_size": 2,
11
+ "grad_accum": 1,
12
+ "effective_batch_size": 8,
13
+ "global_batch_size": 8,
14
+ "lr_schedule": "cosine",
15
+ "optimizer": "adamw",
16
+ "warmup_steps": 1,
17
+ "min_lr": 6e-05,
18
+ "weight_decay": 0.1,
19
+ "adamw_param_groups": "nanogpt",
20
+ "adam_beta1": 0.9,
21
+ "adam_beta2": 0.95,
22
+ "adam_eps": 1e-08,
23
+ "muon_momentum": 0.95,
24
+ "muon_ns_steps": 5,
25
+ "muon_update_scale": 1.0,
26
+ "ema_decay": 0.0,
27
+ "ema_start_step": 0,
28
+ "model_type": "ddit",
29
+ "dual_t": true,
30
+ "corrupt_t_mode": "same",
31
+ "corrupt_min_t": 0.0,
32
+ "corrupt_max_t": 1.0,
33
+ "prefix_block_prob": 0.0,
34
+ "prefix_block_len": 128,
35
+ "dirichlet_endpoint_mode": "categorical_dual_t",
36
+ "dirichlet_semantic_t_mode": "same",
37
+ "dirichlet_semantic_t_value": 0.0,
38
+ "categorical_wrong_from_full_vocab": true,
39
+ "categorical_wrong_from_batch_valid_tokens": false,
40
+ "mask_mixture_original_prob": 0.0,
41
+ "mask_mixture_lowk_prob": 0.0,
42
+ "mask_mixture_lowcorrupt_prob": 0.0,
43
+ "mask_mixture_block_prob": 0.0,
44
+ "mask_mixture_all_prob": 0.0,
45
+ "mask_mixture_lowk_clean_tokens": "1,2,4,8,16,32,64",
46
+ "mask_mixture_lowcorrupt_tokens": "1,2,4,8,16,32,64",
47
+ "mask_mixture_block_tokens": "64,128",
48
+ "simplex_bridge_sampler": "dirichlet",
49
+ "logistic_normal_sigma_min": 0.18,
50
+ "logistic_normal_sigma_max": 2.2,
51
+ "logistic_normal_tau_min": 0.65,
52
+ "logistic_normal_tau_max": 1.15,
53
+ "torch_compile": false,
54
+ "compile_mode": "max-autotune",
55
+ "state_format": "prob",
56
+ "target_loss": "hard_ce",
57
+ "meanflow_weight": 0.0,
58
+ "rollout_train_prob": 0.5,
59
+ "rollout_train_steps": 1,
60
+ "rollout_train_infer_steps": 64,
61
+ "rollout_train_temp": 1.45,
62
+ "rollout_train_max_gamma": 1.0,
63
+ "rollout_train_corrupt_only": true,
64
+ "bridge_noise_init": "logistic_normal",
65
+ "noise_sigma": -1.0,
66
+ "allow_tf32": true,
67
+ "activation_checkpointing": false,
68
+ "activation_checkpoint_interval": 1,
69
+ "ddp_static_graph": false,
70
+ "ddp_gradient_as_bucket_view": true,
71
+ "blocking_data_transfer": false,
72
+ "dataloader_prefetch_factor": 2,
73
+ "full_train_stats": false,
74
+ "record_pad_truncate": false,
75
+ "record_add_eos": false,
76
+ "record_add_special_tokens": false,
77
+ "record_pad_token": "pad",
78
+ "record_shuffle_buffer": 10000,
79
+ "wrap": true,
80
+ "wrap_mode": "stream",
81
+ "wrap_record_buffer_size": 200,
82
+ "owt_cached_chunks": true,
83
+ "owt_chunk_cache_dir": "/e2e-data/evad-tech-vla/wanghan58/data/small_benchmarks/langflow_2604_11748/openwebtext_lta_cached_chunks/gpt2_len1024_train_minus_100k",
84
+ "owt_chunk_cache_rebuild": false,
85
+ "owt_chunk_cache_write_batch": 4096,
86
+ "owt_exact_repeat_per_chunk": 0,
87
+ "online_chunk_shuffle": false,
88
+ "online_chunk_shuffle_buffer": 10000,
89
+ "openwebtext_split": "train_minus_100k",
90
+ "detokenizer": "auto",
91
+ "resolved_detokenizer": null,
92
+ "num_workers": 0,
93
+ "latest_every": 100000,
94
+ "resume_path": ""
95
+ }
96
+ step=1 micro_steps=1 elapsed=1.1s lr=6.000000e-04 acc_all=0.0005 acc_corrupt=0.0011 corrupt_frac=0.4424 loss_all=10.8125 loss_corrupt=10.8125 acc_corrupt_t_0p0_0p2=0.0015 corrupt_frac_t_0p0_0p2=0.7163 acc_corrupt_t_0p8_1p0=0.0000 corrupt_frac_t_0p8_1p0=0.2837 loss=10.8125 loss_recon=10.8125 loss_meanflow=0.0000 mean_model_t=0.5059 mean_corrupt_t=0.5059 mean_loss_t_weight=1.0000 prior_center_loss_beta=0.0000 rollout_train_applied=1.0000 wrong_frac=0.6821 init_acc_corrupt=0.2815 init_gold_top10=0.3013 init_gold_top100=0.3918
97
+ [rank1]: Traceback (most recent call last):
98
+ [rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 1487, in <module>
99
+ [rank1]: main()
100
+ [rank1]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 1372, in main
101
+ [rank1]: logits = trainable_model(loss_state, model_t, bridge.attn_mask)
102
+ [rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
103
+ [rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
104
+ [rank1]: return self._call_impl(*args, **kwargs)
105
+ [rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
106
+ [rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1750, in _call_impl
107
+ [rank1]: return forward_call(*args, **kwargs)
108
+ [rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
109
+ [rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/distributed.py", line 1650, in forward
110
+ [rank1]: inputs, kwargs = self._pre_forward(*inputs, **kwargs)
111
+ [rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
112
+ [rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/distributed.py", line 1539, in _pre_forward
113
+ [rank1]: if torch.is_grad_enabled() and self.reducer._rebuild_buckets():
114
+ [rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
115
+ [rank1]: RuntimeError: Expected to have finished reduction in the prior iteration before starting a new one. This error indicates that your module has parameters that were not used in producing loss. You can enable unused parameter detection by passing the keyword argument `find_unused_parameters=True` to `torch.nn.parallel.DistributedDataParallel`, and by
116
+ [rank1]: making sure all `forward` function outputs participate in calculating loss.
117
+ [rank1]: If you already have done the above, then the distributed data parallel module wasn't able to locate the output tensors in the return value of your module's `forward` function. Please include the loss function and the structure of the return value of `forward` of your module when reporting this issue (e.g. list, dict, iterable).
118
+ [rank1]: Parameter indices which did not receive grad for rank 1: 0 1 2 3 4 6 7 9 10 11 12 13 14 16 17 19 20 21 22 23 24 26 27 29 30 31 32 33 34 36 37 39 40 41 42 43 44 46 47 49 50 51 52 53 54 56 57 59 60 61 62 63 64 66 67 69 70 71 72 73 74 76 77 79 80 81 82 83 84 86 87 89 90 91 92 93 94 96 97 99 100 101 102 103 104 106 107 109 110 111 112 113 114 116 117 119 120 121 122 123 ...
119
+ [rank1]: In addition, you can set the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAIL to print out information about which particular parameters did not receive gradient on this rank as part of this error
120
+ [rank0]: Traceback (most recent call last):
121
+ [rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 1487, in <module>
122
+ [rank0]: main()
123
+ [rank0]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 1372, in main
124
+ [rank0]: logits = trainable_model(loss_state, model_t, bridge.attn_mask)
125
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
126
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
127
+ [rank0]: return self._call_impl(*args, **kwargs)
128
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
129
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1750, in _call_impl
130
+ [rank0]: return forward_call(*args, **kwargs)
131
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
132
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/distributed.py", line 1650, in forward
133
+ [rank0]: inputs, kwargs = self._pre_forward(*inputs, **kwargs)
134
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
135
+ [rank0]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/distributed.py", line 1539, in _pre_forward
136
+ [rank0]: if torch.is_grad_enabled() and self.reducer._rebuild_buckets():
137
+ [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
138
+ [rank0]: RuntimeError: Expected to have finished reduction in the prior iteration before starting a new one. This error indicates that your module has parameters that were not used in producing loss. You can enable unused parameter detection by passing the keyword argument `find_unused_parameters=True` to `torch.nn.parallel.DistributedDataParallel`, and by
139
+ [rank0]: making sure all `forward` function outputs participate in calculating loss.
140
+ [rank0]: If you already have done the above, then the distributed data parallel module wasn't able to locate the output tensors in the return value of your module's `forward` function. Please include the loss function and the structure of the return value of `forward` of your module when reporting this issue (e.g. list, dict, iterable).
141
+ [rank0]: Parameter indices which did not receive grad for rank 0: 0 1 2 3 4 6 7 9 10 11 12 13 14 16 17 19 20 21 22 23 24 26 27 29 30 31 32 33 34 36 37 39 40 41 42 43 44 46 47 49 50 51 52 53 54 56 57 59 60 61 62 63 64 66 67 69 70 71 72 73 74 76 77 79 80 81 82 83 84 86 87 89 90 91 92 93 94 96 97 99 100 101 102 103 104 106 107 109 110 111 112 113 114 116 117 119 120 121 122 123 ...
142
+ [rank0]: In addition, you can set the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAIL to print out information about which particular parameters did not receive gradient on this rank as part of this error
143
+ [rank3]: Traceback (most recent call last):
144
+ [rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 1487, in <module>
145
+ [rank3]: main()
146
+ [rank3]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 1372, in main
147
+ [rank3]: logits = trainable_model(loss_state, model_t, bridge.attn_mask)
148
+ [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
149
+ [rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
150
+ [rank3]: return self._call_impl(*args, **kwargs)
151
+ [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
152
+ [rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1750, in _call_impl
153
+ [rank3]: return forward_call(*args, **kwargs)
154
+ [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
155
+ [rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/distributed.py", line 1650, in forward
156
+ [rank3]: inputs, kwargs = self._pre_forward(*inputs, **kwargs)
157
+ [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
158
+ [rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/distributed.py", line 1539, in _pre_forward
159
+ [rank3]: if torch.is_grad_enabled() and self.reducer._rebuild_buckets():
160
+ [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
161
+ [rank3]: RuntimeError: Expected to have finished reduction in the prior iteration before starting a new one. This error indicates that your module has parameters that were not used in producing loss. You can enable unused parameter detection by passing the keyword argument `find_unused_parameters=True` to `torch.nn.parallel.DistributedDataParallel`, and by
162
+ [rank3]: making sure all `forward` function outputs participate in calculating loss.
163
+ [rank3]: If you already have done the above, then the distributed data parallel module wasn't able to locate the output tensors in the return value of your module's `forward` function. Please include the loss function and the structure of the return value of `forward` of your module when reporting this issue (e.g. list, dict, iterable).
164
+ [rank3]: Parameter indices which did not receive grad for rank 3: 0 1 2 3 4 6 7 9 10 11 12 13 14 16 17 19 20 21 22 23 24 26 27 29 30 31 32 33 34 36 37 39 40 41 42 43 44 46 47 49 50 51 52 53 54 56 57 59 60 61 62 63 64 66 67 69 70 71 72 73 74 76 77 79 80 81 82 83 84 86 87 89 90 91 92 93 94 96 97 99 100 101 102 103 104 106 107 109 110 111 112 113 114 116 117 119 120 121 122 123 ...
165
+ [rank3]: In addition, you can set the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAIL to print out information about which particular parameters did not receive gradient on this rank as part of this error
166
+ [rank2]: Traceback (most recent call last):
167
+ [rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 1487, in <module>
168
+ [rank2]: main()
169
+ [rank2]: File "/e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt/train.py", line 1372, in main
170
+ [rank2]: logits = trainable_model(loss_state, model_t, bridge.attn_mask)
171
+ [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
172
+ [rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
173
+ [rank2]: return self._call_impl(*args, **kwargs)
174
+ [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
175
+ [rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1750, in _call_impl
176
+ [rank2]: return forward_call(*args, **kwargs)
177
+ [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
178
+ [rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/distributed.py", line 1650, in forward
179
+ [rank2]: inputs, kwargs = self._pre_forward(*inputs, **kwargs)
180
+ [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
181
+ [rank2]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/parallel/distributed.py", line 1539, in _pre_forward
182
+ [rank2]: if torch.is_grad_enabled() and self.reducer._rebuild_buckets():
183
+ [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
184
+ [rank2]: RuntimeError: Expected to have finished reduction in the prior iteration before starting a new one. This error indicates that your module has parameters that were not used in producing loss. You can enable unused parameter detection by passing the keyword argument `find_unused_parameters=True` to `torch.nn.parallel.DistributedDataParallel`, and by
185
+ [rank2]: making sure all `forward` function outputs participate in calculating loss.
186
+ [rank2]: If you already have done the above, then the distributed data parallel module wasn't able to locate the output tensors in the return value of your module's `forward` function. Please include the loss function and the structure of the return value of `forward` of your module when reporting this issue (e.g. list, dict, iterable).
187
+ [rank2]: Parameter indices which did not receive grad for rank 2: 0 1 2 3 4 6 7 9 10 11 12 13 14 16 17 19 20 21 22 23 24 26 27 29 30 31 32 33 34 36 37 39 40 41 42 43 44 46 47 49 50 51 52 53 54 56 57 59 60 61 62 63 64 66 67 69 70 71 72 73 74 76 77 79 80 81 82 83 84 86 87 89 90 91 92 93 94 96 97 99 100 101 102 103 104 106 107 109 110 111 112 113 114 116 117 119 120 121 122 123 ...
188
+ [rank2]: In addition, you can set the environment variable TORCH_DISTRIBUTED_DEBUG to either INFO or DETAIL to print out information about which particular parameters did not receive gradient on this rank as part of this error
189
+ [rank0]:[W513 14:38:19.182594534 ProcessGroupNCCL.cpp:1487] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
190
+ W0513 14:38:20.047000 621905 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 621911 closing signal SIGTERM
191
+ W0513 14:38:20.047000 621905 torch/distributed/elastic/multiprocessing/api.py:898] Sending process 621912 closing signal SIGTERM
192
+ E0513 14:38:20.262000 621905 torch/distributed/elastic/multiprocessing/api.py:870] failed (exitcode: 1) local_rank: 0 (pid: 621909) of binary: /usr/bin/python
193
+ Traceback (most recent call last):
194
+ File "<frozen runpy>", line 198, in _run_module_as_main
195
+ File "<frozen runpy>", line 88, in _run_code
196
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 922, in <module>
197
+ main()
198
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
199
+ return f(*args, **kwargs)
200
+ ^^^^^^^^^^^^^^^^^^
201
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 918, in main
202
+ run(args)
203
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 909, in run
204
+ elastic_launch(
205
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
206
+ return launch_agent(self._config, self._entrypoint, list(args))
207
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
208
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent
209
+ raise ChildFailedError(
210
+ torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
211
+ ============================================================
212
+ train.py FAILED
213
+ ------------------------------------------------------------
214
+ Failures:
215
+ [1]:
216
+ time : 2026-05-13_14:38:20
217
+ host : localhost
218
+ rank : 1 (local_rank: 1)
219
+ exitcode : 1 (pid: 621910)
220
+ error_file: <N/A>
221
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
222
+ ------------------------------------------------------------
223
+ Root Cause (first observed failure):
224
+ [0]:
225
+ time : 2026-05-13_14:38:20
226
+ host : localhost
227
+ rank : 0 (local_rank: 0)
228
+ exitcode : 1 (pid: 621909)
229
+ error_file: <N/A>
230
+ traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
231
+ ============================================================
LTA_openwebtext_dualt/logs/lta_owt_t5_absrope_adaln_dirichlet_len1024_Cv_to_2v_mask0p1_1p0_sameT_gbs512_b32_8gpu_1m_save10k_20260525_watcher.sh ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ cd /e2e-data/evad-tech-vla/wanghan58/workspace/LTA_openwebtext_dualt
5
+ export PYTHONPATH="$(pwd)${PYTHONPATH:+:$PYTHONPATH}"
6
+ export TOKENIZERS_PARALLELISM=false
7
+ export PYTHONUNBUFFERED=1
8
+
9
+ : "${RUN_DIR:?RUN_DIR is required}"
10
+ : "${OUT_BASE:?OUT_BASE is required}"
11
+ : "${LOG_DIR:?LOG_DIR is required}"
12
+ : "${TOKENIZER_PATH:?TOKENIZER_PATH is required}"
13
+ : "${SCORER:?SCORER is required}"
14
+
15
+ RUN_STEM="$(basename "${RUN_DIR}")"
16
+ TEMP_TAG="${ENDPOINT_TEMP//./p}"
17
+ PROCESSED_FILE="${LOG_DIR}/processed_${RUN_STEM}_steps${STEPS}_c${CMIN}_${CMAX}_gumbel_t${TEMP_TAG}_n${N_SAMPLES}.txt"
18
+
19
+ mkdir -p "${OUT_BASE}" "${LOG_DIR}"
20
+ touch "${PROCESSED_FILE}"
21
+
22
+ echo "[watch-gumbel] run_dir=${RUN_DIR}"
23
+ echo "[watch-gumbel] out_base=${OUT_BASE}"
24
+ echo "[watch-gumbel] interval=${STEP_INTERVAL} max_len=${MAX_LEN} steps=${STEPS} c=${CMIN}->${CMAX} temp=${ENDPOINT_TEMP} top_p=${ENDPOINT_TOP_P} tau=${GUMBEL_TAU_START}->${GUMBEL_TAU_END} n=${N_SAMPLES}"
25
+
26
+ while true; do
27
+ shopt -s nullglob
28
+ ckpts=("${RUN_DIR}"/step_*.pt)
29
+ shopt -u nullglob
30
+
31
+ if (( ${#ckpts[@]} == 0 )); then
32
+ echo "[watch-gumbel] $(date +%F_%T) no ckpt yet"
33
+ sleep "${SLEEP_SECONDS}"
34
+ continue
35
+ fi
36
+
37
+ printf "%s\n" "${ckpts[@]}" | sort | while read -r ckpt; do
38
+ base="$(basename "${ckpt}")"
39
+ step="${base#step_}"
40
+ step="${step%.pt}"
41
+ step_num=$((10#${step}))
42
+ if (( step_num % STEP_INTERVAL != 0 )); then
43
+ continue
44
+ fi
45
+ if grep -Fxq "${ckpt}" "${PROCESSED_FILE}"; then
46
+ continue
47
+ fi
48
+
49
+ out_dir="${OUT_BASE}/step_${step}"
50
+ log_file="${LOG_DIR}/infer_${RUN_STEM}_step_${step}.log"
51
+ mkdir -p "${out_dir}"
52
+
53
+ echo "[watch-gumbel] $(date +%F_%T) infer ${ckpt} -> ${out_dir}" | tee -a "${log_file}"
54
+ CUDA_VISIBLE_DEVICES="${WATCH_CUDA_VISIBLE_DEVICES}" python scripts/eval_lm1b_c1024_fullycoupled_sde_genppl.py \
55
+ --checkpoint "${ckpt}" \
56
+ --tokenizer_path "${TOKENIZER_PATH}" \
57
+ --scorer "${SCORER}" \
58
+ --out_dir "${out_dir}" \
59
+ --n_samples "${N_SAMPLES}" \
60
+ --max_len "${MAX_LEN}" \
61
+ --steps "${STEPS}" \
62
+ --batch_size "${DECODE_BATCH}" \
63
+ --score_batch "${SCORE_BATCH}" \
64
+ --score_max_length "${SCORE_MAX_LENGTH}" \
65
+ --concentration_min "${CMIN}" \
66
+ --concentration_max "${CMAX}" \
67
+ --endpoint_temp "${ENDPOINT_TEMP}" \
68
+ --endpoint_projection gumbel_softmax \
69
+ --endpoint_top_p "${ENDPOINT_TOP_P}" \
70
+ --gumbel_tau_start "${GUMBEL_TAU_START}" \
71
+ --gumbel_tau_end "${GUMBEL_TAU_END}" \
72
+ --model_t_mode support_t \
73
+ --mean_mode endpoint_only \
74
+ --semantic_power 1.0 \
75
+ --noise_init dirichlet \
76
+ --noise_dirichlet_concentration "${CMIN}" \
77
+ --sde_resample dirichlet \
78
+ --final_from blend_0.5 \
79
+ --seed 20260524 \
80
+ 2>&1 | tee -a "${log_file}"
81
+
82
+ echo "${ckpt}" >> "${PROCESSED_FILE}"
83
+ echo "[watch-gumbel] $(date +%F_%T) done step_${step}" | tee -a "${log_file}"
84
+ done
85
+
86
+ sleep "${SLEEP_SECONDS}"
87
+ done
LTA_openwebtext_dualt/logs/watch_lm1b_len128_Cv2v_sde_20260523.pid ADDED
@@ -0,0 +1 @@
 
 
1
+ 1040038
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/_cl_builtins.py ADDED
@@ -0,0 +1,231 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers._cl_builtins
3
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ ANSI Common Lisp builtins.
6
+
7
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ BUILTIN_FUNCTIONS = { # 638 functions
12
+ '<', '<=', '=', '>', '>=', '-', '/', '/=', '*', '+', '1-', '1+',
13
+ 'abort', 'abs', 'acons', 'acos', 'acosh', 'add-method', 'adjoin',
14
+ 'adjustable-array-p', 'adjust-array', 'allocate-instance',
15
+ 'alpha-char-p', 'alphanumericp', 'append', 'apply', 'apropos',
16
+ 'apropos-list', 'aref', 'arithmetic-error-operands',
17
+ 'arithmetic-error-operation', 'array-dimension', 'array-dimensions',
18
+ 'array-displacement', 'array-element-type', 'array-has-fill-pointer-p',
19
+ 'array-in-bounds-p', 'arrayp', 'array-rank', 'array-row-major-index',
20
+ 'array-total-size', 'ash', 'asin', 'asinh', 'assoc', 'assoc-if',
21
+ 'assoc-if-not', 'atan', 'atanh', 'atom', 'bit', 'bit-and', 'bit-andc1',
22
+ 'bit-andc2', 'bit-eqv', 'bit-ior', 'bit-nand', 'bit-nor', 'bit-not',
23
+ 'bit-orc1', 'bit-orc2', 'bit-vector-p', 'bit-xor', 'boole',
24
+ 'both-case-p', 'boundp', 'break', 'broadcast-stream-streams',
25
+ 'butlast', 'byte', 'byte-position', 'byte-size', 'caaaar', 'caaadr',
26
+ 'caaar', 'caadar', 'caaddr', 'caadr', 'caar', 'cadaar', 'cadadr',
27
+ 'cadar', 'caddar', 'cadddr', 'caddr', 'cadr', 'call-next-method', 'car',
28
+ 'cdaaar', 'cdaadr', 'cdaar', 'cdadar', 'cdaddr', 'cdadr', 'cdar',
29
+ 'cddaar', 'cddadr', 'cddar', 'cdddar', 'cddddr', 'cdddr', 'cddr', 'cdr',
30
+ 'ceiling', 'cell-error-name', 'cerror', 'change-class', 'char', 'char<',
31
+ 'char<=', 'char=', 'char>', 'char>=', 'char/=', 'character',
32
+ 'characterp', 'char-code', 'char-downcase', 'char-equal',
33
+ 'char-greaterp', 'char-int', 'char-lessp', 'char-name',
34
+ 'char-not-equal', 'char-not-greaterp', 'char-not-lessp', 'char-upcase',
35
+ 'cis', 'class-name', 'class-of', 'clear-input', 'clear-output',
36
+ 'close', 'clrhash', 'code-char', 'coerce', 'compile',
37
+ 'compiled-function-p', 'compile-file', 'compile-file-pathname',
38
+ 'compiler-macro-function', 'complement', 'complex', 'complexp',
39
+ 'compute-applicable-methods', 'compute-restarts', 'concatenate',
40
+ 'concatenated-stream-streams', 'conjugate', 'cons', 'consp',
41
+ 'constantly', 'constantp', 'continue', 'copy-alist', 'copy-list',
42
+ 'copy-pprint-dispatch', 'copy-readtable', 'copy-seq', 'copy-structure',
43
+ 'copy-symbol', 'copy-tree', 'cos', 'cosh', 'count', 'count-if',
44
+ 'count-if-not', 'decode-float', 'decode-universal-time', 'delete',
45
+ 'delete-duplicates', 'delete-file', 'delete-if', 'delete-if-not',
46
+ 'delete-package', 'denominator', 'deposit-field', 'describe',
47
+ 'describe-object', 'digit-char', 'digit-char-p', 'directory',
48
+ 'directory-namestring', 'disassemble', 'documentation', 'dpb',
49
+ 'dribble', 'echo-stream-input-stream', 'echo-stream-output-stream',
50
+ 'ed', 'eighth', 'elt', 'encode-universal-time', 'endp',
51
+ 'enough-namestring', 'ensure-directories-exist',
52
+ 'ensure-generic-function', 'eq', 'eql', 'equal', 'equalp', 'error',
53
+ 'eval', 'evenp', 'every', 'exp', 'export', 'expt', 'fboundp',
54
+ 'fceiling', 'fdefinition', 'ffloor', 'fifth', 'file-author',
55
+ 'file-error-pathname', 'file-length', 'file-namestring',
56
+ 'file-position', 'file-string-length', 'file-write-date',
57
+ 'fill', 'fill-pointer', 'find', 'find-all-symbols', 'find-class',
58
+ 'find-if', 'find-if-not', 'find-method', 'find-package', 'find-restart',
59
+ 'find-symbol', 'finish-output', 'first', 'float', 'float-digits',
60
+ 'floatp', 'float-precision', 'float-radix', 'float-sign', 'floor',
61
+ 'fmakunbound', 'force-output', 'format', 'fourth', 'fresh-line',
62
+ 'fround', 'ftruncate', 'funcall', 'function-keywords',
63
+ 'function-lambda-expression', 'functionp', 'gcd', 'gensym', 'gentemp',
64
+ 'get', 'get-decoded-time', 'get-dispatch-macro-character', 'getf',
65
+ 'gethash', 'get-internal-real-time', 'get-internal-run-time',
66
+ 'get-macro-character', 'get-output-stream-string', 'get-properties',
67
+ 'get-setf-expansion', 'get-universal-time', 'graphic-char-p',
68
+ 'hash-table-count', 'hash-table-p', 'hash-table-rehash-size',
69
+ 'hash-table-rehash-threshold', 'hash-table-size', 'hash-table-test',
70
+ 'host-namestring', 'identity', 'imagpart', 'import',
71
+ 'initialize-instance', 'input-stream-p', 'inspect',
72
+ 'integer-decode-float', 'integer-length', 'integerp',
73
+ 'interactive-stream-p', 'intern', 'intersection',
74
+ 'invalid-method-error', 'invoke-debugger', 'invoke-restart',
75
+ 'invoke-restart-interactively', 'isqrt', 'keywordp', 'last', 'lcm',
76
+ 'ldb', 'ldb-test', 'ldiff', 'length', 'lisp-implementation-type',
77
+ 'lisp-implementation-version', 'list', 'list*', 'list-all-packages',
78
+ 'listen', 'list-length', 'listp', 'load',
79
+ 'load-logical-pathname-translations', 'log', 'logand', 'logandc1',
80
+ 'logandc2', 'logbitp', 'logcount', 'logeqv', 'logical-pathname',
81
+ 'logical-pathname-translations', 'logior', 'lognand', 'lognor',
82
+ 'lognot', 'logorc1', 'logorc2', 'logtest', 'logxor', 'long-site-name',
83
+ 'lower-case-p', 'machine-instance', 'machine-type', 'machine-version',
84
+ 'macroexpand', 'macroexpand-1', 'macro-function', 'make-array',
85
+ 'make-broadcast-stream', 'make-concatenated-stream', 'make-condition',
86
+ 'make-dispatch-macro-character', 'make-echo-stream', 'make-hash-table',
87
+ 'make-instance', 'make-instances-obsolete', 'make-list',
88
+ 'make-load-form', 'make-load-form-saving-slots', 'make-package',
89
+ 'make-pathname', 'make-random-state', 'make-sequence', 'make-string',
90
+ 'make-string-input-stream', 'make-string-output-stream', 'make-symbol',
91
+ 'make-synonym-stream', 'make-two-way-stream', 'makunbound', 'map',
92
+ 'mapc', 'mapcan', 'mapcar', 'mapcon', 'maphash', 'map-into', 'mapl',
93
+ 'maplist', 'mask-field', 'max', 'member', 'member-if', 'member-if-not',
94
+ 'merge', 'merge-pathnames', 'method-combination-error',
95
+ 'method-qualifiers', 'min', 'minusp', 'mismatch', 'mod',
96
+ 'muffle-warning', 'name-char', 'namestring', 'nbutlast', 'nconc',
97
+ 'next-method-p', 'nintersection', 'ninth', 'no-applicable-method',
98
+ 'no-next-method', 'not', 'notany', 'notevery', 'nreconc', 'nreverse',
99
+ 'nset-difference', 'nset-exclusive-or', 'nstring-capitalize',
100
+ 'nstring-downcase', 'nstring-upcase', 'nsublis', 'nsubst', 'nsubst-if',
101
+ 'nsubst-if-not', 'nsubstitute', 'nsubstitute-if', 'nsubstitute-if-not',
102
+ 'nth', 'nthcdr', 'null', 'numberp', 'numerator', 'nunion', 'oddp',
103
+ 'open', 'open-stream-p', 'output-stream-p', 'package-error-package',
104
+ 'package-name', 'package-nicknames', 'packagep',
105
+ 'package-shadowing-symbols', 'package-used-by-list', 'package-use-list',
106
+ 'pairlis', 'parse-integer', 'parse-namestring', 'pathname',
107
+ 'pathname-device', 'pathname-directory', 'pathname-host',
108
+ 'pathname-match-p', 'pathname-name', 'pathnamep', 'pathname-type',
109
+ 'pathname-version', 'peek-char', 'phase', 'plusp', 'position',
110
+ 'position-if', 'position-if-not', 'pprint', 'pprint-dispatch',
111
+ 'pprint-fill', 'pprint-indent', 'pprint-linear', 'pprint-newline',
112
+ 'pprint-tab', 'pprint-tabular', 'prin1', 'prin1-to-string', 'princ',
113
+ 'princ-to-string', 'print', 'print-object', 'probe-file', 'proclaim',
114
+ 'provide', 'random', 'random-state-p', 'rassoc', 'rassoc-if',
115
+ 'rassoc-if-not', 'rational', 'rationalize', 'rationalp', 'read',
116
+ 'read-byte', 'read-char', 'read-char-no-hang', 'read-delimited-list',
117
+ 'read-from-string', 'read-line', 'read-preserving-whitespace',
118
+ 'read-sequence', 'readtable-case', 'readtablep', 'realp', 'realpart',
119
+ 'reduce', 'reinitialize-instance', 'rem', 'remhash', 'remove',
120
+ 'remove-duplicates', 'remove-if', 'remove-if-not', 'remove-method',
121
+ 'remprop', 'rename-file', 'rename-package', 'replace', 'require',
122
+ 'rest', 'restart-name', 'revappend', 'reverse', 'room', 'round',
123
+ 'row-major-aref', 'rplaca', 'rplacd', 'sbit', 'scale-float', 'schar',
124
+ 'search', 'second', 'set', 'set-difference',
125
+ 'set-dispatch-macro-character', 'set-exclusive-or',
126
+ 'set-macro-character', 'set-pprint-dispatch', 'set-syntax-from-char',
127
+ 'seventh', 'shadow', 'shadowing-import', 'shared-initialize',
128
+ 'short-site-name', 'signal', 'signum', 'simple-bit-vector-p',
129
+ 'simple-condition-format-arguments', 'simple-condition-format-control',
130
+ 'simple-string-p', 'simple-vector-p', 'sin', 'sinh', 'sixth', 'sleep',
131
+ 'slot-boundp', 'slot-exists-p', 'slot-makunbound', 'slot-missing',
132
+ 'slot-unbound', 'slot-value', 'software-type', 'software-version',
133
+ 'some', 'sort', 'special-operator-p', 'sqrt', 'stable-sort',
134
+ 'standard-char-p', 'store-value', 'stream-element-type',
135
+ 'stream-error-stream', 'stream-external-format', 'streamp', 'string',
136
+ 'string<', 'string<=', 'string=', 'string>', 'string>=', 'string/=',
137
+ 'string-capitalize', 'string-downcase', 'string-equal',
138
+ 'string-greaterp', 'string-left-trim', 'string-lessp',
139
+ 'string-not-equal', 'string-not-greaterp', 'string-not-lessp',
140
+ 'stringp', 'string-right-trim', 'string-trim', 'string-upcase',
141
+ 'sublis', 'subseq', 'subsetp', 'subst', 'subst-if', 'subst-if-not',
142
+ 'substitute', 'substitute-if', 'substitute-if-not', 'subtypep','svref',
143
+ 'sxhash', 'symbol-function', 'symbol-name', 'symbolp', 'symbol-package',
144
+ 'symbol-plist', 'symbol-value', 'synonym-stream-symbol', 'syntax:',
145
+ 'tailp', 'tan', 'tanh', 'tenth', 'terpri', 'third',
146
+ 'translate-logical-pathname', 'translate-pathname', 'tree-equal',
147
+ 'truename', 'truncate', 'two-way-stream-input-stream',
148
+ 'two-way-stream-output-stream', 'type-error-datum',
149
+ 'type-error-expected-type', 'type-of', 'typep', 'unbound-slot-instance',
150
+ 'unexport', 'unintern', 'union', 'unread-char', 'unuse-package',
151
+ 'update-instance-for-different-class',
152
+ 'update-instance-for-redefined-class', 'upgraded-array-element-type',
153
+ 'upgraded-complex-part-type', 'upper-case-p', 'use-package',
154
+ 'user-homedir-pathname', 'use-value', 'values', 'values-list', 'vector',
155
+ 'vectorp', 'vector-pop', 'vector-push', 'vector-push-extend', 'warn',
156
+ 'wild-pathname-p', 'write', 'write-byte', 'write-char', 'write-line',
157
+ 'write-sequence', 'write-string', 'write-to-string', 'yes-or-no-p',
158
+ 'y-or-n-p', 'zerop',
159
+ }
160
+
161
+ SPECIAL_FORMS = {
162
+ 'block', 'catch', 'declare', 'eval-when', 'flet', 'function', 'go', 'if',
163
+ 'labels', 'lambda', 'let', 'let*', 'load-time-value', 'locally', 'macrolet',
164
+ 'multiple-value-call', 'multiple-value-prog1', 'progn', 'progv', 'quote',
165
+ 'return-from', 'setq', 'symbol-macrolet', 'tagbody', 'the', 'throw',
166
+ 'unwind-protect',
167
+ }
168
+
169
+ MACROS = {
170
+ 'and', 'assert', 'call-method', 'case', 'ccase', 'check-type', 'cond',
171
+ 'ctypecase', 'decf', 'declaim', 'defclass', 'defconstant', 'defgeneric',
172
+ 'define-compiler-macro', 'define-condition', 'define-method-combination',
173
+ 'define-modify-macro', 'define-setf-expander', 'define-symbol-macro',
174
+ 'defmacro', 'defmethod', 'defpackage', 'defparameter', 'defsetf',
175
+ 'defstruct', 'deftype', 'defun', 'defvar', 'destructuring-bind', 'do',
176
+ 'do*', 'do-all-symbols', 'do-external-symbols', 'dolist', 'do-symbols',
177
+ 'dotimes', 'ecase', 'etypecase', 'formatter', 'handler-bind',
178
+ 'handler-case', 'ignore-errors', 'incf', 'in-package', 'lambda', 'loop',
179
+ 'loop-finish', 'make-method', 'multiple-value-bind', 'multiple-value-list',
180
+ 'multiple-value-setq', 'nth-value', 'or', 'pop',
181
+ 'pprint-exit-if-list-exhausted', 'pprint-logical-block', 'pprint-pop',
182
+ 'print-unreadable-object', 'prog', 'prog*', 'prog1', 'prog2', 'psetf',
183
+ 'psetq', 'push', 'pushnew', 'remf', 'restart-bind', 'restart-case',
184
+ 'return', 'rotatef', 'setf', 'shiftf', 'step', 'time', 'trace', 'typecase',
185
+ 'unless', 'untrace', 'when', 'with-accessors', 'with-compilation-unit',
186
+ 'with-condition-restarts', 'with-hash-table-iterator',
187
+ 'with-input-from-string', 'with-open-file', 'with-open-stream',
188
+ 'with-output-to-string', 'with-package-iterator', 'with-simple-restart',
189
+ 'with-slots', 'with-standard-io-syntax',
190
+ }
191
+
192
+ LAMBDA_LIST_KEYWORDS = {
193
+ '&allow-other-keys', '&aux', '&body', '&environment', '&key', '&optional',
194
+ '&rest', '&whole',
195
+ }
196
+
197
+ DECLARATIONS = {
198
+ 'dynamic-extent', 'ignore', 'optimize', 'ftype', 'inline', 'special',
199
+ 'ignorable', 'notinline', 'type',
200
+ }
201
+
202
+ BUILTIN_TYPES = {
203
+ 'atom', 'boolean', 'base-char', 'base-string', 'bignum', 'bit',
204
+ 'compiled-function', 'extended-char', 'fixnum', 'keyword', 'nil',
205
+ 'signed-byte', 'short-float', 'single-float', 'double-float', 'long-float',
206
+ 'simple-array', 'simple-base-string', 'simple-bit-vector', 'simple-string',
207
+ 'simple-vector', 'standard-char', 'unsigned-byte',
208
+
209
+ # Condition Types
210
+ 'arithmetic-error', 'cell-error', 'condition', 'control-error',
211
+ 'division-by-zero', 'end-of-file', 'error', 'file-error',
212
+ 'floating-point-inexact', 'floating-point-overflow',
213
+ 'floating-point-underflow', 'floating-point-invalid-operation',
214
+ 'parse-error', 'package-error', 'print-not-readable', 'program-error',
215
+ 'reader-error', 'serious-condition', 'simple-condition', 'simple-error',
216
+ 'simple-type-error', 'simple-warning', 'stream-error', 'storage-condition',
217
+ 'style-warning', 'type-error', 'unbound-variable', 'unbound-slot',
218
+ 'undefined-function', 'warning',
219
+ }
220
+
221
+ BUILTIN_CLASSES = {
222
+ 'array', 'broadcast-stream', 'bit-vector', 'built-in-class', 'character',
223
+ 'class', 'complex', 'concatenated-stream', 'cons', 'echo-stream',
224
+ 'file-stream', 'float', 'function', 'generic-function', 'hash-table',
225
+ 'integer', 'list', 'logical-pathname', 'method-combination', 'method',
226
+ 'null', 'number', 'package', 'pathname', 'ratio', 'rational', 'readtable',
227
+ 'real', 'random-state', 'restart', 'sequence', 'standard-class',
228
+ 'standard-generic-function', 'standard-method', 'standard-object',
229
+ 'string-stream', 'stream', 'string', 'structure-class', 'structure-object',
230
+ 'symbol', 'synonym-stream', 't', 'two-way-stream', 'vector',
231
+ }
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/blueprint.py ADDED
@@ -0,0 +1,173 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers.blueprint
3
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexer for the Blueprint UI markup language.
6
+
7
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ import re
12
+
13
+ from pygments.lexer import RegexLexer, include, bygroups, words
14
+ from pygments.token import (
15
+ Comment,
16
+ Operator,
17
+ Keyword,
18
+ Name,
19
+ String,
20
+ Number,
21
+ Punctuation,
22
+ Whitespace,
23
+ )
24
+
25
+ __all__ = ["BlueprintLexer"]
26
+
27
+
28
+ class BlueprintLexer(RegexLexer):
29
+ """
30
+ For Blueprint UI markup.
31
+ """
32
+
33
+ name = "Blueprint"
34
+ aliases = ["blueprint"]
35
+ filenames = ["*.blp"]
36
+ mimetypes = ["text/x-blueprint"]
37
+ url = "https://gitlab.gnome.org/jwestman/blueprint-compiler"
38
+ version_added = '2.16'
39
+
40
+ flags = re.IGNORECASE
41
+ tokens = {
42
+ "root": [
43
+ include("block-content"),
44
+ ],
45
+ "type": [
46
+ (r"\$\s*[a-z_][a-z0-9_\-]*", Name.Class),
47
+ (r"(?:([a-z_][a-z0-9_\-]*)(\s*)(\.)(\s*))?([a-z_][a-z0-9_\-]*)",
48
+ bygroups(Name.Namespace, Whitespace, Punctuation, Whitespace, Name.Class)),
49
+ ],
50
+ "whitespace": [
51
+ (r"\s+", Whitespace),
52
+ (r"//.*?\n", Comment.Single),
53
+ (r"/\*", Comment.Multiline, "comment-multiline"),
54
+ ],
55
+ "comment-multiline": [
56
+ (r"\*/", Comment.Multiline, "#pop"),
57
+ (r"[^*]+", Comment.Multiline),
58
+ (r"\*", Comment.Multiline),
59
+ ],
60
+ "value": [
61
+ (r"(typeof)(\s*)(<)", bygroups(Keyword, Whitespace, Punctuation), "typeof"),
62
+ (words(("true", "false", "null")), Keyword.Constant),
63
+ (r"[a-z_][a-z0-9_\-]*", Name.Variable),
64
+ (r"\|", Operator),
65
+ (r'".*?"', String.Double),
66
+ (r"\'.*?\'", String.Single),
67
+ (r"0x[\d_]*", Number.Hex),
68
+ (r"[0-9_]+", Number.Integer),
69
+ (r"\d[\d\.a-z_]*", Number),
70
+ ],
71
+ "typeof": [
72
+ include("whitespace"),
73
+ include("type"),
74
+ (r">", Punctuation, "#pop"),
75
+ ],
76
+ "content": [
77
+ include("whitespace"),
78
+ # Keywords
79
+ (words(("after", "bidirectional", "bind-property", "bind", "default",
80
+ "destructive", "disabled", "inverted", "no-sync-create",
81
+ "suggested", "swapped", "sync-create", "template")),
82
+ Keyword),
83
+ # Translated strings
84
+ (r"(C?_)(\s*)(\()",
85
+ bygroups(Name.Function.Builtin, Whitespace, Punctuation),
86
+ "paren-content"),
87
+ # Cast expressions
88
+ (r"(as)(\s*)(<)", bygroups(Keyword, Whitespace, Punctuation), "typeof"),
89
+ # Closures
90
+ (r"(\$?[a-z_][a-z0-9_\-]*)(\s*)(\()",
91
+ bygroups(Name.Function, Whitespace, Punctuation),
92
+ "paren-content"),
93
+ # Objects
94
+ (r"(?:(\$\s*[a-z_][a-z0-9_\-]+)|(?:([a-z_][a-z0-9_\-]*)(\s*)(\.)(\s*))?([a-z_][a-z0-9_\-]*))(?:(\s+)([a-z_][a-z0-9_\-]*))?(\s*)(\{)",
95
+ bygroups(Name.Class, Name.Namespace, Whitespace, Punctuation, Whitespace,
96
+ Name.Class, Whitespace, Name.Variable, Whitespace, Punctuation),
97
+ "brace-block"),
98
+ # Misc
99
+ include("value"),
100
+ (r",|\.", Punctuation),
101
+ ],
102
+ "block-content": [
103
+ # Import statements
104
+ (r"(using)(\s+)([a-z_][a-z0-9_\-]*)(\s+)(\d[\d\.]*)(;)",
105
+ bygroups(Keyword, Whitespace, Name.Namespace, Whitespace,
106
+ Name.Namespace, Punctuation)),
107
+ # Menus
108
+ (r"(menu|section|submenu)(?:(\s+)([a-z_][a-z0-9_\-]*))?(\s*)(\{)",
109
+ bygroups(Keyword, Whitespace, Name.Variable, Whitespace, Punctuation),
110
+ "brace-block"),
111
+ (r"(item)(\s*)(\{)",
112
+ bygroups(Keyword, Whitespace, Punctuation),
113
+ "brace-block"),
114
+ (r"(item)(\s*)(\()",
115
+ bygroups(Keyword, Whitespace, Punctuation),
116
+ "paren-block"),
117
+ # Templates
118
+ (r"template", Keyword.Declaration, "template"),
119
+ # Nested blocks. When extensions are added, this is where they go.
120
+ (r"(responses|items|mime-types|patterns|suffixes|marks|widgets|strings|styles)(\s*)(\[)",
121
+ bygroups(Keyword, Whitespace, Punctuation),
122
+ "bracket-block"),
123
+ (r"(accessibility|setters|layout|item)(\s*)(\{)",
124
+ bygroups(Keyword, Whitespace, Punctuation),
125
+ "brace-block"),
126
+ (r"(condition|mark|item)(\s*)(\()",
127
+ bygroups(Keyword, Whitespace, Punctuation),
128
+ "paren-content"),
129
+ (r"\[", Punctuation, "child-type"),
130
+ # Properties and signals
131
+ (r"([a-z_][a-z0-9_\-]*(?:::[a-z0-9_]+)?)(\s*)(:|=>)",
132
+ bygroups(Name.Property, Whitespace, Punctuation),
133
+ "statement"),
134
+ include("content"),
135
+ ],
136
+ "paren-block": [
137
+ include("block-content"),
138
+ (r"\)", Punctuation, "#pop"),
139
+ ],
140
+ "paren-content": [
141
+ include("content"),
142
+ (r"\)", Punctuation, "#pop"),
143
+ ],
144
+ "bracket-block": [
145
+ include("block-content"),
146
+ (r"\]", Punctuation, "#pop"),
147
+ ],
148
+ "brace-block": [
149
+ include("block-content"),
150
+ (r"\}", Punctuation, "#pop"),
151
+ ],
152
+ "statement": [
153
+ include("content"),
154
+ (r";", Punctuation, "#pop"),
155
+ ],
156
+ "child-type": [
157
+ include("whitespace"),
158
+ (r"(action)(\s+)(response)(\s*)(=)(\s*)",
159
+ bygroups(Keyword, Whitespace, Name.Attribute, Whitespace,
160
+ Punctuation, Whitespace)),
161
+ (words(("default", "internal-child", "response")), Keyword),
162
+ (r"[a-z_][a-z0-9_\-]*", Name.Decorator),
163
+ include("value"),
164
+ (r"=", Punctuation),
165
+ (r"\]", Punctuation, "#pop"),
166
+ ],
167
+ "template": [
168
+ include("whitespace"),
169
+ include("type"),
170
+ (r":", Punctuation),
171
+ (r"\{", Punctuation, ("#pop", "brace-block")),
172
+ ],
173
+ }
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/cddl.py ADDED
@@ -0,0 +1,172 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers.cddl
3
+ ~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexer for the Concise data definition language (CDDL), a notational
6
+ convention to express CBOR and JSON data structures.
7
+
8
+ More information:
9
+ https://datatracker.ietf.org/doc/rfc8610/
10
+
11
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
12
+ :license: BSD, see LICENSE for details.
13
+ """
14
+
15
+ from pygments.lexer import RegexLexer, bygroups, include, words
16
+ from pygments.token import Comment, Error, Keyword, Name, Number, Operator, \
17
+ Punctuation, String, Whitespace
18
+
19
+ __all__ = ['CddlLexer']
20
+
21
+
22
+ class CddlLexer(RegexLexer):
23
+ """
24
+ Lexer for CDDL definitions.
25
+ """
26
+ name = "CDDL"
27
+ url = 'https://datatracker.ietf.org/doc/rfc8610/'
28
+ aliases = ["cddl"]
29
+ filenames = ["*.cddl"]
30
+ mimetypes = ["text/x-cddl"]
31
+ version_added = '2.8'
32
+
33
+ _prelude_types = [
34
+ "any",
35
+ "b64legacy",
36
+ "b64url",
37
+ "bigfloat",
38
+ "bigint",
39
+ "bignint",
40
+ "biguint",
41
+ "bool",
42
+ "bstr",
43
+ "bytes",
44
+ "cbor-any",
45
+ "decfrac",
46
+ "eb16",
47
+ "eb64legacy",
48
+ "eb64url",
49
+ "encoded-cbor",
50
+ "false",
51
+ "float",
52
+ "float16",
53
+ "float16-32",
54
+ "float32",
55
+ "float32-64",
56
+ "float64",
57
+ "int",
58
+ "integer",
59
+ "mime-message",
60
+ "nil",
61
+ "nint",
62
+ "null",
63
+ "number",
64
+ "regexp",
65
+ "tdate",
66
+ "text",
67
+ "time",
68
+ "true",
69
+ "tstr",
70
+ "uint",
71
+ "undefined",
72
+ "unsigned",
73
+ "uri",
74
+ ]
75
+
76
+ _controls = [
77
+ ".and",
78
+ ".bits",
79
+ ".cbor",
80
+ ".cborseq",
81
+ ".default",
82
+ ".eq",
83
+ ".ge",
84
+ ".gt",
85
+ ".le",
86
+ ".lt",
87
+ ".ne",
88
+ ".regexp",
89
+ ".size",
90
+ ".within",
91
+ ]
92
+
93
+ _re_id = (
94
+ r"[$@A-Z_a-z]"
95
+ r"(?:[\-\.]+(?=[$@0-9A-Z_a-z])|[$@0-9A-Z_a-z])*"
96
+
97
+ )
98
+
99
+ # While the spec reads more like "an int must not start with 0" we use a
100
+ # lookahead here that says "after a 0 there must be no digit". This makes the
101
+ # '0' the invalid character in '01', which looks nicer when highlighted.
102
+ _re_uint = r"(?:0b[01]+|0x[0-9a-fA-F]+|[1-9]\d*|0(?!\d))"
103
+ _re_int = r"-?" + _re_uint
104
+
105
+ tokens = {
106
+ "commentsandwhitespace": [(r"\s+", Whitespace), (r";.+$", Comment.Single)],
107
+ "root": [
108
+ include("commentsandwhitespace"),
109
+ # tag types
110
+ (rf"#(\d\.{_re_uint})?", Keyword.Type), # type or any
111
+ # occurrence
112
+ (
113
+ rf"({_re_uint})?(\*)({_re_uint})?",
114
+ bygroups(Number, Operator, Number),
115
+ ),
116
+ (r"\?|\+", Operator), # occurrence
117
+ (r"\^", Operator), # cuts
118
+ (r"(\.\.\.|\.\.)", Operator), # rangeop
119
+ (words(_controls, suffix=r"\b"), Operator.Word), # ctlops
120
+ # into choice op
121
+ (rf"&(?=\s*({_re_id}|\())", Operator),
122
+ (rf"~(?=\s*{_re_id})", Operator), # unwrap op
123
+ (r"//|/(?!/)", Operator), # double und single slash
124
+ (r"=>|/==|/=|=", Operator),
125
+ (r"[\[\]{}\(\),<>:]", Punctuation),
126
+ # Bytestrings
127
+ (r"(b64)(')", bygroups(String.Affix, String.Single), "bstrb64url"),
128
+ (r"(h)(')", bygroups(String.Affix, String.Single), "bstrh"),
129
+ (r"'", String.Single, "bstr"),
130
+ # Barewords as member keys (must be matched before values, types, typenames,
131
+ # groupnames).
132
+ # Token type is String as barewords are always interpreted as such.
133
+ (rf"({_re_id})(\s*)(:)",
134
+ bygroups(String, Whitespace, Punctuation)),
135
+ # predefined types
136
+ (words(_prelude_types, prefix=r"(?![\-_$@])\b", suffix=r"\b(?![\-_$@])"),
137
+ Name.Builtin),
138
+ # user-defined groupnames, typenames
139
+ (_re_id, Name.Class),
140
+ # values
141
+ (r"0b[01]+", Number.Bin),
142
+ (r"0o[0-7]+", Number.Oct),
143
+ (r"0x[0-9a-fA-F]+(\.[0-9a-fA-F]+)?p[+-]?\d+", Number.Hex), # hexfloat
144
+ (r"0x[0-9a-fA-F]+", Number.Hex), # hex
145
+ # Float
146
+ (rf"{_re_int}(?=(\.\d|e[+-]?\d))(?:\.\d+)?(?:e[+-]?\d+)?",
147
+ Number.Float),
148
+ # Int
149
+ (_re_int, Number.Integer),
150
+ (r'"(\\\\|\\"|[^"])*"', String.Double),
151
+ ],
152
+ "bstrb64url": [
153
+ (r"'", String.Single, "#pop"),
154
+ include("commentsandwhitespace"),
155
+ (r"\\.", String.Escape),
156
+ (r"[0-9a-zA-Z\-_=]+", String.Single),
157
+ (r".", Error),
158
+ # (r";.+$", Token.Other),
159
+ ],
160
+ "bstrh": [
161
+ (r"'", String.Single, "#pop"),
162
+ include("commentsandwhitespace"),
163
+ (r"\\.", String.Escape),
164
+ (r"[0-9a-fA-F]+", String.Single),
165
+ (r".", Error),
166
+ ],
167
+ "bstr": [
168
+ (r"'", String.Single, "#pop"),
169
+ (r"\\.", String.Escape),
170
+ (r"[^'\\]+", String.Single),
171
+ ],
172
+ }
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/codeql.py ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers.codeql
3
+ ~~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexer for CodeQL query language.
6
+
7
+ The grammar is originating from:
8
+ https://github.com/github/vscode-codeql/blob/main/syntaxes/README.md
9
+
10
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
11
+ :license: BSD, see LICENSE for details.
12
+ """
13
+
14
+ __all__ = ['CodeQLLexer']
15
+
16
+ import re
17
+
18
+ from pygments.lexer import RegexLexer, words
19
+ from pygments.token import Comment, Operator, Keyword, Name, String, \
20
+ Number, Punctuation, Whitespace
21
+
22
+ class CodeQLLexer(RegexLexer):
23
+ name = 'CodeQL'
24
+ aliases = ['codeql', 'ql']
25
+ filenames = ['*.ql', '*.qll']
26
+ mimetypes = []
27
+ url = 'https://github.com/github/codeql'
28
+ version_added = '2.19'
29
+
30
+ flags = re.MULTILINE | re.UNICODE
31
+
32
+ tokens = {
33
+ 'root': [
34
+ # Whitespace and comments
35
+ (r'\s+', Whitespace),
36
+ (r'//.*?\n', Comment.Single),
37
+ (r'/\*', Comment.Multiline, 'multiline-comments'),
38
+
39
+ # Keywords
40
+ (words((
41
+ 'module', 'import', 'class', 'extends', 'implements',
42
+ 'predicate', 'select', 'where', 'from', 'as', 'and', 'or', 'not',
43
+ 'in', 'if', 'then', 'else', 'exists', 'forall', 'instanceof',
44
+ 'private', 'predicate', 'abstract', 'cached', 'external',
45
+ 'final', 'library', 'override', 'query'
46
+ ), suffix=r'\b'), Keyword.Builtin),
47
+
48
+ # Special Keywords
49
+ (words(('this'), # class related keywords
50
+ prefix=r'\b', suffix=r'\b\??:?'), Name.Builtin.Pseudo),
51
+
52
+ # Types
53
+ (words((
54
+ 'boolean', 'date', 'float', 'int', 'string'
55
+ ), suffix=r'\b'), Keyword.Type),
56
+
57
+ # Literals
58
+ (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
59
+ (r'[0-9]+\.[0-9]+', Number.Float),
60
+ (r'[0-9]+', Number.Integer),
61
+
62
+ # Operators
63
+ (r'<=|>=|<|>|=|!=|\+|-|\*|/', Operator),
64
+
65
+ # Punctuation
66
+ (r'[.,;:\[\]{}()]+', Punctuation),
67
+
68
+ # Identifiers
69
+ (r'@[a-zA-Z_]\w*', Name.Variable), # Variables with @ prefix
70
+ (r'[A-Z][a-zA-Z0-9_]*', Name.Class), # Types and classes
71
+ (r'[a-z][a-zA-Z0-9_]*', Name.Variable), # Variables and predicates
72
+ ],
73
+ 'multiline-comments': [
74
+ (r'[^*/]+', Comment.Multiline),
75
+ (r'/\*', Comment.Multiline, '#push'),
76
+ (r'\*/', Comment.Multiline, '#pop'),
77
+ (r'[*/]', Comment.Multiline),
78
+ ],
79
+
80
+ }
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/data.py ADDED
@@ -0,0 +1,763 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers.data
3
+ ~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexers for data file format.
6
+
7
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import Lexer, ExtendedRegexLexer, LexerContext, \
12
+ include, bygroups
13
+ from pygments.token import Comment, Error, Keyword, Literal, Name, Number, \
14
+ Punctuation, String, Whitespace
15
+
16
+ __all__ = ['YamlLexer', 'JsonLexer', 'JsonBareObjectLexer', 'JsonLdLexer']
17
+
18
+
19
+ class YamlLexerContext(LexerContext):
20
+ """Indentation context for the YAML lexer."""
21
+
22
+ def __init__(self, *args, **kwds):
23
+ super().__init__(*args, **kwds)
24
+ self.indent_stack = []
25
+ self.indent = -1
26
+ self.next_indent = 0
27
+ self.block_scalar_indent = None
28
+
29
+
30
+ class YamlLexer(ExtendedRegexLexer):
31
+ """
32
+ Lexer for YAML, a human-friendly data serialization
33
+ language.
34
+ """
35
+
36
+ name = 'YAML'
37
+ url = 'http://yaml.org/'
38
+ aliases = ['yaml']
39
+ filenames = ['*.yaml', '*.yml']
40
+ mimetypes = ['text/x-yaml']
41
+ version_added = '0.11'
42
+
43
+ def something(token_class):
44
+ """Do not produce empty tokens."""
45
+ def callback(lexer, match, context):
46
+ text = match.group()
47
+ if not text:
48
+ return
49
+ yield match.start(), token_class, text
50
+ context.pos = match.end()
51
+ return callback
52
+
53
+ def reset_indent(token_class):
54
+ """Reset the indentation levels."""
55
+ def callback(lexer, match, context):
56
+ text = match.group()
57
+ context.indent_stack = []
58
+ context.indent = -1
59
+ context.next_indent = 0
60
+ context.block_scalar_indent = None
61
+ yield match.start(), token_class, text
62
+ context.pos = match.end()
63
+ return callback
64
+
65
+ def save_indent(token_class, start=False):
66
+ """Save a possible indentation level."""
67
+ def callback(lexer, match, context):
68
+ text = match.group()
69
+ extra = ''
70
+ if start:
71
+ context.next_indent = len(text)
72
+ if context.next_indent < context.indent:
73
+ while context.next_indent < context.indent:
74
+ context.indent = context.indent_stack.pop()
75
+ if context.next_indent > context.indent:
76
+ extra = text[context.indent:]
77
+ text = text[:context.indent]
78
+ else:
79
+ context.next_indent += len(text)
80
+ if text:
81
+ yield match.start(), token_class, text
82
+ if extra:
83
+ yield match.start()+len(text), token_class.Error, extra
84
+ context.pos = match.end()
85
+ return callback
86
+
87
+ def set_indent(token_class, implicit=False):
88
+ """Set the previously saved indentation level."""
89
+ def callback(lexer, match, context):
90
+ text = match.group()
91
+ if context.indent < context.next_indent:
92
+ context.indent_stack.append(context.indent)
93
+ context.indent = context.next_indent
94
+ if not implicit:
95
+ context.next_indent += len(text)
96
+ yield match.start(), token_class, text
97
+ context.pos = match.end()
98
+ return callback
99
+
100
+ def set_block_scalar_indent(token_class):
101
+ """Set an explicit indentation level for a block scalar."""
102
+ def callback(lexer, match, context):
103
+ text = match.group()
104
+ context.block_scalar_indent = None
105
+ if not text:
106
+ return
107
+ increment = match.group(1)
108
+ if increment:
109
+ current_indent = max(context.indent, 0)
110
+ increment = int(increment)
111
+ context.block_scalar_indent = current_indent + increment
112
+ if text:
113
+ yield match.start(), token_class, text
114
+ context.pos = match.end()
115
+ return callback
116
+
117
+ def parse_block_scalar_empty_line(indent_token_class, content_token_class):
118
+ """Process an empty line in a block scalar."""
119
+ def callback(lexer, match, context):
120
+ text = match.group()
121
+ if (context.block_scalar_indent is None or
122
+ len(text) <= context.block_scalar_indent):
123
+ if text:
124
+ yield match.start(), indent_token_class, text
125
+ else:
126
+ indentation = text[:context.block_scalar_indent]
127
+ content = text[context.block_scalar_indent:]
128
+ yield match.start(), indent_token_class, indentation
129
+ yield (match.start()+context.block_scalar_indent,
130
+ content_token_class, content)
131
+ context.pos = match.end()
132
+ return callback
133
+
134
+ def parse_block_scalar_indent(token_class):
135
+ """Process indentation spaces in a block scalar."""
136
+ def callback(lexer, match, context):
137
+ text = match.group()
138
+ if context.block_scalar_indent is None:
139
+ if len(text) <= max(context.indent, 0):
140
+ context.stack.pop()
141
+ context.stack.pop()
142
+ return
143
+ context.block_scalar_indent = len(text)
144
+ else:
145
+ if len(text) < context.block_scalar_indent:
146
+ context.stack.pop()
147
+ context.stack.pop()
148
+ return
149
+ if text:
150
+ yield match.start(), token_class, text
151
+ context.pos = match.end()
152
+ return callback
153
+
154
+ def parse_plain_scalar_indent(token_class):
155
+ """Process indentation spaces in a plain scalar."""
156
+ def callback(lexer, match, context):
157
+ text = match.group()
158
+ if len(text) <= context.indent:
159
+ context.stack.pop()
160
+ context.stack.pop()
161
+ return
162
+ if text:
163
+ yield match.start(), token_class, text
164
+ context.pos = match.end()
165
+ return callback
166
+
167
+ tokens = {
168
+ # the root rules
169
+ 'root': [
170
+ # ignored whitespaces
171
+ (r'[ ]+(?=#|$)', Whitespace),
172
+ # line breaks
173
+ (r'\n+', Whitespace),
174
+ # a comment
175
+ (r'#[^\n]*', Comment.Single),
176
+ # the '%YAML' directive
177
+ (r'^%YAML(?=[ ]|$)', reset_indent(Name.Tag), 'yaml-directive'),
178
+ # the %TAG directive
179
+ (r'^%TAG(?=[ ]|$)', reset_indent(Name.Tag), 'tag-directive'),
180
+ # document start and document end indicators
181
+ (r'^(?:---|\.\.\.)(?=[ ]|$)', reset_indent(Name.Namespace),
182
+ 'block-line'),
183
+ # indentation spaces
184
+ (r'[ ]*(?!\s|$)', save_indent(Whitespace, start=True),
185
+ ('block-line', 'indentation')),
186
+ ],
187
+
188
+ # trailing whitespaces after directives or a block scalar indicator
189
+ 'ignored-line': [
190
+ # ignored whitespaces
191
+ (r'[ ]+(?=#|$)', Whitespace),
192
+ # a comment
193
+ (r'#[^\n]*', Comment.Single),
194
+ # line break
195
+ (r'\n', Whitespace, '#pop:2'),
196
+ ],
197
+
198
+ # the %YAML directive
199
+ 'yaml-directive': [
200
+ # the version number
201
+ (r'([ ]+)([0-9]+\.[0-9]+)',
202
+ bygroups(Whitespace, Number), 'ignored-line'),
203
+ ],
204
+
205
+ # the %TAG directive
206
+ 'tag-directive': [
207
+ # a tag handle and the corresponding prefix
208
+ (r'([ ]+)(!|![\w-]*!)'
209
+ r'([ ]+)(!|!?[\w;/?:@&=+$,.!~*\'()\[\]%-]+)',
210
+ bygroups(Whitespace, Keyword.Type, Whitespace, Keyword.Type),
211
+ 'ignored-line'),
212
+ ],
213
+
214
+ # block scalar indicators and indentation spaces
215
+ 'indentation': [
216
+ # trailing whitespaces are ignored
217
+ (r'[ ]*$', something(Whitespace), '#pop:2'),
218
+ # whitespaces preceding block collection indicators
219
+ (r'[ ]+(?=[?:-](?:[ ]|$))', save_indent(Whitespace)),
220
+ # block collection indicators
221
+ (r'[?:-](?=[ ]|$)', set_indent(Punctuation.Indicator)),
222
+ # the beginning a block line
223
+ (r'[ ]*', save_indent(Whitespace), '#pop'),
224
+ ],
225
+
226
+ # an indented line in the block context
227
+ 'block-line': [
228
+ # the line end
229
+ (r'[ ]*(?=#|$)', something(Whitespace), '#pop'),
230
+ # whitespaces separating tokens
231
+ (r'[ ]+', Whitespace),
232
+ # key with colon
233
+ (r'''([^#,?\[\]{}"'\n]+)(:)(?=[ ]|$)''',
234
+ bygroups(Name.Tag, set_indent(Punctuation, implicit=True))),
235
+ # tags, anchors and aliases,
236
+ include('descriptors'),
237
+ # block collections and scalars
238
+ include('block-nodes'),
239
+ # flow collections and quoted scalars
240
+ include('flow-nodes'),
241
+ # a plain scalar
242
+ (r'(?=[^\s?:,\[\]{}#&*!|>\'"%@`-]|[?:-]\S)',
243
+ something(Name.Variable),
244
+ 'plain-scalar-in-block-context'),
245
+ ],
246
+
247
+ # tags, anchors, aliases
248
+ 'descriptors': [
249
+ # a full-form tag
250
+ (r'!<[\w#;/?:@&=+$,.!~*\'()\[\]%-]+>', Keyword.Type),
251
+ # a tag in the form '!', '!suffix' or '!handle!suffix'
252
+ (r'!(?:[\w-]+!)?'
253
+ r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]*', Keyword.Type),
254
+ # an anchor
255
+ (r'&[\w-]+', Name.Label),
256
+ # an alias
257
+ (r'\*[\w-]+', Name.Variable),
258
+ ],
259
+
260
+ # block collections and scalars
261
+ 'block-nodes': [
262
+ # implicit key
263
+ (r':(?=[ ]|$)', set_indent(Punctuation.Indicator, implicit=True)),
264
+ # literal and folded scalars
265
+ (r'[|>]', Punctuation.Indicator,
266
+ ('block-scalar-content', 'block-scalar-header')),
267
+ ],
268
+
269
+ # flow collections and quoted scalars
270
+ 'flow-nodes': [
271
+ # a flow sequence
272
+ (r'\[', Punctuation.Indicator, 'flow-sequence'),
273
+ # a flow mapping
274
+ (r'\{', Punctuation.Indicator, 'flow-mapping'),
275
+ # a single-quoted scalar
276
+ (r'\'', String, 'single-quoted-scalar'),
277
+ # a double-quoted scalar
278
+ (r'\"', String, 'double-quoted-scalar'),
279
+ ],
280
+
281
+ # the content of a flow collection
282
+ 'flow-collection': [
283
+ # whitespaces
284
+ (r'[ ]+', Whitespace),
285
+ # line breaks
286
+ (r'\n+', Whitespace),
287
+ # a comment
288
+ (r'#[^\n]*', Comment.Single),
289
+ # simple indicators
290
+ (r'[?:,]', Punctuation.Indicator),
291
+ # tags, anchors and aliases
292
+ include('descriptors'),
293
+ # nested collections and quoted scalars
294
+ include('flow-nodes'),
295
+ # a plain scalar
296
+ (r'(?=[^\s?:,\[\]{}#&*!|>\'"%@`])',
297
+ something(Name.Variable),
298
+ 'plain-scalar-in-flow-context'),
299
+ ],
300
+
301
+ # a flow sequence indicated by '[' and ']'
302
+ 'flow-sequence': [
303
+ # include flow collection rules
304
+ include('flow-collection'),
305
+ # the closing indicator
306
+ (r'\]', Punctuation.Indicator, '#pop'),
307
+ ],
308
+
309
+ # a flow mapping indicated by '{' and '}'
310
+ 'flow-mapping': [
311
+ # key with colon
312
+ (r'''([^,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''',
313
+ bygroups(Name.Tag, Punctuation)),
314
+ # include flow collection rules
315
+ include('flow-collection'),
316
+ # the closing indicator
317
+ (r'\}', Punctuation.Indicator, '#pop'),
318
+ ],
319
+
320
+ # block scalar lines
321
+ 'block-scalar-content': [
322
+ # line break
323
+ (r'\n', Whitespace),
324
+ # empty line
325
+ (r'^[ ]+$',
326
+ parse_block_scalar_empty_line(Whitespace, Name.Constant)),
327
+ # indentation spaces (we may leave the state here)
328
+ (r'^[ ]*', parse_block_scalar_indent(Whitespace)),
329
+ # line content
330
+ (r'[\S\t ]+', Name.Constant),
331
+ ],
332
+
333
+ # the content of a literal or folded scalar
334
+ 'block-scalar-header': [
335
+ # indentation indicator followed by chomping flag
336
+ (r'([1-9])?[+-]?(?=[ ]|$)',
337
+ set_block_scalar_indent(Punctuation.Indicator),
338
+ 'ignored-line'),
339
+ # chomping flag followed by indentation indicator
340
+ (r'[+-]?([1-9])?(?=[ ]|$)',
341
+ set_block_scalar_indent(Punctuation.Indicator),
342
+ 'ignored-line'),
343
+ ],
344
+
345
+ # ignored and regular whitespaces in quoted scalars
346
+ 'quoted-scalar-whitespaces': [
347
+ # leading and trailing whitespaces are ignored
348
+ (r'^[ ]+', Whitespace),
349
+ (r'[ ]+$', Whitespace),
350
+ # line breaks are ignored
351
+ (r'\n+', Whitespace),
352
+ # other whitespaces are a part of the value
353
+ (r'[ ]+', Name.Variable),
354
+ ],
355
+
356
+ # single-quoted scalars
357
+ 'single-quoted-scalar': [
358
+ # include whitespace and line break rules
359
+ include('quoted-scalar-whitespaces'),
360
+ # escaping of the quote character
361
+ (r'\'\'', String.Escape),
362
+ # regular non-whitespace characters
363
+ (r'[^\s\']+', String),
364
+ # the closing quote
365
+ (r'\'', String, '#pop'),
366
+ ],
367
+
368
+ # double-quoted scalars
369
+ 'double-quoted-scalar': [
370
+ # include whitespace and line break rules
371
+ include('quoted-scalar-whitespaces'),
372
+ # escaping of special characters
373
+ (r'\\[0abt\tn\nvfre "\\N_LP]', String),
374
+ # escape codes
375
+ (r'\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})',
376
+ String.Escape),
377
+ # regular non-whitespace characters
378
+ (r'[^\s"\\]+', String),
379
+ # the closing quote
380
+ (r'"', String, '#pop'),
381
+ ],
382
+
383
+ # the beginning of a new line while scanning a plain scalar
384
+ 'plain-scalar-in-block-context-new-line': [
385
+ # empty lines
386
+ (r'^[ ]+$', Whitespace),
387
+ # line breaks
388
+ (r'\n+', Whitespace),
389
+ # document start and document end indicators
390
+ (r'^(?=---|\.\.\.)', something(Name.Namespace), '#pop:3'),
391
+ # indentation spaces (we may leave the block line state here)
392
+ (r'^[ ]*', parse_plain_scalar_indent(Whitespace), '#pop'),
393
+ ],
394
+
395
+ # a plain scalar in the block context
396
+ 'plain-scalar-in-block-context': [
397
+ # the scalar ends with the ':' indicator
398
+ (r'[ ]*(?=:[ ]|:$)', something(Whitespace), '#pop'),
399
+ # the scalar ends with whitespaces followed by a comment
400
+ (r'[ ]+(?=#)', Whitespace, '#pop'),
401
+ # trailing whitespaces are ignored
402
+ (r'[ ]+$', Whitespace),
403
+ # line breaks are ignored
404
+ (r'\n+', Whitespace, 'plain-scalar-in-block-context-new-line'),
405
+ # other whitespaces are a part of the value
406
+ (r'[ ]+', Literal.Scalar.Plain),
407
+ # regular non-whitespace characters
408
+ (r'(?::(?!\s)|[^\s:])+', Literal.Scalar.Plain),
409
+ ],
410
+
411
+ # a plain scalar is the flow context
412
+ 'plain-scalar-in-flow-context': [
413
+ # the scalar ends with an indicator character
414
+ (r'[ ]*(?=[,:?\[\]{}])', something(Whitespace), '#pop'),
415
+ # the scalar ends with a comment
416
+ (r'[ ]+(?=#)', Whitespace, '#pop'),
417
+ # leading and trailing whitespaces are ignored
418
+ (r'^[ ]+', Whitespace),
419
+ (r'[ ]+$', Whitespace),
420
+ # line breaks are ignored
421
+ (r'\n+', Whitespace),
422
+ # other whitespaces are a part of the value
423
+ (r'[ ]+', Name.Variable),
424
+ # regular non-whitespace characters
425
+ (r'[^\s,:?\[\]{}]+', Name.Variable),
426
+ ],
427
+
428
+ }
429
+
430
+ def get_tokens_unprocessed(self, text=None, context=None):
431
+ if context is None:
432
+ context = YamlLexerContext(text, 0)
433
+ return super().get_tokens_unprocessed(text, context)
434
+
435
+
436
+ class JsonLexer(Lexer):
437
+ """
438
+ For JSON data structures.
439
+
440
+ Javascript-style comments are supported (like ``/* */`` and ``//``),
441
+ though comments are not part of the JSON specification.
442
+ This allows users to highlight JSON as it is used in the wild.
443
+
444
+ No validation is performed on the input JSON document.
445
+ """
446
+
447
+ name = 'JSON'
448
+ url = 'https://www.json.org'
449
+ aliases = ['json', 'json-object']
450
+ filenames = ['*.json', '*.jsonl', '*.ndjson', 'Pipfile.lock', '*.module', '*.xc']
451
+ mimetypes = ['application/json', 'application/json-object', 'application/x-ndjson', 'application/jsonl', 'application/json-seq']
452
+ version_added = '1.5'
453
+
454
+ # No validation of integers, floats, or constants is done.
455
+ # As long as the characters are members of the following
456
+ # sets, the token will be considered valid. For example,
457
+ #
458
+ # "--1--" is parsed as an integer
459
+ # "1...eee" is parsed as a float
460
+ # "trustful" is parsed as a constant
461
+ #
462
+ integers = set('-0123456789')
463
+ floats = set('.eE+')
464
+ constants = set('truefalsenull') # true|false|null
465
+ hexadecimals = set('0123456789abcdefABCDEF')
466
+ punctuations = set('{}[],')
467
+ whitespaces = {'\u0020', '\u000a', '\u000d', '\u0009'}
468
+
469
+ def get_tokens_unprocessed(self, text):
470
+ """Parse JSON data."""
471
+
472
+ in_string = False
473
+ in_escape = False
474
+ in_unicode_escape = 0
475
+ in_whitespace = False
476
+ in_constant = False
477
+ in_number = False
478
+ in_float = False
479
+ in_punctuation = False
480
+ in_comment_single = False
481
+ in_comment_multiline = False
482
+ expecting_second_comment_opener = False # // or /*
483
+ expecting_second_comment_closer = False # */
484
+
485
+ start = 0
486
+
487
+ # The queue is used to store data that may need to be tokenized
488
+ # differently based on what follows. In particular, JSON object
489
+ # keys are tokenized differently than string values, but cannot
490
+ # be distinguished until punctuation is encountered outside the
491
+ # string.
492
+ #
493
+ # A ":" character after the string indicates that the string is
494
+ # an object key; any other character indicates the string is a
495
+ # regular string value.
496
+ #
497
+ # The queue holds tuples that contain the following data:
498
+ #
499
+ # (start_index, token_type, text)
500
+ #
501
+ # By default the token type of text in double quotes is
502
+ # String.Double. The token type will be replaced if a colon
503
+ # is encountered after the string closes.
504
+ #
505
+ queue = []
506
+
507
+ for stop, character in enumerate(text):
508
+ if in_string:
509
+ if in_unicode_escape:
510
+ if character in self.hexadecimals:
511
+ in_unicode_escape -= 1
512
+ if not in_unicode_escape:
513
+ in_escape = False
514
+ else:
515
+ in_unicode_escape = 0
516
+ in_escape = False
517
+
518
+ elif in_escape:
519
+ if character == 'u':
520
+ in_unicode_escape = 4
521
+ else:
522
+ in_escape = False
523
+
524
+ elif character == '\\':
525
+ in_escape = True
526
+
527
+ elif character == '"':
528
+ queue.append((start, String.Double, text[start:stop + 1]))
529
+ in_string = False
530
+ in_escape = False
531
+ in_unicode_escape = 0
532
+
533
+ continue
534
+
535
+ elif in_whitespace:
536
+ if character in self.whitespaces:
537
+ continue
538
+
539
+ if queue:
540
+ queue.append((start, Whitespace, text[start:stop]))
541
+ else:
542
+ yield start, Whitespace, text[start:stop]
543
+ in_whitespace = False
544
+ # Fall through so the new character can be evaluated.
545
+
546
+ elif in_constant:
547
+ if character in self.constants:
548
+ continue
549
+
550
+ yield start, Keyword.Constant, text[start:stop]
551
+ in_constant = False
552
+ # Fall through so the new character can be evaluated.
553
+
554
+ elif in_number:
555
+ if character in self.integers:
556
+ continue
557
+ elif character in self.floats:
558
+ in_float = True
559
+ continue
560
+
561
+ if in_float:
562
+ yield start, Number.Float, text[start:stop]
563
+ else:
564
+ yield start, Number.Integer, text[start:stop]
565
+ in_number = False
566
+ in_float = False
567
+ # Fall through so the new character can be evaluated.
568
+
569
+ elif in_punctuation:
570
+ if character in self.punctuations:
571
+ continue
572
+
573
+ yield start, Punctuation, text[start:stop]
574
+ in_punctuation = False
575
+ # Fall through so the new character can be evaluated.
576
+
577
+ elif in_comment_single:
578
+ if character != '\n':
579
+ continue
580
+
581
+ if queue:
582
+ queue.append((start, Comment.Single, text[start:stop]))
583
+ else:
584
+ yield start, Comment.Single, text[start:stop]
585
+
586
+ in_comment_single = False
587
+ # Fall through so the new character can be evaluated.
588
+
589
+ elif in_comment_multiline:
590
+ if character == '*':
591
+ expecting_second_comment_closer = True
592
+ elif expecting_second_comment_closer:
593
+ expecting_second_comment_closer = False
594
+ if character == '/':
595
+ if queue:
596
+ queue.append((start, Comment.Multiline, text[start:stop + 1]))
597
+ else:
598
+ yield start, Comment.Multiline, text[start:stop + 1]
599
+
600
+ in_comment_multiline = False
601
+
602
+ continue
603
+
604
+ elif expecting_second_comment_opener:
605
+ expecting_second_comment_opener = False
606
+ if character == '/':
607
+ in_comment_single = True
608
+ continue
609
+ elif character == '*':
610
+ in_comment_multiline = True
611
+ continue
612
+
613
+ # Exhaust the queue. Accept the existing token types.
614
+ yield from queue
615
+ queue.clear()
616
+
617
+ yield start, Error, text[start:stop]
618
+ # Fall through so the new character can be evaluated.
619
+
620
+ start = stop
621
+
622
+ if character == '"':
623
+ in_string = True
624
+
625
+ elif character in self.whitespaces:
626
+ in_whitespace = True
627
+
628
+ elif character in {'f', 'n', 't'}: # The first letters of true|false|null
629
+ # Exhaust the queue. Accept the existing token types.
630
+ yield from queue
631
+ queue.clear()
632
+
633
+ in_constant = True
634
+
635
+ elif character in self.integers:
636
+ # Exhaust the queue. Accept the existing token types.
637
+ yield from queue
638
+ queue.clear()
639
+
640
+ in_number = True
641
+
642
+ elif character == ':':
643
+ # Yield from the queue. Replace string token types.
644
+ for _start, _token, _text in queue:
645
+ # There can be only three types of tokens before a ':':
646
+ # Whitespace, Comment, or a quoted string.
647
+ #
648
+ # If it's a quoted string we emit Name.Tag.
649
+ # Otherwise, we yield the original token.
650
+ #
651
+ # In all other cases this would be invalid JSON,
652
+ # but this is not a validating JSON lexer, so it's OK.
653
+ if _token is String.Double:
654
+ yield _start, Name.Tag, _text
655
+ else:
656
+ yield _start, _token, _text
657
+ queue.clear()
658
+
659
+ in_punctuation = True
660
+
661
+ elif character in self.punctuations:
662
+ # Exhaust the queue. Accept the existing token types.
663
+ yield from queue
664
+ queue.clear()
665
+
666
+ in_punctuation = True
667
+
668
+ elif character == '/':
669
+ # This is the beginning of a comment.
670
+ expecting_second_comment_opener = True
671
+
672
+ else:
673
+ # Exhaust the queue. Accept the existing token types.
674
+ yield from queue
675
+ queue.clear()
676
+
677
+ yield start, Error, character
678
+
679
+ # Yield any remaining text.
680
+ yield from queue
681
+ if in_string:
682
+ yield start, Error, text[start:]
683
+ elif in_float:
684
+ yield start, Number.Float, text[start:]
685
+ elif in_number:
686
+ yield start, Number.Integer, text[start:]
687
+ elif in_constant:
688
+ yield start, Keyword.Constant, text[start:]
689
+ elif in_whitespace:
690
+ yield start, Whitespace, text[start:]
691
+ elif in_punctuation:
692
+ yield start, Punctuation, text[start:]
693
+ elif in_comment_single:
694
+ yield start, Comment.Single, text[start:]
695
+ elif in_comment_multiline:
696
+ yield start, Error, text[start:]
697
+ elif expecting_second_comment_opener:
698
+ yield start, Error, text[start:]
699
+
700
+
701
+ class JsonBareObjectLexer(JsonLexer):
702
+ """
703
+ For JSON data structures (with missing object curly braces).
704
+
705
+ .. deprecated:: 2.8.0
706
+
707
+ Behaves the same as `JsonLexer` now.
708
+ """
709
+
710
+ name = 'JSONBareObject'
711
+ aliases = []
712
+ filenames = []
713
+ mimetypes = []
714
+ version_added = '2.2'
715
+
716
+
717
+ class JsonLdLexer(JsonLexer):
718
+ """
719
+ For JSON-LD linked data.
720
+ """
721
+
722
+ name = 'JSON-LD'
723
+ url = 'https://json-ld.org/'
724
+ aliases = ['jsonld', 'json-ld']
725
+ filenames = ['*.jsonld']
726
+ mimetypes = ['application/ld+json']
727
+ version_added = '2.0'
728
+
729
+ json_ld_keywords = {
730
+ f'"@{keyword}"'
731
+ for keyword in (
732
+ 'base',
733
+ 'container',
734
+ 'context',
735
+ 'direction',
736
+ 'graph',
737
+ 'id',
738
+ 'import',
739
+ 'included',
740
+ 'index',
741
+ 'json',
742
+ 'language',
743
+ 'list',
744
+ 'nest',
745
+ 'none',
746
+ 'prefix',
747
+ 'propagate',
748
+ 'protected',
749
+ 'reverse',
750
+ 'set',
751
+ 'type',
752
+ 'value',
753
+ 'version',
754
+ 'vocab',
755
+ )
756
+ }
757
+
758
+ def get_tokens_unprocessed(self, text):
759
+ for start, token, value in super().get_tokens_unprocessed(text):
760
+ if token is Name.Tag and value in self.json_ld_keywords:
761
+ yield start, Name.Decorator, value
762
+ else:
763
+ yield start, token, value
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/graph.py ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers.graph
3
+ ~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexers for graph query languages.
6
+
7
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ import re
12
+
13
+ from pygments.lexer import RegexLexer, include, bygroups, using, this, words
14
+ from pygments.token import Keyword, Punctuation, Comment, Operator, Name,\
15
+ String, Number, Whitespace
16
+
17
+
18
+ __all__ = ['CypherLexer']
19
+
20
+
21
+ class CypherLexer(RegexLexer):
22
+ """
23
+ For Cypher Query Language
24
+
25
+ For the Cypher version in Neo4j 3.3
26
+ """
27
+ name = 'Cypher'
28
+ url = 'https://neo4j.com/docs/developer-manual/3.3/cypher/'
29
+ aliases = ['cypher']
30
+ filenames = ['*.cyp', '*.cypher']
31
+ version_added = '2.0'
32
+
33
+ flags = re.MULTILINE | re.IGNORECASE
34
+
35
+ tokens = {
36
+ 'root': [
37
+ include('clauses'),
38
+ include('keywords'),
39
+ include('relations'),
40
+ include('strings'),
41
+ include('whitespace'),
42
+ include('barewords'),
43
+ include('comment'),
44
+ ],
45
+ 'keywords': [
46
+ (r'(create|order|match|limit|set|skip|start|return|with|where|'
47
+ r'delete|foreach|not|by|true|false)\b', Keyword),
48
+ ],
49
+ 'clauses': [
50
+ # based on https://neo4j.com/docs/cypher-refcard/3.3/
51
+ (r'(create)(\s+)(index|unique)\b',
52
+ bygroups(Keyword, Whitespace, Keyword)),
53
+ (r'(drop)(\s+)(contraint|index)(\s+)(on)\b',
54
+ bygroups(Keyword, Whitespace, Keyword, Whitespace, Keyword)),
55
+ (r'(ends)(\s+)(with)\b',
56
+ bygroups(Keyword, Whitespace, Keyword)),
57
+ (r'(is)(\s+)(node)(\s+)(key)\b',
58
+ bygroups(Keyword, Whitespace, Keyword, Whitespace, Keyword)),
59
+ (r'(is)(\s+)(null|unique)\b',
60
+ bygroups(Keyword, Whitespace, Keyword)),
61
+ (r'(load)(\s+)(csv)(\s+)(from)\b',
62
+ bygroups(Keyword, Whitespace, Keyword, Whitespace, Keyword)),
63
+ (r'(on)(\s+)(match|create)\b',
64
+ bygroups(Keyword, Whitespace, Keyword)),
65
+ (r'(optional)(\s+)(match)\b',
66
+ bygroups(Keyword, Whitespace, Keyword)),
67
+ (r'(order)(\s+)(by)\b',
68
+ bygroups(Keyword, Whitespace, Keyword)),
69
+ (r'(starts)(\s+)(with)\b',
70
+ bygroups(Keyword, Whitespace, Keyword)),
71
+ (r'(union)(\s+)(all)\b',
72
+ bygroups(Keyword, Whitespace, Keyword)),
73
+ (r'(using)(\s+)(periodic)(\s+)(commit)\b',
74
+ bygroups(Keyword, Whitespace, Keyword, Whitespace, Keyword)),
75
+ (r'(using)(\s+)(index)\b',
76
+ bygroups(Keyword, Whitespace, Keyword)),
77
+ (r'(using)(\s+)(range|text|point)(\s+)(index)\b',
78
+ bygroups(Keyword, Whitespace, Name, Whitespace, Keyword)),
79
+ (words((
80
+ 'all', 'any', 'as', 'asc', 'ascending', 'assert', 'call', 'case', 'create',
81
+ 'delete', 'desc', 'descending', 'distinct', 'end', 'fieldterminator',
82
+ 'foreach', 'in', 'limit', 'match', 'merge', 'none', 'not', 'null',
83
+ 'remove', 'return', 'set', 'skip', 'single', 'start', 'then', 'union',
84
+ 'unwind', 'yield', 'where', 'when', 'with', 'collect'), suffix=r'\b'), Keyword),
85
+ ],
86
+ 'relations': [
87
+ (r'(-\[)(.*?)(\]->)', bygroups(Operator, using(this), Operator)),
88
+ (r'(<-\[)(.*?)(\]-)', bygroups(Operator, using(this), Operator)),
89
+ (r'(-\[)(.*?)(\]-)', bygroups(Operator, using(this), Operator)),
90
+ (r'-->|<--|\[|\]', Operator),
91
+ (r'<|>|<>|=|<=|=>|\(|\)|\||:|,|;', Punctuation),
92
+ (r'[.*{}]', Punctuation),
93
+ ],
94
+ 'strings': [
95
+ (r'([\'"])(?:\\[tbnrf\'"\\]|[^\\])*?\1', String),
96
+ (r'`(?:``|[^`])+`', Name.Variable),
97
+ ],
98
+ 'whitespace': [
99
+ (r'\s+', Whitespace),
100
+ ],
101
+ 'barewords': [
102
+ (r'[a-z]\w*', Name),
103
+ (r'\d+', Number),
104
+ ],
105
+ 'comment': [
106
+ (r'//.*$', Comment.Single),
107
+ ],
108
+ }
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/julia.py ADDED
@@ -0,0 +1,294 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers.julia
3
+ ~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexers for the Julia language.
6
+
7
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import Lexer, RegexLexer, bygroups, do_insertions, \
12
+ words, include
13
+ from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
14
+ Number, Punctuation, Generic, Whitespace
15
+ from pygments.util import shebang_matches
16
+ from pygments.lexers._julia_builtins import OPERATORS_LIST, DOTTED_OPERATORS_LIST, \
17
+ KEYWORD_LIST, BUILTIN_LIST, LITERAL_LIST
18
+
19
+ __all__ = ['JuliaLexer', 'JuliaConsoleLexer']
20
+
21
+ # see https://docs.julialang.org/en/v1/manual/variables/#Allowed-Variable-Names
22
+ allowed_variable = \
23
+ '(?:[a-zA-Z_\u00A1-\U0010ffff][a-zA-Z_0-9!\u00A1-\U0010ffff]*)'
24
+ # see https://github.com/JuliaLang/julia/blob/master/src/flisp/julia_opsuffs.h
25
+ operator_suffixes = r'[²³¹ʰʲʳʷʸˡˢˣᴬᴮᴰᴱᴳᴴᴵᴶᴷᴸᴹᴺᴼᴾᴿᵀᵁᵂᵃᵇᵈᵉᵍᵏᵐᵒᵖᵗᵘᵛᵝᵞᵟᵠᵡᵢᵣᵤᵥᵦᵧᵨᵩᵪᶜᶠᶥᶦᶫᶰᶸᶻᶿ′″‴‵‶‷⁗⁰ⁱ⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎ₐₑₒₓₕₖₗₘₙₚₛₜⱼⱽ]*'
26
+
27
+ class JuliaLexer(RegexLexer):
28
+ """
29
+ For Julia source code.
30
+ """
31
+
32
+ name = 'Julia'
33
+ url = 'https://julialang.org/'
34
+ aliases = ['julia', 'jl']
35
+ filenames = ['*.jl']
36
+ mimetypes = ['text/x-julia', 'application/x-julia']
37
+ version_added = '1.6'
38
+
39
+ tokens = {
40
+ 'root': [
41
+ (r'\n', Whitespace),
42
+ (r'[^\S\n]+', Whitespace),
43
+ (r'#=', Comment.Multiline, "blockcomment"),
44
+ (r'#.*$', Comment),
45
+ (r'[\[\](),;]', Punctuation),
46
+
47
+ # symbols
48
+ # intercept range expressions first
49
+ (r'(' + allowed_variable + r')(\s*)(:)(' + allowed_variable + ')',
50
+ bygroups(Name, Whitespace, Operator, Name)),
51
+ # then match :name which does not follow closing brackets, digits, or the
52
+ # ::, <:, and :> operators
53
+ (r'(?<![\]):<>\d.])(:' + allowed_variable + ')', String.Symbol),
54
+
55
+ # type assertions - excludes expressions like ::typeof(sin) and ::avec[1]
56
+ (r'(?<=::)(\s*)(' + allowed_variable + r')\b(?![(\[])',
57
+ bygroups(Whitespace, Keyword.Type)),
58
+ # type comparisons
59
+ # - MyType <: A or MyType >: A
60
+ ('(' + allowed_variable + r')(\s*)([<>]:)(\s*)(' + allowed_variable + r')\b(?![(\[])',
61
+ bygroups(Keyword.Type, Whitespace, Operator, Whitespace, Keyword.Type)),
62
+ # - <: B or >: B
63
+ (r'([<>]:)(\s*)(' + allowed_variable + r')\b(?![(\[])',
64
+ bygroups(Operator, Whitespace, Keyword.Type)),
65
+ # - A <: or A >:
66
+ (r'\b(' + allowed_variable + r')(\s*)([<>]:)',
67
+ bygroups(Keyword.Type, Whitespace, Operator)),
68
+
69
+ # operators
70
+ # Suffixes aren't actually allowed on all operators, but we'll ignore that
71
+ # since those cases are invalid Julia code.
72
+ (words([*OPERATORS_LIST, *DOTTED_OPERATORS_LIST],
73
+ suffix=operator_suffixes), Operator),
74
+ (words(['.' + o for o in DOTTED_OPERATORS_LIST],
75
+ suffix=operator_suffixes), Operator),
76
+ (words(['...', '..']), Operator),
77
+
78
+ # NOTE
79
+ # Patterns below work only for definition sites and thus hardly reliable.
80
+ #
81
+ # functions
82
+ # (r'(function)(\s+)(' + allowed_variable + ')',
83
+ # bygroups(Keyword, Text, Name.Function)),
84
+
85
+ # chars
86
+ (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|"
87
+ r"\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'", String.Char),
88
+
89
+ # try to match trailing transpose
90
+ (r'(?<=[.\w)\]])(\'' + operator_suffixes + ')+', Operator),
91
+
92
+ # raw strings
93
+ (r'(raw)(""")', bygroups(String.Affix, String), 'tqrawstring'),
94
+ (r'(raw)(")', bygroups(String.Affix, String), 'rawstring'),
95
+ # regular expressions
96
+ (r'(r)(""")', bygroups(String.Affix, String.Regex), 'tqregex'),
97
+ (r'(r)(")', bygroups(String.Affix, String.Regex), 'regex'),
98
+ # other strings
99
+ (r'(' + allowed_variable + ')?(""")',
100
+ bygroups(String.Affix, String), 'tqstring'),
101
+ (r'(' + allowed_variable + ')?(")',
102
+ bygroups(String.Affix, String), 'string'),
103
+
104
+ # backticks
105
+ (r'(' + allowed_variable + ')?(```)',
106
+ bygroups(String.Affix, String.Backtick), 'tqcommand'),
107
+ (r'(' + allowed_variable + ')?(`)',
108
+ bygroups(String.Affix, String.Backtick), 'command'),
109
+
110
+ # type names
111
+ # - names that begin a curly expression
112
+ ('(' + allowed_variable + r')(\{)',
113
+ bygroups(Keyword.Type, Punctuation), 'curly'),
114
+ # - names as part of bare 'where'
115
+ (r'(where)(\s+)(' + allowed_variable + ')',
116
+ bygroups(Keyword, Whitespace, Keyword.Type)),
117
+ # - curly expressions in general
118
+ (r'(\{)', Punctuation, 'curly'),
119
+ # - names as part of type declaration
120
+ (r'(abstract|primitive)([ \t]+)(type\b)([\s()]+)(' +
121
+ allowed_variable + r')',
122
+ bygroups(Keyword, Whitespace, Keyword, Text, Keyword.Type)),
123
+ (r'(mutable(?=[ \t]))?([ \t]+)?(struct\b)([\s()]+)(' +
124
+ allowed_variable + r')',
125
+ bygroups(Keyword, Whitespace, Keyword, Text, Keyword.Type)),
126
+
127
+ # macros
128
+ (r'@' + allowed_variable, Name.Decorator),
129
+ (words([*OPERATORS_LIST, '..', '.', *DOTTED_OPERATORS_LIST],
130
+ prefix='@', suffix=operator_suffixes), Name.Decorator),
131
+
132
+ # keywords
133
+ (words(KEYWORD_LIST, suffix=r'\b'), Keyword),
134
+ # builtin types
135
+ (words(BUILTIN_LIST, suffix=r'\b'), Keyword.Type),
136
+ # builtin literals
137
+ (words(LITERAL_LIST, suffix=r'\b'), Name.Builtin),
138
+
139
+ # names
140
+ (allowed_variable, Name),
141
+
142
+ # numbers
143
+ (r'(\d+((_\d+)+)?\.(?!\.)(\d+((_\d+)+)?)?|\.\d+((_\d+)+)?)([eEf][+-]?[0-9]+)?', Number.Float),
144
+ (r'\d+((_\d+)+)?[eEf][+-]?[0-9]+', Number.Float),
145
+ (r'0x[a-fA-F0-9]+((_[a-fA-F0-9]+)+)?(\.([a-fA-F0-9]+((_[a-fA-F0-9]+)+)?)?)?p[+-]?\d+', Number.Float),
146
+ (r'0b[01]+((_[01]+)+)?', Number.Bin),
147
+ (r'0o[0-7]+((_[0-7]+)+)?', Number.Oct),
148
+ (r'0x[a-fA-F0-9]+((_[a-fA-F0-9]+)+)?', Number.Hex),
149
+ (r'\d+((_\d+)+)?', Number.Integer),
150
+
151
+ # single dot operator matched last to permit e.g. ".1" as a float
152
+ (words(['.']), Operator),
153
+ ],
154
+
155
+ "blockcomment": [
156
+ (r'[^=#]', Comment.Multiline),
157
+ (r'#=', Comment.Multiline, '#push'),
158
+ (r'=#', Comment.Multiline, '#pop'),
159
+ (r'[=#]', Comment.Multiline),
160
+ ],
161
+
162
+ 'curly': [
163
+ (r'\{', Punctuation, '#push'),
164
+ (r'\}', Punctuation, '#pop'),
165
+ (allowed_variable, Keyword.Type),
166
+ include('root'),
167
+ ],
168
+
169
+ 'tqrawstring': [
170
+ (r'"""', String, '#pop'),
171
+ (r'([^"]|"[^"][^"])+', String),
172
+ ],
173
+ 'rawstring': [
174
+ (r'"', String, '#pop'),
175
+ (r'\\"', String.Escape),
176
+ (r'([^"\\]|\\[^"])+', String),
177
+ ],
178
+
179
+ # Interpolation is defined as "$" followed by the shortest full
180
+ # expression, which is something we can't parse. Include the most
181
+ # common cases here: $word, and $(paren'd expr).
182
+ 'interp': [
183
+ (r'\$' + allowed_variable, String.Interpol),
184
+ (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'),
185
+ ],
186
+ 'in-intp': [
187
+ (r'\(', Punctuation, '#push'),
188
+ (r'\)', Punctuation, '#pop'),
189
+ include('root'),
190
+ ],
191
+
192
+ 'string': [
193
+ (r'(")(' + allowed_variable + r'|\d+)?',
194
+ bygroups(String, String.Affix), '#pop'),
195
+ # FIXME: This escape pattern is not perfect.
196
+ (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape),
197
+ include('interp'),
198
+ # @printf and @sprintf formats
199
+ (r'%[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]',
200
+ String.Interpol),
201
+ (r'[^"$%\\]+', String),
202
+ (r'.', String),
203
+ ],
204
+ 'tqstring': [
205
+ (r'(""")(' + allowed_variable + r'|\d+)?',
206
+ bygroups(String, String.Affix), '#pop'),
207
+ (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape),
208
+ include('interp'),
209
+ (r'[^"$%\\]+', String),
210
+ (r'.', String),
211
+ ],
212
+
213
+ 'regex': [
214
+ (r'(")([imsxa]*)?', bygroups(String.Regex, String.Affix), '#pop'),
215
+ (r'\\"', String.Regex),
216
+ (r'[^\\"]+', String.Regex),
217
+ ],
218
+
219
+ 'tqregex': [
220
+ (r'(""")([imsxa]*)?', bygroups(String.Regex, String.Affix), '#pop'),
221
+ (r'[^"]+', String.Regex),
222
+ ],
223
+
224
+ 'command': [
225
+ (r'(`)(' + allowed_variable + r'|\d+)?',
226
+ bygroups(String.Backtick, String.Affix), '#pop'),
227
+ (r'\\[`$]', String.Escape),
228
+ include('interp'),
229
+ (r'[^\\`$]+', String.Backtick),
230
+ (r'.', String.Backtick),
231
+ ],
232
+ 'tqcommand': [
233
+ (r'(```)(' + allowed_variable + r'|\d+)?',
234
+ bygroups(String.Backtick, String.Affix), '#pop'),
235
+ (r'\\\$', String.Escape),
236
+ include('interp'),
237
+ (r'[^\\`$]+', String.Backtick),
238
+ (r'.', String.Backtick),
239
+ ],
240
+ }
241
+
242
+ def analyse_text(text):
243
+ return shebang_matches(text, r'julia')
244
+
245
+
246
+ class JuliaConsoleLexer(Lexer):
247
+ """
248
+ For Julia console sessions. Modeled after MatlabSessionLexer.
249
+ """
250
+ name = 'Julia console'
251
+ aliases = ['jlcon', 'julia-repl']
252
+ url = 'https://julialang.org/'
253
+ version_added = '1.6'
254
+ _example = "jlcon/console"
255
+
256
+ def get_tokens_unprocessed(self, text):
257
+ jllexer = JuliaLexer(**self.options)
258
+ start = 0
259
+ curcode = ''
260
+ insertions = []
261
+ output = False
262
+ error = False
263
+
264
+ for line in text.splitlines(keepends=True):
265
+ if line.startswith('julia>'):
266
+ insertions.append((len(curcode), [(0, Generic.Prompt, line[:6])]))
267
+ curcode += line[6:]
268
+ output = False
269
+ error = False
270
+ elif line.startswith('help?>') or line.startswith('shell>'):
271
+ yield start, Generic.Prompt, line[:6]
272
+ yield start + 6, Text, line[6:]
273
+ output = False
274
+ error = False
275
+ elif line.startswith(' ') and not output:
276
+ insertions.append((len(curcode), [(0, Whitespace, line[:6])]))
277
+ curcode += line[6:]
278
+ else:
279
+ if curcode:
280
+ yield from do_insertions(
281
+ insertions, jllexer.get_tokens_unprocessed(curcode))
282
+ curcode = ''
283
+ insertions = []
284
+ if line.startswith('ERROR: ') or error:
285
+ yield start, Generic.Error, line
286
+ error = True
287
+ else:
288
+ yield start, Generic.Output, line
289
+ output = True
290
+ start += len(line)
291
+
292
+ if curcode:
293
+ yield from do_insertions(
294
+ insertions, jllexer.get_tokens_unprocessed(curcode))
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/maple.py ADDED
@@ -0,0 +1,291 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers.maple
3
+ ~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexers for Maple.
6
+
7
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import words, bygroups, ExtendedRegexLexer
12
+ from pygments.token import Comment, Name, String, Whitespace, Operator, Punctuation, Number, Keyword
13
+
14
+ __all__ = ['MapleLexer']
15
+
16
+
17
+ class MapleLexer(ExtendedRegexLexer):
18
+ """
19
+ Lexer for Maple.
20
+ """
21
+
22
+ name = 'Maple'
23
+ aliases = ['maple']
24
+ filenames = ['*.mpl', '*.mi', '*.mm']
25
+ mimetypes = ['text/x-maple']
26
+ url = 'https://www.maplesoft.com/products/Maple/'
27
+ version_added = '2.19'
28
+
29
+ keywords = ('and',
30
+ 'assuming',
31
+ 'break',
32
+ 'by',
33
+ 'catch',
34
+ 'description',
35
+ 'do',
36
+ 'done',
37
+ 'elif',
38
+ 'else',
39
+ 'end',
40
+ 'error',
41
+ 'export',
42
+ 'fi',
43
+ 'finally',
44
+ 'for',
45
+ 'from',
46
+ 'global',
47
+ 'if',
48
+ 'implies',
49
+ 'in',
50
+ 'intersect',
51
+ 'local',
52
+ 'minus',
53
+ 'mod',
54
+ 'module',
55
+ 'next',
56
+ 'not',
57
+ 'od',
58
+ 'option',
59
+ 'options',
60
+ 'or',
61
+ 'proc',
62
+ 'quit',
63
+ 'read',
64
+ 'return',
65
+ 'save',
66
+ 'stop',
67
+ 'subset',
68
+ 'then',
69
+ 'to',
70
+ 'try',
71
+ 'union',
72
+ 'use',
73
+ 'uses',
74
+ 'while',
75
+ 'xor')
76
+
77
+ builtins = ('abs',
78
+ 'add',
79
+ 'addressof',
80
+ 'anames',
81
+ 'and',
82
+ 'andmap',
83
+ 'andseq',
84
+ 'appendto',
85
+ 'Array',
86
+ 'array',
87
+ 'ArrayOptions',
88
+ 'assemble',
89
+ 'ASSERT',
90
+ 'assign',
91
+ 'assigned',
92
+ 'attributes',
93
+ 'cat',
94
+ 'ceil',
95
+ 'coeff',
96
+ 'coeffs',
97
+ 'conjugate',
98
+ 'convert',
99
+ 'CopySign',
100
+ 'DEBUG',
101
+ 'debugopts',
102
+ 'Default0',
103
+ 'DefaultOverflow',
104
+ 'DefaultUnderflow',
105
+ 'degree',
106
+ 'denom',
107
+ 'diff',
108
+ 'disassemble',
109
+ 'divide',
110
+ 'done',
111
+ 'entries',
112
+ 'EqualEntries',
113
+ 'eval',
114
+ 'evalb',
115
+ 'evalf',
116
+ 'evalhf',
117
+ 'evalindets',
118
+ 'evaln',
119
+ 'expand',
120
+ 'exports',
121
+ 'factorial',
122
+ 'floor',
123
+ 'frac',
124
+ 'frem',
125
+ 'FromInert',
126
+ 'frontend',
127
+ 'gc',
128
+ 'genpoly',
129
+ 'has',
130
+ 'hastype',
131
+ 'hfarray',
132
+ 'icontent',
133
+ 'igcd',
134
+ 'ilcm',
135
+ 'ilog10',
136
+ 'Im',
137
+ 'implies',
138
+ 'indets',
139
+ 'indices',
140
+ 'intersect',
141
+ 'iolib',
142
+ 'iquo',
143
+ 'irem',
144
+ 'iroot',
145
+ 'iroot',
146
+ 'isqrt',
147
+ 'kernelopts',
148
+ 'lcoeff',
149
+ 'ldegree',
150
+ 'length',
151
+ 'lexorder',
152
+ 'lhs',
153
+ 'lowerbound',
154
+ 'lprint',
155
+ 'macro',
156
+ 'map',
157
+ 'max',
158
+ 'maxnorm',
159
+ 'member',
160
+ 'membertype',
161
+ 'min',
162
+ 'minus',
163
+ 'mod',
164
+ 'modp',
165
+ 'modp1',
166
+ 'modp2',
167
+ 'mods',
168
+ 'mul',
169
+ 'NextAfter',
170
+ 'nops',
171
+ 'normal',
172
+ 'not',
173
+ 'numboccur',
174
+ 'numelems',
175
+ 'numer',
176
+ 'NumericClass',
177
+ 'NumericEvent',
178
+ 'NumericEventHandler',
179
+ 'NumericStatus',
180
+ 'op',
181
+ 'or',
182
+ 'order',
183
+ 'OrderedNE',
184
+ 'ormap',
185
+ 'orseq',
186
+ 'parse',
187
+ 'piecewise',
188
+ 'pointto',
189
+ 'print',
190
+ 'quit',
191
+ 'Re',
192
+ 'readlib',
193
+ 'Record',
194
+ 'remove',
195
+ 'rhs',
196
+ 'round',
197
+ 'rtable',
198
+ 'rtable_elems',
199
+ 'rtable_eval',
200
+ 'rtable_indfns',
201
+ 'rtable_num_elems',
202
+ 'rtable_options',
203
+ 'rtable_redim',
204
+ 'rtable_scanblock',
205
+ 'rtable_set_indfn',
206
+ 'rtable_split_unit',
207
+ 'savelib',
208
+ 'Scale10',
209
+ 'Scale2',
210
+ 'SDMPolynom',
211
+ 'searchtext',
212
+ 'SearchText',
213
+ 'select',
214
+ 'selectremove',
215
+ 'seq',
216
+ 'series',
217
+ 'setattribute',
218
+ 'SFloatExponent',
219
+ 'SFloatMantissa',
220
+ 'sign',
221
+ 'sort',
222
+ 'ssystem',
223
+ 'stop',
224
+ 'String',
225
+ 'subs',
226
+ 'subset',
227
+ 'subsindets',
228
+ 'subsop',
229
+ 'substring',
230
+ 'system',
231
+ 'table',
232
+ 'taylor',
233
+ 'tcoeff',
234
+ 'time',
235
+ 'timelimit',
236
+ 'ToInert',
237
+ 'traperror',
238
+ 'trunc',
239
+ 'type',
240
+ 'typematch',
241
+ 'unames',
242
+ 'unassign',
243
+ 'union',
244
+ 'Unordered',
245
+ 'upperbound',
246
+ 'userinfo',
247
+ 'writeto',
248
+ 'xor',
249
+ 'xormap',
250
+ 'xorseq')
251
+
252
+ def delayed_callback(self, match, ctx):
253
+ yield match.start(1), Punctuation, match.group(1) # quote
254
+
255
+ ctx.pos = match.start(2)
256
+ orig_end = ctx.end
257
+ ctx.end = match.end(2)
258
+
259
+ yield from self.get_tokens_unprocessed(context=ctx)
260
+ yield match.end(2), Punctuation, match.group(1) # quote
261
+
262
+ ctx.pos = match.end()
263
+ ctx.end = orig_end
264
+
265
+ tokens = {
266
+ 'root': [
267
+ (r'#.*\n', Comment.Single),
268
+ (r'\(\*', Comment.Multiline, 'comment'),
269
+ (r'"(\\.|.|\s)*?"', String),
270
+ (r"('+)((.|\n)*?)\1", delayed_callback),
271
+ (r'`(\\`|.)*?`', Name),
272
+ (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword),
273
+ (words(builtins, prefix=r'\b', suffix=r'\b'), Name.Builtin),
274
+ (r'[a-zA-Z_][a-zA-Z0-9_]*', Name),
275
+ (r'(:=|\*\*|@@|<=|>=|<>|->|::|\.\.|&\+|[\+\-\*\.\^\$/@&,:=<>%~])', Operator),
276
+ (r'[;^!@$\(\)\[\]{}|_\\#?]+', Punctuation),
277
+ (r'(\d+)(\.\.)', bygroups(Number.Integer, Punctuation)),
278
+ (r'(\d*\.\d+|\d+\.\d*)([eE][+-]?\d+)?', Number.Float),
279
+ (r'\d+', Number.Integer),
280
+ (r'\s+', Whitespace),
281
+ ],
282
+ 'comment': [
283
+ (r'.*\(\*', Comment.Multiline, '#push'),
284
+ (r'.*\*\)', Comment.Multiline, '#pop'),
285
+ (r'.*\n', Comment.Multiline),
286
+ ]
287
+ }
288
+
289
+ def analyse_text(text):
290
+ if ':=' in text:
291
+ return 0.1
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/ptx.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers.ptx
3
+ ~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexer for other PTX language.
6
+
7
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import RegexLexer, include, words
12
+ from pygments.token import Comment, Keyword, Name, String, Number, \
13
+ Punctuation, Whitespace, Operator
14
+
15
+ __all__ = ["PtxLexer"]
16
+
17
+
18
+ class PtxLexer(RegexLexer):
19
+ """
20
+ For NVIDIA `PTX <https://docs.nvidia.com/cuda/parallel-thread-execution/>`_
21
+ source.
22
+ """
23
+ name = 'PTX'
24
+ url = "https://docs.nvidia.com/cuda/parallel-thread-execution/"
25
+ filenames = ['*.ptx']
26
+ aliases = ['ptx']
27
+ mimetypes = ['text/x-ptx']
28
+ version_added = '2.16'
29
+
30
+ #: optional Comment or Whitespace
31
+ string = r'"[^"]*?"'
32
+ followsym = r'[a-zA-Z0-9_$]'
33
+ identifier = r'([-a-zA-Z$._][\w\-$.]*|' + string + ')'
34
+ block_label = r'(' + identifier + r'|(\d+))'
35
+
36
+ tokens = {
37
+ 'root': [
38
+ include('whitespace'),
39
+
40
+ (block_label + r'\s*:', Name.Label),
41
+
42
+ include('keyword'),
43
+
44
+ (r'%' + identifier, Name.Variable),
45
+ (r'%\d+', Name.Variable.Anonymous),
46
+ (r'c?' + string, String),
47
+ (identifier, Name.Variable),
48
+ (r';', Punctuation),
49
+ (r'[*+-/]', Operator),
50
+
51
+ (r'0[xX][a-fA-F0-9]+', Number),
52
+ (r'-?\d+(?:[.]\d+)?(?:[eE][-+]?\d+(?:[.]\d+)?)?', Number),
53
+
54
+ (r'[=<>{}\[\]()*.,!]|x\b', Punctuation)
55
+
56
+ ],
57
+ 'whitespace': [
58
+ (r'(\n|\s+)+', Whitespace),
59
+ (r'//.*?\n', Comment)
60
+ ],
61
+
62
+ 'keyword': [
63
+ # Instruction keywords
64
+ (words((
65
+ 'abs', 'discard', 'min', 'shf', 'vadd',
66
+ 'activemask', 'div', 'mma', 'shfl', 'vadd2',
67
+ 'add', 'dp2a', 'mov', 'shl', 'vadd4',
68
+ 'addc', 'dp4a', 'movmatrix', 'shr', 'vavrg2',
69
+ 'alloca', 'elect', 'mul', 'sin', 'vavrg4',
70
+ 'and', 'ex2', 'mul24', 'slct', 'vmad',
71
+ 'applypriority', 'exit', 'multimem', 'sqrt', 'vmax',
72
+ 'atom', 'fence', 'nanosleep', 'st', 'vmax2',
73
+ 'bar', 'fma', 'neg', 'stackrestore', 'vmax4',
74
+ 'barrier', 'fns', 'not', 'stacksave', 'vmin',
75
+ 'bfe', 'getctarank', 'or', 'stmatrix', 'vmin2',
76
+ 'bfi', 'griddepcontrol', 'pmevent', 'sub', 'vmin4',
77
+ 'bfind', 'isspacep', 'popc', 'subc', 'vote',
78
+ 'bmsk', 'istypep', 'prefetch', 'suld', 'vset',
79
+ 'bra', 'ld', 'prefetchu', 'suq', 'vset2',
80
+ 'brev', 'ldmatrix', 'prmt', 'sured', 'vset4',
81
+ 'brkpt', 'ldu', 'rcp', 'sust', 'vshl',
82
+ 'brx', 'lg2', 'red', 'szext', 'vshr',
83
+ 'call', 'lop3', 'redux', 'tanh', 'vsub',
84
+ 'clz', 'mad', 'rem', 'testp', 'vsub2',
85
+ 'cnot', 'mad24', 'ret', 'tex', 'vsub4',
86
+ 'copysign', 'madc', 'rsqrt', 'tld4', 'wgmma',
87
+ 'cos', 'mapa', 'sad', 'trap', 'wmma',
88
+ 'cp', 'match', 'selp', 'txq', 'xor',
89
+ 'createpolicy', 'max', 'set', 'vabsdiff', 'cvt',
90
+ 'mbarrier', 'setmaxnreg', 'vabsdiff2', 'cvta',
91
+ 'membar', 'setp', 'vabsdiff4')), Keyword),
92
+ # State Spaces and Suffixes
93
+ (words((
94
+ 'reg', '.sreg', '.const', '.global',
95
+ '.local', '.param', '.shared', '.tex',
96
+ '.wide', '.loc'
97
+ )), Keyword.Pseudo),
98
+ # PTX Directives
99
+ (words((
100
+ '.address_size', '.explicitcluster', '.maxnreg', '.section',
101
+ '.alias', '.extern', '.maxntid', '.shared',
102
+ '.align', '.file', '.minnctapersm', '.sreg',
103
+ '.branchtargets', '.func', '.noreturn', '.target',
104
+ '.callprototype', '.global', '.param', '.tex',
105
+ '.calltargets', '.loc', '.pragma', '.version',
106
+ '.common', '.local', '.reg', '.visible',
107
+ '.const', '.maxclusterrank', '.reqnctapercluster', '.weak',
108
+ '.entry', '.maxnctapersm', '.reqntid')), Keyword.Reserved),
109
+ # Fundamental Types
110
+ (words((
111
+ '.s8', '.s16', '.s32', '.s64',
112
+ '.u8', '.u16', '.u32', '.u64',
113
+ '.f16', '.f16x2', '.f32', '.f64',
114
+ '.b8', '.b16', '.b32', '.b64',
115
+ '.pred'
116
+ )), Keyword.Type)
117
+ ],
118
+
119
+ }
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/scripting.py ADDED
@@ -0,0 +1,1638 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers.scripting
3
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexer for scripting and embedded languages.
6
+
7
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ import re
12
+
13
+ from pygments.lexer import RegexLexer, include, bygroups, default, combined, \
14
+ words
15
+ from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
16
+ Number, Punctuation, Error, Whitespace, Other
17
+ from pygments.util import get_bool_opt, get_list_opt
18
+
19
+ __all__ = ['LuaLexer', 'LuauLexer', 'MoonScriptLexer', 'ChaiscriptLexer', 'LSLLexer',
20
+ 'AppleScriptLexer', 'RexxLexer', 'MOOCodeLexer', 'HybrisLexer',
21
+ 'EasytrieveLexer', 'JclLexer', 'MiniScriptLexer']
22
+
23
+
24
+ def all_lua_builtins():
25
+ from pygments.lexers._lua_builtins import MODULES
26
+ return [w for values in MODULES.values() for w in values]
27
+
28
+ class LuaLexer(RegexLexer):
29
+ """
30
+ For Lua source code.
31
+
32
+ Additional options accepted:
33
+
34
+ `func_name_highlighting`
35
+ If given and ``True``, highlight builtin function names
36
+ (default: ``True``).
37
+ `disabled_modules`
38
+ If given, must be a list of module names whose function names
39
+ should not be highlighted. By default all modules are highlighted.
40
+
41
+ To get a list of allowed modules have a look into the
42
+ `_lua_builtins` module:
43
+
44
+ .. sourcecode:: pycon
45
+
46
+ >>> from pygments.lexers._lua_builtins import MODULES
47
+ >>> MODULES.keys()
48
+ ['string', 'coroutine', 'modules', 'io', 'basic', ...]
49
+ """
50
+
51
+ name = 'Lua'
52
+ url = 'https://www.lua.org/'
53
+ aliases = ['lua']
54
+ filenames = ['*.lua', '*.wlua']
55
+ mimetypes = ['text/x-lua', 'application/x-lua']
56
+ version_added = ''
57
+
58
+ _comment_multiline = r'(?:--\[(?P<level>=*)\[[\w\W]*?\](?P=level)\])'
59
+ _comment_single = r'(?:--.*$)'
60
+ _space = r'(?:\s+(?!\s))'
61
+ _s = rf'(?:{_comment_multiline}|{_comment_single}|{_space})'
62
+ # A lookahead-safe version of _s that avoids catastrophic backtracking.
63
+ # The _comment_multiline pattern contains [\w\W]*? which, when used
64
+ # inside a lookahead with a * quantifier, causes exponential blowup.
65
+ # This version skips only whitespace; comments between an identifier
66
+ # and a following [.:] or ( are rare enough to sacrifice.
67
+ _s_la = r'\s'
68
+ _name = r'(?:[^\W\d]\w*)'
69
+
70
+ tokens = {
71
+ 'root': [
72
+ # Lua allows a file to start with a shebang.
73
+ (r'#!.*', Comment.Preproc),
74
+ default('base'),
75
+ ],
76
+ 'ws': [
77
+ (_comment_multiline, Comment.Multiline),
78
+ (_comment_single, Comment.Single),
79
+ (_space, Whitespace),
80
+ ],
81
+ 'base': [
82
+ include('ws'),
83
+
84
+ (r'(?i)0x[\da-f]*(\.[\da-f]*)?(p[+-]?\d+)?', Number.Hex),
85
+ (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float),
86
+ (r'(?i)\d+e[+-]?\d+', Number.Float),
87
+ (r'\d+', Number.Integer),
88
+
89
+ # multiline strings
90
+ (r'(?s)\[(=*)\[.*?\]\1\]', String),
91
+
92
+ (r'::', Punctuation, 'label'),
93
+ (r'\.{3}', Punctuation),
94
+ (r'[=<>|~&+\-*/%#^]+|\.\.', Operator),
95
+ (r'[\[\]{}().,:;]+', Punctuation),
96
+ (r'(and|or|not)\b', Operator.Word),
97
+
98
+ (words([
99
+ 'break', 'do', 'else', 'elseif', 'end', 'for', 'if', 'in',
100
+ 'repeat', 'return', 'then', 'until', 'while'
101
+ ], suffix=r'\b'), Keyword.Reserved),
102
+ (r'goto\b', Keyword.Reserved, 'goto'),
103
+ (r'(local)\b', Keyword.Declaration),
104
+ (r'(true|false|nil)\b', Keyword.Constant),
105
+
106
+ (r'(function)\b', Keyword.Reserved, 'funcname'),
107
+
108
+ (words(all_lua_builtins(), suffix=r"\b"), Name.Builtin),
109
+ (fr'[A-Za-z_]\w*(?={_s_la}*[.:])', Name.Variable, 'varname'),
110
+ (fr'[A-Za-z_]\w*(?={_s_la}*\()', Name.Function),
111
+ (r'[A-Za-z_]\w*', Name.Variable),
112
+
113
+ ("'", String.Single, combined('stringescape', 'sqs')),
114
+ ('"', String.Double, combined('stringescape', 'dqs'))
115
+ ],
116
+
117
+ 'varname': [
118
+ include('ws'),
119
+ (r'\.\.', Operator, '#pop'),
120
+ (r'[.:]', Punctuation),
121
+ (rf'{_name}(?={_s_la}*[.:])', Name.Property),
122
+ (rf'{_name}(?={_s_la}*\()', Name.Function, '#pop'),
123
+ (_name, Name.Property, '#pop'),
124
+ ],
125
+
126
+ 'funcname': [
127
+ include('ws'),
128
+ (r'[.:]', Punctuation),
129
+ (rf'{_name}(?={_s_la}*[.:])', Name.Class),
130
+ (_name, Name.Function, '#pop'),
131
+ # inline function
132
+ (r'\(', Punctuation, '#pop'),
133
+ ],
134
+
135
+ 'goto': [
136
+ include('ws'),
137
+ (_name, Name.Label, '#pop'),
138
+ ],
139
+
140
+ 'label': [
141
+ include('ws'),
142
+ (r'::', Punctuation, '#pop'),
143
+ (_name, Name.Label),
144
+ ],
145
+
146
+ 'stringescape': [
147
+ (r'\\([abfnrtv\\"\']|[\r\n]{1,2}|z\s*|x[0-9a-fA-F]{2}|\d{1,3}|'
148
+ r'u\{[0-9a-fA-F]+\})', String.Escape),
149
+ ],
150
+
151
+ 'sqs': [
152
+ (r"'", String.Single, '#pop'),
153
+ (r"[^\\']+", String.Single),
154
+ ],
155
+
156
+ 'dqs': [
157
+ (r'"', String.Double, '#pop'),
158
+ (r'[^\\"]+', String.Double),
159
+ ]
160
+ }
161
+
162
+ def __init__(self, **options):
163
+ self.func_name_highlighting = get_bool_opt(
164
+ options, 'func_name_highlighting', True)
165
+ self.disabled_modules = get_list_opt(options, 'disabled_modules', [])
166
+
167
+ self._functions = set()
168
+ if self.func_name_highlighting:
169
+ from pygments.lexers._lua_builtins import MODULES
170
+ for mod, func in MODULES.items():
171
+ if mod not in self.disabled_modules:
172
+ self._functions.update(func)
173
+ RegexLexer.__init__(self, **options)
174
+
175
+ def get_tokens_unprocessed(self, text):
176
+ for index, token, value in \
177
+ RegexLexer.get_tokens_unprocessed(self, text):
178
+ if token is Name.Builtin and value not in self._functions:
179
+ if '.' in value:
180
+ a, b = value.split('.')
181
+ yield index, Name, a
182
+ yield index + len(a), Punctuation, '.'
183
+ yield index + len(a) + 1, Name, b
184
+ else:
185
+ yield index, Name, value
186
+ continue
187
+ yield index, token, value
188
+
189
+ def _luau_make_expression(should_pop, _s, _s_la):
190
+ temp_list = [
191
+ (r'0[xX][\da-fA-F_]*', Number.Hex, '#pop'),
192
+ (r'0[bB][\d_]*', Number.Bin, '#pop'),
193
+ (r'\.?\d[\d_]*(?:\.[\d_]*)?(?:[eE][+-]?[\d_]+)?', Number.Float, '#pop'),
194
+
195
+ (words((
196
+ 'true', 'false', 'nil'
197
+ ), suffix=r'\b'), Keyword.Constant, '#pop'),
198
+
199
+ (r'\[(=*)\[[.\n]*?\]\1\]', String, '#pop'),
200
+
201
+ (r'(\.)([a-zA-Z_]\w*)(?=%s*[({"\'])', bygroups(Punctuation, Name.Function), '#pop'),
202
+ (r'(\.)([a-zA-Z_]\w*)', bygroups(Punctuation, Name.Variable), '#pop'),
203
+
204
+ (rf'[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*(?={_s_la}*[({{"\'])', Name.Other, '#pop'),
205
+ (r'[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*', Name, '#pop'),
206
+ ]
207
+ if should_pop:
208
+ return temp_list
209
+ return [entry[:2] for entry in temp_list]
210
+
211
+ def _luau_make_expression_special(should_pop):
212
+ temp_list = [
213
+ (r'\{', Punctuation, ('#pop', 'closing_brace_base', 'expression')),
214
+ (r'\(', Punctuation, ('#pop', 'closing_parenthesis_base', 'expression')),
215
+
216
+ (r'::?', Punctuation, ('#pop', 'type_end', 'type_start')),
217
+
218
+ (r"'", String.Single, ('#pop', 'string_single')),
219
+ (r'"', String.Double, ('#pop', 'string_double')),
220
+ (r'`', String.Backtick, ('#pop', 'string_interpolated')),
221
+ ]
222
+ if should_pop:
223
+ return temp_list
224
+ return [(entry[0], entry[1], entry[2][1:]) for entry in temp_list]
225
+
226
+ class LuauLexer(RegexLexer):
227
+ """
228
+ For Luau source code.
229
+
230
+ Additional options accepted:
231
+
232
+ `include_luau_builtins`
233
+ If given and ``True``, automatically highlight Luau builtins
234
+ (default: ``True``).
235
+ `include_roblox_builtins`
236
+ If given and ``True``, automatically highlight Roblox-specific builtins
237
+ (default: ``False``).
238
+ `additional_builtins`
239
+ If given, must be a list of additional builtins to highlight.
240
+ `disabled_builtins`
241
+ If given, must be a list of builtins that will not be highlighted.
242
+ """
243
+
244
+ name = 'Luau'
245
+ url = 'https://luau-lang.org/'
246
+ aliases = ['luau']
247
+ filenames = ['*.luau']
248
+ version_added = '2.18'
249
+
250
+ _comment_multiline = r'(?:--\[(?P<level>=*)\[[\w\W]*?\](?P=level)\])'
251
+ _comment_single = r'(?:--.*$)'
252
+ _s = r'(?:{}|{}|{})'.format(_comment_multiline, _comment_single, r'\s+')
253
+ # Lookahead-safe version — avoids catastrophic backtracking from
254
+ # [\w\W]*? inside _comment_multiline when combined with * quantifier.
255
+ _s_la = r'\s'
256
+
257
+ tokens = {
258
+ 'root': [
259
+ (r'#!.*', Comment.Hashbang, 'base'),
260
+ default('base'),
261
+ ],
262
+
263
+ 'ws': [
264
+ (_comment_multiline, Comment.Multiline),
265
+ (_comment_single, Comment.Single),
266
+ (r'\s+', Whitespace),
267
+ ],
268
+
269
+ 'base': [
270
+ include('ws'),
271
+
272
+ *_luau_make_expression_special(False),
273
+ (r'\.\.\.', Punctuation),
274
+
275
+ (rf'type\b(?={_s}+[a-zA-Z_])', Keyword.Reserved, 'type_declaration'),
276
+ (rf'export\b(?={_s}+[a-zA-Z_])', Keyword.Reserved),
277
+
278
+ (r'(?:\.\.|//|[+\-*\/%^<>=])=?', Operator, 'expression'),
279
+ (r'~=', Operator, 'expression'),
280
+
281
+ (words((
282
+ 'and', 'or', 'not'
283
+ ), suffix=r'\b'), Operator.Word, 'expression'),
284
+
285
+ (words((
286
+ 'elseif', 'for', 'if', 'in', 'repeat', 'return', 'until',
287
+ 'while'), suffix=r'\b'), Keyword.Reserved, 'expression'),
288
+ (r'local\b', Keyword.Declaration, 'expression'),
289
+
290
+ (r'function\b', Keyword.Reserved, ('expression', 'func_name')),
291
+
292
+ (r'[\])};]+', Punctuation),
293
+
294
+ include('expression_static'),
295
+ *_luau_make_expression(False, _s, _s_la),
296
+
297
+ (r'[\[.,]', Punctuation, 'expression'),
298
+ ],
299
+ 'expression_static': [
300
+ (words((
301
+ 'break', 'continue', 'do', 'else', 'elseif', 'end', 'for',
302
+ 'if', 'in', 'repeat', 'return', 'then', 'until', 'while'),
303
+ suffix=r'\b'), Keyword.Reserved),
304
+ ],
305
+ 'expression': [
306
+ include('ws'),
307
+
308
+ (r'if\b', Keyword.Reserved, ('ternary', 'expression')),
309
+
310
+ (r'local\b', Keyword.Declaration),
311
+ *_luau_make_expression_special(True),
312
+ (r'\.\.\.', Punctuation, '#pop'),
313
+
314
+ (r'function\b', Keyword.Reserved, 'func_name'),
315
+
316
+ include('expression_static'),
317
+ *_luau_make_expression(True, _s, _s_la),
318
+
319
+ default('#pop'),
320
+ ],
321
+ 'ternary': [
322
+ include('ws'),
323
+
324
+ (r'else\b', Keyword.Reserved, '#pop'),
325
+ (words((
326
+ 'then', 'elseif',
327
+ ), suffix=r'\b'), Operator.Reserved, 'expression'),
328
+
329
+ default('#pop'),
330
+ ],
331
+
332
+ 'closing_brace_pop': [
333
+ (r'\}', Punctuation, '#pop'),
334
+ ],
335
+ 'closing_parenthesis_pop': [
336
+ (r'\)', Punctuation, '#pop'),
337
+ ],
338
+ 'closing_gt_pop': [
339
+ (r'>', Punctuation, '#pop'),
340
+ ],
341
+
342
+ 'closing_parenthesis_base': [
343
+ include('closing_parenthesis_pop'),
344
+ include('base'),
345
+ ],
346
+ 'closing_parenthesis_type': [
347
+ include('closing_parenthesis_pop'),
348
+ include('type'),
349
+ ],
350
+ 'closing_brace_base': [
351
+ include('closing_brace_pop'),
352
+ include('base'),
353
+ ],
354
+ 'closing_brace_type': [
355
+ include('closing_brace_pop'),
356
+ include('type'),
357
+ ],
358
+ 'closing_gt_type': [
359
+ include('closing_gt_pop'),
360
+ include('type'),
361
+ ],
362
+
363
+ 'string_escape': [
364
+ (r'\\z\s*', String.Escape),
365
+ (r'\\(?:[abfnrtvz\\"\'`\{\n])|[\r\n]{1,2}|x[\da-fA-F]{2}|\d{1,3}|'
366
+ r'u\{\}[\da-fA-F]*\}', String.Escape),
367
+ ],
368
+ 'string_single': [
369
+ include('string_escape'),
370
+
371
+ (r"'", String.Single, "#pop"),
372
+ (r"[^\\']+", String.Single),
373
+ ],
374
+ 'string_double': [
375
+ include('string_escape'),
376
+
377
+ (r'"', String.Double, "#pop"),
378
+ (r'[^\\"]+', String.Double),
379
+ ],
380
+ 'string_interpolated': [
381
+ include('string_escape'),
382
+
383
+ (r'\{', Punctuation, ('closing_brace_base', 'expression')),
384
+
385
+ (r'`', String.Backtick, "#pop"),
386
+ (r'[^\\`\{]+', String.Backtick),
387
+ ],
388
+
389
+ 'func_name': [
390
+ include('ws'),
391
+
392
+ (r'[.:]', Punctuation),
393
+ (rf'[a-zA-Z_]\w*(?={_s_la}*[.:])', Name.Class),
394
+ (r'[a-zA-Z_]\w*', Name.Function),
395
+
396
+ (r'<', Punctuation, 'closing_gt_type'),
397
+
398
+ (r'\(', Punctuation, '#pop'),
399
+ ],
400
+
401
+ 'type': [
402
+ include('ws'),
403
+
404
+ (r'\(', Punctuation, 'closing_parenthesis_type'),
405
+ (r'\{', Punctuation, 'closing_brace_type'),
406
+ (r'<', Punctuation, 'closing_gt_type'),
407
+
408
+ (r"'", String.Single, 'string_single'),
409
+ (r'"', String.Double, 'string_double'),
410
+
411
+ (r'[|&\.,\[\]:=]+', Punctuation),
412
+ (r'->', Punctuation),
413
+
414
+ (r'typeof\(', Name.Builtin, ('closing_parenthesis_base',
415
+ 'expression')),
416
+ (r'[a-zA-Z_]\w*', Name.Class),
417
+ ],
418
+ 'type_start': [
419
+ include('ws'),
420
+
421
+ (r'\(', Punctuation, ('#pop', 'closing_parenthesis_type')),
422
+ (r'\{', Punctuation, ('#pop', 'closing_brace_type')),
423
+ (r'<', Punctuation, ('#pop', 'closing_gt_type')),
424
+
425
+ (r"'", String.Single, ('#pop', 'string_single')),
426
+ (r'"', String.Double, ('#pop', 'string_double')),
427
+
428
+ (r'typeof\(', Name.Builtin, ('#pop', 'closing_parenthesis_base',
429
+ 'expression')),
430
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
431
+ ],
432
+ 'type_end': [
433
+ include('ws'),
434
+
435
+ (r'[|&\.]', Punctuation, 'type_start'),
436
+ (r'->', Punctuation, 'type_start'),
437
+
438
+ (r'<', Punctuation, 'closing_gt_type'),
439
+
440
+ default('#pop'),
441
+ ],
442
+ 'type_declaration': [
443
+ include('ws'),
444
+
445
+ (r'[a-zA-Z_]\w*', Name.Class),
446
+ (r'<', Punctuation, 'closing_gt_type'),
447
+
448
+ (r'=', Punctuation, ('#pop', 'type_end', 'type_start')),
449
+ ],
450
+ }
451
+
452
+ def __init__(self, **options):
453
+ self.include_luau_builtins = get_bool_opt(
454
+ options, 'include_luau_builtins', True)
455
+ self.include_roblox_builtins = get_bool_opt(
456
+ options, 'include_roblox_builtins', False)
457
+ self.additional_builtins = get_list_opt(options, 'additional_builtins', [])
458
+ self.disabled_builtins = get_list_opt(options, 'disabled_builtins', [])
459
+
460
+ self._builtins = set(self.additional_builtins)
461
+ if self.include_luau_builtins:
462
+ from pygments.lexers._luau_builtins import LUAU_BUILTINS
463
+ self._builtins.update(LUAU_BUILTINS)
464
+ if self.include_roblox_builtins:
465
+ from pygments.lexers._luau_builtins import ROBLOX_BUILTINS
466
+ self._builtins.update(ROBLOX_BUILTINS)
467
+ if self.additional_builtins:
468
+ self._builtins.update(self.additional_builtins)
469
+ self._builtins.difference_update(self.disabled_builtins)
470
+
471
+ RegexLexer.__init__(self, **options)
472
+
473
+ def get_tokens_unprocessed(self, text):
474
+ for index, token, value in \
475
+ RegexLexer.get_tokens_unprocessed(self, text):
476
+ if token is Name or token is Name.Other:
477
+ split_value = value.split('.')
478
+ complete_value = []
479
+ new_index = index
480
+ for position in range(len(split_value), 0, -1):
481
+ potential_string = '.'.join(split_value[:position])
482
+ if potential_string in self._builtins:
483
+ yield index, Name.Builtin, potential_string
484
+ new_index += len(potential_string)
485
+
486
+ if complete_value:
487
+ yield new_index, Punctuation, '.'
488
+ new_index += 1
489
+ break
490
+ complete_value.insert(0, split_value[position - 1])
491
+
492
+ for position, substring in enumerate(complete_value):
493
+ if position + 1 == len(complete_value):
494
+ if token is Name:
495
+ yield new_index, Name.Variable, substring
496
+ continue
497
+ yield new_index, Name.Function, substring
498
+ continue
499
+ yield new_index, Name.Variable, substring
500
+ new_index += len(substring)
501
+ yield new_index, Punctuation, '.'
502
+ new_index += 1
503
+
504
+ continue
505
+ yield index, token, value
506
+
507
+ class MoonScriptLexer(LuaLexer):
508
+ """
509
+ For MoonScript source code.
510
+ """
511
+
512
+ name = 'MoonScript'
513
+ url = 'http://moonscript.org'
514
+ aliases = ['moonscript', 'moon']
515
+ filenames = ['*.moon']
516
+ mimetypes = ['text/x-moonscript', 'application/x-moonscript']
517
+ version_added = '1.5'
518
+
519
+ tokens = {
520
+ 'root': [
521
+ (r'#!(.*?)$', Comment.Preproc),
522
+ default('base'),
523
+ ],
524
+ 'base': [
525
+ ('--.*$', Comment.Single),
526
+ (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number.Float),
527
+ (r'(?i)\d+e[+-]?\d+', Number.Float),
528
+ (r'(?i)0x[0-9a-f]*', Number.Hex),
529
+ (r'\d+', Number.Integer),
530
+ (r'\n', Whitespace),
531
+ (r'[^\S\n]+', Text),
532
+ (r'(?s)\[(=*)\[.*?\]\1\]', String),
533
+ (r'(->|=>)', Name.Function),
534
+ (r':[a-zA-Z_]\w*', Name.Variable),
535
+ (r'(==|!=|~=|<=|>=|\.\.\.|\.\.|[=+\-*/%^<>#!.\\:])', Operator),
536
+ (r'[;,]', Punctuation),
537
+ (r'[\[\]{}()]', Keyword.Type),
538
+ (r'[a-zA-Z_]\w*:', Name.Variable),
539
+ (words((
540
+ 'class', 'extends', 'if', 'then', 'super', 'do', 'with',
541
+ 'import', 'export', 'while', 'elseif', 'return', 'for', 'in',
542
+ 'from', 'when', 'using', 'else', 'and', 'or', 'not', 'switch',
543
+ 'break'), suffix=r'\b'),
544
+ Keyword),
545
+ (r'(true|false|nil)\b', Keyword.Constant),
546
+ (r'(and|or|not)\b', Operator.Word),
547
+ (r'(self)\b', Name.Builtin.Pseudo),
548
+ (r'@@?([a-zA-Z_]\w*)?', Name.Variable.Class),
549
+ (r'[A-Z]\w*', Name.Class), # proper name
550
+ (words(all_lua_builtins(), suffix=r"\b"), Name.Builtin),
551
+ (r'[A-Za-z_]\w*', Name),
552
+ ("'", String.Single, combined('stringescape', 'sqs')),
553
+ ('"', String.Double, combined('stringescape', 'dqs'))
554
+ ],
555
+ 'stringescape': [
556
+ (r'''\\([abfnrtv\\"']|\d{1,3})''', String.Escape)
557
+ ],
558
+ 'strings': [
559
+ (r'[^#\\\'"]+', String),
560
+ # note that strings are multi-line.
561
+ # hashmarks, quotes and backslashes must be parsed one at a time
562
+ ],
563
+ 'interpoling_string': [
564
+ (r'\}', String.Interpol, "#pop"),
565
+ include('base')
566
+ ],
567
+ 'dqs': [
568
+ (r'"', String.Double, '#pop'),
569
+ (r'\\.|\'', String), # double-quoted string don't need ' escapes
570
+ (r'#\{', String.Interpol, "interpoling_string"),
571
+ (r'#', String),
572
+ include('strings')
573
+ ],
574
+ 'sqs': [
575
+ (r"'", String.Single, '#pop'),
576
+ (r'#|\\.|"', String), # single quoted strings don't need " escapses
577
+ include('strings')
578
+ ]
579
+ }
580
+
581
+ def get_tokens_unprocessed(self, text):
582
+ # set . as Operator instead of Punctuation
583
+ for index, token, value in LuaLexer.get_tokens_unprocessed(self, text):
584
+ if token == Punctuation and value == ".":
585
+ token = Operator
586
+ yield index, token, value
587
+
588
+
589
+ class ChaiscriptLexer(RegexLexer):
590
+ """
591
+ For ChaiScript source code.
592
+ """
593
+
594
+ name = 'ChaiScript'
595
+ url = 'http://chaiscript.com/'
596
+ aliases = ['chaiscript', 'chai']
597
+ filenames = ['*.chai']
598
+ mimetypes = ['text/x-chaiscript', 'application/x-chaiscript']
599
+ version_added = '2.0'
600
+
601
+ flags = re.DOTALL | re.MULTILINE
602
+
603
+ tokens = {
604
+ 'commentsandwhitespace': [
605
+ (r'\s+', Text),
606
+ (r'//.*?\n', Comment.Single),
607
+ (r'/\*.*?\*/', Comment.Multiline),
608
+ (r'^\#.*?\n', Comment.Single)
609
+ ],
610
+ 'slashstartsregex': [
611
+ include('commentsandwhitespace'),
612
+ (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
613
+ r'([gim]+\b|\B)', String.Regex, '#pop'),
614
+ (r'(?=/)', Text, ('#pop', 'badregex')),
615
+ default('#pop')
616
+ ],
617
+ 'badregex': [
618
+ (r'\n', Text, '#pop')
619
+ ],
620
+ 'root': [
621
+ include('commentsandwhitespace'),
622
+ (r'\n', Text),
623
+ (r'[^\S\n]+', Text),
624
+ (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|\.\.'
625
+ r'(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
626
+ (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
627
+ (r'[})\].]', Punctuation),
628
+ (r'[=+\-*/]', Operator),
629
+ (r'(for|in|while|do|break|return|continue|if|else|'
630
+ r'throw|try|catch'
631
+ r')\b', Keyword, 'slashstartsregex'),
632
+ (r'(var)\b', Keyword.Declaration, 'slashstartsregex'),
633
+ (r'(attr|def|fun)\b', Keyword.Reserved),
634
+ (r'(true|false)\b', Keyword.Constant),
635
+ (r'(eval|throw)\b', Name.Builtin),
636
+ (r'`\S+`', Name.Builtin),
637
+ (r'[$a-zA-Z_]\w*', Name.Other),
638
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
639
+ (r'0x[0-9a-fA-F]+', Number.Hex),
640
+ (r'[0-9]+', Number.Integer),
641
+ (r'"', String.Double, 'dqstring'),
642
+ (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
643
+ ],
644
+ 'dqstring': [
645
+ (r'\$\{[^"}]+?\}', String.Interpol),
646
+ (r'\$', String.Double),
647
+ (r'\\\\', String.Double),
648
+ (r'\\"', String.Double),
649
+ (r'[^\\"$]+', String.Double),
650
+ (r'"', String.Double, '#pop'),
651
+ ],
652
+ }
653
+
654
+
655
+ class LSLLexer(RegexLexer):
656
+ """
657
+ For Second Life's Linden Scripting Language source code.
658
+ """
659
+
660
+ name = 'LSL'
661
+ aliases = ['lsl']
662
+ filenames = ['*.lsl']
663
+ mimetypes = ['text/x-lsl']
664
+ url = 'https://wiki.secondlife.com/wiki/Linden_Scripting_Language'
665
+ version_added = '2.0'
666
+
667
+ flags = re.MULTILINE
668
+
669
+ lsl_keywords = r'\b(?:do|else|for|if|jump|return|while)\b'
670
+ lsl_types = r'\b(?:float|integer|key|list|quaternion|rotation|string|vector)\b'
671
+ lsl_states = r'\b(?:(?:state)\s+\w+|default)\b'
672
+ lsl_events = r'\b(?:state_(?:entry|exit)|touch(?:_(?:start|end))?|(?:land_)?collision(?:_(?:start|end))?|timer|listen|(?:no_)?sensor|control|(?:not_)?at_(?:rot_)?target|money|email|run_time_permissions|changed|attach|dataserver|moving_(?:start|end)|link_message|(?:on|object)_rez|remote_data|http_re(?:sponse|quest)|path_update|transaction_result)\b'
673
+ lsl_functions_builtin = r'\b(?:ll(?:ReturnObjectsBy(?:ID|Owner)|Json(?:2List|[GS]etValue|ValueType)|Sin|Cos|Tan|Atan2|Sqrt|Pow|Abs|Fabs|Frand|Floor|Ceil|Round|Vec(?:Mag|Norm|Dist)|Rot(?:Between|2(?:Euler|Fwd|Left|Up))|(?:Euler|Axes)2Rot|Whisper|(?:Region|Owner)?Say|Shout|Listen(?:Control|Remove)?|Sensor(?:Repeat|Remove)?|Detected(?:Name|Key|Owner|Type|Pos|Vel|Grab|Rot|Group|LinkNumber)|Die|Ground|Wind|(?:[GS]et)(?:AnimationOverride|MemoryLimit|PrimMediaParams|ParcelMusicURL|Object(?:Desc|Name)|PhysicsMaterial|Status|Scale|Color|Alpha|Texture|Pos|Rot|Force|Torque)|ResetAnimationOverride|(?:Scale|Offset|Rotate)Texture|(?:Rot)?Target(?:Remove)?|(?:Stop)?MoveToTarget|Apply(?:Rotational)?Impulse|Set(?:KeyframedMotion|ContentType|RegionPos|(?:Angular)?Velocity|Buoyancy|HoverHeight|ForceAndTorque|TimerEvent|ScriptState|Damage|TextureAnim|Sound(?:Queueing|Radius)|Vehicle(?:Type|(?:Float|Vector|Rotation)Param)|(?:Touch|Sit)?Text|Camera(?:Eye|At)Offset|PrimitiveParams|ClickAction|Link(?:Alpha|Color|PrimitiveParams(?:Fast)?|Texture(?:Anim)?|Camera|Media)|RemoteScriptAccessPin|PayPrice|LocalRot)|ScaleByFactor|Get(?:(?:Max|Min)ScaleFactor|ClosestNavPoint|StaticPath|SimStats|Env|PrimitiveParams|Link(?:PrimitiveParams|Number(?:OfSides)?|Key|Name|Media)|HTTPHeader|FreeURLs|Object(?:Details|PermMask|PrimCount)|Parcel(?:MaxPrims|Details|Prim(?:Count|Owners))|Attached|(?:SPMax|Free|Used)Memory|Region(?:Name|TimeDilation|FPS|Corner|AgentCount)|Root(?:Position|Rotation)|UnixTime|(?:Parcel|Region)Flags|(?:Wall|GMT)clock|SimulatorHostname|BoundingBox|GeometricCenter|Creator|NumberOf(?:Prims|NotecardLines|Sides)|Animation(?:List)?|(?:Camera|Local)(?:Pos|Rot)|Vel|Accel|Omega|Time(?:stamp|OfDay)|(?:Object|CenterOf)?Mass|MassMKS|Energy|Owner|(?:Owner)?Key|SunDirection|Texture(?:Offset|Scale|Rot)|Inventory(?:Number|Name|Key|Type|Creator|PermMask)|Permissions(?:Key)?|StartParameter|List(?:Length|EntryType)|Date|Agent(?:Size|Info|Language|List)|LandOwnerAt|NotecardLine|Script(?:Name|State))|(?:Get|Reset|GetAndReset)Time|PlaySound(?:Slave)?|LoopSound(?:Master|Slave)?|(?:Trigger|Stop|Preload)Sound|(?:(?:Get|Delete)Sub|Insert)String|To(?:Upper|Lower)|Give(?:InventoryList|Money)|RezObject|(?:Stop)?LookAt|Sleep|CollisionFilter|(?:Take|Release)Controls|DetachFromAvatar|AttachToAvatar(?:Temp)?|InstantMessage|(?:GetNext)?Email|StopHover|MinEventDelay|RotLookAt|String(?:Length|Trim)|(?:Start|Stop)Animation|TargetOmega|RequestPermissions|(?:Create|Break)Link|BreakAllLinks|(?:Give|Remove)Inventory|Water|PassTouches|Request(?:Agent|Inventory)Data|TeleportAgent(?:Home|GlobalCoords)?|ModifyLand|CollisionSound|ResetScript|MessageLinked|PushObject|PassCollisions|AxisAngle2Rot|Rot2(?:Axis|Angle)|A(?:cos|sin)|AngleBetween|AllowInventoryDrop|SubStringIndex|List2(?:CSV|Integer|Json|Float|String|Key|Vector|Rot|List(?:Strided)?)|DeleteSubList|List(?:Statistics|Sort|Randomize|(?:Insert|Find|Replace)List)|EdgeOfWorld|AdjustSoundVolume|Key2Name|TriggerSoundLimited|EjectFromLand|(?:CSV|ParseString)2List|OverMyLand|SameGroup|UnSit|Ground(?:Slope|Normal|Contour)|GroundRepel|(?:Set|Remove)VehicleFlags|(?:AvatarOn)?(?:Link)?SitTarget|Script(?:Danger|Profiler)|Dialog|VolumeDetect|ResetOtherScript|RemoteLoadScriptPin|(?:Open|Close)RemoteDataChannel|SendRemoteData|RemoteDataReply|(?:Integer|String)ToBase64|XorBase64|Log(?:10)?|Base64To(?:String|Integer)|ParseStringKeepNulls|RezAtRoot|RequestSimulatorData|ForceMouselook|(?:Load|Release|(?:E|Une)scape)URL|ParcelMedia(?:CommandList|Query)|ModPow|MapDestination|(?:RemoveFrom|AddTo|Reset)Land(?:Pass|Ban)List|(?:Set|Clear)CameraParams|HTTP(?:Request|Response)|TextBox|DetectedTouch(?:UV|Face|Pos|(?:N|Bin)ormal|ST)|(?:MD5|SHA1|DumpList2)String|Request(?:Secure)?URL|Clear(?:Prim|Link)Media|(?:Link)?ParticleSystem|(?:Get|Request)(?:Username|DisplayName)|RegionSayTo|CastRay|GenerateKey|TransferLindenDollars|ManageEstateAccess|(?:Create|Delete)Character|ExecCharacterCmd|Evade|FleeFrom|NavigateTo|PatrolPoints|Pursue|UpdateCharacter|WanderWithin))\b'
674
+ lsl_constants_float = r'\b(?:DEG_TO_RAD|PI(?:_BY_TWO)?|RAD_TO_DEG|SQRT2|TWO_PI)\b'
675
+ lsl_constants_integer = r'\b(?:JSON_APPEND|STATUS_(?:PHYSICS|ROTATE_[XYZ]|PHANTOM|SANDBOX|BLOCK_GRAB(?:_OBJECT)?|(?:DIE|RETURN)_AT_EDGE|CAST_SHADOWS|OK|MALFORMED_PARAMS|TYPE_MISMATCH|BOUNDS_ERROR|NOT_(?:FOUND|SUPPORTED)|INTERNAL_ERROR|WHITELIST_FAILED)|AGENT(?:_(?:BY_(?:LEGACY_|USER)NAME|FLYING|ATTACHMENTS|SCRIPTED|MOUSELOOK|SITTING|ON_OBJECT|AWAY|WALKING|IN_AIR|TYPING|CROUCHING|BUSY|ALWAYS_RUN|AUTOPILOT|LIST_(?:PARCEL(?:_OWNER)?|REGION)))?|CAMERA_(?:PITCH|DISTANCE|BEHINDNESS_(?:ANGLE|LAG)|(?:FOCUS|POSITION)(?:_(?:THRESHOLD|LOCKED|LAG))?|FOCUS_OFFSET|ACTIVE)|ANIM_ON|LOOP|REVERSE|PING_PONG|SMOOTH|ROTATE|SCALE|ALL_SIDES|LINK_(?:ROOT|SET|ALL_(?:OTHERS|CHILDREN)|THIS)|ACTIVE|PASSIVE|SCRIPTED|CONTROL_(?:FWD|BACK|(?:ROT_)?(?:LEFT|RIGHT)|UP|DOWN|(?:ML_)?LBUTTON)|PERMISSION_(?:RETURN_OBJECTS|DEBIT|OVERRIDE_ANIMATIONS|SILENT_ESTATE_MANAGEMENT|TAKE_CONTROLS|TRIGGER_ANIMATION|ATTACH|CHANGE_LINKS|(?:CONTROL|TRACK)_CAMERA|TELEPORT)|INVENTORY_(?:TEXTURE|SOUND|OBJECT|SCRIPT|LANDMARK|CLOTHING|NOTECARD|BODYPART|ANIMATION|GESTURE|ALL|NONE)|CHANGED_(?:INVENTORY|COLOR|SHAPE|SCALE|TEXTURE|LINK|ALLOWED_DROP|OWNER|REGION(?:_START)?|TELEPORT|MEDIA)|OBJECT_(?:(?:PHYSICS|SERVER|STREAMING)_COST|UNKNOWN_DETAIL|CHARACTER_TIME|PHANTOM|PHYSICS|TEMP_ON_REZ|NAME|DESC|POS|PRIM_EQUIVALENCE|RETURN_(?:PARCEL(?:_OWNER)?|REGION)|ROO?T|VELOCITY|OWNER|GROUP|CREATOR|ATTACHED_POINT|RENDER_WEIGHT|PATHFINDING_TYPE|(?:RUNNING|TOTAL)_SCRIPT_COUNT|SCRIPT_(?:MEMORY|TIME))|TYPE_(?:INTEGER|FLOAT|STRING|KEY|VECTOR|ROTATION|INVALID)|(?:DEBUG|PUBLIC)_CHANNEL|ATTACH_(?:AVATAR_CENTER|CHEST|HEAD|BACK|PELVIS|MOUTH|CHIN|NECK|NOSE|BELLY|[LR](?:SHOULDER|HAND|FOOT|EAR|EYE|[UL](?:ARM|LEG)|HIP)|(?:LEFT|RIGHT)_PEC|HUD_(?:CENTER_[12]|TOP_(?:RIGHT|CENTER|LEFT)|BOTTOM(?:_(?:RIGHT|LEFT))?))|LAND_(?:LEVEL|RAISE|LOWER|SMOOTH|NOISE|REVERT)|DATA_(?:ONLINE|NAME|BORN|SIM_(?:POS|STATUS|RATING)|PAYINFO)|PAYMENT_INFO_(?:ON_FILE|USED)|REMOTE_DATA_(?:CHANNEL|REQUEST|REPLY)|PSYS_(?:PART_(?:BF_(?:ZERO|ONE(?:_MINUS_(?:DEST_COLOR|SOURCE_(ALPHA|COLOR)))?|DEST_COLOR|SOURCE_(ALPHA|COLOR))|BLEND_FUNC_(DEST|SOURCE)|FLAGS|(?:START|END)_(?:COLOR|ALPHA|SCALE|GLOW)|MAX_AGE|(?:RIBBON|WIND|INTERP_(?:COLOR|SCALE)|BOUNCE|FOLLOW_(?:SRC|VELOCITY)|TARGET_(?:POS|LINEAR)|EMISSIVE)_MASK)|SRC_(?:MAX_AGE|PATTERN|ANGLE_(?:BEGIN|END)|BURST_(?:RATE|PART_COUNT|RADIUS|SPEED_(?:MIN|MAX))|ACCEL|TEXTURE|TARGET_KEY|OMEGA|PATTERN_(?:DROP|EXPLODE|ANGLE(?:_CONE(?:_EMPTY)?)?)))|VEHICLE_(?:REFERENCE_FRAME|TYPE_(?:NONE|SLED|CAR|BOAT|AIRPLANE|BALLOON)|(?:LINEAR|ANGULAR)_(?:FRICTION_TIMESCALE|MOTOR_DIRECTION)|LINEAR_MOTOR_OFFSET|HOVER_(?:HEIGHT|EFFICIENCY|TIMESCALE)|BUOYANCY|(?:LINEAR|ANGULAR)_(?:DEFLECTION_(?:EFFICIENCY|TIMESCALE)|MOTOR_(?:DECAY_)?TIMESCALE)|VERTICAL_ATTRACTION_(?:EFFICIENCY|TIMESCALE)|BANKING_(?:EFFICIENCY|MIX|TIMESCALE)|FLAG_(?:NO_DEFLECTION_UP|LIMIT_(?:ROLL_ONLY|MOTOR_UP)|HOVER_(?:(?:WATER|TERRAIN|UP)_ONLY|GLOBAL_HEIGHT)|MOUSELOOK_(?:STEER|BANK)|CAMERA_DECOUPLED))|PRIM_(?:TYPE(?:_(?:BOX|CYLINDER|PRISM|SPHERE|TORUS|TUBE|RING|SCULPT))?|HOLE_(?:DEFAULT|CIRCLE|SQUARE|TRIANGLE)|MATERIAL(?:_(?:STONE|METAL|GLASS|WOOD|FLESH|PLASTIC|RUBBER))?|SHINY_(?:NONE|LOW|MEDIUM|HIGH)|BUMP_(?:NONE|BRIGHT|DARK|WOOD|BARK|BRICKS|CHECKER|CONCRETE|TILE|STONE|DISKS|GRAVEL|BLOBS|SIDING|LARGETILE|STUCCO|SUCTION|WEAVE)|TEXGEN_(?:DEFAULT|PLANAR)|SCULPT_(?:TYPE_(?:SPHERE|TORUS|PLANE|CYLINDER|MASK)|FLAG_(?:MIRROR|INVERT))|PHYSICS(?:_(?:SHAPE_(?:CONVEX|NONE|PRIM|TYPE)))?|(?:POS|ROT)_LOCAL|SLICE|TEXT|FLEXIBLE|POINT_LIGHT|TEMP_ON_REZ|PHANTOM|POSITION|SIZE|ROTATION|TEXTURE|NAME|OMEGA|DESC|LINK_TARGET|COLOR|BUMP_SHINY|FULLBRIGHT|TEXGEN|GLOW|MEDIA_(?:ALT_IMAGE_ENABLE|CONTROLS|(?:CURRENT|HOME)_URL|AUTO_(?:LOOP|PLAY|SCALE|ZOOM)|FIRST_CLICK_INTERACT|(?:WIDTH|HEIGHT)_PIXELS|WHITELIST(?:_ENABLE)?|PERMS_(?:INTERACT|CONTROL)|PARAM_MAX|CONTROLS_(?:STANDARD|MINI)|PERM_(?:NONE|OWNER|GROUP|ANYONE)|MAX_(?:URL_LENGTH|WHITELIST_(?:SIZE|COUNT)|(?:WIDTH|HEIGHT)_PIXELS)))|MASK_(?:BASE|OWNER|GROUP|EVERYONE|NEXT)|PERM_(?:TRANSFER|MODIFY|COPY|MOVE|ALL)|PARCEL_(?:MEDIA_COMMAND_(?:STOP|PAUSE|PLAY|LOOP|TEXTURE|URL|TIME|AGENT|UNLOAD|AUTO_ALIGN|TYPE|SIZE|DESC|LOOP_SET)|FLAG_(?:ALLOW_(?:FLY|(?:GROUP_)?SCRIPTS|LANDMARK|TERRAFORM|DAMAGE|CREATE_(?:GROUP_)?OBJECTS)|USE_(?:ACCESS_(?:GROUP|LIST)|BAN_LIST|LAND_PASS_LIST)|LOCAL_SOUND_ONLY|RESTRICT_PUSHOBJECT|ALLOW_(?:GROUP|ALL)_OBJECT_ENTRY)|COUNT_(?:TOTAL|OWNER|GROUP|OTHER|SELECTED|TEMP)|DETAILS_(?:NAME|DESC|OWNER|GROUP|AREA|ID|SEE_AVATARS))|LIST_STAT_(?:MAX|MIN|MEAN|MEDIAN|STD_DEV|SUM(?:_SQUARES)?|NUM_COUNT|GEOMETRIC_MEAN|RANGE)|PAY_(?:HIDE|DEFAULT)|REGION_FLAG_(?:ALLOW_DAMAGE|FIXED_SUN|BLOCK_TERRAFORM|SANDBOX|DISABLE_(?:COLLISIONS|PHYSICS)|BLOCK_FLY|ALLOW_DIRECT_TELEPORT|RESTRICT_PUSHOBJECT)|HTTP_(?:METHOD|MIMETYPE|BODY_(?:MAXLENGTH|TRUNCATED)|CUSTOM_HEADER|PRAGMA_NO_CACHE|VERBOSE_THROTTLE|VERIFY_CERT)|STRING_(?:TRIM(?:_(?:HEAD|TAIL))?)|CLICK_ACTION_(?:NONE|TOUCH|SIT|BUY|PAY|OPEN(?:_MEDIA)?|PLAY|ZOOM)|TOUCH_INVALID_FACE|PROFILE_(?:NONE|SCRIPT_MEMORY)|RC_(?:DATA_FLAGS|DETECT_PHANTOM|GET_(?:LINK_NUM|NORMAL|ROOT_KEY)|MAX_HITS|REJECT_(?:TYPES|AGENTS|(?:NON)?PHYSICAL|LAND))|RCERR_(?:CAST_TIME_EXCEEDED|SIM_PERF_LOW|UNKNOWN)|ESTATE_ACCESS_(?:ALLOWED_(?:AGENT|GROUP)_(?:ADD|REMOVE)|BANNED_AGENT_(?:ADD|REMOVE))|DENSITY|FRICTION|RESTITUTION|GRAVITY_MULTIPLIER|KFM_(?:COMMAND|CMD_(?:PLAY|STOP|PAUSE|SET_MODE)|MODE|FORWARD|LOOP|PING_PONG|REVERSE|DATA|ROTATION|TRANSLATION)|ERR_(?:GENERIC|PARCEL_PERMISSIONS|MALFORMED_PARAMS|RUNTIME_PERMISSIONS|THROTTLED)|CHARACTER_(?:CMD_(?:(?:SMOOTH_)?STOP|JUMP)|DESIRED_(?:TURN_)?SPEED|RADIUS|STAY_WITHIN_PARCEL|LENGTH|ORIENTATION|ACCOUNT_FOR_SKIPPED_FRAMES|AVOIDANCE_MODE|TYPE(?:_(?:[A-D]|NONE))?|MAX_(?:DECEL|TURN_RADIUS|(?:ACCEL|SPEED)))|PURSUIT_(?:OFFSET|FUZZ_FACTOR|GOAL_TOLERANCE|INTERCEPT)|REQUIRE_LINE_OF_SIGHT|FORCE_DIRECT_PATH|VERTICAL|HORIZONTAL|AVOID_(?:CHARACTERS|DYNAMIC_OBSTACLES|NONE)|PU_(?:EVADE_(?:HIDDEN|SPOTTED)|FAILURE_(?:DYNAMIC_PATHFINDING_DISABLED|INVALID_(?:GOAL|START)|NO_(?:NAVMESH|VALID_DESTINATION)|OTHER|TARGET_GONE|(?:PARCEL_)?UNREACHABLE)|(?:GOAL|SLOWDOWN_DISTANCE)_REACHED)|TRAVERSAL_TYPE(?:_(?:FAST|NONE|SLOW))?|CONTENT_TYPE_(?:ATOM|FORM|HTML|JSON|LLSD|RSS|TEXT|XHTML|XML)|GCNP_(?:RADIUS|STATIC)|(?:PATROL|WANDER)_PAUSE_AT_WAYPOINTS|OPT_(?:AVATAR|CHARACTER|EXCLUSION_VOLUME|LEGACY_LINKSET|MATERIAL_VOLUME|OTHER|STATIC_OBSTACLE|WALKABLE)|SIM_STAT_PCT_CHARS_STEPPED)\b'
676
+ lsl_constants_integer_boolean = r'\b(?:FALSE|TRUE)\b'
677
+ lsl_constants_rotation = r'\b(?:ZERO_ROTATION)\b'
678
+ lsl_constants_string = r'\b(?:EOF|JSON_(?:ARRAY|DELETE|FALSE|INVALID|NULL|NUMBER|OBJECT|STRING|TRUE)|NULL_KEY|TEXTURE_(?:BLANK|DEFAULT|MEDIA|PLYWOOD|TRANSPARENT)|URL_REQUEST_(?:GRANTED|DENIED))\b'
679
+ lsl_constants_vector = r'\b(?:TOUCH_INVALID_(?:TEXCOORD|VECTOR)|ZERO_VECTOR)\b'
680
+ lsl_invalid_broken = r'\b(?:LAND_(?:LARGE|MEDIUM|SMALL)_BRUSH)\b'
681
+ lsl_invalid_deprecated = r'\b(?:ATTACH_[LR]PEC|DATA_RATING|OBJECT_ATTACHMENT_(?:GEOMETRY_BYTES|SURFACE_AREA)|PRIM_(?:CAST_SHADOWS|MATERIAL_LIGHT|TYPE_LEGACY)|PSYS_SRC_(?:INNER|OUTER)ANGLE|VEHICLE_FLAG_NO_FLY_UP|ll(?:Cloud|Make(?:Explosion|Fountain|Smoke|Fire)|RemoteDataSetRegion|Sound(?:Preload)?|XorBase64Strings(?:Correct)?))\b'
682
+ lsl_invalid_illegal = r'\b(?:event)\b'
683
+ lsl_invalid_unimplemented = r'\b(?:CHARACTER_(?:MAX_ANGULAR_(?:ACCEL|SPEED)|TURN_SPEED_MULTIPLIER)|PERMISSION_(?:CHANGE_(?:JOINTS|PERMISSIONS)|RELEASE_OWNERSHIP|REMAP_CONTROLS)|PRIM_PHYSICS_MATERIAL|PSYS_SRC_OBJ_REL_MASK|ll(?:CollisionSprite|(?:Stop)?PointAt|(?:(?:Refresh|Set)Prim)URL|(?:Take|Release)Camera|RemoteLoadScript))\b'
684
+ lsl_reserved_godmode = r'\b(?:ll(?:GodLikeRezObject|Set(?:Inventory|Object)PermMask))\b'
685
+ lsl_reserved_log = r'\b(?:print)\b'
686
+ lsl_operators = r'\+\+|\-\-|<<|>>|&&?|\|\|?|\^|~|[!%<>=*+\-/]=?'
687
+
688
+ tokens = {
689
+ 'root':
690
+ [
691
+ (r'//.*?\n', Comment.Single),
692
+ (r'/\*', Comment.Multiline, 'comment'),
693
+ (r'"', String.Double, 'string'),
694
+ (lsl_keywords, Keyword),
695
+ (lsl_types, Keyword.Type),
696
+ (lsl_states, Name.Class),
697
+ (lsl_events, Name.Builtin),
698
+ (lsl_functions_builtin, Name.Function),
699
+ (lsl_constants_float, Keyword.Constant),
700
+ (lsl_constants_integer, Keyword.Constant),
701
+ (lsl_constants_integer_boolean, Keyword.Constant),
702
+ (lsl_constants_rotation, Keyword.Constant),
703
+ (lsl_constants_string, Keyword.Constant),
704
+ (lsl_constants_vector, Keyword.Constant),
705
+ (lsl_invalid_broken, Error),
706
+ (lsl_invalid_deprecated, Error),
707
+ (lsl_invalid_illegal, Error),
708
+ (lsl_invalid_unimplemented, Error),
709
+ (lsl_reserved_godmode, Keyword.Reserved),
710
+ (lsl_reserved_log, Keyword.Reserved),
711
+ (r'\b([a-zA-Z_]\w*)\b', Name.Variable),
712
+ (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d*', Number.Float),
713
+ (r'(\d+\.\d*|\.\d+)', Number.Float),
714
+ (r'0[xX][0-9a-fA-F]+', Number.Hex),
715
+ (r'\d+', Number.Integer),
716
+ (lsl_operators, Operator),
717
+ (r':=?', Error),
718
+ (r'[,;{}()\[\]]', Punctuation),
719
+ (r'\n+', Whitespace),
720
+ (r'\s+', Whitespace)
721
+ ],
722
+ 'comment':
723
+ [
724
+ (r'[^*/]+', Comment.Multiline),
725
+ (r'/\*', Comment.Multiline, '#push'),
726
+ (r'\*/', Comment.Multiline, '#pop'),
727
+ (r'[*/]', Comment.Multiline)
728
+ ],
729
+ 'string':
730
+ [
731
+ (r'\\([nt"\\])', String.Escape),
732
+ (r'"', String.Double, '#pop'),
733
+ (r'\\.', Error),
734
+ (r'[^"\\]+', String.Double),
735
+ ]
736
+ }
737
+
738
+
739
+ class AppleScriptLexer(RegexLexer):
740
+ """
741
+ For AppleScript source code,
742
+ including `AppleScript Studio
743
+ <http://developer.apple.com/documentation/AppleScript/
744
+ Reference/StudioReference>`_.
745
+ Contributed by Andreas Amann <aamann@mac.com>.
746
+ """
747
+
748
+ name = 'AppleScript'
749
+ url = 'https://developer.apple.com/library/archive/documentation/AppleScript/Conceptual/AppleScriptLangGuide/introduction/ASLR_intro.html'
750
+ aliases = ['applescript']
751
+ filenames = ['*.applescript']
752
+ version_added = '1.0'
753
+
754
+ flags = re.MULTILINE | re.DOTALL
755
+
756
+ Identifiers = r'[a-zA-Z]\w*'
757
+
758
+ # XXX: use words() for all of these
759
+ Literals = ('AppleScript', 'current application', 'false', 'linefeed',
760
+ 'missing value', 'pi', 'quote', 'result', 'return', 'space',
761
+ 'tab', 'text item delimiters', 'true', 'version')
762
+ Classes = ('alias ', 'application ', 'boolean ', 'class ', 'constant ',
763
+ 'date ', 'file ', 'integer ', 'list ', 'number ', 'POSIX file ',
764
+ 'real ', 'record ', 'reference ', 'RGB color ', 'script ',
765
+ 'text ', 'unit types', '(?:Unicode )?text', 'string')
766
+ BuiltIn = ('attachment', 'attribute run', 'character', 'day', 'month',
767
+ 'paragraph', 'word', 'year')
768
+ HandlerParams = ('about', 'above', 'against', 'apart from', 'around',
769
+ 'aside from', 'at', 'below', 'beneath', 'beside',
770
+ 'between', 'for', 'given', 'instead of', 'on', 'onto',
771
+ 'out of', 'over', 'since')
772
+ Commands = ('ASCII (character|number)', 'activate', 'beep', 'choose URL',
773
+ 'choose application', 'choose color', 'choose file( name)?',
774
+ 'choose folder', 'choose from list',
775
+ 'choose remote application', 'clipboard info',
776
+ 'close( access)?', 'copy', 'count', 'current date', 'delay',
777
+ 'delete', 'display (alert|dialog)', 'do shell script',
778
+ 'duplicate', 'exists', 'get eof', 'get volume settings',
779
+ 'info for', 'launch', 'list (disks|folder)', 'load script',
780
+ 'log', 'make', 'mount volume', 'new', 'offset',
781
+ 'open( (for access|location))?', 'path to', 'print', 'quit',
782
+ 'random number', 'read', 'round', 'run( script)?',
783
+ 'say', 'scripting components',
784
+ 'set (eof|the clipboard to|volume)', 'store script',
785
+ 'summarize', 'system attribute', 'system info',
786
+ 'the clipboard', 'time to GMT', 'write', 'quoted form')
787
+ References = ('(in )?back of', '(in )?front of', '[0-9]+(st|nd|rd|th)',
788
+ 'first', 'second', 'third', 'fourth', 'fifth', 'sixth',
789
+ 'seventh', 'eighth', 'ninth', 'tenth', 'after', 'back',
790
+ 'before', 'behind', 'every', 'front', 'index', 'last',
791
+ 'middle', 'some', 'that', 'through', 'thru', 'where', 'whose')
792
+ Operators = ("and", "or", "is equal", "equals", "(is )?equal to", "is not",
793
+ "isn't", "isn't equal( to)?", "is not equal( to)?",
794
+ "doesn't equal", "does not equal", "(is )?greater than",
795
+ "comes after", "is not less than or equal( to)?",
796
+ "isn't less than or equal( to)?", "(is )?less than",
797
+ "comes before", "is not greater than or equal( to)?",
798
+ "isn't greater than or equal( to)?",
799
+ "(is )?greater than or equal( to)?", "is not less than",
800
+ "isn't less than", "does not come before",
801
+ "doesn't come before", "(is )?less than or equal( to)?",
802
+ "is not greater than", "isn't greater than",
803
+ "does not come after", "doesn't come after", "starts? with",
804
+ "begins? with", "ends? with", "contains?", "does not contain",
805
+ "doesn't contain", "is in", "is contained by", "is not in",
806
+ "is not contained by", "isn't contained by", "div", "mod",
807
+ "not", "(a )?(ref( to)?|reference to)", "is", "does")
808
+ Control = ('considering', 'else', 'error', 'exit', 'from', 'if',
809
+ 'ignoring', 'in', 'repeat', 'tell', 'then', 'times', 'to',
810
+ 'try', 'until', 'using terms from', 'while', 'whith',
811
+ 'with timeout( of)?', 'with transaction', 'by', 'continue',
812
+ 'end', 'its?', 'me', 'my', 'return', 'of', 'as')
813
+ Declarations = ('global', 'local', 'prop(erty)?', 'set', 'get')
814
+ Reserved = ('but', 'put', 'returning', 'the')
815
+ StudioClasses = ('action cell', 'alert reply', 'application', 'box',
816
+ 'browser( cell)?', 'bundle', 'button( cell)?', 'cell',
817
+ 'clip view', 'color well', 'color-panel',
818
+ 'combo box( item)?', 'control',
819
+ 'data( (cell|column|item|row|source))?', 'default entry',
820
+ 'dialog reply', 'document', 'drag info', 'drawer',
821
+ 'event', 'font(-panel)?', 'formatter',
822
+ 'image( (cell|view))?', 'matrix', 'menu( item)?', 'item',
823
+ 'movie( view)?', 'open-panel', 'outline view', 'panel',
824
+ 'pasteboard', 'plugin', 'popup button',
825
+ 'progress indicator', 'responder', 'save-panel',
826
+ 'scroll view', 'secure text field( cell)?', 'slider',
827
+ 'sound', 'split view', 'stepper', 'tab view( item)?',
828
+ 'table( (column|header cell|header view|view))',
829
+ 'text( (field( cell)?|view))?', 'toolbar( item)?',
830
+ 'user-defaults', 'view', 'window')
831
+ StudioEvents = ('accept outline drop', 'accept table drop', 'action',
832
+ 'activated', 'alert ended', 'awake from nib', 'became key',
833
+ 'became main', 'begin editing', 'bounds changed',
834
+ 'cell value', 'cell value changed', 'change cell value',
835
+ 'change item value', 'changed', 'child of item',
836
+ 'choose menu item', 'clicked', 'clicked toolbar item',
837
+ 'closed', 'column clicked', 'column moved',
838
+ 'column resized', 'conclude drop', 'data representation',
839
+ 'deminiaturized', 'dialog ended', 'document nib name',
840
+ 'double clicked', 'drag( (entered|exited|updated))?',
841
+ 'drop', 'end editing', 'exposed', 'idle', 'item expandable',
842
+ 'item value', 'item value changed', 'items changed',
843
+ 'keyboard down', 'keyboard up', 'launched',
844
+ 'load data representation', 'miniaturized', 'mouse down',
845
+ 'mouse dragged', 'mouse entered', 'mouse exited',
846
+ 'mouse moved', 'mouse up', 'moved',
847
+ 'number of browser rows', 'number of items',
848
+ 'number of rows', 'open untitled', 'opened', 'panel ended',
849
+ 'parameters updated', 'plugin loaded', 'prepare drop',
850
+ 'prepare outline drag', 'prepare outline drop',
851
+ 'prepare table drag', 'prepare table drop',
852
+ 'read from file', 'resigned active', 'resigned key',
853
+ 'resigned main', 'resized( sub views)?',
854
+ 'right mouse down', 'right mouse dragged',
855
+ 'right mouse up', 'rows changed', 'scroll wheel',
856
+ 'selected tab view item', 'selection changed',
857
+ 'selection changing', 'should begin editing',
858
+ 'should close', 'should collapse item',
859
+ 'should end editing', 'should expand item',
860
+ 'should open( untitled)?',
861
+ 'should quit( after last window closed)?',
862
+ 'should select column', 'should select item',
863
+ 'should select row', 'should select tab view item',
864
+ 'should selection change', 'should zoom', 'shown',
865
+ 'update menu item', 'update parameters',
866
+ 'update toolbar item', 'was hidden', 'was miniaturized',
867
+ 'will become active', 'will close', 'will dismiss',
868
+ 'will display browser cell', 'will display cell',
869
+ 'will display item cell', 'will display outline cell',
870
+ 'will finish launching', 'will hide', 'will miniaturize',
871
+ 'will move', 'will open', 'will pop up', 'will quit',
872
+ 'will resign active', 'will resize( sub views)?',
873
+ 'will select tab view item', 'will show', 'will zoom',
874
+ 'write to file', 'zoomed')
875
+ StudioCommands = ('animate', 'append', 'call method', 'center',
876
+ 'close drawer', 'close panel', 'display',
877
+ 'display alert', 'display dialog', 'display panel', 'go',
878
+ 'hide', 'highlight', 'increment', 'item for',
879
+ 'load image', 'load movie', 'load nib', 'load panel',
880
+ 'load sound', 'localized string', 'lock focus', 'log',
881
+ 'open drawer', 'path for', 'pause', 'perform action',
882
+ 'play', 'register', 'resume', 'scroll', 'select( all)?',
883
+ 'show', 'size to fit', 'start', 'step back',
884
+ 'step forward', 'stop', 'synchronize', 'unlock focus',
885
+ 'update')
886
+ StudioProperties = ('accepts arrow key', 'action method', 'active',
887
+ 'alignment', 'allowed identifiers',
888
+ 'allows branch selection', 'allows column reordering',
889
+ 'allows column resizing', 'allows column selection',
890
+ 'allows customization',
891
+ 'allows editing text attributes',
892
+ 'allows empty selection', 'allows mixed state',
893
+ 'allows multiple selection', 'allows reordering',
894
+ 'allows undo', 'alpha( value)?', 'alternate image',
895
+ 'alternate increment value', 'alternate title',
896
+ 'animation delay', 'associated file name',
897
+ 'associated object', 'auto completes', 'auto display',
898
+ 'auto enables items', 'auto repeat',
899
+ 'auto resizes( outline column)?',
900
+ 'auto save expanded items', 'auto save name',
901
+ 'auto save table columns', 'auto saves configuration',
902
+ 'auto scroll', 'auto sizes all columns to fit',
903
+ 'auto sizes cells', 'background color', 'bezel state',
904
+ 'bezel style', 'bezeled', 'border rect', 'border type',
905
+ 'bordered', 'bounds( rotation)?', 'box type',
906
+ 'button returned', 'button type',
907
+ 'can choose directories', 'can choose files',
908
+ 'can draw', 'can hide',
909
+ 'cell( (background color|size|type))?', 'characters',
910
+ 'class', 'click count', 'clicked( data)? column',
911
+ 'clicked data item', 'clicked( data)? row',
912
+ 'closeable', 'collating', 'color( (mode|panel))',
913
+ 'command key down', 'configuration',
914
+ 'content(s| (size|view( margins)?))?', 'context',
915
+ 'continuous', 'control key down', 'control size',
916
+ 'control tint', 'control view',
917
+ 'controller visible', 'coordinate system',
918
+ 'copies( on scroll)?', 'corner view', 'current cell',
919
+ 'current column', 'current( field)? editor',
920
+ 'current( menu)? item', 'current row',
921
+ 'current tab view item', 'data source',
922
+ 'default identifiers', 'delta (x|y|z)',
923
+ 'destination window', 'directory', 'display mode',
924
+ 'displayed cell', 'document( (edited|rect|view))?',
925
+ 'double value', 'dragged column', 'dragged distance',
926
+ 'dragged items', 'draws( cell)? background',
927
+ 'draws grid', 'dynamically scrolls', 'echos bullets',
928
+ 'edge', 'editable', 'edited( data)? column',
929
+ 'edited data item', 'edited( data)? row', 'enabled',
930
+ 'enclosing scroll view', 'ending page',
931
+ 'error handling', 'event number', 'event type',
932
+ 'excluded from windows menu', 'executable path',
933
+ 'expanded', 'fax number', 'field editor', 'file kind',
934
+ 'file name', 'file type', 'first responder',
935
+ 'first visible column', 'flipped', 'floating',
936
+ 'font( panel)?', 'formatter', 'frameworks path',
937
+ 'frontmost', 'gave up', 'grid color', 'has data items',
938
+ 'has horizontal ruler', 'has horizontal scroller',
939
+ 'has parent data item', 'has resize indicator',
940
+ 'has shadow', 'has sub menu', 'has vertical ruler',
941
+ 'has vertical scroller', 'header cell', 'header view',
942
+ 'hidden', 'hides when deactivated', 'highlights by',
943
+ 'horizontal line scroll', 'horizontal page scroll',
944
+ 'horizontal ruler view', 'horizontally resizable',
945
+ 'icon image', 'id', 'identifier',
946
+ 'ignores multiple clicks',
947
+ 'image( (alignment|dims when disabled|frame style|scaling))?',
948
+ 'imports graphics', 'increment value',
949
+ 'indentation per level', 'indeterminate', 'index',
950
+ 'integer value', 'intercell spacing', 'item height',
951
+ 'key( (code|equivalent( modifier)?|window))?',
952
+ 'knob thickness', 'label', 'last( visible)? column',
953
+ 'leading offset', 'leaf', 'level', 'line scroll',
954
+ 'loaded', 'localized sort', 'location', 'loop mode',
955
+ 'main( (bunde|menu|window))?', 'marker follows cell',
956
+ 'matrix mode', 'maximum( content)? size',
957
+ 'maximum visible columns',
958
+ 'menu( form representation)?', 'miniaturizable',
959
+ 'miniaturized', 'minimized image', 'minimized title',
960
+ 'minimum column width', 'minimum( content)? size',
961
+ 'modal', 'modified', 'mouse down state',
962
+ 'movie( (controller|file|rect))?', 'muted', 'name',
963
+ 'needs display', 'next state', 'next text',
964
+ 'number of tick marks', 'only tick mark values',
965
+ 'opaque', 'open panel', 'option key down',
966
+ 'outline table column', 'page scroll', 'pages across',
967
+ 'pages down', 'palette label', 'pane splitter',
968
+ 'parent data item', 'parent window', 'pasteboard',
969
+ 'path( (names|separator))?', 'playing',
970
+ 'plays every frame', 'plays selection only', 'position',
971
+ 'preferred edge', 'preferred type', 'pressure',
972
+ 'previous text', 'prompt', 'properties',
973
+ 'prototype cell', 'pulls down', 'rate',
974
+ 'released when closed', 'repeated',
975
+ 'requested print time', 'required file type',
976
+ 'resizable', 'resized column', 'resource path',
977
+ 'returns records', 'reuses columns', 'rich text',
978
+ 'roll over', 'row height', 'rulers visible',
979
+ 'save panel', 'scripts path', 'scrollable',
980
+ 'selectable( identifiers)?', 'selected cell',
981
+ 'selected( data)? columns?', 'selected data items?',
982
+ 'selected( data)? rows?', 'selected item identifier',
983
+ 'selection by rect', 'send action on arrow key',
984
+ 'sends action when done editing', 'separates columns',
985
+ 'separator item', 'sequence number', 'services menu',
986
+ 'shared frameworks path', 'shared support path',
987
+ 'sheet', 'shift key down', 'shows alpha',
988
+ 'shows state by', 'size( mode)?',
989
+ 'smart insert delete enabled', 'sort case sensitivity',
990
+ 'sort column', 'sort order', 'sort type',
991
+ 'sorted( data rows)?', 'sound', 'source( mask)?',
992
+ 'spell checking enabled', 'starting page', 'state',
993
+ 'string value', 'sub menu', 'super menu', 'super view',
994
+ 'tab key traverses cells', 'tab state', 'tab type',
995
+ 'tab view', 'table view', 'tag', 'target( printer)?',
996
+ 'text color', 'text container insert',
997
+ 'text container origin', 'text returned',
998
+ 'tick mark position', 'time stamp',
999
+ 'title(d| (cell|font|height|position|rect))?',
1000
+ 'tool tip', 'toolbar', 'trailing offset', 'transparent',
1001
+ 'treat packages as directories', 'truncated labels',
1002
+ 'types', 'unmodified characters', 'update views',
1003
+ 'use sort indicator', 'user defaults',
1004
+ 'uses data source', 'uses ruler',
1005
+ 'uses threaded animation',
1006
+ 'uses title from previous column', 'value wraps',
1007
+ 'version',
1008
+ 'vertical( (line scroll|page scroll|ruler view))?',
1009
+ 'vertically resizable', 'view',
1010
+ 'visible( document rect)?', 'volume', 'width', 'window',
1011
+ 'windows menu', 'wraps', 'zoomable', 'zoomed')
1012
+
1013
+ tokens = {
1014
+ 'root': [
1015
+ (r'\s+', Text),
1016
+ (r'¬\n', String.Escape),
1017
+ (r"'s\s+", Text), # This is a possessive, consider moving
1018
+ (r'(--|#).*?$', Comment),
1019
+ (r'\(\*', Comment.Multiline, 'comment'),
1020
+ (r'[(){}!,.:]', Punctuation),
1021
+ (r'(«)([^»]+)(»)',
1022
+ bygroups(Text, Name.Builtin, Text)),
1023
+ (r'\b((?:considering|ignoring)\s*)'
1024
+ r'(application responses|case|diacriticals|hyphens|'
1025
+ r'numeric strings|punctuation|white space)',
1026
+ bygroups(Keyword, Name.Builtin)),
1027
+ (r'(-|\*|\+|&|≠|>=?|<=?|=|≥|≤|/|÷|\^)', Operator),
1028
+ (r"\b({})\b".format('|'.join(Operators)), Operator.Word),
1029
+ (r'^(\s*(?:on|end)\s+)'
1030
+ r'({})'.format('|'.join(StudioEvents[::-1])),
1031
+ bygroups(Keyword, Name.Function)),
1032
+ (r'^(\s*)(in|on|script|to)(\s+)', bygroups(Text, Keyword, Text)),
1033
+ (r'\b(as )({})\b'.format('|'.join(Classes)),
1034
+ bygroups(Keyword, Name.Class)),
1035
+ (r'\b({})\b'.format('|'.join(Literals)), Name.Constant),
1036
+ (r'\b({})\b'.format('|'.join(Commands)), Name.Builtin),
1037
+ (r'\b({})\b'.format('|'.join(Control)), Keyword),
1038
+ (r'\b({})\b'.format('|'.join(Declarations)), Keyword),
1039
+ (r'\b({})\b'.format('|'.join(Reserved)), Name.Builtin),
1040
+ (r'\b({})s?\b'.format('|'.join(BuiltIn)), Name.Builtin),
1041
+ (r'\b({})\b'.format('|'.join(HandlerParams)), Name.Builtin),
1042
+ (r'\b({})\b'.format('|'.join(StudioProperties)), Name.Attribute),
1043
+ (r'\b({})s?\b'.format('|'.join(StudioClasses)), Name.Builtin),
1044
+ (r'\b({})\b'.format('|'.join(StudioCommands)), Name.Builtin),
1045
+ (r'\b({})\b'.format('|'.join(References)), Name.Builtin),
1046
+ (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
1047
+ (rf'\b({Identifiers})\b', Name.Variable),
1048
+ (r'[-+]?(\d+\.\d*|\d*\.\d+)(E[-+][0-9]+)?', Number.Float),
1049
+ (r'[-+]?\d+', Number.Integer),
1050
+ ],
1051
+ 'comment': [
1052
+ (r'\(\*', Comment.Multiline, '#push'),
1053
+ (r'\*\)', Comment.Multiline, '#pop'),
1054
+ ('[^*(]+', Comment.Multiline),
1055
+ ('[*(]', Comment.Multiline),
1056
+ ],
1057
+ }
1058
+
1059
+
1060
+ class RexxLexer(RegexLexer):
1061
+ """
1062
+ Rexx is a scripting language available for
1063
+ a wide range of different platforms with its roots found on mainframe
1064
+ systems. It is popular for I/O- and data based tasks and can act as glue
1065
+ language to bind different applications together.
1066
+ """
1067
+ name = 'Rexx'
1068
+ url = 'http://www.rexxinfo.org/'
1069
+ aliases = ['rexx', 'arexx']
1070
+ filenames = ['*.rexx', '*.rex', '*.rx', '*.arexx']
1071
+ mimetypes = ['text/x-rexx']
1072
+ version_added = '2.0'
1073
+ flags = re.IGNORECASE
1074
+
1075
+ tokens = {
1076
+ 'root': [
1077
+ (r'\s+', Whitespace),
1078
+ (r'/\*', Comment.Multiline, 'comment'),
1079
+ (r'"', String, 'string_double'),
1080
+ (r"'", String, 'string_single'),
1081
+ (r'[0-9]+(\.[0-9]+)?(e[+-]?[0-9])?', Number),
1082
+ (r'([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b',
1083
+ bygroups(Name.Function, Whitespace, Operator, Whitespace,
1084
+ Keyword.Declaration)),
1085
+ (r'([a-z_]\w*)(\s*)(:)',
1086
+ bygroups(Name.Label, Whitespace, Operator)),
1087
+ include('function'),
1088
+ include('keyword'),
1089
+ include('operator'),
1090
+ (r'[a-z_]\w*', Text),
1091
+ ],
1092
+ 'function': [
1093
+ (words((
1094
+ 'abbrev', 'abs', 'address', 'arg', 'b2x', 'bitand', 'bitor', 'bitxor',
1095
+ 'c2d', 'c2x', 'center', 'charin', 'charout', 'chars', 'compare',
1096
+ 'condition', 'copies', 'd2c', 'd2x', 'datatype', 'date', 'delstr',
1097
+ 'delword', 'digits', 'errortext', 'form', 'format', 'fuzz', 'insert',
1098
+ 'lastpos', 'left', 'length', 'linein', 'lineout', 'lines', 'max',
1099
+ 'min', 'overlay', 'pos', 'queued', 'random', 'reverse', 'right', 'sign',
1100
+ 'sourceline', 'space', 'stream', 'strip', 'substr', 'subword', 'symbol',
1101
+ 'time', 'trace', 'translate', 'trunc', 'value', 'verify', 'word',
1102
+ 'wordindex', 'wordlength', 'wordpos', 'words', 'x2b', 'x2c', 'x2d',
1103
+ 'xrange'), suffix=r'(\s*)(\()'),
1104
+ bygroups(Name.Builtin, Whitespace, Operator)),
1105
+ ],
1106
+ 'keyword': [
1107
+ (r'(address|arg|by|call|do|drop|else|end|exit|for|forever|if|'
1108
+ r'interpret|iterate|leave|nop|numeric|off|on|options|parse|'
1109
+ r'pull|push|queue|return|say|select|signal|to|then|trace|until|'
1110
+ r'while)\b', Keyword.Reserved),
1111
+ ],
1112
+ 'operator': [
1113
+ (r'(-|//|/|\(|\)|\*\*|\*|\\<<|\\<|\\==|\\=|\\>>|\\>|\\|\|\||\||'
1114
+ r'&&|&|%|\+|<<=|<<|<=|<>|<|==|=|><|>=|>>=|>>|>|¬<<|¬<|¬==|¬=|'
1115
+ r'¬>>|¬>|¬|\.|,)', Operator),
1116
+ ],
1117
+ 'string_double': [
1118
+ (r'[^"\n]+', String),
1119
+ (r'""', String),
1120
+ (r'"', String, '#pop'),
1121
+ (r'\n', Text, '#pop'), # Stray linefeed also terminates strings.
1122
+ ],
1123
+ 'string_single': [
1124
+ (r'[^\'\n]+', String),
1125
+ (r'\'\'', String),
1126
+ (r'\'', String, '#pop'),
1127
+ (r'\n', Text, '#pop'), # Stray linefeed also terminates strings.
1128
+ ],
1129
+ 'comment': [
1130
+ (r'[^*]+', Comment.Multiline),
1131
+ (r'\*/', Comment.Multiline, '#pop'),
1132
+ (r'\*', Comment.Multiline),
1133
+ ]
1134
+ }
1135
+
1136
+ def _c(s):
1137
+ return re.compile(s, re.MULTILINE)
1138
+ _ADDRESS_COMMAND_PATTERN = _c(r'^\s*address\s+command\b')
1139
+ _ADDRESS_PATTERN = _c(r'^\s*address\s+')
1140
+ _DO_WHILE_PATTERN = _c(r'^\s*do\s+while\b')
1141
+ _IF_THEN_DO_PATTERN = _c(r'^\s*if\b.+\bthen\s+do\s*$')
1142
+ _PROCEDURE_PATTERN = _c(r'^\s*([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b')
1143
+ _ELSE_DO_PATTERN = _c(r'\belse\s+do\s*$')
1144
+ _PARSE_ARG_PATTERN = _c(r'^\s*parse\s+(upper\s+)?(arg|value)\b')
1145
+ PATTERNS_AND_WEIGHTS = (
1146
+ (_ADDRESS_COMMAND_PATTERN, 0.2),
1147
+ (_ADDRESS_PATTERN, 0.05),
1148
+ (_DO_WHILE_PATTERN, 0.1),
1149
+ (_ELSE_DO_PATTERN, 0.1),
1150
+ (_IF_THEN_DO_PATTERN, 0.1),
1151
+ (_PROCEDURE_PATTERN, 0.5),
1152
+ (_PARSE_ARG_PATTERN, 0.2),
1153
+ )
1154
+
1155
+ def analyse_text(text):
1156
+ """
1157
+ Check for initial comment and patterns that distinguish Rexx from other
1158
+ C-like languages.
1159
+ """
1160
+ if re.search(r'/\*\**\s*rexx', text, re.IGNORECASE):
1161
+ # Header matches MVS Rexx requirements, this is certainly a Rexx
1162
+ # script.
1163
+ return 1.0
1164
+ elif text.startswith('/*'):
1165
+ # Header matches general Rexx requirements; the source code might
1166
+ # still be any language using C comments such as C++, C# or Java.
1167
+ lowerText = text.lower()
1168
+ result = sum(weight
1169
+ for (pattern, weight) in RexxLexer.PATTERNS_AND_WEIGHTS
1170
+ if pattern.search(lowerText)) + 0.01
1171
+ return min(result, 1.0)
1172
+
1173
+
1174
+ class MOOCodeLexer(RegexLexer):
1175
+ """
1176
+ For MOOCode (the MOO scripting language).
1177
+ """
1178
+ name = 'MOOCode'
1179
+ url = 'http://www.moo.mud.org/'
1180
+ filenames = ['*.moo']
1181
+ aliases = ['moocode', 'moo']
1182
+ mimetypes = ['text/x-moocode']
1183
+ version_added = '0.9'
1184
+
1185
+ tokens = {
1186
+ 'root': [
1187
+ # Numbers
1188
+ (r'(0|[1-9][0-9_]*)', Number.Integer),
1189
+ # Strings
1190
+ (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
1191
+ # exceptions
1192
+ (r'(E_PERM|E_DIV)', Name.Exception),
1193
+ # db-refs
1194
+ (r'((#[-0-9]+)|(\$\w+))', Name.Entity),
1195
+ # Keywords
1196
+ (r'\b(if|else|elseif|endif|for|endfor|fork|endfork|while'
1197
+ r'|endwhile|break|continue|return|try'
1198
+ r'|except|endtry|finally|in)\b', Keyword),
1199
+ # builtins
1200
+ (r'(random|length)', Name.Builtin),
1201
+ # special variables
1202
+ (r'(player|caller|this|args)', Name.Variable.Instance),
1203
+ # skip whitespace
1204
+ (r'\s+', Text),
1205
+ (r'\n', Text),
1206
+ # other operators
1207
+ (r'([!;=,{}&|:.\[\]@()<>?]+)', Operator),
1208
+ # function call
1209
+ (r'(\w+)(\()', bygroups(Name.Function, Operator)),
1210
+ # variables
1211
+ (r'(\w+)', Text),
1212
+ ]
1213
+ }
1214
+
1215
+
1216
+ class HybrisLexer(RegexLexer):
1217
+ """
1218
+ For Hybris source code.
1219
+ """
1220
+
1221
+ name = 'Hybris'
1222
+ aliases = ['hybris']
1223
+ filenames = ['*.hyb']
1224
+ mimetypes = ['text/x-hybris', 'application/x-hybris']
1225
+ url = 'https://github.com/evilsocket/hybris'
1226
+ version_added = '1.4'
1227
+
1228
+ flags = re.MULTILINE | re.DOTALL
1229
+
1230
+ tokens = {
1231
+ 'root': [
1232
+ # method names
1233
+ (r'^(\s*(?:function|method|operator\s+)+?)'
1234
+ r'([a-zA-Z_]\w*)'
1235
+ r'(\s*)(\()', bygroups(Keyword, Name.Function, Text, Operator)),
1236
+ (r'[^\S\n]+', Text),
1237
+ (r'//.*?\n', Comment.Single),
1238
+ (r'/\*.*?\*/', Comment.Multiline),
1239
+ (r'@[a-zA-Z_][\w.]*', Name.Decorator),
1240
+ (r'(break|case|catch|next|default|do|else|finally|for|foreach|of|'
1241
+ r'unless|if|new|return|switch|me|throw|try|while)\b', Keyword),
1242
+ (r'(extends|private|protected|public|static|throws|function|method|'
1243
+ r'operator)\b', Keyword.Declaration),
1244
+ (r'(true|false|null|__FILE__|__LINE__|__VERSION__|__LIB_PATH__|'
1245
+ r'__INC_PATH__)\b', Keyword.Constant),
1246
+ (r'(class|struct)(\s+)',
1247
+ bygroups(Keyword.Declaration, Text), 'class'),
1248
+ (r'(import|include)(\s+)',
1249
+ bygroups(Keyword.Namespace, Text), 'import'),
1250
+ (words((
1251
+ 'gc_collect', 'gc_mm_items', 'gc_mm_usage', 'gc_collect_threshold',
1252
+ 'urlencode', 'urldecode', 'base64encode', 'base64decode', 'sha1', 'crc32',
1253
+ 'sha2', 'md5', 'md5_file', 'acos', 'asin', 'atan', 'atan2', 'ceil', 'cos',
1254
+ 'cosh', 'exp', 'fabs', 'floor', 'fmod', 'log', 'log10', 'pow', 'sin',
1255
+ 'sinh', 'sqrt', 'tan', 'tanh', 'isint', 'isfloat', 'ischar', 'isstring',
1256
+ 'isarray', 'ismap', 'isalias', 'typeof', 'sizeof', 'toint', 'tostring',
1257
+ 'fromxml', 'toxml', 'binary', 'pack', 'load', 'eval', 'var_names',
1258
+ 'var_values', 'user_functions', 'dyn_functions', 'methods', 'call',
1259
+ 'call_method', 'mknod', 'mkfifo', 'mount', 'umount2', 'umount', 'ticks',
1260
+ 'usleep', 'sleep', 'time', 'strtime', 'strdate', 'dllopen', 'dlllink',
1261
+ 'dllcall', 'dllcall_argv', 'dllclose', 'env', 'exec', 'fork', 'getpid',
1262
+ 'wait', 'popen', 'pclose', 'exit', 'kill', 'pthread_create',
1263
+ 'pthread_create_argv', 'pthread_exit', 'pthread_join', 'pthread_kill',
1264
+ 'smtp_send', 'http_get', 'http_post', 'http_download', 'socket', 'bind',
1265
+ 'listen', 'accept', 'getsockname', 'getpeername', 'settimeout', 'connect',
1266
+ 'server', 'recv', 'send', 'close', 'print', 'println', 'printf', 'input',
1267
+ 'readline', 'serial_open', 'serial_fcntl', 'serial_get_attr',
1268
+ 'serial_get_ispeed', 'serial_get_ospeed', 'serial_set_attr',
1269
+ 'serial_set_ispeed', 'serial_set_ospeed', 'serial_write', 'serial_read',
1270
+ 'serial_close', 'xml_load', 'xml_parse', 'fopen', 'fseek', 'ftell',
1271
+ 'fsize', 'fread', 'fwrite', 'fgets', 'fclose', 'file', 'readdir',
1272
+ 'pcre_replace', 'size', 'pop', 'unmap', 'has', 'keys', 'values',
1273
+ 'length', 'find', 'substr', 'replace', 'split', 'trim', 'remove',
1274
+ 'contains', 'join'), suffix=r'\b'),
1275
+ Name.Builtin),
1276
+ (words((
1277
+ 'MethodReference', 'Runner', 'Dll', 'Thread', 'Pipe', 'Process',
1278
+ 'Runnable', 'CGI', 'ClientSocket', 'Socket', 'ServerSocket',
1279
+ 'File', 'Console', 'Directory', 'Exception'), suffix=r'\b'),
1280
+ Keyword.Type),
1281
+ (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
1282
+ (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char),
1283
+ (r'(\.)([a-zA-Z_]\w*)',
1284
+ bygroups(Operator, Name.Attribute)),
1285
+ (r'[a-zA-Z_]\w*:', Name.Label),
1286
+ (r'[a-zA-Z_$]\w*', Name),
1287
+ (r'[~^*!%&\[\](){}<>|+=:;,./?\-@]+', Operator),
1288
+ (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
1289
+ (r'0x[0-9a-f]+', Number.Hex),
1290
+ (r'[0-9]+L?', Number.Integer),
1291
+ (r'\n', Text),
1292
+ ],
1293
+ 'class': [
1294
+ (r'[a-zA-Z_]\w*', Name.Class, '#pop')
1295
+ ],
1296
+ 'import': [
1297
+ (r'[\w.]+\*?', Name.Namespace, '#pop')
1298
+ ],
1299
+ }
1300
+
1301
+ def analyse_text(text):
1302
+ """public method and private method don't seem to be quite common
1303
+ elsewhere."""
1304
+ result = 0
1305
+ if re.search(r'\b(?:public|private)\s+method\b', text):
1306
+ result += 0.01
1307
+ return result
1308
+
1309
+
1310
+
1311
+ class EasytrieveLexer(RegexLexer):
1312
+ """
1313
+ Easytrieve Plus is a programming language for extracting, filtering and
1314
+ converting sequential data. Furthermore it can layout data for reports.
1315
+ It is mainly used on mainframe platforms and can access several of the
1316
+ mainframe's native file formats. It is somewhat comparable to awk.
1317
+ """
1318
+ name = 'Easytrieve'
1319
+ aliases = ['easytrieve']
1320
+ filenames = ['*.ezt', '*.mac']
1321
+ mimetypes = ['text/x-easytrieve']
1322
+ url = 'https://www.broadcom.com/products/mainframe/application-development/easytrieve-report-generator'
1323
+ version_added = '2.1'
1324
+ flags = 0
1325
+
1326
+ # Note: We cannot use r'\b' at the start and end of keywords because
1327
+ # Easytrieve Plus delimiter characters are:
1328
+ #
1329
+ # * space ( )
1330
+ # * apostrophe (')
1331
+ # * period (.)
1332
+ # * comma (,)
1333
+ # * parenthesis ( and )
1334
+ # * colon (:)
1335
+ #
1336
+ # Additionally words end once a '*' appears, indicatins a comment.
1337
+ _DELIMITERS = r' \'.,():\n'
1338
+ _DELIMITERS_OR_COMENT = _DELIMITERS + '*'
1339
+ _DELIMITER_PATTERN = '[' + _DELIMITERS + ']'
1340
+ _DELIMITER_PATTERN_CAPTURE = '(' + _DELIMITER_PATTERN + ')'
1341
+ _NON_DELIMITER_OR_COMMENT_PATTERN = '[^' + _DELIMITERS_OR_COMENT + ']'
1342
+ _OPERATORS_PATTERN = '[.+\\-/=\\[\\](){}<>;,&%¬]'
1343
+ _KEYWORDS = [
1344
+ 'AFTER-BREAK', 'AFTER-LINE', 'AFTER-SCREEN', 'AIM', 'AND', 'ATTR',
1345
+ 'BEFORE', 'BEFORE-BREAK', 'BEFORE-LINE', 'BEFORE-SCREEN', 'BUSHU',
1346
+ 'BY', 'CALL', 'CASE', 'CHECKPOINT', 'CHKP', 'CHKP-STATUS', 'CLEAR',
1347
+ 'CLOSE', 'COL', 'COLOR', 'COMMIT', 'CONTROL', 'COPY', 'CURSOR', 'D',
1348
+ 'DECLARE', 'DEFAULT', 'DEFINE', 'DELETE', 'DENWA', 'DISPLAY', 'DLI',
1349
+ 'DO', 'DUPLICATE', 'E', 'ELSE', 'ELSE-IF', 'END', 'END-CASE',
1350
+ 'END-DO', 'END-IF', 'END-PROC', 'ENDPAGE', 'ENDTABLE', 'ENTER', 'EOF',
1351
+ 'EQ', 'ERROR', 'EXIT', 'EXTERNAL', 'EZLIB', 'F1', 'F10', 'F11', 'F12',
1352
+ 'F13', 'F14', 'F15', 'F16', 'F17', 'F18', 'F19', 'F2', 'F20', 'F21',
1353
+ 'F22', 'F23', 'F24', 'F25', 'F26', 'F27', 'F28', 'F29', 'F3', 'F30',
1354
+ 'F31', 'F32', 'F33', 'F34', 'F35', 'F36', 'F4', 'F5', 'F6', 'F7',
1355
+ 'F8', 'F9', 'FETCH', 'FILE-STATUS', 'FILL', 'FINAL', 'FIRST',
1356
+ 'FIRST-DUP', 'FOR', 'GE', 'GET', 'GO', 'GOTO', 'GQ', 'GR', 'GT',
1357
+ 'HEADING', 'HEX', 'HIGH-VALUES', 'IDD', 'IDMS', 'IF', 'IN', 'INSERT',
1358
+ 'JUSTIFY', 'KANJI-DATE', 'KANJI-DATE-LONG', 'KANJI-TIME', 'KEY',
1359
+ 'KEY-PRESSED', 'KOKUGO', 'KUN', 'LAST-DUP', 'LE', 'LEVEL', 'LIKE',
1360
+ 'LINE', 'LINE-COUNT', 'LINE-NUMBER', 'LINK', 'LIST', 'LOW-VALUES',
1361
+ 'LQ', 'LS', 'LT', 'MACRO', 'MASK', 'MATCHED', 'MEND', 'MESSAGE',
1362
+ 'MOVE', 'MSTART', 'NE', 'NEWPAGE', 'NOMASK', 'NOPRINT', 'NOT',
1363
+ 'NOTE', 'NOVERIFY', 'NQ', 'NULL', 'OF', 'OR', 'OTHERWISE', 'PA1',
1364
+ 'PA2', 'PA3', 'PAGE-COUNT', 'PAGE-NUMBER', 'PARM-REGISTER',
1365
+ 'PATH-ID', 'PATTERN', 'PERFORM', 'POINT', 'POS', 'PRIMARY', 'PRINT',
1366
+ 'PROCEDURE', 'PROGRAM', 'PUT', 'READ', 'RECORD', 'RECORD-COUNT',
1367
+ 'RECORD-LENGTH', 'REFRESH', 'RELEASE', 'RENUM', 'REPEAT', 'REPORT',
1368
+ 'REPORT-INPUT', 'RESHOW', 'RESTART', 'RETRIEVE', 'RETURN-CODE',
1369
+ 'ROLLBACK', 'ROW', 'S', 'SCREEN', 'SEARCH', 'SECONDARY', 'SELECT',
1370
+ 'SEQUENCE', 'SIZE', 'SKIP', 'SOKAKU', 'SORT', 'SQL', 'STOP', 'SUM',
1371
+ 'SYSDATE', 'SYSDATE-LONG', 'SYSIN', 'SYSIPT', 'SYSLST', 'SYSPRINT',
1372
+ 'SYSSNAP', 'SYSTIME', 'TALLY', 'TERM-COLUMNS', 'TERM-NAME',
1373
+ 'TERM-ROWS', 'TERMINATION', 'TITLE', 'TO', 'TRANSFER', 'TRC',
1374
+ 'UNIQUE', 'UNTIL', 'UPDATE', 'UPPERCASE', 'USER', 'USERID', 'VALUE',
1375
+ 'VERIFY', 'W', 'WHEN', 'WHILE', 'WORK', 'WRITE', 'X', 'XDM', 'XRST'
1376
+ ]
1377
+
1378
+ tokens = {
1379
+ 'root': [
1380
+ (r'\*.*\n', Comment.Single),
1381
+ (r'\n+', Whitespace),
1382
+ # Macro argument
1383
+ (r'&' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+\.', Name.Variable,
1384
+ 'after_macro_argument'),
1385
+ # Macro call
1386
+ (r'%' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name.Variable),
1387
+ (r'(FILE|MACRO|REPORT)(\s+)',
1388
+ bygroups(Keyword.Declaration, Whitespace), 'after_declaration'),
1389
+ (r'(JOB|PARM)' + r'(' + _DELIMITER_PATTERN + r')',
1390
+ bygroups(Keyword.Declaration, Operator)),
1391
+ (words(_KEYWORDS, suffix=_DELIMITER_PATTERN_CAPTURE),
1392
+ bygroups(Keyword.Reserved, Operator)),
1393
+ (_OPERATORS_PATTERN, Operator),
1394
+ # Procedure declaration
1395
+ (r'(' + _NON_DELIMITER_OR_COMMENT_PATTERN + r'+)(\s*)(\.?)(\s*)(PROC)(\s*\n)',
1396
+ bygroups(Name.Function, Whitespace, Operator, Whitespace,
1397
+ Keyword.Declaration, Whitespace)),
1398
+ (r'[0-9]+\.[0-9]*', Number.Float),
1399
+ (r'[0-9]+', Number.Integer),
1400
+ (r"'(''|[^'])*'", String),
1401
+ (r'\s+', Whitespace),
1402
+ # Everything else just belongs to a name
1403
+ (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name),
1404
+ ],
1405
+ 'after_declaration': [
1406
+ (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name.Function),
1407
+ default('#pop'),
1408
+ ],
1409
+ 'after_macro_argument': [
1410
+ (r'\*.*\n', Comment.Single, '#pop'),
1411
+ (r'\s+', Whitespace, '#pop'),
1412
+ (_OPERATORS_PATTERN, Operator, '#pop'),
1413
+ (r"'(''|[^'])*'", String, '#pop'),
1414
+ # Everything else just belongs to a name
1415
+ (_NON_DELIMITER_OR_COMMENT_PATTERN + r'+', Name),
1416
+ ],
1417
+ }
1418
+ _COMMENT_LINE_REGEX = re.compile(r'^\s*\*')
1419
+ _MACRO_HEADER_REGEX = re.compile(r'^\s*MACRO')
1420
+
1421
+ def analyse_text(text):
1422
+ """
1423
+ Perform a structural analysis for basic Easytrieve constructs.
1424
+ """
1425
+ result = 0.0
1426
+ lines = text.split('\n')
1427
+ hasEndProc = False
1428
+ hasHeaderComment = False
1429
+ hasFile = False
1430
+ hasJob = False
1431
+ hasProc = False
1432
+ hasParm = False
1433
+ hasReport = False
1434
+
1435
+ def isCommentLine(line):
1436
+ return EasytrieveLexer._COMMENT_LINE_REGEX.match(lines[0]) is not None
1437
+
1438
+ def isEmptyLine(line):
1439
+ return not bool(line.strip())
1440
+
1441
+ # Remove possible empty lines and header comments.
1442
+ while lines and (isEmptyLine(lines[0]) or isCommentLine(lines[0])):
1443
+ if not isEmptyLine(lines[0]):
1444
+ hasHeaderComment = True
1445
+ del lines[0]
1446
+
1447
+ if EasytrieveLexer._MACRO_HEADER_REGEX.match(lines[0]):
1448
+ # Looks like an Easytrieve macro.
1449
+ result = 0.4
1450
+ if hasHeaderComment:
1451
+ result += 0.4
1452
+ else:
1453
+ # Scan the source for lines starting with indicators.
1454
+ for line in lines:
1455
+ words = line.split()
1456
+ if (len(words) >= 2):
1457
+ firstWord = words[0]
1458
+ if not hasReport:
1459
+ if not hasJob:
1460
+ if not hasFile:
1461
+ if not hasParm:
1462
+ if firstWord == 'PARM':
1463
+ hasParm = True
1464
+ if firstWord == 'FILE':
1465
+ hasFile = True
1466
+ if firstWord == 'JOB':
1467
+ hasJob = True
1468
+ elif firstWord == 'PROC':
1469
+ hasProc = True
1470
+ elif firstWord == 'END-PROC':
1471
+ hasEndProc = True
1472
+ elif firstWord == 'REPORT':
1473
+ hasReport = True
1474
+
1475
+ # Weight the findings.
1476
+ if hasJob and (hasProc == hasEndProc):
1477
+ if hasHeaderComment:
1478
+ result += 0.1
1479
+ if hasParm:
1480
+ if hasProc:
1481
+ # Found PARM, JOB and PROC/END-PROC:
1482
+ # pretty sure this is Easytrieve.
1483
+ result += 0.8
1484
+ else:
1485
+ # Found PARAM and JOB: probably this is Easytrieve
1486
+ result += 0.5
1487
+ else:
1488
+ # Found JOB and possibly other keywords: might be Easytrieve
1489
+ result += 0.11
1490
+ if hasParm:
1491
+ # Note: PARAM is not a proper English word, so this is
1492
+ # regarded a much better indicator for Easytrieve than
1493
+ # the other words.
1494
+ result += 0.2
1495
+ if hasFile:
1496
+ result += 0.01
1497
+ if hasReport:
1498
+ result += 0.01
1499
+ assert 0.0 <= result <= 1.0
1500
+ return result
1501
+
1502
+
1503
+ class JclLexer(RegexLexer):
1504
+ """
1505
+ Job Control Language (JCL)
1506
+ is a scripting language used on mainframe platforms to instruct the system
1507
+ on how to run a batch job or start a subsystem. It is somewhat
1508
+ comparable to MS DOS batch and Unix shell scripts.
1509
+ """
1510
+ name = 'JCL'
1511
+ aliases = ['jcl']
1512
+ filenames = ['*.jcl']
1513
+ mimetypes = ['text/x-jcl']
1514
+ url = 'https://en.wikipedia.org/wiki/Job_Control_Language'
1515
+ version_added = '2.1'
1516
+
1517
+ flags = re.IGNORECASE
1518
+
1519
+ tokens = {
1520
+ 'root': [
1521
+ (r'//\*.*\n', Comment.Single),
1522
+ (r'//', Keyword.Pseudo, 'statement'),
1523
+ (r'/\*', Keyword.Pseudo, 'jes2_statement'),
1524
+ # TODO: JES3 statement
1525
+ (r'.*\n', Other) # Input text or inline code in any language.
1526
+ ],
1527
+ 'statement': [
1528
+ (r'\s*\n', Whitespace, '#pop'),
1529
+ (r'([a-z]\w*)(\s+)(exec|job)(\s*)',
1530
+ bygroups(Name.Label, Whitespace, Keyword.Reserved, Whitespace),
1531
+ 'option'),
1532
+ (r'[a-z]\w*', Name.Variable, 'statement_command'),
1533
+ (r'\s+', Whitespace, 'statement_command'),
1534
+ ],
1535
+ 'statement_command': [
1536
+ (r'\s+(command|cntl|dd|endctl|endif|else|include|jcllib|'
1537
+ r'output|pend|proc|set|then|xmit)\s+', Keyword.Reserved, 'option'),
1538
+ include('option')
1539
+ ],
1540
+ 'jes2_statement': [
1541
+ (r'\s*\n', Whitespace, '#pop'),
1542
+ (r'\$', Keyword, 'option'),
1543
+ (r'\b(jobparam|message|netacct|notify|output|priority|route|'
1544
+ r'setup|signoff|xeq|xmit)\b', Keyword, 'option'),
1545
+ ],
1546
+ 'option': [
1547
+ # (r'\n', Text, 'root'),
1548
+ (r'\*', Name.Builtin),
1549
+ (r'[\[\](){}<>;,]', Punctuation),
1550
+ (r'[-+*/=&%]', Operator),
1551
+ (r'[a-z_]\w*', Name),
1552
+ (r'\d+\.\d*', Number.Float),
1553
+ (r'\.\d+', Number.Float),
1554
+ (r'\d+', Number.Integer),
1555
+ (r"'", String, 'option_string'),
1556
+ (r'[ \t]+', Whitespace, 'option_comment'),
1557
+ (r'\.', Punctuation),
1558
+ ],
1559
+ 'option_string': [
1560
+ (r"(\n)(//)", bygroups(Text, Keyword.Pseudo)),
1561
+ (r"''", String),
1562
+ (r"[^']", String),
1563
+ (r"'", String, '#pop'),
1564
+ ],
1565
+ 'option_comment': [
1566
+ # (r'\n', Text, 'root'),
1567
+ (r'.+', Comment.Single),
1568
+ ]
1569
+ }
1570
+
1571
+ _JOB_HEADER_PATTERN = re.compile(r'^//[a-z#$@][a-z0-9#$@]{0,7}\s+job(\s+.*)?$',
1572
+ re.IGNORECASE)
1573
+
1574
+ def analyse_text(text):
1575
+ """
1576
+ Recognize JCL job by header.
1577
+ """
1578
+ result = 0.0
1579
+ lines = text.split('\n')
1580
+ if len(lines) > 0:
1581
+ if JclLexer._JOB_HEADER_PATTERN.match(lines[0]):
1582
+ result = 1.0
1583
+ assert 0.0 <= result <= 1.0
1584
+ return result
1585
+
1586
+
1587
+ class MiniScriptLexer(RegexLexer):
1588
+ """
1589
+ For MiniScript source code.
1590
+ """
1591
+
1592
+ name = 'MiniScript'
1593
+ url = 'https://miniscript.org'
1594
+ aliases = ['miniscript', 'ms']
1595
+ filenames = ['*.ms']
1596
+ mimetypes = ['text/x-minicript', 'application/x-miniscript']
1597
+ version_added = '2.6'
1598
+
1599
+ tokens = {
1600
+ 'root': [
1601
+ (r'#!(.*?)$', Comment.Preproc),
1602
+ default('base'),
1603
+ ],
1604
+ 'base': [
1605
+ ('//.*$', Comment.Single),
1606
+ (r'(?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?', Number),
1607
+ (r'(?i)\d+e[+-]?\d+', Number),
1608
+ (r'\d+', Number),
1609
+ (r'\n', Text),
1610
+ (r'[^\S\n]+', Text),
1611
+ (r'"', String, 'string_double'),
1612
+ (r'(==|!=|<=|>=|[=+\-*/%^<>.:])', Operator),
1613
+ (r'[;,\[\]{}()]', Punctuation),
1614
+ (words((
1615
+ 'break', 'continue', 'else', 'end', 'for', 'function', 'if',
1616
+ 'in', 'isa', 'then', 'repeat', 'return', 'while'), suffix=r'\b'),
1617
+ Keyword),
1618
+ (words((
1619
+ 'abs', 'acos', 'asin', 'atan', 'ceil', 'char', 'cos', 'floor',
1620
+ 'log', 'round', 'rnd', 'pi', 'sign', 'sin', 'sqrt', 'str', 'tan',
1621
+ 'hasIndex', 'indexOf', 'len', 'val', 'code', 'remove', 'lower',
1622
+ 'upper', 'replace', 'split', 'indexes', 'values', 'join', 'sum',
1623
+ 'sort', 'shuffle', 'push', 'pop', 'pull', 'range',
1624
+ 'print', 'input', 'time', 'wait', 'locals', 'globals', 'outer',
1625
+ 'yield'), suffix=r'\b'),
1626
+ Name.Builtin),
1627
+ (r'(true|false|null)\b', Keyword.Constant),
1628
+ (r'(and|or|not|new)\b', Operator.Word),
1629
+ (r'(self|super|__isa)\b', Name.Builtin.Pseudo),
1630
+ (r'[a-zA-Z_]\w*', Name.Variable)
1631
+ ],
1632
+ 'string_double': [
1633
+ (r'[^"\n]+', String),
1634
+ (r'""', String),
1635
+ (r'"', String, '#pop'),
1636
+ (r'\n', Text, '#pop'), # Stray linefeed also terminates strings.
1637
+ ]
1638
+ }
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/tact.py ADDED
@@ -0,0 +1,303 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers.tact
3
+ ~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexers for Tact.
6
+
7
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import RegexLexer, include, bygroups, words
12
+ from pygments.token import Comment, Operator, Keyword, Name, String, \
13
+ Number, Whitespace, Punctuation
14
+
15
+ __all__ = ['TactLexer']
16
+
17
+
18
+ class TactLexer(RegexLexer):
19
+ """For Tact source code."""
20
+
21
+ name = 'Tact'
22
+ aliases = ['tact']
23
+ filenames = ['*.tact']
24
+ url = "https://tact-lang.org"
25
+ version_added = '2.18'
26
+
27
+ tokens = {
28
+ 'root': [
29
+ (r'\s+', Whitespace),
30
+ (r'[.;(),\[\]{}]', Punctuation),
31
+ (r'\?|!!', Operator),
32
+ include('comments'),
33
+ include('import-in'),
34
+ include('struct-in'),
35
+ include('contract-or-trait-in'),
36
+ include('annotation-in'),
37
+ include('fun-declaration-in'),
38
+ include('const-declaration-in'),
39
+ include('statements'),
40
+ ],
41
+ 'import-in': [
42
+ (r'((?<=\.\.\.)|(?<![.$]))\b(import)\b(\s*)', bygroups(Punctuation, Keyword, Whitespace), 'import'),
43
+ ],
44
+ 'import': [
45
+ (r';', Punctuation, '#pop'),
46
+ include('comments'),
47
+ include('string-in'),
48
+ (r'\s+', Whitespace),
49
+ ],
50
+ 'struct-in': [
51
+ (r'((?<=\.\.\.)|(?<![.$]))\b(struct|message)\b', bygroups(Punctuation, Keyword), 'struct'),
52
+ ],
53
+ 'struct': [
54
+ include('comments'),
55
+ include('struct-header'),
56
+ include('struct-body-in'),
57
+ (r'\s+', Whitespace),
58
+ ],
59
+ 'struct-header': [
60
+ include('comments'),
61
+ (r'\b\w+', Name.Class),
62
+ (r'(\()((?:\b0[xX])[0-9a-fA-F][0-9a-fA-F_]*\b)(\))', bygroups(Punctuation, Number.Hex, Punctuation)),
63
+ (r'(\()((?:\b[0-9]+\b))(\))', bygroups(Punctuation, Number.Integer, Punctuation)),
64
+ ],
65
+ 'struct-body-in': [
66
+ (r'\{', Punctuation, 'struct-body'),
67
+ ],
68
+ 'struct-body': [
69
+ (r'\}', Punctuation, '#pop:2'),
70
+ include('comments'),
71
+ include('field-declaration-in'),
72
+ ],
73
+ 'contract-or-trait-in': [
74
+ (r'((?<=\.\.\.)|(?<![.$]))\b(contract|trait)\b', Keyword, 'contract-or-trait'),
75
+ ],
76
+ 'contract-or-trait': [
77
+ include('comments'),
78
+ (r'with', Keyword),
79
+ (r'\b\w+', Name.Class),
80
+ include('contract-or-trait-body-in'),
81
+ (r'\s+', Whitespace),
82
+ (r',', Punctuation),
83
+ ],
84
+ 'contract-or-trait-body-in': [
85
+ (r'\{', Punctuation, 'contract-or-trait-body'),
86
+ ],
87
+ 'contract-or-trait-body': [
88
+ (r'\}', Punctuation, '#pop:2'),
89
+ include('comments'),
90
+ include('init-declaration-in'),
91
+ include('receive-declaration-in'),
92
+ include('bounce-declaration-in'),
93
+ include('fun-declaration-in'),
94
+ include('const-declaration-in'),
95
+ include('field-declaration-in'),
96
+ (r'\s+', Whitespace),
97
+ ],
98
+ 'field-declaration-in': [
99
+ (r'\b\w+', Name.Property, 'field-declaration'),
100
+ ],
101
+ 'field-declaration': [
102
+ (r';', Punctuation, '#pop'),
103
+ include('comments'),
104
+ include('type-annotation-in'),
105
+ include('variable-init-in'),
106
+ ],
107
+ 'const-declaration-in': [
108
+ (r'(?=\b(?:(?:get|native|extends|mutates|virtual|override|inline|abstract)\s*)*const\b)', Keyword, 'const-declaration'),
109
+ ],
110
+ 'const-declaration': [
111
+ (r'(;)', Punctuation, '#pop'),
112
+ (r'const', Keyword),
113
+ (words(('get', 'native', 'extends', 'mutates', 'virtual', 'override', 'inline', 'abstract'), suffix=r'\b'), Keyword),
114
+ (r'\b\w+\b', Name.Constant),
115
+ include('comments'),
116
+ include('type-annotation-in'),
117
+ include('variable-init-in'),
118
+ (r'\s+', Whitespace),
119
+ ],
120
+ 'init-declaration-in': [
121
+ (r'(init)', Keyword, 'init-declaration')
122
+ ],
123
+ 'init-declaration': [
124
+ (r'(?<=\})', Punctuation, '#pop'),
125
+ include('comments'),
126
+ include('fun-arguments-in'),
127
+ include('block-declaration-in'),
128
+ (r'\s+', Whitespace),
129
+ ],
130
+ 'receive-declaration-in': [
131
+ (r'(receive|exernal)', Keyword, 'receive-declaration')
132
+ ],
133
+ 'receive-declaration': [
134
+ (r'(?<=\})', Punctuation, '#pop'),
135
+ include('comments'),
136
+ include('fun-arguments-in'),
137
+ include('block-declaration-in'),
138
+ ],
139
+ 'bounce-declaration-in': [
140
+ (r'(bounced)', Keyword, 'bounce-declaration')
141
+ ],
142
+ 'bounce-declaration': [
143
+ (r'(?<=\})', Punctuation, '#pop'),
144
+ include('comments'),
145
+ include('fun-arguments-in'),
146
+ include('block-declaration-in'),
147
+ ],
148
+ 'fun-declaration-in': [
149
+ (r'(?=\b(?:(?:get|native|extends|mutates|virtual|override|inline|abstract)\s*)*fun\b)', Keyword, 'fun-declaration')
150
+ ],
151
+ 'fun-declaration': [
152
+ (r'(?<=\}|\;)', Punctuation, '#pop'),
153
+ (r'fun', Keyword),
154
+ (r'\b(get|native|extends|mutates|virtual|override|inline|abstract)\b', Keyword),
155
+ (r'\b[\w]+', Name.Function),
156
+ include('fun-declaration-body'),
157
+ (r'[,;]', Punctuation),
158
+ ],
159
+ 'fun-declaration-body': [
160
+ include('comments'),
161
+ include('fun-arguments-in'),
162
+ include('type-annotation-in'),
163
+ include('block-declaration-in'),
164
+ (r'\s+', Whitespace),
165
+ ],
166
+ 'fun-arguments-in': [
167
+ (r'\(', Punctuation, 'fun-arguments'),
168
+ ],
169
+ 'fun-arguments': [
170
+ (r'\)', Punctuation, '#pop'),
171
+ include('comments'),
172
+ include('string-in'),
173
+ include('type-annotation-in'),
174
+ (r'(self)|(\b[\w]+\b)', bygroups(Name.Variable.Instance, Name.Variable)),
175
+ (r',', Punctuation),
176
+ (r'\s+', Whitespace),
177
+ ],
178
+ 'block-declaration-in': [
179
+ (r'\{', Punctuation, 'block-declaration')
180
+ ],
181
+ 'block-declaration': [
182
+ (r'\}', Punctuation, '#pop'),
183
+ include('statements'),
184
+ ],
185
+ 'statements': [
186
+ include('comments'),
187
+ include('block-declaration-in'),
188
+ include('expressions'),
189
+ ],
190
+ 'annotation-in': [
191
+ (r'(@)(\w+)(\()', bygroups(Keyword.Pseudo, Keyword, Punctuation), 'annotation')
192
+ ],
193
+ 'annotation': [
194
+ (r'\)', Punctuation, '#pop'),
195
+ include('annotation-argument'),
196
+ (r'\s+', Whitespace),
197
+ ],
198
+ 'annotation-argument': [
199
+ (r'\w+', Name.Function.Magic),
200
+ ],
201
+ 'expressions': [
202
+ include('comments'),
203
+ include('type-annotation-in'),
204
+ include('keywords'),
205
+ include('numeric'),
206
+ include('string-in'),
207
+ include('variable'),
208
+ include('function-call'),
209
+ include('struct-init-in'),
210
+ ],
211
+ 'struct-init-in': [
212
+ (r'(\b\w+)(\s*)(\{)', bygroups(Name.Class, Whitespace, Punctuation), 'struct-init')
213
+ ],
214
+ 'struct-init': [
215
+ (r'(\})', Punctuation, '#pop'),
216
+ include('comments'),
217
+ include('struct-property-in'),
218
+ (r'\s+', Whitespace),
219
+ (r',', Punctuation),
220
+ ],
221
+ 'struct-property-in': [
222
+ (r'(\b[\w]+)(\s*)(:)', bygroups(Name.Property, Whitespace, Punctuation), 'struct-property')
223
+ ],
224
+ 'struct-property': [
225
+ (r'(?=\}|\,)', Punctuation, '#pop'),
226
+ include('comments'),
227
+ include('expressions'),
228
+ (r'\s+', Whitespace),
229
+ ],
230
+ 'variable-init-in': [
231
+ (r'(=)', Operator, 'variable-init')
232
+ ],
233
+ 'variable-init': [
234
+ (r'(?=\}|\{|\,|\;)',Punctuation, '#pop'),
235
+ include('comments'),
236
+ include('expressions'),
237
+ (r'\s+', Whitespace),
238
+ ],
239
+ 'type-annotation-in': [
240
+ (r'(:)(\s+)', bygroups(Punctuation, Whitespace), 'type-annotation')
241
+ ],
242
+ 'type-annotation': [
243
+ (r'(?=\{|\;|\=|\,|\))', Punctuation, '#pop'),
244
+ include('comments'),
245
+ include('type-as-in'),
246
+ include('type-generic-in'),
247
+ (r'\?', Operator),
248
+ (r'\b\w+', Keyword.Type),
249
+ (r'\s+', Whitespace),
250
+ ],
251
+ 'type-generic-in': [
252
+ (r'<', Punctuation, 'type-generic'),
253
+ ],
254
+ 'type-generic': [
255
+ (r'>', Punctuation, '#pop'),
256
+ include('comments'),
257
+ include('type-as-in'),
258
+ (r'\b\w+', Keyword.Type),
259
+ (r'\s+', Whitespace),
260
+ (r',', Punctuation),
261
+ ],
262
+ 'type-as-in': [
263
+ (r'\b(as)(\s+)', bygroups(Keyword, Whitespace), 'type-as'),
264
+ ],
265
+ 'type-as': [
266
+ (r'(?=\{|\;|\=|\,|\)|\>)', Punctuation, '#pop'),
267
+ include('comments'),
268
+ (r'\b\w+', Keyword.Type),
269
+ (r'\s+', Whitespace),
270
+ ],
271
+ 'keywords': [
272
+ (words(('if', 'else', 'while', 'do', 'until', 'repeat', 'return', 'extends', 'mutates', 'virtual', 'override', 'inline', 'native', 'let', 'const', 'fun', 'self', 'is', 'initOf', 'map', 'bounced', 'get', 'as'), prefix=r'\b', suffix=r'\b'), Keyword),
273
+ (r'(<=>|>=|<=|!=|==|\^>>|~>>|>>|<<|\/%|\^%|~%|\^\/|~\/|\+=|-=|\*=|\/=|~\/=|\^\/=|%=|\^%=|<<=|>>=|~>>=|\^>>=|&=|\|=|\^=|\^|=|~|\/|%|-|\*|\+|>|<|&|\||:|\?)', Operator),
274
+ (words(('true', 'false'), prefix=r'\b', suffix=r'\b'), Keyword.Constant),
275
+ ],
276
+ 'string-in': [
277
+ (r'"', String, 'string'),
278
+ ],
279
+ 'string': [
280
+ (r'"', String, '#pop'),
281
+ (r'\\.', String.Escape),
282
+ (r'[^\\"]+', String.Double),
283
+ ],
284
+ 'numeric': [
285
+ (r'(?:\b0[xX])[0-9a-fA-F][0-9a-fA-F_]*\b', Number.Hex),
286
+ (r'(?:\b[0-9]+\b)', Number.Integer),
287
+ ],
288
+ 'comments': [
289
+ (r'//.*', Comment.Single),
290
+ (r'/\*', Comment.Multiline, 'comments-multiline'),
291
+ ],
292
+ 'comments-multiline': [
293
+ (r'\*/', Comment.Multiline, '#pop'),
294
+ (r'[^*]+', Comment.Multiline),
295
+ (r'[*]', Comment.Multiline),
296
+ ],
297
+ 'variable': [
298
+ (r'\b\w+\b(?!\s*\()(?!\s*\{)', Name.Variable)
299
+ ],
300
+ 'function-call': [
301
+ (r'\b\w+\b(?=\s*\()(?!\s*\{)', Name.Function)
302
+ ],
303
+ }
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/thingsdb.py ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers.thingsdb
3
+ ~~~~~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexers for the ThingsDB language.
6
+
7
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import RegexLexer, include, bygroups
12
+ from pygments.token import Comment, Keyword, Name, Number, String, Text, \
13
+ Operator, Punctuation, Whitespace
14
+
15
+ __all__ = ['ThingsDBLexer']
16
+
17
+
18
+ class ThingsDBLexer(RegexLexer):
19
+ """
20
+ Lexer for the ThingsDB programming language.
21
+ """
22
+ name = 'ThingsDB'
23
+ aliases = ['ti', 'thingsdb']
24
+ filenames = ['*.ti']
25
+ url = 'https://www.thingsdb.net'
26
+ version_added = '2.9'
27
+
28
+ tokens = {
29
+ 'root': [
30
+ include('expression'),
31
+ ],
32
+ 'expression': [
33
+ include('comments'),
34
+ include('whitespace'),
35
+
36
+ # numbers
37
+ (r'[-+]?0b[01]+', Number.Bin),
38
+ (r'[-+]?0o[0-8]+', Number.Oct),
39
+ (r'([-+]?0x[0-9a-fA-F]+)', Number.Hex),
40
+ (r'[-+]?[0-9]+', Number.Integer),
41
+ (r'[-+]?((inf|nan)([^0-9A-Za-z_]|$)|[0-9]*\.[0-9]+(e[+-][0-9]+)?)',
42
+ Number.Float),
43
+
44
+ # strings
45
+ (r'(?:"(?:[^"]*)")+', String.Double),
46
+ (r"(?:'(?:[^']*)')+", String.Single),
47
+ (r"(?:`(?:[^`]*)`)+", String.Backtick),
48
+
49
+ # literals
50
+ (r'(true|false|nil)\b', Keyword.Constant),
51
+
52
+ # name constants
53
+ (r'(FULL|USER|GRANT|CHANGE|JOIN|RUN|QUERY|'
54
+ r'DEBUG|INFO|WARNING|ERROR|CRITICAL|'
55
+ r'NO_IDS|INT_MIN|INT_MAX|MATH_E|MATH_PI)\b', Name.Constant),
56
+
57
+ # regular expressions
58
+ (r'(/[^/\\]*(?:\\.[^/\\]*)*/i?)', String.Regex),
59
+
60
+ # name, assignments and functions
61
+ include('names'),
62
+
63
+ (r'[(){}\[\],;]', Punctuation),
64
+ (r'[+\-*/%&|<>^!~@=:?]', Operator),
65
+ ],
66
+ 'names': [
67
+ (r'(\.)'
68
+ r'(first|last|then|else|load|at|again_in|again_at|err|cancel|'
69
+ r'closure|set_closure|args|set_args|owner|set_owner|equals|copy|'
70
+ r'dup|assign|week|weekday|yday|zone|len|call|doc|emit|extract|'
71
+ r'choice|code|format|msg|each|every|extend|extend_unique|filter|'
72
+ r'find|flat|find_index|has|index_of|count|sum|is_unique|unique|'
73
+ r'join|map|map_id|map_wrap|map_type|vmap|move|pop|push|fill|'
74
+ r'remove|replace|restrict|restriction|shift|sort|splice|to|add|'
75
+ r'one|clear|contains|ends_with|name|lower|replace|reverse|'
76
+ r'starts_with|split|test|trim|trim_left|trim_right|upper|del|ren|'
77
+ r'to_type|to_thing|get|id|keys|reduce|set|some|value|values|wrap|'
78
+ r'unshift|unwrap|search|set_name|bit_count)'
79
+ r'(\()',
80
+ bygroups(Name.Function, Name.Function, Punctuation), 'arguments'),
81
+ (r'(alt_raise|assert|base64_encode|base64_decode|bool|bytes|'
82
+ r'closure|datetime|deep|future|is_future|del_enum|del_type|room|'
83
+ r'is_room|task|tasks|is_task|is_email|is_url|is_tel|is_time_zone|'
84
+ r'timeit|enum|enum_info|enum_map|enums_info|err|regex|is_regex|'
85
+ r'change_id|float|has_enum|has_type|int|is_array|is_ascii|'
86
+ r'is_float|is_bool|is_bytes|is_closure|is_datetime|is_enum|'
87
+ r'is_err|is_mpdata|is_inf|is_int|is_list|is_nan|is_nil|is_raw|'
88
+ r'is_set|is_str|is_thing|is_timeval|is_tuple|is_utf8|json_dump|'
89
+ r'json_load|list|log|import|export|root|mod_enum|mod_type|new|'
90
+ r'new_type|now|raise|rand|range|randint|randstr|refs|rename_enum|'
91
+ r'set|set_enum|set_type|str|thing|timeval|try|type|type_assert|'
92
+ r'type_all|type_count|type_info|types_info|nse|wse|backup_info|'
93
+ r'backups_info|backups_ok|counters|del_backup|has_backup|'
94
+ r'new_backup|node_info|nodes_info|reset_counters|restart_module|'
95
+ r'set_log_level|shutdown|has_module|del_module|module_info|'
96
+ r'modules_info|new_module|deploy_module|rename_module|'
97
+ r'refresh_module|set_module_conf|set_module_scope|'
98
+ r'collections_info|del_collection|del_expired|del_node|del_token|'
99
+ r'del_user|grant|has_collection|has_node|has_token|has_user|'
100
+ r'new_collection|new_node|new_token|new_user|rename_collection|'
101
+ r'rename_user|restore|revoke|set_password|set_time_zone|'
102
+ r'set_default_deep|time_zones_info|user_info|users_info|'
103
+ r'del_procedure|has_procedure|new_procedure|mod_procedure|'
104
+ r'procedure_doc|procedure_info|procedures_info|rename_procedure|'
105
+ r'run|assert_err|auth_err|bad_data_err|cancelled_err|'
106
+ r'rename_type|forbidden_err|lookup_err|max_quota_err|node_err|'
107
+ r'num_arguments_err|operation_err|overflow_err|syntax_err|'
108
+ r'collection_info|type_err|value_err|zero_div_err|whitelist_add|'
109
+ r'whitelist_del|round|abs|ceil|cos|exp|floor|log10|log2|'
110
+ r'loge|pow|sin|sqrt|tan|is_module|commit|history|set_history|'
111
+ r'del_history|ano)'
112
+ r'(\()',
113
+ bygroups(Name.Function, Punctuation),
114
+ 'arguments'),
115
+ (r'(\.[A-Za-z_][0-9A-Za-z_]*)'
116
+ r'(\s*)(=)',
117
+ bygroups(Name.Attribute, Text, Operator)),
118
+ (r'\.[A-Za-z_][0-9A-Za-z_]*', Name.Attribute),
119
+ (r'([A-Za-z_][0-9A-Za-z_]*)(\s*)(=)',
120
+ bygroups(Name.Variable, Text, Operator)),
121
+ (r'[A-Za-z_][0-9A-Za-z_]*', Name.Variable),
122
+ ],
123
+ 'whitespace': [
124
+ (r'\n', Whitespace),
125
+ (r'\s+', Whitespace),
126
+ ],
127
+ 'comments': [
128
+ (r'//(.*?)\n', Comment.Single),
129
+ (r'/\*', Comment.Multiline, 'comment'),
130
+ ],
131
+ 'comment': [
132
+ (r'[^*/]+', Comment.Multiline),
133
+ (r'/\*', Comment.Multiline, '#push'),
134
+ (r'\*/', Comment.Multiline, '#pop'),
135
+ (r'[*/]', Comment.Multiline),
136
+ ],
137
+ 'arguments': [
138
+ include('expression'),
139
+ (',', Punctuation),
140
+ (r'\(', Punctuation, '#push'),
141
+ (r'\)', Punctuation, '#pop'),
142
+ ],
143
+ }
LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/pygments/lexers/yara.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ pygments.lexers.yara
3
+ ~~~~~~~~~~~~~~~~~~~~
4
+
5
+ Lexers for YARA.
6
+
7
+ :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8
+ :license: BSD, see LICENSE for details.
9
+ """
10
+
11
+ from pygments.lexer import RegexLexer, words
12
+ from pygments.token import Comment, String, Name, Text, Punctuation, \
13
+ Operator, Keyword, Whitespace, Number
14
+
15
+ __all__ = ['YaraLexer']
16
+
17
+
18
+ class YaraLexer(RegexLexer):
19
+ """
20
+ For YARA rules
21
+ """
22
+
23
+ name = 'YARA'
24
+ url = 'https://virustotal.github.io/yara/'
25
+ aliases = ['yara', 'yar']
26
+ filenames = ['*.yar']
27
+ mimetypes = ['text/x-yara']
28
+ version_added = '2.16'
29
+
30
+ tokens = {
31
+ 'root': [
32
+ (r'\s+', Whitespace),
33
+ (r'//.*?$', Comment.Single),
34
+ (r'\#.*?$', Comment.Single),
35
+ (r'/\*', Comment.Multiline, 'comment'),
36
+ (words(('rule', 'private', 'global', 'import', 'include'),
37
+ prefix=r'\b', suffix=r'\b'),
38
+ Keyword.Declaration),
39
+ (words(('strings', 'condition', 'meta'), prefix=r'\b', suffix=r'\b'),
40
+ Keyword),
41
+ (words(('ascii', 'at', 'base64', 'base64wide', 'condition',
42
+ 'contains', 'endswith', 'entrypoint', 'filesize', 'for',
43
+ 'fullword', 'icontains', 'iendswith', 'iequals', 'in',
44
+ 'include', 'int16', 'int16be', 'int32', 'int32be', 'int8',
45
+ 'int8be', 'istartswith', 'matches', 'meta', 'nocase',
46
+ 'none', 'of', 'startswith', 'strings', 'them', 'uint16',
47
+ 'uint16be', 'uint32', 'uint32be', 'uint8', 'uint8be',
48
+ 'wide', 'xor', 'defined'),
49
+ prefix=r'\b', suffix=r'\b'),
50
+ Name.Builtin),
51
+ (r'(true|false)\b', Keyword.Constant),
52
+ (r'(and|or|not|any|all)\b', Operator.Word),
53
+ (r'(\$\w+)', Name.Variable),
54
+ (r'"[^"]*"', String.Double),
55
+ (r'\'[^\']*\'', String.Single),
56
+ (r'\{.*?\}$', Number.Hex),
57
+ (r'(/.*?/)', String.Regex),
58
+ (r'[a-z_]\w*', Name),
59
+ (r'[$(){}[\].?+*|]', Punctuation),
60
+ (r'[:=,;]', Punctuation),
61
+ (r'.', Text)
62
+ ],
63
+ 'comment': [
64
+ (r'[^*/]+', Comment.Multiline),
65
+ (r'/\*', Comment.Multiline, '#push'),
66
+ (r'\*/', Comment.Multiline, '#pop'),
67
+ (r'[*/]', Comment.Multiline)
68
+ ]
69
+ }