Add files using upload-large-folder tool

LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0017000_logistic_normal_t1p45.log

@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_00:08:31 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0017000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0017000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0017000.pt
+[ckpt] step=17000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0017000.pt",
+  "step": 17000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 32.96868844968726,
+    "nll_per_token": 3.4955582761797803,
+    "tokens": 34392,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 44.511234516933115,
+    "nll_per_token": 3.795741618363942,
+    "tokens": 28655,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.516973326930281,
+    "unique_tokens": 1775,
+    "token_count": 32768,
+    "distinct_1": 0.054168701171875,
+    "distinct_2": 0.27303764763779526,
+    "top_token_mass": 0.1605224609375
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0017000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_00:09:58 done step_0017000

LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0045000_logistic_normal_t1p45.log

@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_02:45:03 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0045000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0045000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0045000.pt
+[ckpt] step=45000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0045000.pt",
+  "step": 45000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 30.422828271197886,
+    "nll_per_token": 3.4151932565789473,
+    "tokens": 36974,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 43.95881792699332,
+    "nll_per_token": 3.78325323943306,
+    "tokens": 30292,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.7343144847775447,
+    "unique_tokens": 1830,
+    "token_count": 32768,
+    "distinct_1": 0.05584716796875,
+    "distinct_2": 0.30164247047244097,
+    "top_token_mass": 0.089263916015625
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0045000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_02:46:31 done step_0045000

LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0080000_logistic_normal_t1p45.log

@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_06:00:36 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0080000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0080000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0080000.pt
+[ckpt] step=80000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0080000.pt",
+  "step": 80000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 31.097837718960108,
+    "nll_per_token": 3.437138290046554,
+    "tokens": 36369,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 44.05213155429666,
+    "nll_per_token": 3.785373740639305,
+    "tokens": 29929,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.7593508905065947,
+    "unique_tokens": 1830,
+    "token_count": 32768,
+    "distinct_1": 0.05584716796875,
+    "distinct_2": 0.3041031003937008,
+    "top_token_mass": 0.090576171875
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0080000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_06:02:04 done step_0080000

LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0095000_logistic_normal_t1p45.log

@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_07:24:16 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0095000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0095000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0095000.pt
+[ckpt] step=95000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0095000.pt",
+  "step": 95000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 35.29678225364393,
+    "nll_per_token": 3.5637918054967392,
+    "tokens": 34388,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 45.884026994283424,
+    "nll_per_token": 3.8261170607346755,
+    "tokens": 29120,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.5066169703973267,
+    "unique_tokens": 2268,
+    "token_count": 32768,
+    "distinct_1": 0.0692138671875,
+    "distinct_2": 0.34639517716535434,
+    "top_token_mass": 0.137786865234375
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0095000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_07:25:43 done step_0095000

LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0115000_logistic_normal_t1p45.log

@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_09:15:41 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0115000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0115000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0115000.pt
+[ckpt] step=115000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0115000.pt",
+  "step": 115000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 31.88784973151671,
+    "nll_per_token": 3.4622250510758907,
+    "tokens": 37559,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 44.994357822922595,
+    "nll_per_token": 3.8065371001965445,
+    "tokens": 31068,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.9007762593098327,
+    "unique_tokens": 2532,
+    "token_count": 32768,
+    "distinct_1": 0.0772705078125,
+    "distinct_2": 0.3822896161417323,
+    "top_token_mass": 0.07061767578125
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0115000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_09:17:09 done step_0115000

LTA_openwebtext_dualt/logs/lm1b_len128_lognormal_atoms_sde_watch/infer_lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522_step_0119000_logistic_normal_t1p45.log

@@ -0,0 +1,76 @@
+[watch-lognormal-sde] 2026-05-23_09:38:06 infer runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0119000.pt -> docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0119000
+[load] runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0119000.pt
+[ckpt] step=119000
+[sde] generated 16/256
+[sde] generated 32/256
+[sde] generated 48/256
+[sde] generated 64/256
+[sde] generated 80/256
+[sde] generated 96/256
+[sde] generated 112/256
+[sde] generated 128/256
+[sde] generated 144/256
+[sde] generated 160/256
+[sde] generated 176/256
+[sde] generated 192/256
+[sde] generated 208/256
+[sde] generated 224/256
+[sde] generated 240/256
+[sde] generated 256/256
+[score] loading scorer: /e2e-data/evad-tech-vla/wanghan58/models/flowtext_scorers/gpt2-large-standard
+[summary] {
+  "type": "summary",
+  "checkpoint": "runs/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0119000.pt",
+  "step": 119000,
+  "decode": {
+    "decode_rule": "logistic_normal_resample_sde",
+    "steps": 128,
+    "model_t_mode": "const0.5",
+    "mean_mode": "anchor_semantic",
+    "endpoint_floor": 0.0,
+    "concentration_min": 1.0,
+    "concentration_max": 1024.0,
+    "endpoint_temp": 1.45,
+    "support_power": 1.0,
+    "semantic_power": 1.0,
+    "noise_init": "logistic_normal",
+    "noise_sigma": 3.0,
+    "noise_dirichlet_concentration": 1.0,
+    "sde_resample": "logistic_normal",
+    "logistic_normal_sigma_min": 0.18,
+    "logistic_normal_sigma_max": 3.0,
+    "logistic_normal_tau_min": 0.65,
+    "logistic_normal_tau_max": 1.0,
+    "final_from": "blend_0.5",
+    "n_samples": 256,
+    "seed": 20260522
+  },
+  "raw_genppl": {
+    "ppl": 35.481425020285705,
+    "nll_per_token": 3.5690093206789544,
+    "tokens": 32597,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "stripped_genppl": {
+    "ppl": 48.60573987367665,
+    "nll_per_token": 3.883741628329312,
+    "tokens": 27042,
+    "kept_samples": 256,
+    "total_samples": 256,
+    "empty_rate": 0.0,
+    "skipped_samples": 0
+  },
+  "diversity": {
+    "sample_entropy": 3.401137109533905,
+    "unique_tokens": 2040,
+    "token_count": 32768,
+    "distinct_1": 0.062255859375,
+    "distinct_2": 0.3155142716535433,
+    "top_token_mass": 0.198883056640625
+  }
+}
+[done] docs/lta_samples/metrics_20260522/lm1b_len128_lognormal_atoms_every1k_logistic_normal_sde_t1p45_steps128_n256/lta_lm1b_classic_c1024_len128_lognormalatoms_gbs512_4gpu_20260522/step_0119000/sde_steps128_samples256_scored.jsonl
+[watch-lognormal-sde] 2026-05-23_09:39:33 done step_0119000

LTA_openwebtext_dualt/mini_owt_logdirichlet/.deps/muon_optimizer-0.1.0.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

LTA_openwebtext_dualt/mini_owt_logdirichlet/.deps/muon_optimizer-0.1.0.dist-info/METADATA

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: muon-optimizer
+Version: 0.1.0
+Summary: Muon opimizer
+Home-page: https://github.com/KellerJordan/Muon
+Author: Keller Jordan
+Author-email: kjordan4077@gmail.com
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Image Recognition
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: summary
+
+# Muon: An optimizer for the hidden layers of neural networks
+
+This repo contains an implementation of the `Muon` optimizer originally described in [this thread](https://x.com/kellerjordan0/status/1842300916864844014) and [this writeup](https://kellerjordan.github.io/posts/muon/).
+
+## Installation
+
+```
+pip install git+https://github.com/KellerJordan/Muon
+```
+
+## Usage
+
+Muon is intended to optimize only the internal ≥2D parameters of a network.
+Embeddings, classifier heads, and internal gains/biases should be optimized using AdamW.
+
+```python
+# optimizer = torch.optim.AdamW(model.parameters(), lr=3e-4, betas=(0.90, 0.95), weight_decay=0.01)
+
+from muon import MuonWithAuxAdam
+# Find ≥2D parameters in the body of the network -- these should be optimized by Muon
+hidden_weights = [p for p in model.body.parameters() if p.ndim >= 2]
+# Find everything else -- these should be optimized by AdamW
+hidden_gains_biases = [p for p in model.body.parameters() if p.ndim < 2]
+exterior_weights = [*model.head.parameters(), *model.embed.parameters()])
+# Create the optimizer
+# Note: you can also use multiple groups of each type with different hparams if you want.
+muon_group = dict(params=hidden_weights, lr=0.02, weight_decay=0.01, use_muon=True)
+adam_group = dict(params=hidden_gains_biases+exterior_weights, lr=3e-4,
+                  betas=(0.9, 0.95), weight_decay=0.01, use_muon=False)
+optimizer = MuonWithAuxAdam([muon_group, adam_group])
+```
+
+You'll have to replace `model.body`, `model.head`, and `model.embed` with whatever subset is appropriate for your model.
+E.g., for a ConvNet, Muon should optimize all the convolutional filters except the first one, and AdamW should optimize everything else.
+
+## Example usage
+
+[Example use in the NanoGPT speedrun](https://github.com/KellerJordan/modded-nanogpt/blob/master/records/052525_MuonWithAuxAdamExample/b01550f9-03d8-4a9c-86fe-4ab434f1c5e0.txt#L470)
+
+[Example use in the CIFAR-10 speedrun](https://github.com/KellerJordan/cifar10-airbench/blob/28bff5f5b31e95aa45b5b20e1f48baf1ed98d5f6/airbench94_muon.py#L362)
+
+## Hyperparameter tuning
+
+Typically, the default values of momentum (0.95), nesterov (True), and ns_steps (5) work well. The only hyperparameter which must be tuned is the learning rate.
+It should have constant muP scaling, that is, as you scale up the model size, you shouldn't need to retune the learning rate.
+
+## Benchmarks
+
+For a comparison between AdamW, Shampoo, SOAP, and Muon for training a 124M-parameter transformer, see [here](https://github.com/KellerJordan/modded-nanogpt/tree/master/records/102924_Optimizers).
+
+## Accomplishments
+
+* [Lowered the record for training to 94% on CIFAR-10 from 3.3 A100-seconds to 2.6 A100-seconds](https://github.com/KellerJordan/cifar10-airbench)
+* [Used to train a transformer to GPT-2 (XL) performance in $175 of compute](https://x.com/kellerjordan0/status/1850995958697308307)
+* [Improved the training speed record for attaining GPT-2 (small) performance by a factor of 1.35x](https://x.com/kellerjordan0/status/1842300916864844014)
+* [Used by the Kimi.ai frontier lab for scaled LLM training](https://x.com/Kimi_Moonshot/status/1893379158472044623)
+
+## More learning resources and results about Muon
+
+* [Blog post on Muon by Jialin Su (the creator of RoPE)](https://kexue.fm/archives/10592)
+* [Blog post by Jeremy Bernstein on theoretical background of Muon](https://jeremybernste.in/writing/deriving-muon)
+* [Tech report by Kimi.ai on using Muon for scaled training](https://arxiv.org/abs/2502.16982v1)
+* [Why we chose Muon: Our chain of thought (by Jianlin Su at Kimi.ai)](https://x.com/Kimi_Moonshot/status/1897929976948965870)
+
+## Citation
+
+```bibtex
+@misc{jordan2024muon,
+  author       = {Keller Jordan and Yuchen Jin and Vlado Boza and You Jiacheng and
+                  Franz Cesista and Laker Newhouse and Jeremy Bernstein},
+  title        = {Muon: An optimizer for hidden layers in neural networks},
+  year         = {2024},
+  url          = {https://kellerjordan.github.io/posts/muon/}
+}
+```

LTA_openwebtext_dualt/mini_owt_logdirichlet/.deps/muon_optimizer-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+__pycache__/muon.cpython-312.pyc,,
+muon.py,sha256=xxTO43XZxzFcsN_g1yPPNX0xJNPGboRGwVDrsGhutIw,11624
+muon_optimizer-0.1.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+muon_optimizer-0.1.0.dist-info/METADATA,sha256=EIFIEtZFlCwJomkFsjDfIJol-VW5vsf6z7lMMDRBfa4,5099
+muon_optimizer-0.1.0.dist-info/RECORD,,
+muon_optimizer-0.1.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+muon_optimizer-0.1.0.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+muon_optimizer-0.1.0.dist-info/licenses/LICENSE,sha256=jI0XOY7M490_yxu3e10mikJ_glDzZKxlXJs431tpP2s,1070
+muon_optimizer-0.1.0.dist-info/top_level.txt,sha256=HwisYzr2fexMeo6u2BZYrJruXGC9r7Bw4v7p2GE1z0I,5

LTA_openwebtext_dualt/mini_owt_logdirichlet/.deps/muon_optimizer-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.8.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

LTA_openwebtext_dualt/mini_owt_logdirichlet/.deps/muon_optimizer-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Keller Jordan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LTA_openwebtext_dualt/mini_owt_logdirichlet/.deps/muon_optimizer-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+muon

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/bloom/configuration_bloom.py

@@ -0,0 +1,82 @@
+# Copyright 2022 the Big Science Workshop and HuggingFace Inc. team.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Bloom configuration"""
+
+from huggingface_hub.dataclasses import strict
+
+from ...configuration_utils import PreTrainedConfig
+from ...utils import auto_docstring
+
+
+@auto_docstring(checkpoint="bigscience/bloom")
+@strict
+class BloomConfig(PreTrainedConfig):
+    r"""
+    apply_residual_connection_post_layernorm (`bool`, *optional*, defaults to `False`):
+        If enabled, use the layer norm of the hidden states as the residual in the transformer blocks
+    slow_but_exact (`bool`, *optional*, defaults to `False`):
+        Experimental feature. Whether to use slow but exact implementation of the attention mechanism. While
+        merging the TP rank tensors, due to slicing operations the results may be slightly different between the
+        model trained on Megatron and our model. Please refer to [this
+        issue](https://github.com/pytorch/pytorch/issues/76232). A solution to obtain more accurate results is to
+        enable this feature. Enabling this will hurt the computational time of the inference. Will be probably
+        resolved in the future once the main model has been fine-tuned with TP_rank=1.
+
+    Example:
+
+    ```python
+    >>> from transformers import BloomConfig, BloomModel
+
+    >>> # Initializing a Bloom configuration
+    >>> configuration = BloomConfig()
+
+    >>> # Initializing a model (with random weights) from the configuration
+    >>> model = BloomModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "bloom"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    attribute_map = {
+        "num_hidden_layers": "n_layer",
+        "num_attention_heads": "n_head",
+    }
+
+    vocab_size: int = 250880
+    hidden_size: int = 64
+    n_layer: int = 2
+    n_head: int = 8
+    layer_norm_epsilon: float = 1e-5
+    initializer_range: float = 0.02
+    use_cache: bool = True
+    bos_token_id: int | None = 1
+    eos_token_id: int | list[int] | None = 2
+    pad_token_id: int | None = None
+    apply_residual_connection_post_layernorm: bool = False
+    hidden_dropout: float | int = 0.0
+    attention_dropout: float | int = 0.0
+    pretraining_tp: int = 1  # TP rank used when training with megatro
+    slow_but_exact: bool = False
+    tie_word_embeddings: bool = True
+
+    def __post_init__(self, **kwargs):
+        # Backward compatibility with n_embed kwarg
+        n_embed = kwargs.pop("n_embed", None)
+        self.hidden_size = self.hidden_size if n_embed is None else n_embed
+        super().__post_init__(**kwargs)
+
+
+__all__ = ["BloomConfig"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/got_ocr2/image_processing_pil_got_ocr2.py

@@ -0,0 +1,310 @@
+# Copyright 2025 HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image processor class for Got-OCR-2."""
+
+from functools import lru_cache
+
+import numpy as np
+
+from ...image_processing_backends import PilBackend
+from ...image_processing_utils import BatchFeature
+from ...image_transforms import to_channel_dimension_format
+from ...image_utils import (
+    OPENAI_CLIP_MEAN,
+    OPENAI_CLIP_STD,
+    ChannelDimension,
+    PILImageResampling,
+    SizeDict,
+    get_image_size,
+    infer_channel_dimension_format,
+)
+from ...processing_utils import ImagesKwargs, Unpack
+from ...utils import (
+    TensorType,
+    auto_docstring,
+)
+
+
+# Adapted from transformers.models.got_ocr2.image_processing_got_ocr2.GotOcr2ImageProcessorKwargs
+class GotOcr2ImageProcessorKwargs(ImagesKwargs, total=False):
+    r"""
+    crop_to_patches (`bool`, *optional*, defaults to `self.crop_to_patches`):
+        Whether to crop the image to patches. Can be overridden by the `crop_to_patches` parameter in the
+        `preprocess` method.
+    min_patches (`int`, *optional*, defaults to `self.min_patches`):
+        The minimum number of patches to be extracted from the image. Only has an effect if `crop_to_patches` is
+        set to `True`. Can be overridden by the `min_patches` parameter in the `preprocess` method.
+    max_patches (`int`, *optional*, defaults to `self.max_patches`):
+        The maximum number of patches to be extracted from the image. Only has an effect if `crop_to_patches` is
+        set to `True`. Can be overridden by the `max_patches` parameter in the `preprocess` method.
+    """
+
+    crop_to_patches: bool
+    min_patches: int
+    max_patches: int
+
+
+# Adapted from transformers.models.got_ocr2.image_processing_got_ocr2.get_all_supported_aspect_ratios
+@lru_cache(maxsize=10)
+def get_all_supported_aspect_ratios(min_image_tiles: int, max_image_tiles: int) -> list[tuple[int, int]]:
+    """
+    Computes all allowed aspect ratios for a given minimum and maximum number of input tiles.
+
+    This function calculates all possible arrangements of tiles that can be formed
+    within the constraint of the minimum and maximum number of tiles. Each arrangement is
+    represented by its aspect ratio (width/height) and the corresponding tile configuration.
+
+    Args:
+        min_image_tiles (`int`):
+            The minimum number of tiles allowed.
+        max_image_tiles (`int`):
+            The maximum number of tiles allowed.
+
+    Returns:
+        `list[tuple[int, int]]`: A list of tuples, each tuple representing a valid (width, height)
+        configuration in terms of number of tiles.
+
+    Example:
+        >>> get_all_supported_aspect_ratios(1, 4)
+        [(1, 1), (1, 2), (2, 1), (1, 3), (3, 1), (1, 4), (2, 2), (4, 1)]
+
+    """
+    aspect_ratios = []
+    for width in range(1, max_image_tiles + 1):
+        for height in range(1, max_image_tiles + 1):
+            if width * height <= max_image_tiles and width * height >= min_image_tiles:
+                aspect_ratios.append((width, height))
+
+    aspect_ratios = sorted(aspect_ratios, key=lambda x: x[0] * x[1])
+
+    return aspect_ratios
+
+
+# Adapted from transformers.models.got_ocr2.image_processing_got_ocr2.get_optimal_tiled_canvas
+@lru_cache(maxsize=100)
+def get_optimal_tiled_canvas(
+    original_image_size: tuple[int, int],
+    target_tile_size: tuple[int, int],
+    min_image_tiles: int,
+    max_image_tiles: int,
+) -> tuple[int, int]:
+    """
+    Given a minimum and maximum number of tiles, find the canvas with the closest aspect ratio to the
+    original image aspect ratio.
+    In case of tie-breaking condition when two canvases have the same aspect ratio difference, we favor the canvas with
+    more tiles, until the area covered by the tiles is more than twice the target area, in order to avoid unnecessarily
+    excessive tiling.
+    """
+    possible_tile_arrangements = get_all_supported_aspect_ratios(min_image_tiles, max_image_tiles)
+
+    original_height, original_width = original_image_size
+    target_tile_height, target_tile_width = target_tile_size
+    aspect_ratio = original_width / original_height
+    area = original_width * original_height
+
+    # find the grid with the best aspect ratio
+    best_ratio_diff = float("inf")
+    best_grid = (1, 1)
+    for grid in possible_tile_arrangements:
+        grid_aspect_ratio = grid[0] / grid[1]
+        ratio_diff = abs(aspect_ratio - grid_aspect_ratio)
+        if ratio_diff < best_ratio_diff:
+            best_ratio_diff = ratio_diff
+            best_grid = grid
+        elif ratio_diff == best_ratio_diff:
+            # if the aspect ratio difference is the same, we favor the grid with more patches
+            # until the area covered by the patches is more than twice the original image area
+            if area > 0.5 * target_tile_height * target_tile_width * grid[0] * grid[1]:
+                best_grid = grid
+
+    return best_grid
+
+
+@auto_docstring
+class GotOcr2ImageProcessorPil(PilBackend):
+    valid_kwargs = GotOcr2ImageProcessorKwargs
+    resample = PILImageResampling.BICUBIC
+    image_mean = OPENAI_CLIP_MEAN
+    image_std = OPENAI_CLIP_STD
+    size = {"height": 384, "width": 384}
+    do_resize = True
+    do_rescale = True
+    do_normalize = True
+    do_convert_rgb = True
+    crop_to_patches = False
+    min_patches = 1
+    max_patches = 12
+
+    def __init__(self, **kwargs: Unpack[GotOcr2ImageProcessorKwargs]):
+        super().__init__(**kwargs)
+
+    def crop_image_to_patches(
+        self,
+        image: np.ndarray,
+        min_patches: int,
+        max_patches: int,
+        use_thumbnail: bool = True,
+        patch_size: SizeDict | None = None,
+        resample: "PILImageResampling | None" = None,
+    ):
+        """
+        Crop the image to patches and return a list of cropped images.
+        The number of patches and their grid arrangement are determined by the original image size,
+        the target patch size and the minimum and maximum number of patches.
+        The aspect ratio of the patches grid is chosen to be the closest to the original image aspect ratio.
+
+        Args:
+            image (`np.ndarray`):
+                The image to be cropped.
+            min_patches (`int`):
+                The minimum number of patches to be extracted from the image.
+            max_patches (`int`):
+                The maximum number of patches to be extracted from the image.
+            use_thumbnail (`bool`, *optional*, defaults to `True`):
+                Whether to add a thumbnail image to the list of cropped patches.
+            patch_size (`SizeDict`, *optional*):
+                The size of the output patches.
+            resample (`PILImageResampling | int | None`, *optional*):
+                Resampling filter to use when resizing.
+        """
+        # Ensure image is in CHW format for processing
+        input_data_format = infer_channel_dimension_format(image)
+        image = to_channel_dimension_format(image, ChannelDimension.FIRST, input_data_format)
+
+        patch_size_height, patch_size_width = patch_size.height, patch_size.width
+        original_height, original_width = get_image_size(image, channel_dim=ChannelDimension.FIRST)
+        # find the closest aspect ratio to the target
+        num_columns, num_rows = get_optimal_tiled_canvas(
+            (original_height, original_width), (patch_size_height, patch_size_width), min_patches, max_patches
+        )
+
+        # calculate the target width and height
+        target_width = patch_size_width * num_columns
+        target_height = patch_size_height * num_rows
+        num_blocks = num_columns * num_rows
+
+        # resize the image so that each patch is of patch_size
+        resized_image = self.resize(image, SizeDict(height=target_height, width=target_width), resample=resample)
+        # split the image into patches
+        processed_images = []
+        for i in range(num_blocks):
+            column = i % num_columns
+            row = i // num_columns
+            box = (
+                column * patch_size_width,
+                row * patch_size_height,
+                (column + 1) * patch_size_width,
+                (row + 1) * patch_size_height,
+            )
+            # split the image (images are CHW format)
+            patch_image = resized_image[..., box[1] : box[3], box[0] : box[2]]
+            # Convert back to original format
+            patch_image = to_channel_dimension_format(patch_image, input_data_format, ChannelDimension.FIRST)
+            processed_images.append(patch_image)
+
+        if use_thumbnail and len(processed_images) != 1:
+            thumbnail_img = self.resize(image, patch_size, resample=resample)
+            thumbnail_img = to_channel_dimension_format(thumbnail_img, input_data_format, ChannelDimension.FIRST)
+            processed_images.append(thumbnail_img)
+
+        return processed_images
+
+    def _preprocess(
+        self,
+        images: list[np.ndarray],
+        do_resize: bool,
+        size: SizeDict,
+        resample: "PILImageResampling | None",
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: float | list[float] | None,
+        image_std: float | list[float] | None,
+        return_tensors: str | TensorType | None,
+        crop_to_patches: bool = False,
+        min_patches: int = 1,
+        max_patches: int = 12,
+        **kwargs,
+    ) -> BatchFeature:
+        num_patches = []
+        processed_images = []
+
+        for image in images:
+            if crop_to_patches and max_patches > 1:
+                patches = self.crop_image_to_patches(
+                    image,
+                    min_patches,
+                    max_patches,
+                    patch_size=size,
+                    resample=resample,
+                )
+                num_patches.append(len(patches))
+                # Normalize and rescale patches
+                for patch in patches:
+                    if do_rescale:
+                        patch = self.rescale(patch, rescale_factor)
+                    if do_normalize:
+                        patch = self.normalize(patch, image_mean, image_std)
+                    processed_images.append(patch)
+            else:
+                num_patches.append(1)
+                if do_resize:
+                    image = self.resize(image, size, resample)
+                if do_rescale:
+                    image = self.rescale(image, rescale_factor)
+                if do_normalize:
+                    image = self.normalize(image, image_mean, image_std)
+                processed_images.append(image)
+
+        return BatchFeature(
+            data={"pixel_values": processed_images, "num_patches": num_patches}, tensor_type=return_tensors
+        )
+
+    def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None):
+        """
+        A utility that returns number patches for a given image size.
+
+        Args:
+            height (`int`):
+                Height of the input image.
+            width (`int`):
+                Width of the input image.
+            images_kwargs (`dict`, *optional*)
+                Any kwargs to override defaults of the image processor.
+        Returns:
+            `int`: Number of patches per image.
+        """
+        min_patches = images_kwargs.get("min_patches", self.min_patches) if images_kwargs else self.min_patches
+        max_patches = images_kwargs.get("max_patches", self.max_patches) if images_kwargs else self.max_patches
+        patch_size = images_kwargs.get("patch_size", self.size) if images_kwargs else self.size
+        crop_to_patches = (
+            images_kwargs.get("crop_to_patches", self.crop_to_patches) if images_kwargs else self.crop_to_patches
+        )
+
+        num_patches = 1
+        if crop_to_patches and max_patches > 1:
+            if isinstance(patch_size, dict):
+                patch_height, patch_width = patch_size["height"], patch_size["width"]
+            else:
+                patch_height, patch_width = patch_size.height, patch_size.width
+            num_columns, num_rows = get_optimal_tiled_canvas(
+                (height, width), (patch_height, patch_width), min_patches, max_patches
+            )
+            if num_columns * num_rows > 1:
+                num_patches += num_columns * num_rows
+
+        return num_patches
+
+
+__all__ = ["GotOcr2ImageProcessorPil"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/got_ocr2/modular_got_ocr2.py

@@ -0,0 +1,388 @@
+# Copyright 2024 HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import torch
+import torch.nn as nn
+from huggingface_hub.dataclasses import strict
+
+from ... import initialization as init
+from ...cache_utils import Cache
+from ...configuration_utils import PreTrainedConfig
+from ...modeling_outputs import BaseModelOutputWithPooling
+from ...modeling_utils import PreTrainedModel
+from ...processing_utils import Unpack
+from ...utils import TransformersKwargs, auto_docstring, can_return_tuple, logging
+from ..auto import CONFIG_MAPPING, AutoConfig
+from ..llava.modeling_llava import (
+    LlavaCausalLMOutputWithPast,
+    LlavaForConditionalGeneration,
+    LlavaModel,
+    LlavaModelOutputWithPast,
+    LlavaPreTrainedModel,
+)
+from ..sam.modeling_sam import (
+    SamMLPBlock,
+    SamPreTrainedModel,
+    SamVisionAttention,
+    SamVisionEncoder,
+    SamVisionLayer,
+)
+
+
+logger = logging.get_logger(__name__)
+
+
+@auto_docstring(checkpoint="facebook/sam-vit-huge")
+@strict
+class GotOcr2VisionConfig(PreTrainedConfig):
+    r"""
+    output_channels (`int`, *optional*, defaults to 256):
+        Dimensionality of the output channels in the Patch Encoder.
+    use_abs_pos (`bool`, *optional*, defaults to `True`):
+        Whether to use absolute position embedding.
+    use_rel_pos (`bool`, *optional*, defaults to `True`):
+        Whether to use relative position embedding.
+    window_size (`int`, *optional*, defaults to 14):
+        Window size for relative position.
+    global_attn_indexes (`list[int]`, *optional*, defaults to `[2, 5, 8, 11]`):
+        The indexes of the global attention layers.
+    mlp_dim (`int`, *optional*, defaults to 3072):
+        The dimensionality of the MLP layer in the Transformer encoder.
+    """
+
+    base_config_key = "vision_config"
+    hidden_size: int = 768
+    output_channels: int = 256
+    num_hidden_layers: int = 12
+    num_attention_heads: int = 12
+    num_channels: int = 3
+    image_size: int | list[int] | tuple[int, int] = 1024
+    patch_size: int | list[int] | tuple[int, int] = 16
+    hidden_act: str = "gelu"
+    layer_norm_eps: float = 1e-06
+    attention_dropout: float | int = 0.0
+    initializer_range: float = 1e-10
+    qkv_bias: bool = True
+    use_abs_pos: bool = True
+    use_rel_pos: bool = True
+    window_size: int = 14
+    global_attn_indexes: list[int] | tuple[int, ...] = (2, 5, 8, 11)
+    mlp_dim: int = 3072
+
+
+@auto_docstring(checkpoint="facebook/sam-vit-huge")
+@strict
+class GotOcr2Config(PreTrainedConfig):
+    r"""
+    Example:
+
+    ```python
+    >>> from transformers import GotOcr2ForConditionalGeneration, GotOcr2Config
+
+    >>> # Initializing a GotOcr2 style configuration
+    >>> configuration = GotOcr2Config()
+
+    >>> # Initializing a model from the Qwen2-VL-7B style configuration
+    >>> model = GotOcr2ForConditionalGeneration(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "got_ocr2"
+    attribute_map = {
+        "image_token_id": "image_token_index",
+    }
+    sub_configs = {"text_config": AutoConfig, "vision_config": GotOcr2VisionConfig}
+
+    vision_config: dict | PreTrainedConfig | None = None
+    text_config: dict | PreTrainedConfig | None = None
+    image_token_index: int = 151859
+    image_seq_length: int = 576
+    tie_word_embeddings: bool = True
+
+    def __post_init__(self, **kwargs):
+        if self.vision_config is None:
+            self.vision_config = GotOcr2VisionConfig()
+        elif isinstance(self.vision_config, dict):
+            self.vision_config = GotOcr2VisionConfig(**self.vision_config)
+
+        if isinstance(self.text_config, dict):
+            self.text_config["model_type"] = self.text_config.get("model_type", "qwen2")
+            self.text_config = CONFIG_MAPPING[self.text_config["model_type"]](**self.text_config)
+        elif self.text_config is None:
+            self.text_config = CONFIG_MAPPING["qwen2"](
+                vocab_size=151860,
+                hidden_size=1024,
+                intermediate_size=2816,
+                num_hidden_layers=24,
+                num_attention_heads=16,
+                num_key_value_heads=16,
+                hidden_act="silu",
+                max_position_embeddings=32768,
+                initializer_range=0.02,
+                rms_norm_eps=1e-6,
+                use_cache=True,
+                tie_word_embeddings=self.tie_word_embeddings,
+                rope_theta=1000000.0,
+                rope_parameters=None,
+                use_sliding_window=False,
+                sliding_window=4096,
+                max_window_layers=21,
+                attention_dropout=0.0,
+            )
+
+        super().__post_init__(**kwargs)
+
+
+class GotOcr2MLPBlock(SamMLPBlock):
+    pass
+
+
+class GotOcr2VisionAttention(SamVisionAttention):
+    pass
+
+
+class GotOcr2VisionLayer(SamVisionLayer):
+    def __init__(self, config, window_size):
+        super().__init__(config, window_size)
+        self.layer_norm1 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.attn = GotOcr2VisionAttention(config, window_size)
+        self.layer_norm2 = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.mlp = GotOcr2MLPBlock(config)
+        self.window_size = window_size
+
+
+class GotOcr2PreTrainedModel(SamPreTrainedModel):
+    input_modalities = ("image", "text")
+
+
+class GotOcr2VisionEncoder(SamVisionEncoder, GotOcr2PreTrainedModel):
+    input_modalities = ("image",)
+
+
+class GotOcr2MultiModalProjector(nn.Module):
+    def __init__(self, config: GotOcr2Config):
+        super().__init__()
+        vision_output_channels = config.vision_config.output_channels
+        language_hidden_size = config.text_config.hidden_size
+        self.conv_upsampler1 = nn.Conv2d(
+            vision_output_channels, vision_output_channels * 2, kernel_size=3, stride=2, padding=1, bias=False
+        )
+        self.conv_upsampler2 = nn.Conv2d(
+            vision_output_channels * 2, language_hidden_size, kernel_size=3, stride=2, padding=1, bias=False
+        )
+        self.multimodal_projector = nn.Linear(language_hidden_size, language_hidden_size)
+
+    def forward(self, vision_embeddings: torch.Tensor) -> torch.Tensor:
+        hidden_state = self.conv_upsampler1(vision_embeddings)
+        hidden_state = self.conv_upsampler2(hidden_state)
+        hidden_state = hidden_state.flatten(2).permute(0, 2, 1)
+        hidden_state = self.multimodal_projector(hidden_state)
+        return hidden_state
+
+
+class GotOcr2CausalLMOutputWithPast(LlavaCausalLMOutputWithPast):
+    pass
+
+
+class GotOcr2ModelOutputWithPast(LlavaModelOutputWithPast):
+    pass
+
+
+class GotOcr2PreTrainedModel(LlavaPreTrainedModel):
+    _supports_flash_attn = False
+    _supports_sdpa = False
+    _supports_flex_attn = False
+
+    @torch.no_grad()
+    def _init_weights(self, module):
+        PreTrainedModel._init_weights(self, module)
+        if isinstance(module, GotOcr2VisionAttention):
+            if module.use_rel_pos:
+                init.zeros_(module.rel_pos_h)
+                init.zeros_(module.rel_pos_w)
+        elif isinstance(module, GotOcr2VisionEncoder):
+            if module.pos_embed is not None:
+                init.zeros_(module.pos_embed)
+
+
+class GotOcr2Model(LlavaModel):
+    def __init__(self, config: GotOcr2Config):
+        super().__init__(config)
+        self.vision_tower = GotOcr2VisionEncoder(config.vision_config)
+
+    @can_return_tuple
+    @auto_docstring(
+        custom_intro="Obtains image last hidden states from the vision tower and apply multimodal projection."
+    )
+    def get_image_features(
+        self,
+        pixel_values: torch.FloatTensor,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | BaseModelOutputWithPooling:
+        image_outputs = self.vision_tower(pixel_values, return_dict=True, **kwargs)
+        last_hidden_state = image_outputs.last_hidden_state
+        image_outputs.pooler_output = self.multi_modal_projector(last_hidden_state)
+
+        return image_outputs
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        pixel_values: torch.FloatTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        use_cache: bool | None = None,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | GotOcr2ModelOutputWithPast:
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        if pixel_values is not None:
+            image_features = self.get_image_features(
+                pixel_values=pixel_values.to(inputs_embeds.dtype), return_dict=True
+            ).pooler_output
+            image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype)
+            special_image_mask = self.get_placeholder_mask(
+                input_ids, inputs_embeds=inputs_embeds, image_features=image_features
+            )
+            inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features)
+
+        outputs = self.language_model(
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            return_dict=True,
+            **kwargs,
+        )
+
+        return GotOcr2ModelOutputWithPast(
+            last_hidden_state=outputs.last_hidden_state,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            image_hidden_states=image_features if pixel_values is not None else None,
+        )
+
+
+class GotOcr2ForConditionalGeneration(LlavaForConditionalGeneration):
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        pixel_values: torch.FloatTensor | None = None,
+        attention_mask: torch.Tensor | None = None,
+        position_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        inputs_embeds: torch.FloatTensor | None = None,
+        labels: torch.LongTensor | None = None,
+        use_cache: bool | None = None,
+        logits_to_keep: int | torch.Tensor = 0,
+        **kwargs: Unpack[TransformersKwargs],
+    ) -> tuple | GotOcr2CausalLMOutputWithPast:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        Example:
+
+        ```python
+        >>> from PIL import Image
+        >>> import httpx
+        >>> from io import BytesIO
+        >>> from transformers import AutoProcessor, GotOcr2ForConditionalGeneration, TextStreamer
+
+        >>> model = GotOcr2ForConditionalGeneration.from_pretrained("stepfun-ai/GOT-OCR-2.0-hf").to("cuda")
+        >>> processor = AutoProcessor.from_pretrained("stepfun-ai/GOT-OCR-2.0-hf")
+
+        >>> url = "https://huggingface.co/datasets/hf-internal-testing/fixtures_got_ocr/resolve/main/multi_box.png"
+        >>> with httpx.stream("GET", url) as response:
+        ...     image = Image.open(BytesIO(response.read()))
+
+        >>> inputs = processor(image, return_tensors="pt", color="green").to("cuda")
+
+        >>> # Generate
+        >>> streamer = TextStreamer(processor.tokenizer, skip_prompt=True, skip_special_tokens=True)
+        >>> generate_ids = model.generate(
+        ...     **inputs,
+        ...     do_sample=False,
+        ...     tokenizer = processor.tokenizer,
+        ...     stop_strings='<|im_end|>',
+        ...     streamer=streamer,
+        ...     max_new_tokens=4096,
+        ... )
+        "You should keep in mind what features from the module should be used, especially
+        when you're planning to sell a template."
+        ```"""
+        outputs = self.model(
+            input_ids=input_ids,
+            pixel_values=pixel_values,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            return_dict=True,
+            logits_to_keep=logits_to_keep,
+            **kwargs,
+        )
+
+        hidden_states = outputs[0]
+        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(
+                logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **kwargs
+            )
+
+        return GotOcr2CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            image_hidden_states=outputs.image_hidden_states,
+        )
+
+    @auto_docstring
+    def get_image_features(
+        self, pixel_values: torch.FloatTensor, **kwargs: Unpack[TransformersKwargs]
+    ) -> tuple | BaseModelOutputWithPooling:
+        return self.model.get_image_features(pixel_values=pixel_values, **kwargs)
+
+
+__all__ = [
+    "GotOcr2VisionConfig",
+    "GotOcr2Config",
+    "GotOcr2PreTrainedModel",
+    "GotOcr2Model",
+    "GotOcr2ForConditionalGeneration",
+]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/got_ocr2/processing_got_ocr2.py

@@ -0,0 +1,231 @@
+# Copyright 2024 HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import numpy as np
+
+from ...image_processing_utils import BatchFeature
+from ...image_utils import ImageInput
+from ...processing_utils import ImagesKwargs, ProcessingKwargs, ProcessorMixin, TextKwargs, Unpack
+from ...tokenization_utils_base import PreTokenizedInput, TextInput
+from ...utils import auto_docstring, is_vision_available, logging
+
+
+if is_vision_available():
+    from ...image_utils import load_images
+
+logger = logging.get_logger(__name__)
+
+
+class GotOcr2TextKwargs(TextKwargs, total=False):
+    """
+    format (`bool`, *optional*, defaults to `False`):
+        Whether to request formatted output from the OCR model. When enabled, the model is instructed to return
+        structured and formatted text output rather than raw OCR results.
+    """
+
+    format: bool | None
+
+
+class GotOcr2ImagesKwargs(ImagesKwargs, total=False):
+    """
+    crop_to_patches (`bool`, *optional*, defaults to `False`):
+        Whether to crop images into patches before processing. When enabled, large images are divided into
+        smaller patches for more efficient OCR processing.
+    min_patches (`int`, *optional*, defaults to `1`):
+        Minimum number of patches to generate when cropping images. This ensures that even small images are
+        processed with at least this many patches.
+    max_patches (`int`, *optional*, defaults to `12`):
+        Maximum number of patches to generate when cropping images. Large images will be divided into at most
+        this many patches to control computational complexity.
+    box (`list`, `tuple[float, float]`, or `tuple[float, float, float, float]`, *optional*):
+        Bounding box coordinates for OCR region of interest. Can be specified as a single box `[x1, y1, x2, y2]`
+        or a list of boxes. Coordinates are normalized to the range [0, 1000] based on the image dimensions.
+        If not provided, OCR is performed on the entire image.
+    color (`str`, *optional*):
+        Color filter specification for OCR. When provided, the OCR query is prefixed with the color information
+        to focus on text of a specific color (e.g., "red", "blue").
+    num_image_tokens (`int`, *optional*, defaults to `256`):
+        Number of image tokens (patches) to use per image. This controls the resolution of the image representation
+        passed to the model. Higher values provide more detail but increase computational cost.
+    multi_page (`bool`, *optional*, defaults to `False`):
+        Whether the input consists of multi-page documents. When enabled, images can be provided as nested lists
+        where each inner list represents a page, and OCR is performed across all pages with appropriate handling
+        of page boundaries.
+    """
+
+    crop_to_patches: bool
+    min_patches: int
+    max_patches: int
+    box: list | tuple[float, float] | tuple[float, float, float, float] | None
+    color: str | None
+    num_image_tokens: int
+    multi_page: bool
+
+
+class GotOcr2ProcessorKwargs(ProcessingKwargs, total=False):
+    text_kwargs: GotOcr2TextKwargs
+    images_kwargs: GotOcr2ImagesKwargs
+    _defaults = {
+        "text_kwargs": {
+            "padding": False,
+            "format": False,
+        },
+        "images_kwargs": {
+            "num_image_tokens": 256,
+            "multi_page": False,
+            "crop_to_patches": False,
+            "min_patches": 1,
+            "max_patches": 12,
+        },
+    }
+
+
+def preprocess_box_annotation(box: list | tuple, image_size: tuple[int, int]) -> list:
+    """
+    Convert box annotation to the format [x1, y1, x2, y2] in the range [0, 1000].
+    """
+    width, height = image_size
+    if len(box) == 4:
+        box[0] = int(box[0] / width * 1000)
+        box[1] = int(box[1] / height * 1000)
+        box[2] = int(box[2] / width * 1000)
+        box[3] = int(box[3] / height * 1000)
+    else:
+        raise ValueError("Box must be a list or tuple of lists in the form [x1, y1, x2, y2].")
+
+    return list(box)
+
+
+@auto_docstring
+class GotOcr2Processor(ProcessorMixin):
+    def __init__(self, image_processor=None, tokenizer=None, chat_template=None, **kwargs):
+        super().__init__(image_processor, tokenizer, chat_template=chat_template)
+
+        self.message_start_token = "<|im_start|>"
+        self.message_end_token = "<|im_end|>"
+        self.img_start_token = "<img>"
+        self.img_end_token = "</img>"
+        self.img_pad_token = "<imgpad>"
+        self.image_token = "<imgpad>"  # keep the above for BC, but we need to call it `image_token`
+        self.image_token_id = tokenizer.convert_tokens_to_ids(self.image_token)
+        self.system_query = "system\nYou should follow the instructions carefully and explain your answers in detail."
+
+    def _make_list_of_inputs(self, images, text, box, color, multi_page):
+        if not isinstance(images, (list, tuple)):
+            images = [images]
+            if multi_page:
+                logger.warning("Multi-page inference is enabled but only one image is passed.")
+                images = [images]
+        elif isinstance(images[0], (list, tuple)) and not multi_page:
+            raise ValueError("Nested images are only supported with `multi_page` set to `True`.")
+        elif not isinstance(images[0], (list, tuple)) and multi_page:
+            images = [images]
+
+        if isinstance(text, str):
+            text = [text]
+
+        if not isinstance(box[0], (list, tuple)):
+            # Use the same box for all images
+            box = [box for _ in range(len(images))]
+        if not isinstance(color, (list, tuple)):
+            color = [color for _ in range(len(images))]
+
+        return images, text, box, color
+
+    @auto_docstring
+    def __call__(
+        self,
+        images: ImageInput | None = None,
+        text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
+        **kwargs: Unpack[GotOcr2ProcessorKwargs],
+    ) -> BatchFeature:
+        r"""
+        Returns:
+            [`BatchFeature`]: A [`BatchFeature`] with the following fields:
+
+            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
+            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
+              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
+              `None`).
+            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
+        """
+
+        output_kwargs = self._merge_kwargs(
+            GotOcr2ProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+        format_output = output_kwargs["text_kwargs"].pop("format")
+        num_image_tokens = output_kwargs["images_kwargs"].pop("num_image_tokens")
+        box = output_kwargs["images_kwargs"].pop("box", [None])
+        color = output_kwargs["images_kwargs"].pop("color", None)
+        multi_page = output_kwargs["images_kwargs"].pop("multi_page")
+
+        crop_to_patches = output_kwargs["images_kwargs"].get("crop_to_patches")
+        images, text, box, color = self._make_list_of_inputs(images, text, box, color, multi_page)
+        if multi_page:
+            # save the number of pages per batch
+            num_pages_per_batch = [len(image_group) for image_group in images]
+            # flatten the list of images
+            images = [image for image_group in images for image in image_group]
+        else:
+            num_pages_per_batch = [1 for _ in range(len(images))]
+        # Load images as we need to know the image size
+        images = load_images(images)
+        image_sizes = [image.size for image in images]
+        image_inputs = self.image_processor(images=images, **output_kwargs["images_kwargs"])
+        num_patches_array = image_inputs.pop("num_patches")
+        if text is None:
+            text = []
+            patch_indices = np.cumsum(num_pages_per_batch)
+            for index, (num_pages, box_single, color_single) in enumerate(zip(num_pages_per_batch, box, color)):
+                current_patch_index = patch_indices[index - 1] if index > 0 else 0
+                num_patches = sum(num_patches_array[current_patch_index : current_patch_index + num_pages])
+                if box_single[0] is not None:
+                    box_single = preprocess_box_annotation(box_single, image_sizes[index])
+                query = (
+                    f"{f'[{color_single}] ' if color_single is not None else ''}"
+                    f"{str(box_single) if box_single[0] is not None else ''} "
+                    "OCR"
+                    f"{' with format' if format_output else ''}"
+                    f"{' across multi pages' if multi_page else ''}"
+                    f"{' upon the patch reference' if crop_to_patches else ''}"
+                    ": "
+                )
+                prompt = (
+                    self.message_start_token
+                    + self.system_query
+                    + self.message_end_token
+                    + self.message_start_token
+                    + "user\n"
+                    + self.img_start_token
+                    + self.img_pad_token * num_image_tokens * num_patches
+                    + self.img_end_token
+                    + "\n"
+                    + query
+                    + self.message_end_token
+                    + self.message_start_token
+                    + "assistant\n"
+                )
+                text.append(prompt)
+
+        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
+        text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
+        self._check_special_mm_tokens(text, text_inputs, modalities=["image"])
+
+        return BatchFeature(data={**text_inputs, **image_inputs}, tensor_type=return_tensors)
+
+
+__all__ = ["GotOcr2Processor"]

LTA_openwebtext_dualt/mini_owt_logdirichlet/.venv_qwen35_uv/lib/python3.12/site-packages/transformers/models/moonshine/__init__.py

@@ -0,0 +1,27 @@
+# Copyright 2025 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import TYPE_CHECKING
+
+from ...utils import _LazyModule
+from ...utils.import_utils import define_import_structure
+
+
+if TYPE_CHECKING:
+    from .configuration_moonshine import *
+    from .modeling_moonshine import *
+else:
+    import sys
+
+    _file = globals()["__file__"]
+    sys.modules[__name__] = _LazyModule(__name__, _file, define_import_structure(_file), module_spec=__spec__)

LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/dirichlet_numeric_sim_old_vs_log_v32100_s128.csv

@@ -0,0 +1,13 @@
+vocab=32100 samples=128 C=1.0->1024.0 device=cuda target=1
+t,c,alpha_target,alpha_non_target,old_raw_g_below_1e-8,old_p_zero_frac,log_g_nonfinite_frac,log_p_zero_frac,old_target_mean,log_target_mean,old_entropy_mean,log_entropy_mean,old_top_target_frac,log_top_target_frac,old_emb_l2_to_mean,log_emb_l2_to_mean,mean_emb_l2_old_vs_log
+0.000,1,3.11526e-05,3.11526e-05,0.999433,0.000000,0.000000,0.996774,0.000000,0.000000,1.051059,0.977972,0.000000,0.000000,0.686867,0.700737,0.989373
+0.001,1.00696,0.00103829,3.1338e-05,0.999429,0.000000,0.000000,0.996751,0.000947,0.000017,1.056590,0.981897,0.000000,0.000000,0.685100,0.699690,0.987289
+0.010,1.07177,0.0107508,3.30547e-05,0.999395,0.000000,0.000000,0.996564,0.007713,0.003211,1.087266,1.023427,0.000000,0.000000,0.673804,0.688418,0.971353
+0.050,1.41421,0.0707525,4.18537e-05,0.999210,0.000000,0.000000,0.995653,0.025586,0.048411,1.254443,1.211471,0.007812,0.031250,0.627183,0.633667,0.899046
+0.100,2,0.200056,5.60748e-05,0.998959,0.000000,0.000000,0.994213,0.069044,0.099894,1.499430,1.436666,0.039062,0.125000,0.561897,0.569334,0.806829
+0.250,5.65685,1.41435,0.000132169,0.997642,0.000000,0.000000,0.986620,0.252017,0.245821,2.090487,2.048243,0.507812,0.445312,0.365040,0.378998,0.491217
+0.500,32,16.0005,0.000498442,0.991163,0.000000,0.000000,0.951413,0.498356,0.492434,2.366313,2.380998,1.000000,1.000000,0.148980,0.152542,0.188174
+0.750,181.019,135.766,0.00140981,0.975175,0.000000,0.000000,0.870699,0.750778,0.746908,1.651484,1.670976,1.000000,1.000000,0.048614,0.048720,0.063642
+0.900,512,460.802,0.00159502,0.971988,0.000000,0.000000,0.856373,0.900468,0.898712,0.771760,0.782816,1.000000,1.000000,0.018904,0.018869,0.025504
+0.990,955.426,945.872,0.00029764,0.994694,0.000000,0.000000,0.971638,0.989913,0.989733,0.085018,0.085953,1.000000,1.000000,0.004177,0.004542,0.005748
+0.999,1016.93,1015.91,3.168e-05,0.999395,0.000000,0.000000,0.996928,0.998994,0.998903,0.008652,0.009190,1.000000,1.000000,0.001085,0.001234,0.001510

LTA_openwebtext_dualt/mini_owt_logdirichlet/runs/dirichlet_numeric_sim_v32100_s128.csv

@@ -0,0 +1,13 @@
+vocab=32100 samples=128 C=1.0->1024.0 device=cuda target=1
+t,c,alpha_target,alpha_non_target,native_g_zero_frac,native_p_zero_frac,log_g_nonfinite_frac,log_p_zero_frac,native_target_mean,log_target_mean,native_entropy_mean,log_entropy_mean,native_top_target_frac,log_top_target_frac,native_emb_l2_to_mean,log_emb_l2_to_mean,mean_emb_l2_native_vs_log
+0.000,1,3.11526e-05,3.11526e-05,0.000000,0.000000,0.000000,0.996774,0.000000,0.000000,1.007299,0.977972,0.000000,0.000000,0.689184,0.700737,0.991104
+0.001,1.00696,0.00103829,3.1338e-05,0.000000,0.000000,0.000000,0.996751,0.000947,0.000017,1.013896,0.981897,0.000000,0.000000,0.687334,0.699690,0.988957
+0.010,1.07177,0.0107508,3.30547e-05,0.000000,0.000000,0.000000,0.996564,0.007714,0.003211,1.052980,1.023427,0.000000,0.000000,0.675541,0.688418,0.972662
+0.050,1.41421,0.0707525,4.18537e-05,0.000000,0.000000,0.000000,0.995653,0.025615,0.048411,1.240227,1.211471,0.007812,0.031250,0.627777,0.633667,0.899489
+0.100,2,0.200056,5.60748e-05,0.000000,0.000000,0.000000,0.994213,0.069085,0.099894,1.493398,1.436666,0.039062,0.125000,0.562099,0.569334,0.806974
+0.250,5.65685,1.41435,0.000132169,0.000000,0.000000,0.000000,0.986620,0.252034,0.245821,2.089214,2.048243,0.507812,0.445312,0.365058,0.378998,0.491233
+0.500,32,16.0005,0.000498442,0.000000,0.000000,0.000000,0.951413,0.498361,0.492434,2.366103,2.380998,1.000000,1.000000,0.148984,0.152542,0.188176
+0.750,181.019,135.766,0.00140981,0.000000,0.000000,0.000000,0.870699,0.750780,0.746908,1.651444,1.670976,1.000000,1.000000,0.048614,0.048720,0.063642
+0.900,512,460.802,0.00159502,0.000000,0.000000,0.000000,0.856373,0.900468,0.898712,0.771744,0.782816,1.000000,1.000000,0.018904,0.018869,0.025504
+0.990,955.426,945.872,0.00029764,0.000000,0.000000,0.000000,0.971638,0.989913,0.989733,0.085010,0.085953,1.000000,1.000000,0.004177,0.004542,0.005748
+0.999,1016.93,1015.91,3.168e-05,0.000000,0.000000,0.000000,0.996928,0.998994,0.998903,0.008644,0.009190,1.000000,1.000000,0.001085,0.001234,0.001510