diff --git a/README.md b/README.md index b33756349b760a91b7889e97c45b48f942df834f..d436d2696330e2eff7fbebd42348475371228543 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,9 @@ --- license: apache-2.0 -tags: [masked-diffusion, discrete-diffusion, length-generalization, positional-encoding, from-scratch] +tags: [masked-diffusion, parallel-decoding, text8, qm9, from-scratch] --- -# Length Generalization in Masked Diffusion Language Models - -From-scratch masked diffusion language models (MDLM-style absorbing state) and -matched autoregressive baselines, trained on arithmetic and algorithmic tasks and -evaluated far beyond the lengths seen in training. - -**No pretrained weights or tokenizers are used anywhere.** +# Two Failure Modes of Parallel Decoding in Masked Diffusion Code: https://github.com/GIND123/MSML612 @@ -40,19 +34,28 @@ Answers are unique but only ~10 characters with shallow, local dependencies. One | english | 95.3 | 95.2 | -0.1 | | finnish | 50.9 | 55.8 | +4.9 | | finnish | 48.6 | 56.6 | +8.0 | +| finnish | 48.3 | 53.5 | +5.3 | +| finnish | 50.6 | 55.6 | +4.9 | | finnish | 50.9 | 55.8 | +4.9 | | finnish | 52.9 | 57.9 | +4.9 | +| finnish | 50.3 | 57.0 | +6.7 | +| finnish | 53.6 | 60.9 | +7.3 | | finnish | 48.6 | 56.6 | +8.0 | | finnish | 54.3 | 61.7 | +7.4 | | georgian | 94.3 | 94.1 | -0.2 | | georgian | 95.4 | 95.3 | -0.1 | +| georgian | 94.8 | 94.5 | -0.3 | +| georgian | 93.9 | 93.8 | -0.1 | | georgian | 94.3 | 94.1 | -0.2 | | georgian | 93.8 | 94.0 | +0.1 | +| georgian | 94.4 | 94.9 | +0.5 | +| georgian | 94.1 | 93.8 | -0.3 | | georgian | 95.4 | 95.3 | -0.1 | | georgian | 93.9 | 93.9 | +0.0 | | german | 78.9 | 79.3 | +0.4 | | german | 75.6 | 79.5 | +3.9 | | german | 78.6 | 78.9 | +0.3 | +| german | 77.9 | 78.5 | +0.7 | | german | 78.9 | 79.3 | +0.4 | | german | 77.9 | 78.0 | +0.1 | | german | 79.4 | 81.0 | +1.7 | @@ -62,22 +65,34 @@ Answers are unique but only ~10 characters with shallow, local dependencies. One | german | 77.2 | 78.3 | +1.0 | | navajo | 29.5 | 33.8 | +4.3 | | navajo | 31.4 | 40.2 | +8.9 | +| navajo | 33.7 | 39.0 | +5.2 | +| navajo | 24.2 | 30.8 | +6.6 | | navajo | 29.5 | 33.8 | +4.3 | | navajo | 24.3 | 29.0 | +4.8 | +| navajo | 31.1 | 39.3 | +8.2 | +| navajo | 26.3 | 34.0 | +7.7 | | navajo | 31.4 | 40.2 | +8.9 | | navajo | 24.6 | 31.7 | +7.1 | | russian | 72.7 | 72.0 | -0.7 | | russian | 69.1 | 71.7 | +2.6 | +| russian | 73.5 | 73.0 | -0.5 | +| russian | 74.2 | 74.5 | +0.4 | | russian | 72.7 | 72.0 | -0.7 | | russian | 71.9 | 72.0 | +0.1 | +| russian | 73.1 | 73.3 | +0.3 | +| russian | 76.3 | 76.2 | -0.1 | | russian | 69.1 | 71.7 | +2.6 | | russian | 74.1 | 75.9 | +1.8 | | spanish | 84.9 | 84.7 | -0.2 | | spanish | 83.9 | 84.1 | +0.1 | | turkish | 80.2 | 81.2 | +1.0 | | turkish | 77.5 | 82.2 | +4.7 | +| turkish | 77.3 | 78.9 | +1.6 | +| turkish | 83.1 | 83.9 | +0.8 | | turkish | 80.2 | 81.2 | +1.0 | | turkish | 84.5 | 85.1 | +0.7 | +| turkish | 77.0 | 80.9 | +3.9 | +| turkish | 80.4 | 83.8 | +3.4 | | turkish | 77.5 | 82.2 | +4.7 | | turkish | 79.9 | 82.2 | +2.3 | | turkish | 76.4 | 81.8 | +5.4 | @@ -112,19 +127,73 @@ The account in THEORY.md predicts minimum depth growing as log2(n), since the ca | digits | 2L | 3L | 4L | 6L | 8L | 12L | log2(n) | min depth >90% | |---|---|---|---|---|---|---|---|---| | **4** | 100 | 100 | 100 | 100 | 100 | 100 | 2.0 | 2 | -| **8** | 99 | 100 | 100 | 100 | 100 | – | 3.0 | 2 | -| **12** | 48 | 100 | 100 | – | – | – | 3.6 | 3 | +| **8** | 99 | 100 | 100 | 100 | 100 | 100 | 3.0 | 2 | +| **12** | 48 | 100 | 100 | 100 | – | – | 3.6 | 3 | +| **16** | 0 | 50 | – | – | – | – | 4.0 | – | ## Table 7 — Which component carries the gain (inflection) | language | mdlm | matched | distill | full | (gap 1→32) | |---|---|---|---|---|---| -| finnish | 51.4 | – | – | 51.9 | +7.7 / – / – / +4.9 | -| georgian | 94.6 | – | – | 94.1 | -0.0 / – / – / -0.0 | -| german | 76.0 | 77.9 | 78.6 | 78.4 | +3.2 / +1.4 / +0.3 / +0.2 | -| navajo | 28.0 | – | – | 26.9 | +8.0 / – / – / +4.5 | -| russian | 71.6 | – | – | 72.3 | +2.2 / – / – / -0.3 | -| turkish | 78.7 | – | – | 82.3 | +3.5 / – / – / +0.8 | +| finnish | 51.4 | 52.0 | 49.4 | 51.9 | +7.7 / +7.0 / +5.1 / +4.9 | +| georgian | 94.6 | 94.3 | 94.3 | 94.1 | -0.0 / +0.1 / -0.2 / -0.0 | +| german | 76.0 | 77.9 | 78.2 | 78.4 | +3.2 / +1.4 / +0.5 / +0.2 | +| navajo | 28.0 | 28.7 | 29.0 | 26.9 | +8.0 / +8.0 / +5.9 / +4.5 | +| russian | 71.6 | 74.7 | 73.9 | 72.3 | +2.2 / +0.1 / -0.1 / -0.3 | +| turkish | 78.7 | 78.7 | 80.2 | 82.3 | +3.5 / +3.7 / +1.2 / +0.8 | + +## Table 8 — text8: quality against compute + +Generative perplexity under a from-scratch autoregressive character evaluator, against the number of function evaluations actually used. Entropy and distinct-4-gram rate are reported beside it because generative perplexity alone is gamed by degenerate repetition; the real-text reference row is the target, not zero. + +Real text8 reference — ppl 2.86, entropy 2.855, distinct-4 0.177 + +| training | decoding | NFE | gen ppl | entropy | distinct-4 | valid BPC | +|---|---|---|---|---|---|---| +| mdlm | conf_0.5 | 234.5 | 32.99 | 2.871 | 0.360 | 1.7225 | +| mdlm | conf_0.7 | 248.5 | 16.99 | 2.866 | 0.295 | 1.7225 | +| mdlm | conf_0.9 | 252.8 | 9.78 | 2.848 | 0.245 | 1.7225 | +| mdlm | conf_0.95 | 254.0 | 9.08 | 2.856 | 0.238 | 1.7225 | +| mdlm | conf_0.99 | 255.5 | 8.74 | 2.851 | 0.236 | 1.7225 | +| mdlm | entbudget_0.05 | 256.0 | 9.47 | 2.854 | 0.241 | 1.7225 | +| mdlm | entbudget_0.1 | 255.5 | 8.79 | 2.861 | 0.241 | 1.7225 | +| mdlm | entbudget_0.25 | 255.0 | 8.93 | 2.858 | 0.239 | 1.7225 | +| mdlm | entbudget_0.5 | 253.2 | 9.39 | 2.858 | 0.243 | 1.7225 | +| mdlm | entbudget_1.0 | 248.2 | 10.39 | 2.857 | 0.248 | 1.7225 | +| mdlm | entbudget_2.0 | 245.0 | 15.44 | 2.861 | 0.279 | 1.7225 | +| mdlm | entbudget_4.0 | 223.2 | 51.18 | 2.882 | 0.401 | 1.7225 | +| mdlm | entbudget_8.0 | 96.5 | 111.68 | 2.888 | 0.520 | 1.7225 | +| mdlm | fixedK_1 | 1.0 | 646.96 | 2.860 | 0.630 | 1.7225 | +| mdlm | fixedK_128 | 128.0 | 44.33 | 2.867 | 0.410 | 1.7225 | +| mdlm | fixedK_16 | 16.0 | 126.50 | 2.878 | 0.532 | 1.7225 | +| mdlm | fixedK_2 | 2.0 | 310.58 | 2.890 | 0.625 | 1.7225 | +| mdlm | fixedK_256 | 256.0 | 8.98 | 2.854 | 0.241 | 1.7225 | +| mdlm | fixedK_32 | 32.0 | 95.60 | 2.875 | 0.497 | 1.7225 | +| mdlm | fixedK_4 | 4.0 | 155.18 | 2.881 | 0.550 | 1.7225 | +| mdlm | fixedK_64 | 64.0 | 86.83 | 2.882 | 0.490 | 1.7225 | +| mdlm | fixedK_8 | 8.0 | 121.66 | 2.879 | 0.527 | 1.7225 | +| matched | conf_0.5 | 237.5 | 32.44 | 2.873 | 0.366 | 1.6946 | +| matched | conf_0.7 | 242.0 | 14.86 | 2.854 | 0.282 | 1.6946 | +| matched | conf_0.9 | 249.2 | 9.67 | 2.856 | 0.248 | 1.6946 | +| matched | conf_0.95 | 251.2 | 8.65 | 2.855 | 0.241 | 1.6946 | +| matched | conf_0.99 | 254.8 | 8.34 | 2.853 | 0.236 | 1.6946 | +| matched | entbudget_0.05 | 256.0 | 8.39 | 2.857 | 0.241 | 1.6946 | +| matched | entbudget_0.1 | 256.0 | 8.50 | 2.855 | 0.238 | 1.6946 | +| matched | entbudget_0.25 | 255.0 | 8.72 | 2.857 | 0.243 | 1.6946 | +| matched | entbudget_0.5 | 253.2 | 9.00 | 2.863 | 0.247 | 1.6946 | +| matched | entbudget_1.0 | 251.5 | 10.02 | 2.861 | 0.255 | 1.6946 | +| matched | entbudget_2.0 | 247.2 | 14.98 | 2.862 | 0.284 | 1.6946 | +| matched | entbudget_4.0 | 235.2 | 47.92 | 2.885 | 0.403 | 1.6946 | +| matched | entbudget_8.0 | 98.0 | 104.65 | 2.895 | 0.526 | 1.6946 | +| matched | fixedK_1 | 1.0 | 651.67 | 2.860 | 0.631 | 1.6946 | +| matched | fixedK_128 | 128.0 | 42.38 | 2.869 | 0.407 | 1.6946 | +| matched | fixedK_16 | 16.0 | 94.52 | 2.868 | 0.502 | 1.6946 | +| matched | fixedK_2 | 2.0 | 295.79 | 2.881 | 0.620 | 1.6946 | +| matched | fixedK_256 | 256.0 | 8.19 | 2.852 | 0.235 | 1.6946 | +| matched | fixedK_32 | 32.0 | 93.33 | 2.882 | 0.510 | 1.6946 | +| matched | fixedK_4 | 4.0 | 146.64 | 2.880 | 0.558 | 1.6946 | +| matched | fixedK_64 | 64.0 | 76.89 | 2.884 | 0.484 | 1.6946 | +| matched | fixedK_8 | 8.0 | 123.33 | 2.876 | 0.540 | 1.6946 | ## Table 9 — Did the baseline fail to parallelise, or fail to learn? diff --git a/figures/beat_d12.json b/figures/beat_d12.json index 8f2b694402e326a679dffe0cdff07a1fc0744eeb..4a11235a181f18d16ffa8dd3914d31b7114bc132 100644 --- a/figures/beat_d12.json +++ b/figures/beat_d12.json @@ -31,10 +31,10 @@ ], "random": [ 0.766875, - 0.764375, 0.768125, - 0.770625, - 0.76625 + 0.77125, + 0.764375, + 0.77 ] }, "ours": { @@ -46,10 +46,61 @@ 0.999375 ] }, - "significance": {}, + "significance": { + "confidence": { + "diff": 0.233125, + "ci95": [ + 0.226875, + 0.23875 + ], + "p_value": 0.0, + "n": 400, + "their_best_budget": 1 + }, + "entropy": { + "diff": 0.233125, + "ci95": [ + 0.226875, + 0.23875 + ], + "p_value": 0.0, + "n": 400, + "their_best_budget": 1 + }, + "entropy-gated": { + "diff": 0.233125, + "ci95": [ + 0.226875, + 0.23875 + ], + "p_value": 0.0, + "n": 400, + "their_best_budget": 1 + }, + "margin": { + "diff": 0.233125, + "ci95": [ + 0.226875, + 0.23875 + ], + "p_value": 0.0, + "n": 400, + "their_best_budget": 1 + }, + "random": { + "diff": 0.22875, + "ci95": [ + 0.221875, + 0.235625 + ], + "p_value": 0.0, + "n": 400, + "their_best_budget": 4 + } + }, "summary": { "best_inference_1pass": 0.766875, - "best_inference_any_budget": 0.770625, + "best_inference_any_budget": 0.77125, "ours_1pass": 1.0 } } \ No newline at end of file diff --git a/figures/beat_d16.json b/figures/beat_d16.json index 2f89bed54fc52dc42ffd7f89fdcd7a0934fea231..a4e9ff9eaa4612a62bc6cacef2823c78c7159e80 100644 --- a/figures/beat_d16.json +++ b/figures/beat_d16.json @@ -33,8 +33,8 @@ 0.25, 0.25, 0.25, - 0.249375, - 0.25 + 0.25, + 0.249375 ] }, "ours": { @@ -46,7 +46,58 @@ 0.749375 ] }, - "significance": {}, + "significance": { + "confidence": { + "diff": 0.5, + "ci95": [ + 0.5, + 0.5 + ], + "p_value": 0.0, + "n": 400, + "their_best_budget": 1 + }, + "entropy": { + "diff": 0.5, + "ci95": [ + 0.5, + 0.5 + ], + "p_value": 0.0, + "n": 400, + "their_best_budget": 1 + }, + "entropy-gated": { + "diff": 0.5, + "ci95": [ + 0.5, + 0.5 + ], + "p_value": 0.0, + "n": 400, + "their_best_budget": 1 + }, + "margin": { + "diff": 0.5, + "ci95": [ + 0.5, + 0.5 + ], + "p_value": 0.0, + "n": 400, + "their_best_budget": 1 + }, + "random": { + "diff": 0.5, + "ci95": [ + 0.5, + 0.5 + ], + "p_value": 0.0, + "n": 400, + "their_best_budget": 1 + } + }, "summary": { "best_inference_1pass": 0.25, "best_inference_any_budget": 0.25, diff --git a/figures/fig3_depth_scaling.png b/figures/fig3_depth_scaling.png index 8d28742fb1166882e1fd2c90ed07983648f5ed47..b345f9573c4d78ca3f3e3a5bfdd25875efca31ec 100644 Binary files a/figures/fig3_depth_scaling.png and b/figures/fig3_depth_scaling.png differ diff --git a/figures/fig4_text8_quality_vs_compute.png b/figures/fig4_text8_quality_vs_compute.png new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/figures/graph_ceiling.json b/figures/graph_ceiling.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/figures/summary.md b/figures/summary.md index f6095fc7c18e679de0f863745ab8d4a1203d662c..7f8615511c00beed4fc6ceccad93dfcb59f2b525 100644 --- a/figures/summary.md +++ b/figures/summary.md @@ -23,19 +23,28 @@ Answers are unique but only ~10 characters with shallow, local dependencies. One | english | 95.3 | 95.2 | -0.1 | | finnish | 50.9 | 55.8 | +4.9 | | finnish | 48.6 | 56.6 | +8.0 | +| finnish | 48.3 | 53.5 | +5.3 | +| finnish | 50.6 | 55.6 | +4.9 | | finnish | 50.9 | 55.8 | +4.9 | | finnish | 52.9 | 57.9 | +4.9 | +| finnish | 50.3 | 57.0 | +6.7 | +| finnish | 53.6 | 60.9 | +7.3 | | finnish | 48.6 | 56.6 | +8.0 | | finnish | 54.3 | 61.7 | +7.4 | | georgian | 94.3 | 94.1 | -0.2 | | georgian | 95.4 | 95.3 | -0.1 | +| georgian | 94.8 | 94.5 | -0.3 | +| georgian | 93.9 | 93.8 | -0.1 | | georgian | 94.3 | 94.1 | -0.2 | | georgian | 93.8 | 94.0 | +0.1 | +| georgian | 94.4 | 94.9 | +0.5 | +| georgian | 94.1 | 93.8 | -0.3 | | georgian | 95.4 | 95.3 | -0.1 | | georgian | 93.9 | 93.9 | +0.0 | | german | 78.9 | 79.3 | +0.4 | | german | 75.6 | 79.5 | +3.9 | | german | 78.6 | 78.9 | +0.3 | +| german | 77.9 | 78.5 | +0.7 | | german | 78.9 | 79.3 | +0.4 | | german | 77.9 | 78.0 | +0.1 | | german | 79.4 | 81.0 | +1.7 | @@ -45,22 +54,34 @@ Answers are unique but only ~10 characters with shallow, local dependencies. One | german | 77.2 | 78.3 | +1.0 | | navajo | 29.5 | 33.8 | +4.3 | | navajo | 31.4 | 40.2 | +8.9 | +| navajo | 33.7 | 39.0 | +5.2 | +| navajo | 24.2 | 30.8 | +6.6 | | navajo | 29.5 | 33.8 | +4.3 | | navajo | 24.3 | 29.0 | +4.8 | +| navajo | 31.1 | 39.3 | +8.2 | +| navajo | 26.3 | 34.0 | +7.7 | | navajo | 31.4 | 40.2 | +8.9 | | navajo | 24.6 | 31.7 | +7.1 | | russian | 72.7 | 72.0 | -0.7 | | russian | 69.1 | 71.7 | +2.6 | +| russian | 73.5 | 73.0 | -0.5 | +| russian | 74.2 | 74.5 | +0.4 | | russian | 72.7 | 72.0 | -0.7 | | russian | 71.9 | 72.0 | +0.1 | +| russian | 73.1 | 73.3 | +0.3 | +| russian | 76.3 | 76.2 | -0.1 | | russian | 69.1 | 71.7 | +2.6 | | russian | 74.1 | 75.9 | +1.8 | | spanish | 84.9 | 84.7 | -0.2 | | spanish | 83.9 | 84.1 | +0.1 | | turkish | 80.2 | 81.2 | +1.0 | | turkish | 77.5 | 82.2 | +4.7 | +| turkish | 77.3 | 78.9 | +1.6 | +| turkish | 83.1 | 83.9 | +0.8 | | turkish | 80.2 | 81.2 | +1.0 | | turkish | 84.5 | 85.1 | +0.7 | +| turkish | 77.0 | 80.9 | +3.9 | +| turkish | 80.4 | 83.8 | +3.4 | | turkish | 77.5 | 82.2 | +4.7 | | turkish | 79.9 | 82.2 | +2.3 | | turkish | 76.4 | 81.8 | +5.4 | @@ -95,28 +116,61 @@ The account in THEORY.md predicts minimum depth growing as log2(n), since the ca | digits | 2L | 3L | 4L | 6L | 8L | 12L | log2(n) | min depth >90% | |---|---|---|---|---|---|---|---|---| | **4** | 100 | 100 | 100 | 100 | 100 | 100 | 2.0 | 2 | -| **8** | 99 | 100 | 100 | 100 | 100 | – | 3.0 | 2 | -| **12** | 48 | 100 | 100 | – | – | – | 3.6 | 3 | +| **8** | 99 | 100 | 100 | 100 | 100 | 100 | 3.0 | 2 | +| **12** | 48 | 100 | 100 | 100 | – | – | 3.6 | 3 | +| **16** | 0 | 50 | – | – | – | – | 4.0 | – | ## Table 7 — Which component carries the gain (inflection) | language | mdlm | matched | distill | full | (gap 1→32) | |---|---|---|---|---|---| -| finnish | 51.4 | – | – | 51.9 | +7.7 / – / – / +4.9 | -| georgian | 94.6 | – | – | 94.1 | -0.0 / – / – / -0.0 | -| german | 76.0 | 77.9 | 78.6 | 78.4 | +3.2 / +1.4 / +0.3 / +0.2 | -| navajo | 28.0 | – | – | 26.9 | +8.0 / – / – / +4.5 | -| russian | 71.6 | – | – | 72.3 | +2.2 / – / – / -0.3 | -| turkish | 78.7 | – | – | 82.3 | +3.5 / – / – / +0.8 | - -## Table 9 — Did the baseline fail to parallelise, or fail to learn? - -A 20-digit answer occupies 21 slots, so the standard 16-pass grid is not a sequential decode. Re-decoded at one token per pass, the two explanations separate. - -| digits | mode | @1 pass | fully sequential | reading | -|---|---|---|---|---| -| 20 | full | 99.8 | 85.7 | learned it and parallelises | -| 20 | mdlm | 0.0 | 0.0 | never learned the task | -| 24 | full | 33.7 | 33.3 | learned it and parallelises | -| 24 | mdlm | 0.0 | 0.0 | never learned the task | - +| finnish | 51.4 | 52.0 | 49.4 | 51.9 | +7.7 / +7.0 / +5.1 / +4.9 | +| georgian | 94.6 | 94.3 | 94.3 | 94.1 | -0.0 / +0.1 / -0.2 / -0.0 | +| german | 76.0 | 77.9 | 78.2 | 78.4 | +3.2 / +1.4 / +0.5 / +0.2 | +| navajo | 28.0 | 28.7 | 29.0 | 26.9 | +8.0 / +8.0 / +5.9 / +4.5 | +| russian | 71.6 | 74.7 | 73.9 | 72.3 | +2.2 / +0.1 / -0.1 / -0.3 | +| turkish | 78.7 | 78.7 | 80.2 | 82.3 | +3.5 / +3.7 / +1.2 / +0.8 | + +## Table 8 — text8: quality against compute + +Generative perplexity under a from-scratch autoregressive character evaluator, against the number of function evaluations actually used. Entropy and distinct-4-gram rate are reported beside it because generative perplexity alone is gamed by degenerate repetition; the real-text reference row is the target, not zero. + +Real text8 reference — ppl 2.86, entropy 2.855, distinct-4 0.177 + +| training | decoding | NFE | gen ppl | entropy | distinct-4 | valid BPC | +|---|---|---|---|---|---|---| +| mdlm | conf_0.5 | 234.5 | 32.99 | 2.871 | 0.360 | 1.7225 | +| mdlm | conf_0.7 | 248.5 | 16.99 | 2.866 | 0.295 | 1.7225 | +| mdlm | conf_0.9 | 252.8 | 9.78 | 2.848 | 0.245 | 1.7225 | +| mdlm | conf_0.95 | 254.0 | 9.08 | 2.856 | 0.238 | 1.7225 | +| mdlm | conf_0.99 | 255.5 | 8.74 | 2.851 | 0.236 | 1.7225 | +| mdlm | entbudget_0.05 | 256.0 | 9.47 | 2.854 | 0.241 | 1.7225 | +| mdlm | entbudget_0.1 | 255.5 | 8.79 | 2.861 | 0.241 | 1.7225 | +| mdlm | entbudget_0.25 | 255.0 | 8.93 | 2.858 | 0.239 | 1.7225 | +| mdlm | entbudget_0.5 | 253.2 | 9.39 | 2.858 | 0.243 | 1.7225 | +| mdlm | entbudget_1.0 | 248.2 | 10.39 | 2.857 | 0.248 | 1.7225 | +| mdlm | entbudget_2.0 | 245.0 | 15.44 | 2.861 | 0.279 | 1.7225 | +| mdlm | entbudget_4.0 | 223.2 | 51.18 | 2.882 | 0.401 | 1.7225 | +| mdlm | entbudget_8.0 | 96.5 | 111.68 | 2.888 | 0.520 | 1.7225 | +| mdlm | fixedK_1 | 1.0 | 646.96 | 2.860 | 0.630 | 1.7225 | +| mdlm | fixedK_128 | 128.0 | 44.33 | 2.867 | 0.410 | 1.7225 | +| mdlm | fixedK_16 | 16.0 | 126.50 | 2.878 | 0.532 | 1.7225 | +| mdlm | fixedK_2 | 2.0 | 310.58 | 2.890 | 0.625 | 1.7225 | +| mdlm | fixedK_256 | 256.0 | 8.98 | 2.854 | 0.241 | 1.7225 | +| mdlm | fixedK_32 | 32.0 | 95.60 | 2.875 | 0.497 | 1.7225 | +| mdlm | fixedK_4 | 4.0 | 155.18 | 2.881 | 0.550 | 1.7225 | +| mdlm | fixedK_64 | 64.0 | 86.83 | 2.882 | 0.490 | 1.7225 | +| mdlm | fixedK_8 | 8.0 | 121.66 | 2.879 | 0.527 | 1.7225 | +| matched | conf_0.5 | 237.5 | 32.44 | 2.873 | 0.366 | 1.6946 | +| matched | conf_0.7 | 242.0 | 14.86 | 2.854 | 0.282 | 1.6946 | +| matched | conf_0.9 | 249.2 | 9.67 | 2.856 | 0.248 | 1.6946 | +| matched | conf_0.95 | 251.2 | 8.65 | 2.855 | 0.241 | 1.6946 | +| matched | conf_0.99 | 254.8 | 8.34 | 2.853 | 0.236 | 1.6946 | +| matched | entbudget_0.05 | 256.0 | 8.39 | 2.857 | 0.241 | 1.6946 | +| matched | entbudget_0.1 | 256.0 | 8.50 | 2.855 | 0.238 | 1.6946 | +| matched | entbudget_0.25 | 255.0 | 8.72 | 2.857 | 0.243 | 1.6946 | +| matched | entbudget_0.5 | 253.2 | 9.00 | 2.863 | 0.247 | 1.6946 | +| matched | entbudget_1.0 | 251.5 | 10.02 | 2.861 | 0.255 | 1.6946 | +| matched | entbudget_2.0 | 247.2 | 14.98 | 2.862 | 0.284 | 1.6946 | +| matched | entbudget_4.0 | 235.2 | 47.92 | 2.885 | 0.403 | 1.6946 | +| matched \ No newline at end of file diff --git a/results/evallm.json b/results/evallm.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g14-d12-L6-s0.json b/results/g14-d12-L6-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g14-d12-L6-s1.json b/results/g14-d12-L6-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g14-d16-L2-s0.json b/results/g14-d16-L2-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g14-d16-L2-s1.json b/results/g14-d16-L2-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g14-d16-L3-s0.json b/results/g14-d16-L3-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g14-d16-L3-s1.json b/results/g14-d16-L3-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g14-d8-L12-s0.json b/results/g14-d8-L12-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g14-d8-L12-s1.json b/results/g14-d8-L12-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-finnish-distill-s0.json b/results/g15-finnish-distill-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-finnish-distill-s1.json b/results/g15-finnish-distill-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-finnish-matched-s0.json b/results/g15-finnish-matched-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-finnish-matched-s1.json b/results/g15-finnish-matched-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-georgian-distill-s0.json b/results/g15-georgian-distill-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-georgian-distill-s1.json b/results/g15-georgian-distill-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-georgian-matched-s0.json b/results/g15-georgian-matched-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-georgian-matched-s1.json b/results/g15-georgian-matched-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-german-distill-s1.json b/results/g15-german-distill-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-navajo-distill-s0.json b/results/g15-navajo-distill-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-navajo-distill-s1.json b/results/g15-navajo-distill-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-navajo-matched-s0.json b/results/g15-navajo-matched-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-navajo-matched-s1.json b/results/g15-navajo-matched-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-russian-distill-s0.json b/results/g15-russian-distill-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-russian-distill-s1.json b/results/g15-russian-distill-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-russian-matched-s0.json b/results/g15-russian-matched-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-russian-matched-s1.json b/results/g15-russian-matched-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-turkish-distill-s0.json b/results/g15-turkish-distill-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-turkish-distill-s1.json b/results/g15-turkish-distill-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-turkish-matched-s0.json b/results/g15-turkish-matched-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g15-turkish-matched-s1.json b/results/g15-turkish-matched-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g17-d16-mdlm2x-s0.json b/results/g17-d16-mdlm2x-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g17-d16-mdlm2x-s1.json b/results/g17-d16-mdlm2x-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g17-d20-mdlm2x-s0.json b/results/g17-d20-mdlm2x-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g17-d20-mdlm2x-s1.json b/results/g17-d20-mdlm2x-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g18-d20-anybudget-s0.json b/results/g18-d20-anybudget-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g18-d20-anybudget-s1.json b/results/g18-d20-anybudget-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g18-d20-anybudget-s2.json b/results/g18-d20-anybudget-s2.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g18-d20-anybudget_nc-s0.json b/results/g18-d20-anybudget_nc-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g18-d20-anybudget_nc-s1.json b/results/g18-d20-anybudget_nc-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/g18-d20-anybudget_nc-s2.json b/results/g18-d20-anybudget_nc-s2.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-2regular-anybudget-s0.json b/results/gr1-2regular-anybudget-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-2regular-anybudget-s1.json b/results/gr1-2regular-anybudget-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-2regular-mdlm-s0.json b/results/gr1-2regular-mdlm-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-2regular-mdlm-s1.json b/results/gr1-2regular-mdlm-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-bipartite-anybudget-s0.json b/results/gr1-bipartite-anybudget-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-bipartite-anybudget-s1.json b/results/gr1-bipartite-anybudget-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-bipartite-mdlm-s0.json b/results/gr1-bipartite-mdlm-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-bipartite-mdlm-s1.json b/results/gr1-bipartite-mdlm-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-matching-anybudget-s0.json b/results/gr1-matching-anybudget-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-matching-anybudget-s1.json b/results/gr1-matching-anybudget-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-matching-mdlm-s0.json b/results/gr1-matching-mdlm-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-matching-mdlm-s1.json b/results/gr1-matching-mdlm-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-tree-anybudget-s0.json b/results/gr1-tree-anybudget-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/gr1-tree-mdlm-s1.json b/results/gr1-tree-mdlm-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/t8-anybudget-s0.json b/results/t8-anybudget-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/t8-anybudget-s1.json b/results/t8-anybudget-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/t8-matched-s1.json b/results/t8-matched-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/results/t8-mdlm-s0.json b/results/t8-mdlm-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/slurm/finalize.sh b/slurm/finalize.sh index 3f850dc44f60bfecc0d845835d2cab9a0c5a8288..e5956b387bc909c5f61c634be2d97c078574752d 100644 --- a/slurm/finalize.sh +++ b/slurm/finalize.sh @@ -43,16 +43,23 @@ for d in $STAR/runs/*/; do done echo " results backed up: $(ls $HOME_BK/results | wc -l)" -echo "=== 3. back up weights to SHELL (not purged) ===" -if mkdir -p $SHELL_BK 2>/dev/null; then - for d in $STAR/runs/*/; do - n=$(basename $d) - [ -f "$d/model.pt" ] && cp -f "$d/model.pt" "$SHELL_BK/$n.pt" 2>/dev/null - done - echo " weights backed up: $(ls $SHELL_BK 2>/dev/null | wc -l)" -else - echo " SHELL not writable from this node - weights remain on scratch" -fi +echo "=== 3. back up weights to HOME (nightly-backed-up) ===" +# SHELL was the original target but /afs/shell.umd.edu/project/msml612 is +# drwx------ and owned by root, so it is unwritable from BOTH login and compute +# nodes - every previous run skipped this step silently. HOME is the tier that +# is actually backed up nightly and has ample free space for the weights. +mkdir -p $HOME_BK/weights +n=0 +for d in $STAR/runs/*/; do + b=$(basename $d) + if [ -f "$d/model.pt" ]; then + # copy only when missing or newer, so repeated finalize runs stay cheap + if [ ! -f "$HOME_BK/weights/$b.pt" ] || [ "$d/model.pt" -nt "$HOME_BK/weights/$b.pt" ]; then + cp -f "$d/model.pt" "$HOME_BK/weights/$b.pt" && n=$((n+1)) + fi + fi +done +echo " weights copied this run: $n (total: $(ls $HOME_BK/weights 2>/dev/null | wc -l), $(du -sh $HOME_BK/weights 2>/dev/null | cut -f1))" echo "=== 4. push to Hugging Face (with retries) ===" # A transient network failure must not mean the results never reach the Hub, so diff --git a/slurm/gr1.sh b/slurm/gr1.sh new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/slurm/gr2.sh b/slurm/gr2.sh new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/audit.py b/src/audit.py index 7a13c9b23fd933d14f71bc0ab0f2ffd6a7babb89..8ea0a28a9ffa3481d0aa344f465079d1c356edd5 100644 --- a/src/audit.py +++ b/src/audit.py @@ -172,7 +172,10 @@ except FileNotFoundError: except Exception as e: check("text8 loader", False, f"{type(e).__name__}: {e}") -print(f"\n=== AUDIT: {len(FAIL)} failure(s) ===") -for f in FAIL: - print(" !", f) -sys.exit(1 if FAIL else 0) +print("\n=== 9. graph families: enumeration matches known combinatorics ===") +# These families have closed-form counts, so the enumerator can be checked +# against mathematics rather than against itself. If enumeration is wrong then +# every marginal, every entropy and V* are wrong, and the central proposition +# would be "verified" against a fiction. +try: + from graphs import enumerate_family, exact_stats, \ No newline at end of file diff --git a/src/diffusion.py b/src/diffusion.py index 7723c1b37410ebfb5ca7c77d35ca03f74b07276c..6e88bd4323657a00146f6ac0878a0c8fa1973f1f 100644 --- a/src/diffusion.py +++ b/src/diffusion.py @@ -514,25 +514,33 @@ def _commit(x, take, samp): return torch.where(take, samp, x) -def real_probs(logits, tok): - """Distribution over REAL tokens only. +def real_probs(logits, tok, allowed=None): + """Distribution over REAL tokens only, optionally position-typed. The absorbing state is an input symbol, never an output: in the SUBS parameterisation the denoiser places zero probability on [MASK]. Sampling from the raw softmax instead lets a position be "committed" as [MASK], so it stays masked, decoding silently stalls, and the adaptive rules stop being monotone in their budget. The audit caught exactly this. + + `allowed` is an optional (L, V) boolean mask of which symbols each position + may take. Graph data needs it: a node slot must take an atom type and an + edge slot a bond type, and that is a property of the representation rather + than something the model should have to spend capacity rediscovering. """ logits = logits.clone() logits[..., tok.mask] = -float("inf") if getattr(tok, "pad", tok.mask) != tok.mask: logits[..., tok.pad] = -float("inf") + if allowed is not None: + logits = logits.masked_fill(~allowed.to(logits.device), -float("inf")) return logits.softmax(-1) @torch.no_grad() def entropy_budget_decode(model, x, tok, device, budget=0.5, max_steps=256, - temperature=1.0, fillable=None, budget_cond=None): + temperature=1.0, fillable=None, budget_cond=None, + allowed=None): """FIX (ii): commit a set whose summed conditional entropy stays under `budget`. At each pass, rank the still-masked positions by conditional entropy and @@ -560,7 +568,7 @@ def entropy_budget_decode(model, x, tok, device, budget=0.5, max_steps=256, nfe += 1 if temperature != 1.0: logits = logits / temperature - probs = real_probs(logits, tok) + probs = real_probs(logits, tok, allowed) H = -(probs * probs.clamp_min(1e-9).log()).sum(-1) samp = torch.multinomial(probs.view(-1, probs.size(-1)), 1).view(B, L) @@ -578,7 +586,8 @@ def entropy_budget_decode(model, x, tok, device, budget=0.5, max_steps=256, @torch.no_grad() def confidence_threshold_decode(model, x, tok, device, tau=0.9, max_steps=256, - temperature=1.0, fillable=None, budget_cond=None): + temperature=1.0, fillable=None, budget_cond=None, + allowed=None): """The published inference-time competitor (confidence-threshold family). Commit every masked position whose top-1 probability exceeds `tau`. This is @@ -601,7 +610,7 @@ def confidence_threshold_decode(model, x, tok, device, tau=0.9, max_steps=256, nfe += 1 if temperature != 1.0: logits = logits / temperature - probs = real_probs(logits, tok) + probs = real_probs(logits, tok, allowed) conf = probs.max(-1).values samp = torch.multinomial(probs.view(-1, probs.size(-1)), 1).view(B, L) @@ -617,7 +626,7 @@ def confidence_threshold_decode(model, x, tok, device, tau=0.9, max_steps=256, @torch.no_grad() def fixed_k_decode(model, x, tok, device, steps, temperature=1.0, fillable=None, - budget_cond=None): + budget_cond=None, allowed=None): """Plain K-pass decoding: reveal an equal share of positions per pass, highest-confidence first. The non-adaptive reference point, and the setting in which the addition results are reported.""" @@ -636,7 +645,7 @@ def fixed_k_decode(model, x, tok, device, steps, temperature=1.0, fillable=None, nfe += 1 if temperature != 1.0: logits = logits / temperature - probs = real_probs(logits, tok) + probs = real_probs(logits, tok, allowed) conf = probs.max(-1).values.masked_fill(~masked, -float("inf")) samp = torch.multinomial(probs.view(-1, probs.size(-1)), 1).view(B, L) diff --git a/src/graph_analysis.py b/src/graph_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/graphs.py b/src/graphs.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/marginal_check.py b/src/marginal_check.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/qm9.py b/src/qm9.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/train_graph.py b/src/train_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391