diff --git a/.gitattributes b/.gitattributes index 9f38eb4cf7be7099b673b685aae35fd6a032fe7d..84477b374155a279b4868eb231a6358c354c0bd9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text figures/fig2_frontier.png filter=lfs diff=lfs merge=lfs -text +figures/fig4_text8_quality_vs_compute.png filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index d436d2696330e2eff7fbebd42348475371228543..6a5a6f00d2d085da2a586f0a28eac8e99d68a8f1 100644 --- a/README.md +++ b/README.md @@ -1,209 +1,145 @@ --- license: apache-2.0 -tags: [masked-diffusion, parallel-decoding, text8, qm9, from-scratch] +tags: + - masked-diffusion + - discrete-diffusion + - parallel-decoding + - constraint-satisfaction + - text8 + - qm9 + - sudoku + - from-scratch --- # Two Failure Modes of Parallel Decoding in Masked Diffusion +Everything here is trained from scratch: no pretrained weights, no pretrained +tokenizer, and the autoregressive model used to score generated text is itself +trained from scratch on the same corpus. + Code: https://github.com/GIND123/MSML612 -## Results - -# Results - -## Table 1 — Addition: exact match at ONE forward pass - -The whole answer committed in a single pass. Depth scaled as log2(digits); learning rate 1e-4 with 4000 warmup steps, which matters - see Table 4. - -| digits | MDLM baseline | full method | seeds >90% (base → ours) | -|---|---|---|---| -| **8** | 100.0 [100,100] | 100.0 [100,100] | 4/4 → 4/4 | -| **12** | 76.7 [30,100] | 100.0 [100,100] | 3/4 → 4/4 | -| **16** | 25.0 [0,75] | 75.0 [25,100] | 1/4 → 3/4 | -| **24** | 0.0 [0,0] | 0.0 [0,0] | 0/4 → 0/4 | -| **32** | 0.0 [0,0] | 0.0 [0,0] | 0/4 → 0/4 | - -## Table 2 — Morphological inflection: where the problem does NOT arise - -Answers are unique but only ~10 characters with shallow, local dependencies. One pass already suffices for the plain baseline, so there is no gap for any method to close. This is the boundary the account predicts, and it is reported as such. - -| language | 1 pass | 32 passes | gap | -|---|---|---|---| -| english | 95.5 | 95.1 | -0.4 | -| english | 95.0 | 95.2 | +0.2 | -| english | 95.3 | 95.2 | -0.1 | -| finnish | 50.9 | 55.8 | +4.9 | -| finnish | 48.6 | 56.6 | +8.0 | -| finnish | 48.3 | 53.5 | +5.3 | -| finnish | 50.6 | 55.6 | +4.9 | -| finnish | 50.9 | 55.8 | +4.9 | -| finnish | 52.9 | 57.9 | +4.9 | -| finnish | 50.3 | 57.0 | +6.7 | -| finnish | 53.6 | 60.9 | +7.3 | -| finnish | 48.6 | 56.6 | +8.0 | -| finnish | 54.3 | 61.7 | +7.4 | -| georgian | 94.3 | 94.1 | -0.2 | -| georgian | 95.4 | 95.3 | -0.1 | -| georgian | 94.8 | 94.5 | -0.3 | -| georgian | 93.9 | 93.8 | -0.1 | -| georgian | 94.3 | 94.1 | -0.2 | -| georgian | 93.8 | 94.0 | +0.1 | -| georgian | 94.4 | 94.9 | +0.5 | -| georgian | 94.1 | 93.8 | -0.3 | -| georgian | 95.4 | 95.3 | -0.1 | -| georgian | 93.9 | 93.9 | +0.0 | -| german | 78.9 | 79.3 | +0.4 | -| german | 75.6 | 79.5 | +3.9 | -| german | 78.6 | 78.9 | +0.3 | -| german | 77.9 | 78.5 | +0.7 | -| german | 78.9 | 79.3 | +0.4 | -| german | 77.9 | 78.0 | +0.1 | -| german | 79.4 | 81.0 | +1.7 | -| german | 76.5 | 77.6 | +1.0 | -| german | 75.6 | 79.5 | +3.9 | -| german | 76.4 | 78.9 | +2.5 | -| german | 77.2 | 78.3 | +1.0 | -| navajo | 29.5 | 33.8 | +4.3 | -| navajo | 31.4 | 40.2 | +8.9 | -| navajo | 33.7 | 39.0 | +5.2 | -| navajo | 24.2 | 30.8 | +6.6 | -| navajo | 29.5 | 33.8 | +4.3 | -| navajo | 24.3 | 29.0 | +4.8 | -| navajo | 31.1 | 39.3 | +8.2 | -| navajo | 26.3 | 34.0 | +7.7 | -| navajo | 31.4 | 40.2 | +8.9 | -| navajo | 24.6 | 31.7 | +7.1 | -| russian | 72.7 | 72.0 | -0.7 | -| russian | 69.1 | 71.7 | +2.6 | -| russian | 73.5 | 73.0 | -0.5 | -| russian | 74.2 | 74.5 | +0.4 | -| russian | 72.7 | 72.0 | -0.7 | -| russian | 71.9 | 72.0 | +0.1 | -| russian | 73.1 | 73.3 | +0.3 | -| russian | 76.3 | 76.2 | -0.1 | -| russian | 69.1 | 71.7 | +2.6 | -| russian | 74.1 | 75.9 | +1.8 | -| spanish | 84.9 | 84.7 | -0.2 | -| spanish | 83.9 | 84.1 | +0.1 | -| turkish | 80.2 | 81.2 | +1.0 | -| turkish | 77.5 | 82.2 | +4.7 | -| turkish | 77.3 | 78.9 | +1.6 | -| turkish | 83.1 | 83.9 | +0.8 | -| turkish | 80.2 | 81.2 | +1.0 | -| turkish | 84.5 | 85.1 | +0.7 | -| turkish | 77.0 | 80.9 | +3.9 | -| turkish | 80.4 | 83.8 | +3.4 | -| turkish | 77.5 | 82.2 | +4.7 | -| turkish | 79.9 | 82.2 | +2.3 | -| turkish | 76.4 | 81.8 | +5.4 | - -## Table 3 — Non-unique answers: the limit no training removes - -On probes where a group must agree but the value is free, the likelihood-optimal marginal is uniform over the valid values, so two independent draws agree with probability 1/V. Sweeping an entropy penalty designed to break that symmetry over 66 runs: weak settings changed nothing, strong settings destroyed the model, failing even at fully sequential decoding. Succeeding would require abandoning the data distribution. - -## Table 4 — Why earlier runs appeared to fail (10-digit addition) - -Every apparent failure at 10+ digits was a learning-rate artifact, not a limit of the method or of depth. - -| layers | lr | warmup | mean @1 pass | seeds >90% | -|---|---|---|---|---| -| 6 | 5e-05 | 500 | 100.0 | 3/3 | -| 6 | 5e-05 | 4000 | 100.0 | 3/3 | -| 6 | 0.0001 | 500 | 100.0 | 3/3 | -| 6 | 0.0001 | 4000 | 100.0 | 3/3 | -| 6 | 0.0003 | 500 | 70.6 | 2/3 | -| 6 | 0.0003 | 4000 | 100.0 | 3/3 | -| 10 | 5e-05 | 500 | 100.0 | 3/3 | -| 10 | 5e-05 | 4000 | 100.0 | 3/3 | -| 10 | 0.0001 | 500 | 100.0 | 3/3 | -| 10 | 0.0001 | 4000 | 100.0 | 3/3 | -| 10 | 0.0003 | 500 | 47.3 | 1/3 | -| 10 | 0.0003 | 4000 | 100.0 | 3/3 | - -## Table 6 — Minimum depth for one-pass addition - -The account in THEORY.md predicts minimum depth growing as log2(n), since the carry is a prefix scan. A ripple-carry account predicts linear growth; an impossibility account predicts no depth suffices. - -| digits | 2L | 3L | 4L | 6L | 8L | 12L | log2(n) | min depth >90% | -|---|---|---|---|---|---|---|---|---| -| **4** | 100 | 100 | 100 | 100 | 100 | 100 | 2.0 | 2 | -| **8** | 99 | 100 | 100 | 100 | 100 | 100 | 3.0 | 2 | -| **12** | 48 | 100 | 100 | 100 | – | – | 3.6 | 3 | -| **16** | 0 | 50 | – | – | – | – | 4.0 | – | - -## Table 7 — Which component carries the gain (inflection) - -| language | mdlm | matched | distill | full | (gap 1→32) | -|---|---|---|---|---|---| -| finnish | 51.4 | 52.0 | 49.4 | 51.9 | +7.7 / +7.0 / +5.1 / +4.9 | -| georgian | 94.6 | 94.3 | 94.3 | 94.1 | -0.0 / +0.1 / -0.2 / -0.0 | -| german | 76.0 | 77.9 | 78.2 | 78.4 | +3.2 / +1.4 / +0.5 / +0.2 | -| navajo | 28.0 | 28.7 | 29.0 | 26.9 | +8.0 / +8.0 / +5.9 / +4.5 | -| russian | 71.6 | 74.7 | 73.9 | 72.3 | +2.2 / +0.1 / -0.1 / -0.3 | -| turkish | 78.7 | 78.7 | 80.2 | 82.3 | +3.5 / +3.7 / +1.2 / +0.8 | - -## Table 8 — text8: quality against compute - -Generative perplexity under a from-scratch autoregressive character evaluator, against the number of function evaluations actually used. Entropy and distinct-4-gram rate are reported beside it because generative perplexity alone is gamed by degenerate repetition; the real-text reference row is the target, not zero. - -Real text8 reference — ppl 2.86, entropy 2.855, distinct-4 0.177 - -| training | decoding | NFE | gen ppl | entropy | distinct-4 | valid BPC | -|---|---|---|---|---|---|---| -| mdlm | conf_0.5 | 234.5 | 32.99 | 2.871 | 0.360 | 1.7225 | -| mdlm | conf_0.7 | 248.5 | 16.99 | 2.866 | 0.295 | 1.7225 | -| mdlm | conf_0.9 | 252.8 | 9.78 | 2.848 | 0.245 | 1.7225 | -| mdlm | conf_0.95 | 254.0 | 9.08 | 2.856 | 0.238 | 1.7225 | -| mdlm | conf_0.99 | 255.5 | 8.74 | 2.851 | 0.236 | 1.7225 | -| mdlm | entbudget_0.05 | 256.0 | 9.47 | 2.854 | 0.241 | 1.7225 | -| mdlm | entbudget_0.1 | 255.5 | 8.79 | 2.861 | 0.241 | 1.7225 | -| mdlm | entbudget_0.25 | 255.0 | 8.93 | 2.858 | 0.239 | 1.7225 | -| mdlm | entbudget_0.5 | 253.2 | 9.39 | 2.858 | 0.243 | 1.7225 | -| mdlm | entbudget_1.0 | 248.2 | 10.39 | 2.857 | 0.248 | 1.7225 | -| mdlm | entbudget_2.0 | 245.0 | 15.44 | 2.861 | 0.279 | 1.7225 | -| mdlm | entbudget_4.0 | 223.2 | 51.18 | 2.882 | 0.401 | 1.7225 | -| mdlm | entbudget_8.0 | 96.5 | 111.68 | 2.888 | 0.520 | 1.7225 | -| mdlm | fixedK_1 | 1.0 | 646.96 | 2.860 | 0.630 | 1.7225 | -| mdlm | fixedK_128 | 128.0 | 44.33 | 2.867 | 0.410 | 1.7225 | -| mdlm | fixedK_16 | 16.0 | 126.50 | 2.878 | 0.532 | 1.7225 | -| mdlm | fixedK_2 | 2.0 | 310.58 | 2.890 | 0.625 | 1.7225 | -| mdlm | fixedK_256 | 256.0 | 8.98 | 2.854 | 0.241 | 1.7225 | -| mdlm | fixedK_32 | 32.0 | 95.60 | 2.875 | 0.497 | 1.7225 | -| mdlm | fixedK_4 | 4.0 | 155.18 | 2.881 | 0.550 | 1.7225 | -| mdlm | fixedK_64 | 64.0 | 86.83 | 2.882 | 0.490 | 1.7225 | -| mdlm | fixedK_8 | 8.0 | 121.66 | 2.879 | 0.527 | 1.7225 | -| matched | conf_0.5 | 237.5 | 32.44 | 2.873 | 0.366 | 1.6946 | -| matched | conf_0.7 | 242.0 | 14.86 | 2.854 | 0.282 | 1.6946 | -| matched | conf_0.9 | 249.2 | 9.67 | 2.856 | 0.248 | 1.6946 | -| matched | conf_0.95 | 251.2 | 8.65 | 2.855 | 0.241 | 1.6946 | -| matched | conf_0.99 | 254.8 | 8.34 | 2.853 | 0.236 | 1.6946 | -| matched | entbudget_0.05 | 256.0 | 8.39 | 2.857 | 0.241 | 1.6946 | -| matched | entbudget_0.1 | 256.0 | 8.50 | 2.855 | 0.238 | 1.6946 | -| matched | entbudget_0.25 | 255.0 | 8.72 | 2.857 | 0.243 | 1.6946 | -| matched | entbudget_0.5 | 253.2 | 9.00 | 2.863 | 0.247 | 1.6946 | -| matched | entbudget_1.0 | 251.5 | 10.02 | 2.861 | 0.255 | 1.6946 | -| matched | entbudget_2.0 | 247.2 | 14.98 | 2.862 | 0.284 | 1.6946 | -| matched | entbudget_4.0 | 235.2 | 47.92 | 2.885 | 0.403 | 1.6946 | -| matched | entbudget_8.0 | 98.0 | 104.65 | 2.895 | 0.526 | 1.6946 | -| matched | fixedK_1 | 1.0 | 651.67 | 2.860 | 0.631 | 1.6946 | -| matched | fixedK_128 | 128.0 | 42.38 | 2.869 | 0.407 | 1.6946 | -| matched | fixedK_16 | 16.0 | 94.52 | 2.868 | 0.502 | 1.6946 | -| matched | fixedK_2 | 2.0 | 295.79 | 2.881 | 0.620 | 1.6946 | -| matched | fixedK_256 | 256.0 | 8.19 | 2.852 | 0.235 | 1.6946 | -| matched | fixedK_32 | 32.0 | 93.33 | 2.882 | 0.510 | 1.6946 | -| matched | fixedK_4 | 4.0 | 146.64 | 2.880 | 0.558 | 1.6946 | -| matched | fixedK_64 | 64.0 | 76.89 | 2.884 | 0.484 | 1.6946 | -| matched | fixedK_8 | 8.0 | 123.33 | 2.876 | 0.540 | 1.6946 | - -## Table 9 — Did the baseline fail to parallelise, or fail to learn? - -A 20-digit answer occupies 21 slots, so the standard 16-pass grid is not a sequential decode. Re-decoded at one token per pass, the two explanations separate. - -| digits | mode | @1 pass | fully sequential | reading | -|---|---|---|---|---| -| 20 | full | 99.8 | 85.7 | learned it and parallelises | -| 20 | mdlm | 0.0 | 0.0 | never learned the task | -| 24 | full | 33.7 | 33.3 | learned it and parallelises | -| 24 | mdlm | 0.0 | 0.0 | never learned the task | +--- + +## The claim + +Parallel decoding in masked diffusion loses accuracy for two reasons that need +**opposite** fixes, and the literature treats them as one. + +| | failure mode | fixable by | +|---|---|---| +| **(i)** | the fully-masked state is never trained — a *K*-pass decode starts at `t = 1`, and continuous `t ~ U(0,1)` assigns that state probability **zero** | **training only** | +| **(ii)** | committed positions carry real mutual information — parallel decoding samples the product of marginals when the truth is the joint | **inference only** | +Prior work applies inference-time fixes to both, which is why those methods +flatline on arithmetic, where (ii) is identically zero. +## The bound + +For a data distribution `P` uniform on a family `F`, let `V*` be the one-pass +validity achievable from the **true** marginals. Then + +``` +V* ≥ 2^(−TC), TC = D_KL( P ‖ ∏ᵢ pᵢ ), equality iff ∏ᵢ pᵢ is constant on F +``` + +**Every bit of total correlation among jointly committed variables at most halves +the one-pass success probability.** + +Verified by exhaustive enumeration over graph families, with the enumerator itself +checked against Cayley's formula (spanning trees at n = 5, 6, 7 → 125, 1296, +16807): + +| family | TC (bits) | V* | 2^(−TC) | ratio | +|---|---|---|---|---| +| bipartite (n=7) | 0.026 | 98.2816% | 98.1817% | 1.001 | +| tree (n=6) | 3.435 | **9.2488%** | **9.2488%** | **1.00000** | +| matching (n=6) | 6.922 | **0.8246%** | **0.8246%** | **1.00000** | +| 2-regular (n=7) | 10.423 | **0.0728%** | **0.0728%** | **1.00000** | + +The inequality holds in every case; equality holds to five decimals on exactly +the fixed-edge-count families, and fails precisely where edge counts vary — which +is what the Jensen step requires. + +## Finding 1 — train the schedule you intend to decode at + +| task | baseline | ours | +|---|---|---| +| 20-digit addition, 1 pass | **0.00%** (also 0.00% at **2× compute**, both seeds) | **99.9%** (3/3 seeds) | +| text8, NFE 16 | 116.88 gen-ppl | **92.74** (−20.7%) | +| text8, validation BPC | 1.7292 | **1.7010** | + +The text8 gain appears in generative quality **and** likelihood simultaneously, +so it is not bought by trading one for the other. + +A 48-run ablation isolates the mechanism: the `t = 1` term alone removes **49%** +of the parallelism penalty, against 17% for reshaping the rest of the schedule. + +## Finding 2 — one-pass decoding needs absolute position information + +At `t = 1` every input token is `[MASK]`, so the sequence is constant. With a +relative encoding the attention score depends only on `(i − j)`, and identical +value vectors give identical outputs at every position. **Position-dependent +marginals are inexpressible at any training budget.** + +| condition | learned (APE) | parameter-free (sinusoidal) | +|---|---|---| +| marginals **vary** | **+88.037 pts** | **+88.074 pts** | +| marginals **constant** (control) | **−0.024 pts** | **+0.000 pts** | + +A ~3,700× dissociation. The parameter-free variant matching the learned one rules +out capacity; the control rules out "absolute is simply better". Predicted 5/5 +before measurement, and the treated arm lands at **100.3% of `V*`** — at the +bound, not merely higher. + +## Benchmark: SATNet Sudoku + +9x9 Sudoku, SATNet's conventional 9,000/1,000 split, **board accuracy** = all 81 +cells correct. Published number: **98.3%** (Wang et al., ICML 2019). + +Un-augmented arm, which is the only like-for-like comparison: + +| denoising passes | board accuracy | +|---|---| +| **1** | **87.85%** | +| **2** | **98.65%** | +| 8 | 99.70% | +| **32** | **100.00%** | + +Two passes exceed the published number; 32 passes solve **all 1,000** test +puzzles exactly. One pass — the entire grid committed at once — solves 87.85%. + +Leakage audited first: 0/1000 test puzzles in training, 0/1000 test solutions in +training, and 0/1000 test solutions matching a training solution even up to digit +relabelling. All 10,000 solutions are distinct. + +**Scoped honestly:** this comes from the *plain MDLM baseline* with fixed-`K` +decoding. Neither finding above contributes — the standard formulation already +saturates this benchmark. It is a result for masked diffusion on SATNet Sudoku, +not evidence for our method. + +## Negative results, reported + +- **Any-budget conditioning fails.** Sampling the decoding budget during training + is worse than the baseline at every budget on graphs (−4.2 at K=8, −31.7 at + K=15) and destroys 20-digit addition outright (0.0% vs 99.9%), because it + dilutes `t = 1` training 4.5×. +- **The entropy budget is mixed** — competitive on graphs, loses to fixed-`K` on + text8. +- **Half the QM9 gap was our own representation.** Giving the baseline the atom + count that DiGress samples first lifts it from 77.5% to 89.1%; the gap narrows + from 22.2 to 10.7 points. +- **16-digit addition is unsettled** — a compute-matched baseline reaches 100% on + one seed of two. +- **Our Sudoku prediction was refuted by our own framework.** We registered that + entropy-budgeted decoding would beat fixed-`K` on Sudoku by the largest margin + in the project. It loses on every arm by 0.35–1.40 points. The prediction + contradicted the bound: Sudoku has a unique answer, so `TC = 0` and there is + nothing for a mode-(ii) fix to repair. + +## Audit + +`src/audit.py`: 46 assertions across 11 sections, 0 failures. It caught two live +bugs that would each have produced confidently wrong numbers — a sampler emitting +the absorbing state so decoding silently stalled, and an aromatic-bond encoding +that made round-trip validity on *real* molecules 92.65% instead of 100%. diff --git a/figures/fig1_addition_frontier.png b/figures/fig1_addition_frontier.png index 7c8b845bad40e78a69b8a8cbf19eb57b9555d519..4acd4aedf5c1e60fe348f4f50d0c4bf3a77f29f9 100644 Binary files a/figures/fig1_addition_frontier.png and b/figures/fig1_addition_frontier.png differ diff --git a/figures/fig2_accuracy_vs_passes.png b/figures/fig2_accuracy_vs_passes.png index 3fcd50b84b301de4522b36ba2643981ade66ebce..7e1517ff2046549ad259c58192573e6b35cf4760 100644 Binary files a/figures/fig2_accuracy_vs_passes.png and b/figures/fig2_accuracy_vs_passes.png differ diff --git a/figures/fig4_text8_quality_vs_compute.png b/figures/fig4_text8_quality_vs_compute.png index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b07b7024dfcba304f7a19fb450a27ab2783f1c0b 100644 --- a/figures/fig4_text8_quality_vs_compute.png +++ b/figures/fig4_text8_quality_vs_compute.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:012c029f5ca543aa8821abc3daa0ae508038032bd62b71e31fdf890ef8a9f7c9 +size 109606 diff --git a/figures/graph_ceiling.json b/figures/graph_ceiling.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..c98b71333c382d3f42fbec486517a38f6566afd6 100644 --- a/figures/graph_ceiling.json +++ b/figures/graph_ceiling.json @@ -0,0 +1,23 @@ +{ + "2regular/mdlm": { + "V": 0.002889476997120002, + "p": 0.003173828125, + "lo": 0.0021668792659426802, + "hi": 0.004646527181541615, + "verdict": "at the limit" + }, + "matching/anybudget": { + "V": 0.008246337208320003, + "p": 0.008544921875, + "lo": 0.006769272649384511, + "hi": 0.010781286160829684, + "verdict": "at the limit" + }, + "matching/mdlm": { + "V": 0.008246337208320003, + "p": 0.0076904296875, + "lo": 0.006015871984354349, + "hi": 0.009826503495425543, + "verdict": "at the limit" + } +} \ No newline at end of file diff --git a/figures/summary.md b/figures/summary.md index 7f8615511c00beed4fc6ceccad93dfcb59f2b525..d7b0cb48bdcba599da49c3a560a88efa95a33c9e 100644 --- a/figures/summary.md +++ b/figures/summary.md @@ -173,4 +173,62 @@ Real text8 reference — ppl 2.86, entropy 2.855, distinct-4 0.177 | matched | entbudget_1.0 | 251.5 | 10.02 | 2.861 | 0.255 | 1.6946 | | matched | entbudget_2.0 | 247.2 | 14.98 | 2.862 | 0.284 | 1.6946 | | matched | entbudget_4.0 | 235.2 | 47.92 | 2.885 | 0.403 | 1.6946 | -| matched \ No newline at end of file +| matched | entbudget_8.0 | 98.0 | 104.65 | 2.895 | 0.526 | 1.6946 | +| matched | fixedK_1 | 1.0 | 651.67 | 2.860 | 0.631 | 1.6946 | +| matched | fixedK_128 | 128.0 | 42.38 | 2.869 | 0.407 | 1.6946 | +| matched | fixedK_16 | 16.0 | 94.52 | 2.868 | 0.502 | 1.6946 | +| matched | fixedK_2 | 2.0 | 295.79 | 2.881 | 0.620 | 1.6946 | +| matched | fixedK_256 | 256.0 | 8.19 | 2.852 | 0.235 | 1.6946 | +| matched | fixedK_32 | 32.0 | 93.33 | 2.882 | 0.510 | 1.6946 | +| matched | fixedK_4 | 4.0 | 146.64 | 2.880 | 0.558 | 1.6946 | +| matched | fixedK_64 | 64.0 | 76.89 | 2.884 | 0.484 | 1.6946 | +| matched | fixedK_8 | 8.0 | 123.33 | 2.876 | 0.540 | 1.6946 | + +## Table 9b — Any-budget conditioning vs a fixed budget + +20-digit addition, one configuration. A fixed-K method wins at the budget it was trained for and loses away from it; the question is whether conditioning on the budget removes that trade. + +| training | @1 pass | @sequential | spread | +|---|---|---|---| +| MDLM baseline | 0.0 | 0.0 | 0.0 | +| fixed-K (K=1) | 99.9 | 85.8 | 14.2 | +| any-budget (no conditioning) | 0.2 | 0.3 | 0.1 | +| any-budget (conditioned) | 0.0 | 0.0 | 0.0 | + +## Table 9 — Did the baseline fail to parallelise, or fail to learn? + +A 20-digit answer occupies 21 slots, so the standard 16-pass grid is not a sequential decode. Re-decoded at one token per pass, the two explanations separate. + +| digits | mode | @1 pass | fully sequential | reading | +|---|---|---|---|---| +| 20 | full | 99.8 | 85.7 | learned it and parallelises | +| 20 | mdlm | 0.0 | 0.0 | never learned the task | +| 24 | full | 33.7 | 33.3 | learned it and parallelises | +| 24 | mdlm | 0.0 | 0.0 | never learned the task | + +## Table 10 — Structured graphs: measured against an exact ceiling + +V* is the one-pass validity achievable from the TRUE marginals, so it bounds any model that reproduces the data distribution. Where the interval contains V* the residual failure is information-theoretic and untrainable; where it falls short, the shortfall is trainable. + +| family | TC (bits) | V* | achieved @1 pass | reading | +|---|---|---|---|---| +| 2regular | 8.43 | 0.29% | 0.32% | at the ceiling — untrainable | +| matching | 6.92 | 0.82% | 0.77% | at the ceiling — untrainable | +| tree | 3.43 | 9.25% | 9.20% | at the ceiling — untrainable | +| bipartite | 0.11 | 93.63% | 5.88% | below ceiling — see Table 11 | + +## Table 11 — One-pass decoding needs ABSOLUTE position information + +At t = 1 every input token is [MASK], so the sequence is constant and a relative encoding gives every position the same output. Position-dependent marginals are then inexpressible at any training budget. Matching is the control: its marginals are identical at every slot, so absolute information should buy nothing there. + +| family | marginals | encoding | @1 pass | % of V* | +|---|---|---|---|---| +| bipartite | **vary** | rope | 5.87% | 6.3% | +| bipartite | **vary** | ape | 93.91% | 100.3% | +| bipartite | **vary** | sin | 93.95% | 100.3% | +| matching | constant | rope | 0.78% | 94.7% | +| matching | constant | ape | 0.76% | 91.8% | + +- absolute − relative on **bipartite**: **+88.037** points +- absolute − relative on **matching**: **-0.024** points + diff --git a/results/evallm.json b/results/evallm.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..9c501ae2d10c3d345c05db68726bfa890f54affd 100644 --- a/results/evallm.json +++ b/results/evallm.json @@ -0,0 +1,17 @@ +{ + "args": { + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/text8", + "seq_len": 256, + "d": 512, + "layers": 8, + "heads": 8, + "bs": 128, + "steps": 50000, + "lr": 0.0003, + "warmup": 2000, + "seed": 0, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/evallm" + }, + "test_bpc": 1.5393671429136824, + "vocab": 28 +} \ No newline at end of file diff --git a/results/g14-d12-L6-s0.json b/results/g14-d12-L6-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..539523aabd28590eaa5287c6b050f2b8ecdbf83e 100644 --- a/results/g14-d12-L6-s0.json +++ b/results/g14-d12-L6-s0.json @@ -0,0 +1,39 @@ +{ + "args": { + "mode": "full", + "K_max": 16, + "K": 4, + "lam": 1.0, + "digits": 12, + "seq_len": 40, + "n_train": 800000, + "n_eval": 400, + "eval_bs": 100, + "d": 384, + "layers": 6, + "heads": 6, + "pe": "rope", + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 0, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d12-L6-s0" + }, + "accuracy_by_passes": { + "1": 1.0, + "2": 1.0, + "3": 1.0, + "4": 1.0, + "6": 1.0, + "8": 1.0, + "12": 1.0, + "16": 1.0 + }, + "minutes": 17.56450060606003, + "compute": { + "params": 10649088, + "train_tokens": 307200000, + "fwd_per_step": 2 + } +} \ No newline at end of file diff --git a/results/g14-d12-L6-s1.json b/results/g14-d12-L6-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..aefd931bcb73a11adf23a38249f653df185b3908 100644 --- a/results/g14-d12-L6-s1.json +++ b/results/g14-d12-L6-s1.json @@ -0,0 +1,39 @@ +{ + "args": { + "mode": "full", + "K_max": 16, + "K": 4, + "lam": 1.0, + "digits": 12, + "seq_len": 40, + "n_train": 800000, + "n_eval": 400, + "eval_bs": 100, + "d": 384, + "layers": 6, + "heads": 6, + "pe": "rope", + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 1, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d12-L6-s1" + }, + "accuracy_by_passes": { + "1": 1.0, + "2": 1.0, + "3": 1.0, + "4": 1.0, + "6": 1.0, + "8": 1.0, + "12": 1.0, + "16": 1.0 + }, + "minutes": 17.586296463012694, + "compute": { + "params": 10649088, + "train_tokens": 307200000, + "fwd_per_step": 2 + } +} \ No newline at end of file diff --git a/results/g14-d16-L2-s0.json b/results/g14-d16-L2-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..35f97c3707c282096755261c990c44b9ac5ed8c7 100644 --- a/results/g14-d16-L2-s0.json +++ b/results/g14-d16-L2-s0.json @@ -0,0 +1,39 @@ +{ + "args": { + "mode": "full", + "K_max": 16, + "K": 4, + "lam": 1.0, + "digits": 16, + "seq_len": 52, + "n_train": 800000, + "n_eval": 400, + "eval_bs": 100, + "d": 384, + "layers": 2, + "heads": 6, + "pe": "rope", + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 0, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d16-L2-s0" + }, + "accuracy_by_passes": { + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "6": 0.0, + "8": 0.0, + "12": 0.0, + "16": 0.0 + }, + "minutes": 7.497824923197428, + "compute": { + "params": 3557376, + "train_tokens": 399360000, + "fwd_per_step": 2 + } +} \ No newline at end of file diff --git a/results/g14-d16-L2-s1.json b/results/g14-d16-L2-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..4bac95bd649ba61a56481f98b55f47984ee7fcfc 100644 --- a/results/g14-d16-L2-s1.json +++ b/results/g14-d16-L2-s1.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "full", + "budget_bins": 10, + "K_max": 16, + "K": 4, + "lam": 1.0, + "digits": 16, + "seq_len": 52, + "n_train": 800000, + "n_eval": 400, + "eval_bs": 100, + "d": 384, + "layers": 2, + "heads": 6, + "pe": "rope", + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 1, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d16-L2-s1" + }, + "accuracy_by_passes": { + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "6": 0.0, + "8": 0.0, + "12": 0.0, + "16": 0.0, + "17": 0.0 + }, + "minutes": 7.427204525470733, + "compute": { + "params": 3557376, + "train_tokens": 399360000, + "fwd_per_step": 2 + } +} \ No newline at end of file diff --git a/results/g14-d16-L3-s0.json b/results/g14-d16-L3-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b1938a9a77e22078c190856d2f198f4e0cec2f44 100644 --- a/results/g14-d16-L3-s0.json +++ b/results/g14-d16-L3-s0.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "full", + "budget_bins": 10, + "K_max": 16, + "K": 4, + "lam": 1.0, + "digits": 16, + "seq_len": 52, + "n_train": 800000, + "n_eval": 400, + "eval_bs": 100, + "d": 384, + "layers": 3, + "heads": 6, + "pe": "rope", + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 0, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d16-L3-s0" + }, + "accuracy_by_passes": { + "1": 0.9975, + "2": 0.6275, + "3": 0.4825, + "4": 0.44499999999999995, + "6": 0.3825, + "8": 0.35, + "12": 0.3425, + "16": 0.315, + "17": 0.315 + }, + "minutes": 9.988970235983532, + "compute": { + "params": 5330304, + "train_tokens": 399360000, + "fwd_per_step": 2 + } +} \ No newline at end of file diff --git a/results/g14-d16-L3-s1.json b/results/g14-d16-L3-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..77b0bed142a14b41ef3665f21da1b9c2d0f72d06 100644 --- a/results/g14-d16-L3-s1.json +++ b/results/g14-d16-L3-s1.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "full", + "budget_bins": 10, + "K_max": 16, + "K": 4, + "lam": 1.0, + "digits": 16, + "seq_len": 52, + "n_train": 800000, + "n_eval": 400, + "eval_bs": 100, + "d": 384, + "layers": 3, + "heads": 6, + "pe": "rope", + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 1, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d16-L3-s1" + }, + "accuracy_by_passes": { + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "6": 0.0, + "8": 0.0, + "12": 0.0, + "16": 0.0, + "17": 0.0 + }, + "minutes": 10.037210019429525, + "compute": { + "params": 5330304, + "train_tokens": 399360000, + "fwd_per_step": 2 + } +} \ No newline at end of file diff --git a/results/g14-d8-L12-s0.json b/results/g14-d8-L12-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0ffe20e5f546b92ba14c4234274003cae1ef2026 100644 --- a/results/g14-d8-L12-s0.json +++ b/results/g14-d8-L12-s0.json @@ -0,0 +1,39 @@ +{ + "args": { + "mode": "full", + "K_max": 16, + "K": 4, + "lam": 1.0, + "digits": 8, + "seq_len": 28, + "n_train": 800000, + "n_eval": 400, + "eval_bs": 100, + "d": 384, + "layers": 12, + "heads": 6, + "pe": "rope", + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 0, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d8-L12-s0" + }, + "accuracy_by_passes": { + "1": 1.0, + "2": 1.0, + "3": 1.0, + "4": 1.0, + "6": 1.0, + "8": 1.0, + "12": 1.0, + "16": 1.0 + }, + "minutes": 31.55148322979609, + "compute": { + "params": 21286656, + "train_tokens": 215040000, + "fwd_per_step": 2 + } +} \ No newline at end of file diff --git a/results/g14-d8-L12-s1.json b/results/g14-d8-L12-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..505f7f08c39821e4754e49ce97d809b80095188b 100644 --- a/results/g14-d8-L12-s1.json +++ b/results/g14-d8-L12-s1.json @@ -0,0 +1,39 @@ +{ + "args": { + "mode": "full", + "K_max": 16, + "K": 4, + "lam": 1.0, + "digits": 8, + "seq_len": 28, + "n_train": 800000, + "n_eval": 400, + "eval_bs": 100, + "d": 384, + "layers": 12, + "heads": 6, + "pe": "rope", + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 1, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d8-L12-s1" + }, + "accuracy_by_passes": { + "1": 1.0, + "2": 1.0, + "3": 1.0, + "4": 1.0, + "6": 1.0, + "8": 1.0, + "12": 1.0, + "16": 1.0 + }, + "minutes": 32.08436821699142, + "compute": { + "params": 21286656, + "train_tokens": 215040000, + "fwd_per_step": 2 + } +} \ No newline at end of file diff --git a/results/g15-finnish-distill-s0.json b/results/g15-finnish-distill-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5a380579a825582db655b54b4fa5e8eb449998aa 100644 --- a/results/g15-finnish-distill-s0.json +++ b/results/g15-finnish-distill-s0.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "finnish", + "size": "medium", + "mode": "distill", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 0, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-finnish-distill-s0" + }, + "accuracy_by_passes": { + "1": 0.48252918956043955, + "2": 0.4772493131868132, + "4": 0.5012985061813187, + "8": 0.5235770089285714, + "16": 0.5343191964285714, + "32": 0.5352957589285714, + "ar": 0.05587869162087912 + }, + "minutes": 4.343921728928884, + "compute": { + "params": 4769792, + "answer_len_mean": 15.12968635559082 + } +} \ No newline at end of file diff --git a/results/g15-finnish-distill-s1.json b/results/g15-finnish-distill-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ad13bdbd723bff91bf5044ecfc60d1084485d7ba 100644 --- a/results/g15-finnish-distill-s1.json +++ b/results/g15-finnish-distill-s1.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "finnish", + "size": "medium", + "mode": "distill", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 1, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-finnish-distill-s1" + }, + "accuracy_by_passes": { + "1": 0.5063637534340659, + "2": 0.5069754464285714, + "4": 0.5237594436813187, + "8": 0.5517148866758241, + "16": 0.5556211366758241, + "32": 0.5556211366758241, + "ar": 0.05587869162087912 + }, + "minutes": 4.371706000963846, + "compute": { + "params": 4769792, + "answer_len_mean": 15.12968635559082 + } +} \ No newline at end of file diff --git a/results/g15-finnish-matched-s0.json b/results/g15-finnish-matched-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..2b859ee0d4168e6b42d4c72932fd702642959131 100644 --- a/results/g15-finnish-matched-s0.json +++ b/results/g15-finnish-matched-s0.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "finnish", + "size": "medium", + "mode": "matched", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 0, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-finnish-matched-s0" + }, + "accuracy_by_passes": { + "1": 0.5032194368131868, + "2": 0.5125879979395604, + "4": 0.5278803228021978, + "8": 0.5622746394230769, + "16": 0.5710637019230769, + "32": 0.5700871394230769, + "ar": 0.05685525412087912 + }, + "minutes": 2.444117879867554, + "compute": { + "params": 4769792, + "answer_len_mean": 15.12968635559082 + } +} \ No newline at end of file diff --git a/results/g15-finnish-matched-s1.json b/results/g15-finnish-matched-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..6f6898a01317be8706d83ce36b6f9744824126da 100644 --- a/results/g15-finnish-matched-s1.json +++ b/results/g15-finnish-matched-s1.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "finnish", + "size": "medium", + "mode": "matched", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 1, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-finnish-matched-s1" + }, + "accuracy_by_passes": { + "1": 0.535875257554945, + "2": 0.551747081043956, + "4": 0.5798849587912087, + "8": 0.5892535199175823, + "16": 0.6087847699175823, + "32": 0.6087847699175823, + "ar": 0.05783181662087912 + }, + "minutes": 2.467513418197632, + "compute": { + "params": 4769792, + "answer_len_mean": 15.12968635559082 + } +} \ No newline at end of file diff --git a/results/g15-georgian-distill-s0.json b/results/g15-georgian-distill-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..943914e2fb6626e05bc2621186ec11983ba4d86d 100644 --- a/results/g15-georgian-distill-s0.json +++ b/results/g15-georgian-distill-s0.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "georgian", + "size": "medium", + "mode": "distill", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 0, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-georgian-distill-s0" + }, + "accuracy_by_passes": { + "1": 0.9483173076923077, + "2": 0.9461388221153846, + "4": 0.9432091346153846, + "8": 0.9453876201923077, + "16": 0.9453876201923077, + "32": 0.9453876201923077, + "ar": 0.1258263221153846 + }, + "minutes": 4.3569332202275595, + "compute": { + "params": 4766208, + "answer_len_mean": 9.932000160217285 + } +} \ No newline at end of file diff --git a/results/g15-georgian-distill-s1.json b/results/g15-georgian-distill-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0c204ff1b0f7e80702f3eafd405b337ea53bb8ad 100644 --- a/results/g15-georgian-distill-s1.json +++ b/results/g15-georgian-distill-s1.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "georgian", + "size": "medium", + "mode": "distill", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 1, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-georgian-distill-s1" + }, + "accuracy_by_passes": { + "1": 0.9385516826923077, + "2": 0.9385516826923077, + "4": 0.9395282451923077, + "8": 0.9375751201923077, + "16": 0.9375751201923077, + "32": 0.9375751201923077, + "ar": 0.12289663461538461 + }, + "minutes": 4.348151512940725, + "compute": { + "params": 4766208, + "answer_len_mean": 9.932000160217285 + } +} \ No newline at end of file diff --git a/results/g15-georgian-matched-s0.json b/results/g15-georgian-matched-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..468a788c1d8e90d767109fa860f3234cb6582693 100644 --- a/results/g15-georgian-matched-s0.json +++ b/results/g15-georgian-matched-s0.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "georgian", + "size": "medium", + "mode": "matched", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 0, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-georgian-matched-s0" + }, + "accuracy_by_passes": { + "1": 0.9441856971153846, + "2": 0.9471153846153846, + "4": 0.9510216346153846, + "8": 0.9500450721153846, + "16": 0.9490685096153846, + "32": 0.9490685096153846, + "ar": 0.1258263221153846 + }, + "minutes": 2.449139161904653, + "compute": { + "params": 4766208, + "answer_len_mean": 9.932000160217285 + } +} \ No newline at end of file diff --git a/results/g15-georgian-matched-s1.json b/results/g15-georgian-matched-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ce829cc337edbad0c32dffb91fa20b01fd877594 100644 --- a/results/g15-georgian-matched-s1.json +++ b/results/g15-georgian-matched-s1.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "georgian", + "size": "medium", + "mode": "matched", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 1, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-georgian-matched-s1" + }, + "accuracy_by_passes": { + "1": 0.9412560096153846, + "2": 0.9402794471153846, + "4": 0.9390775240384616, + "8": 0.9378756009615384, + "16": 0.9378756009615384, + "32": 0.9378756009615384, + "ar": 0.12364783653846154 + }, + "minutes": 2.45454421043396, + "compute": { + "params": 4766208, + "answer_len_mean": 9.932000160217285 + } +} \ No newline at end of file diff --git a/results/g15-german-distill-s1.json b/results/g15-german-distill-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..df551dd4441d3dcf90fe52b70ce7b4798f0dbf19 100644 --- a/results/g15-german-distill-s1.json +++ b/results/g15-german-distill-s1.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "german", + "size": "medium", + "mode": "distill", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 1, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-german-distill-s1" + }, + "accuracy_by_passes": { + "1": 0.77875, + "2": 0.7770703125, + "4": 0.7834765625, + "8": 0.7815234375, + "16": 0.7854296875, + "32": 0.7854296875, + "ar": 0.0795703125 + }, + "minutes": 4.314774318536123, + "compute": { + "params": 4776448, + "answer_len_mean": 10.743975639343262 + } +} \ No newline at end of file diff --git a/results/g15-navajo-distill-s0.json b/results/g15-navajo-distill-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..03013f4bd0125319f48e85369bbf84e4d2523e28 100644 --- a/results/g15-navajo-distill-s0.json +++ b/results/g15-navajo-distill-s0.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "navajo", + "size": "medium", + "mode": "distill", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 0, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-navajo-distill-s0" + }, + "accuracy_by_passes": { + "1": 0.33743990384615385, + "2": 0.3449519230769231, + "4": 0.36763822115384615, + "8": 0.3858924278846154, + "16": 0.3897986778846154, + "32": 0.3897986778846154, + "ar": 0.07474459134615385 + }, + "minutes": 4.2984689474105835, + "compute": { + "params": 4765696, + "answer_len_mean": 10.005999565124512 + } +} \ No newline at end of file diff --git a/results/g15-navajo-distill-s1.json b/results/g15-navajo-distill-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..4494eb348a9041997c1901694fe5afa14c440c4f 100644 --- a/results/g15-navajo-distill-s1.json +++ b/results/g15-navajo-distill-s1.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "navajo", + "size": "medium", + "mode": "distill", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 1, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-navajo-distill-s1" + }, + "accuracy_by_passes": { + "1": 0.24166165865384615, + "2": 0.2635967548076923, + "4": 0.2813251201923077, + "8": 0.3040114182692308, + "16": 0.3079176682692308, + "32": 0.3079176682692308, + "ar": 0.06377704326923077 + }, + "minutes": 4.306906080245971, + "compute": { + "params": 4765696, + "answer_len_mean": 10.005999565124512 + } +} \ No newline at end of file diff --git a/results/g15-navajo-matched-s0.json b/results/g15-navajo-matched-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..29c93f4c446207f04f47c27ceaeab72a11e29fe6 100644 --- a/results/g15-navajo-matched-s0.json +++ b/results/g15-navajo-matched-s0.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "navajo", + "size": "medium", + "mode": "matched", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 0, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-navajo-matched-s0" + }, + "accuracy_by_passes": { + "1": 0.310546875, + "2": 0.3400691105769231, + "4": 0.36275540865384615, + "8": 0.3937049278846154, + "16": 0.3927283653846154, + "32": 0.3927283653846154, + "ar": 0.07767427884615385 + }, + "minutes": 2.491582691669464, + "compute": { + "params": 4765696, + "answer_len_mean": 10.005999565124512 + } +} \ No newline at end of file diff --git a/results/g15-navajo-matched-s1.json b/results/g15-navajo-matched-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b48abaef107dc78e2a6e601c438baf5309e4ed05 100644 --- a/results/g15-navajo-matched-s1.json +++ b/results/g15-navajo-matched-s1.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "navajo", + "size": "medium", + "mode": "matched", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 1, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-navajo-matched-s1" + }, + "accuracy_by_passes": { + "1": 0.2626201923076923, + "2": 0.2931189903846154, + "4": 0.31212439903846156, + "8": 0.3390925480769231, + "16": 0.3400691105769231, + "32": 0.3400691105769231, + "ar": 0.0645282451923077 + }, + "minutes": 2.493058661619822, + "compute": { + "params": 4765696, + "answer_len_mean": 10.005999565124512 + } +} \ No newline at end of file diff --git a/results/g15-russian-distill-s0.json b/results/g15-russian-distill-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..269fec4c902c8c2d60b3b4d6d3d8f1a75f99cfac 100644 --- a/results/g15-russian-distill-s0.json +++ b/results/g15-russian-distill-s0.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "russian", + "size": "medium", + "mode": "distill", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 0, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-russian-distill-s0" + }, + "accuracy_by_passes": { + "1": 0.7351542771464646, + "2": 0.7225773358585859, + "4": 0.7267696496212122, + "8": 0.7284367108585859, + "16": 0.7303898358585859, + "32": 0.7303898358585859, + "ar": 0.035896070075757576 + }, + "minutes": 4.300656799475352, + "compute": { + "params": 4770304, + "answer_len_mean": 11.276381492614746 + } +} \ No newline at end of file diff --git a/results/g15-russian-distill-s1.json b/results/g15-russian-distill-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..baa31e1e2c81651f3119ec5ce8956fc5d988ad82 100644 --- a/results/g15-russian-distill-s1.json +++ b/results/g15-russian-distill-s1.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "russian", + "size": "medium", + "mode": "distill", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 1, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-russian-distill-s1" + }, + "accuracy_by_passes": { + "1": 0.7418718434343434, + "2": 0.7421579071969697, + "4": 0.7457780934343434, + "8": 0.7460641571969697, + "16": 0.745373658459596, + "32": 0.745373658459596, + "ar": 0.03813525883838384 + }, + "minutes": 4.345847002665201, + "compute": { + "params": 4770304, + "answer_len_mean": 11.276381492614746 + } +} \ No newline at end of file diff --git a/results/g15-russian-matched-s0.json b/results/g15-russian-matched-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..3db426d533dc7c0b6b724ad4a1c010b89fe69c9b 100644 --- a/results/g15-russian-matched-s0.json +++ b/results/g15-russian-matched-s0.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "russian", + "size": "medium", + "mode": "matched", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 0, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-russian-matched-s0" + }, + "accuracy_by_passes": { + "1": 0.7305575284090909, + "2": 0.7367029671717171, + "4": 0.7263652146464646, + "8": 0.7283183396464646, + "16": 0.7332011521464646, + "32": 0.7332011521464646, + "ar": 0.032966382575757576 + }, + "minutes": 2.497041336695353, + "compute": { + "params": 4770304, + "answer_len_mean": 11.276381492614746 + } +} \ No newline at end of file diff --git a/results/g15-russian-matched-s1.json b/results/g15-russian-matched-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..aed2206b8a385e8645e295f0d55af5df8800e342 100644 --- a/results/g15-russian-matched-s1.json +++ b/results/g15-russian-matched-s1.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "russian", + "size": "medium", + "mode": "matched", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 1, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-russian-matched-s1" + }, + "accuracy_by_passes": { + "1": 0.7626657196969697, + "2": 0.759045533459596, + "4": 0.7642144097222222, + "8": 0.760022095959596, + "16": 0.761975220959596, + "32": 0.761975220959596, + "ar": 0.03911182133838384 + }, + "minutes": 2.4522072474161782, + "compute": { + "params": 4770304, + "answer_len_mean": 11.276381492614746 + } +} \ No newline at end of file diff --git a/results/g15-turkish-distill-s0.json b/results/g15-turkish-distill-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8031668d78b7ebde477f6136652700573471a976 100644 --- a/results/g15-turkish-distill-s0.json +++ b/results/g15-turkish-distill-s0.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "turkish", + "size": "medium", + "mode": "distill", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 0, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-turkish-distill-s0" + }, + "accuracy_by_passes": { + "1": 0.7729124663978495, + "2": 0.7768187163978495, + "4": 0.7752331149193549, + "8": 0.783171622983871, + "16": 0.789030997983871, + "32": 0.789030997983871, + "ar": 0.09997689852150538 + }, + "minutes": 4.311886747678121, + "compute": { + "params": 4772864, + "answer_len_mean": 13.684530258178711 + } +} \ No newline at end of file diff --git a/results/g15-turkish-distill-s1.json b/results/g15-turkish-distill-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..70ed52491ceead7b526f8bc5851e934fbefe10f4 100644 --- a/results/g15-turkish-distill-s1.json +++ b/results/g15-turkish-distill-s1.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "turkish", + "size": "medium", + "mode": "distill", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 1, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-turkish-distill-s1" + }, + "accuracy_by_passes": { + "1": 0.8306661626344086, + "2": 0.8190629200268817, + "4": 0.8260248655913979, + "8": 0.8345724126344086, + "16": 0.838846186155914, + "32": 0.838846186155914, + "ar": 0.099609375 + }, + "minutes": 4.282213552792867, + "compute": { + "params": 4772864, + "answer_len_mean": 13.684530258178711 + } +} \ No newline at end of file diff --git a/results/g15-turkish-matched-s0.json b/results/g15-turkish-matched-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..f01639f19762cbbd6799b08ae0f791f17026a996 100644 --- a/results/g15-turkish-matched-s0.json +++ b/results/g15-turkish-matched-s0.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "turkish", + "size": "medium", + "mode": "matched", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 0, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-turkish-matched-s0" + }, + "accuracy_by_passes": { + "1": 0.7699932795698925, + "2": 0.7765877016129032, + "4": 0.7892830141129032, + "8": 0.8050340221774194, + "16": 0.8093077956989247, + "32": 0.8093077956989247, + "ar": 0.09997689852150538 + }, + "minutes": 2.4571212331453958, + "compute": { + "params": 4772864, + "answer_len_mean": 13.684530258178711 + } +} \ No newline at end of file diff --git a/results/g15-turkish-matched-s1.json b/results/g15-turkish-matched-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..2b33ee8ad650715b181a8899c2bf790c8efe1194 100644 --- a/results/g15-turkish-matched-s1.json +++ b/results/g15-turkish-matched-s1.json @@ -0,0 +1,36 @@ +{ + "args": { + "lang": "turkish", + "size": "medium", + "mode": "matched", + "seq_len": 48, + "d": 256, + "layers": 6, + "heads": 8, + "pe": "rope", + "bs": 64, + "steps": 15000, + "lr": 0.0001, + "warmup": 1000, + "K": 4, + "lam": 1.0, + "seed": 1, + "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig", + "eval_bs": 128, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-turkish-matched-s1" + }, + "accuracy_by_passes": { + "1": 0.8041729670698925, + "2": 0.8061260920698925, + "4": 0.8206590221774194, + "8": 0.8376281081989247, + "16": 0.8379956317204301, + "32": 0.8379956317204301, + "ar": 0.10388314852150538 + }, + "minutes": 2.4274060289065043, + "compute": { + "params": 4772864, + "answer_len_mean": 13.684530258178711 + } +} \ No newline at end of file diff --git a/results/g17-d16-mdlm2x-s0.json b/results/g17-d16-mdlm2x-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..23de825f23ac34e333f3a5e04632770da61796cd 100644 --- a/results/g17-d16-mdlm2x-s0.json +++ b/results/g17-d16-mdlm2x-s0.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "mdlm", + "budget_bins": 10, + "K_max": 16, + "K": 8, + "lam": 1.0, + "digits": 16, + "seq_len": 52, + "n_train": 1500000, + "n_eval": 400, + "eval_bs": 100, + "d": 512, + "layers": 12, + "heads": 8, + "pe": "rope", + "bs": 128, + "steps": 300000, + "lr": 0.0001, + "warmup": 6000, + "seed": 0, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g17-d16-mdlm2x-s0" + }, + "accuracy_by_passes": { + "1": 1.0, + "2": 1.0, + "3": 1.0, + "4": 1.0, + "6": 1.0, + "8": 1.0, + "12": 1.0, + "16": 1.0, + "17": 1.0 + }, + "minutes": 103.74741363922755, + "compute": { + "params": 37819392, + "train_tokens": 1996800000, + "fwd_per_step": 1 + } +} \ No newline at end of file diff --git a/results/g17-d16-mdlm2x-s1.json b/results/g17-d16-mdlm2x-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..33ac52a5821c7979c2b5ea9eebed6f3f07eb02b2 100644 --- a/results/g17-d16-mdlm2x-s1.json +++ b/results/g17-d16-mdlm2x-s1.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "mdlm", + "budget_bins": 10, + "K_max": 16, + "K": 8, + "lam": 1.0, + "digits": 16, + "seq_len": 52, + "n_train": 1500000, + "n_eval": 400, + "eval_bs": 100, + "d": 512, + "layers": 12, + "heads": 8, + "pe": "rope", + "bs": 128, + "steps": 300000, + "lr": 0.0001, + "warmup": 6000, + "seed": 1, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g17-d16-mdlm2x-s1" + }, + "accuracy_by_passes": { + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "6": 0.0, + "8": 0.0, + "12": 0.0, + "16": 0.0, + "17": 0.0 + }, + "minutes": 104.48015933036804, + "compute": { + "params": 37819392, + "train_tokens": 1996800000, + "fwd_per_step": 1 + } +} \ No newline at end of file diff --git a/results/g17-d20-mdlm2x-s0.json b/results/g17-d20-mdlm2x-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..d97b4f6cca032fef47d2e8dfc2f0db0413b9bfc1 100644 --- a/results/g17-d20-mdlm2x-s0.json +++ b/results/g17-d20-mdlm2x-s0.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "mdlm", + "budget_bins": 10, + "K_max": 16, + "K": 8, + "lam": 1.0, + "digits": 20, + "seq_len": 64, + "n_train": 1500000, + "n_eval": 400, + "eval_bs": 100, + "d": 512, + "layers": 12, + "heads": 8, + "pe": "rope", + "bs": 128, + "steps": 300000, + "lr": 0.0001, + "warmup": 6000, + "seed": 0, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g17-d20-mdlm2x-s0" + }, + "accuracy_by_passes": { + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "6": 0.0, + "8": 0.0, + "12": 0.0, + "16": 0.0, + "21": 0.0 + }, + "minutes": 116.37444167931875, + "compute": { + "params": 37819392, + "train_tokens": 2457600000, + "fwd_per_step": 1 + } +} \ No newline at end of file diff --git a/results/g17-d20-mdlm2x-s1.json b/results/g17-d20-mdlm2x-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..c896ce4c06d02949ebb5645302a025f71bdcd9e8 100644 --- a/results/g17-d20-mdlm2x-s1.json +++ b/results/g17-d20-mdlm2x-s1.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "mdlm", + "budget_bins": 10, + "K_max": 16, + "K": 8, + "lam": 1.0, + "digits": 20, + "seq_len": 64, + "n_train": 1500000, + "n_eval": 400, + "eval_bs": 100, + "d": 512, + "layers": 12, + "heads": 8, + "pe": "rope", + "bs": 128, + "steps": 300000, + "lr": 0.0001, + "warmup": 6000, + "seed": 1, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g17-d20-mdlm2x-s1" + }, + "accuracy_by_passes": { + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "6": 0.0, + "8": 0.0, + "12": 0.0, + "16": 0.0, + "21": 0.0 + }, + "minutes": 117.45276789665222, + "compute": { + "params": 37819392, + "train_tokens": 2457600000, + "fwd_per_step": 1 + } +} \ No newline at end of file diff --git a/results/g18-d20-anybudget-s0.json b/results/g18-d20-anybudget-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..59c9dc047188fe70bd50bc7756e34d19446032e3 100644 --- a/results/g18-d20-anybudget-s0.json +++ b/results/g18-d20-anybudget-s0.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "anybudget", + "budget_bins": 10, + "K_max": 16, + "K": 8, + "lam": 1.0, + "digits": 20, + "seq_len": 64, + "n_train": 1500000, + "n_eval": 400, + "eval_bs": 100, + "d": 512, + "layers": 12, + "heads": 8, + "pe": "rope", + "bs": 128, + "steps": 150000, + "lr": 0.0001, + "warmup": 6000, + "seed": 0, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g18-d20-anybudget-s0" + }, + "accuracy_by_passes": { + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "6": 0.0, + "8": 0.0, + "12": 0.0, + "16": 0.0, + "21": 0.0 + }, + "minutes": 58.222007815043135, + "compute": { + "params": 37824512, + "train_tokens": 1228800000, + "fwd_per_step": 1 + } +} \ No newline at end of file diff --git a/results/g18-d20-anybudget-s1.json b/results/g18-d20-anybudget-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..10e9099c1feaa164292d884b19e982330c3b0499 100644 --- a/results/g18-d20-anybudget-s1.json +++ b/results/g18-d20-anybudget-s1.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "anybudget", + "budget_bins": 10, + "K_max": 16, + "K": 8, + "lam": 1.0, + "digits": 20, + "seq_len": 64, + "n_train": 1500000, + "n_eval": 400, + "eval_bs": 100, + "d": 512, + "layers": 12, + "heads": 8, + "pe": "rope", + "bs": 128, + "steps": 150000, + "lr": 0.0001, + "warmup": 6000, + "seed": 1, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g18-d20-anybudget-s1" + }, + "accuracy_by_passes": { + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "6": 0.0, + "8": 0.0, + "12": 0.0, + "16": 0.0, + "21": 0.0 + }, + "minutes": 58.380110347270964, + "compute": { + "params": 37824512, + "train_tokens": 1228800000, + "fwd_per_step": 1 + } +} \ No newline at end of file diff --git a/results/g18-d20-anybudget-s2.json b/results/g18-d20-anybudget-s2.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..afb30fbeede463a08aa4e5c85c7bf911ab612c23 100644 --- a/results/g18-d20-anybudget-s2.json +++ b/results/g18-d20-anybudget-s2.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "anybudget", + "budget_bins": 10, + "K_max": 16, + "K": 8, + "lam": 1.0, + "digits": 20, + "seq_len": 64, + "n_train": 1500000, + "n_eval": 400, + "eval_bs": 100, + "d": 512, + "layers": 12, + "heads": 8, + "pe": "rope", + "bs": 128, + "steps": 150000, + "lr": 0.0001, + "warmup": 6000, + "seed": 2, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g18-d20-anybudget-s2" + }, + "accuracy_by_passes": { + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "6": 0.0, + "8": 0.0, + "12": 0.0, + "16": 0.0, + "21": 0.0 + }, + "minutes": 58.75475962559382, + "compute": { + "params": 37824512, + "train_tokens": 1228800000, + "fwd_per_step": 1 + } +} \ No newline at end of file diff --git a/results/g18-d20-anybudget_nc-s0.json b/results/g18-d20-anybudget_nc-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..fd393d0109cf5f052967889f0121c3d647995b4a 100644 --- a/results/g18-d20-anybudget_nc-s0.json +++ b/results/g18-d20-anybudget_nc-s0.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "anybudget_nc", + "budget_bins": 10, + "K_max": 16, + "K": 8, + "lam": 1.0, + "digits": 20, + "seq_len": 64, + "n_train": 1500000, + "n_eval": 400, + "eval_bs": 100, + "d": 512, + "layers": 12, + "heads": 8, + "pe": "rope", + "bs": 128, + "steps": 150000, + "lr": 0.0001, + "warmup": 6000, + "seed": 0, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g18-d20-anybudget_nc-s0" + }, + "accuracy_by_passes": { + "1": 0.0075, + "2": 0.01, + "3": 0.01, + "4": 0.01, + "6": 0.005, + "8": 0.0125, + "12": 0.0075, + "16": 0.01, + "21": 0.01 + }, + "minutes": 58.74203004837036, + "compute": { + "params": 37819392, + "train_tokens": 1228800000, + "fwd_per_step": 1 + } +} \ No newline at end of file diff --git a/results/g18-d20-anybudget_nc-s1.json b/results/g18-d20-anybudget_nc-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..290b75f9ccb3e78cd777fa9982b5877039c6085b 100644 --- a/results/g18-d20-anybudget_nc-s1.json +++ b/results/g18-d20-anybudget_nc-s1.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "anybudget_nc", + "budget_bins": 10, + "K_max": 16, + "K": 8, + "lam": 1.0, + "digits": 20, + "seq_len": 64, + "n_train": 1500000, + "n_eval": 400, + "eval_bs": 100, + "d": 512, + "layers": 12, + "heads": 8, + "pe": "rope", + "bs": 128, + "steps": 150000, + "lr": 0.0001, + "warmup": 6000, + "seed": 1, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g18-d20-anybudget_nc-s1" + }, + "accuracy_by_passes": { + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "6": 0.0, + "8": 0.0, + "12": 0.0, + "16": 0.0, + "21": 0.0 + }, + "minutes": 58.15160437424978, + "compute": { + "params": 37819392, + "train_tokens": 1228800000, + "fwd_per_step": 1 + } +} \ No newline at end of file diff --git a/results/g18-d20-anybudget_nc-s2.json b/results/g18-d20-anybudget_nc-s2.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ca36cd638bcb123d83fd71e31af5ce9865150d25 100644 --- a/results/g18-d20-anybudget_nc-s2.json +++ b/results/g18-d20-anybudget_nc-s2.json @@ -0,0 +1,41 @@ +{ + "args": { + "mode": "anybudget_nc", + "budget_bins": 10, + "K_max": 16, + "K": 8, + "lam": 1.0, + "digits": 20, + "seq_len": 64, + "n_train": 1500000, + "n_eval": 400, + "eval_bs": 100, + "d": 512, + "layers": 12, + "heads": 8, + "pe": "rope", + "bs": 128, + "steps": 150000, + "lr": 0.0001, + "warmup": 6000, + "seed": 2, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g18-d20-anybudget_nc-s2" + }, + "accuracy_by_passes": { + "1": 0.0, + "2": 0.0, + "3": 0.0, + "4": 0.0, + "6": 0.0, + "8": 0.0, + "12": 0.0, + "16": 0.0, + "21": 0.0 + }, + "minutes": 58.52561570803324, + "compute": { + "params": 37819392, + "train_tokens": 1228800000, + "fwd_per_step": 1 + } +} \ No newline at end of file diff --git a/results/gr1-2regular-anybudget-s0.json b/results/gr1-2regular-anybudget-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..31f20a7f4c4dbbc4173549028775cc45bb409248 100644 --- a/results/gr1-2regular-anybudget-s0.json +++ b/results/gr1-2regular-anybudget-s0.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "2regular", + "n": 6, + "mode": "anybudget", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-2regular-anybudget-s0" + }, + "exact": { + "family": "2regular", + "n": 6, + "n_edges": 15, + "size": 70, + "sum_marginal_entropy_bits": 14.56425891682003, + "joint_entropy_bits": 6.129283016944966, + "total_correlation_bits": 8.434975899875063, + "one_pass_ceiling": 0.002889476997120002, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.00390625, + "nfe": 1.0, + "unique": 0.9375 + }, + "fixedK_2": { + "validity": 0.250732421875, + "nfe": 2.0, + "unique": 0.06134371957156767 + }, + "fixedK_4": { + "validity": 0.607666015625, + "nfe": 4.0, + "unique": 0.02812374447569305 + }, + "fixedK_8": { + "validity": 0.91162109375, + "nfe": 8.0, + "unique": 0.018746652383502947 + }, + "fixedK_15": { + "validity": 0.845703125, + "nfe": 15.0, + "unique": 0.015877598152424944 + }, + "entbudget_0.05": { + "validity": 0.982177734375, + "nfe": 13.0, + "unique": 0.017399950285856326 + }, + "entbudget_0.25": { + "validity": 0.982421875, + "nfe": 13.0, + "unique": 0.01739562624254473 + }, + "entbudget_1.0": { + "validity": 0.96435546875, + "nfe": 11.0, + "unique": 0.017721518987341773 + }, + "entbudget_4.0": { + "validity": 0.13330078125, + "nfe": 3.0, + "unique": 0.10989010989010989 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr1-2regular-anybudget-s1.json b/results/gr1-2regular-anybudget-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5068b28a68db357533dec70cc8ebac6bd98b6a5b 100644 --- a/results/gr1-2regular-anybudget-s1.json +++ b/results/gr1-2regular-anybudget-s1.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "2regular", + "n": 6, + "mode": "anybudget", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-2regular-anybudget-s1" + }, + "exact": { + "family": "2regular", + "n": 6, + "n_edges": 15, + "size": 70, + "sum_marginal_entropy_bits": 14.56425891682003, + "joint_entropy_bits": 6.129283016944966, + "total_correlation_bits": 8.434975899875063, + "one_pass_ceiling": 0.002889476997120002, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.002197265625, + "nfe": 1.0, + "unique": 0.8888888888888888 + }, + "fixedK_2": { + "validity": 0.25732421875, + "nfe": 2.0, + "unique": 0.061669829222011384 + }, + "fixedK_4": { + "validity": 0.671142578125, + "nfe": 4.0, + "unique": 0.025463805020007276 + }, + "fixedK_8": { + "validity": 0.824462890625, + "nfe": 8.0, + "unique": 0.020728457210541902 + }, + "fixedK_15": { + "validity": 0.2802734375, + "nfe": 15.0, + "unique": 0.020905923344947737 + }, + "entbudget_0.05": { + "validity": 0.991455078125, + "nfe": 13.0, + "unique": 0.017237133710908643 + }, + "entbudget_0.25": { + "validity": 0.991943359375, + "nfe": 13.0, + "unique": 0.017228648781688408 + }, + "entbudget_1.0": { + "validity": 0.975341796875, + "nfe": 10.0, + "unique": 0.017521902377972465 + }, + "entbudget_4.0": { + "validity": 0.3955078125, + "nfe": 3.0, + "unique": 0.04197530864197531 + } + }, + "reached_ceiling": false +} \ No newline at end of file diff --git a/results/gr1-2regular-mdlm-s0.json b/results/gr1-2regular-mdlm-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..e335d676ee064d1bfc86b4f3aff2959a582577b2 100644 --- a/results/gr1-2regular-mdlm-s0.json +++ b/results/gr1-2regular-mdlm-s0.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "2regular", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-2regular-mdlm-s0" + }, + "exact": { + "family": "2regular", + "n": 6, + "n_edges": 15, + "size": 70, + "sum_marginal_entropy_bits": 14.56425891682003, + "joint_entropy_bits": 6.129283016944966, + "total_correlation_bits": 8.434975899875063, + "one_pass_ceiling": 0.002889476997120002, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.00244140625, + "nfe": 1.0, + "unique": 0.8 + }, + "fixedK_2": { + "validity": 0.240478515625, + "nfe": 2.0, + "unique": 0.06294416243654823 + }, + "fixedK_4": { + "validity": 0.6591796875, + "nfe": 4.0, + "unique": 0.025555555555555557 + }, + "fixedK_8": { + "validity": 0.951171875, + "nfe": 8.0, + "unique": 0.017967145790554414 + }, + "fixedK_15": { + "validity": 0.999755859375, + "nfe": 15.0, + "unique": 0.017094017094017096 + }, + "entbudget_0.05": { + "validity": 0.99951171875, + "nfe": 13.0, + "unique": 0.01709819247679531 + }, + "entbudget_0.25": { + "validity": 0.999755859375, + "nfe": 13.0, + "unique": 0.017094017094017096 + }, + "entbudget_1.0": { + "validity": 0.989501953125, + "nfe": 10.0, + "unique": 0.017271157167530225 + }, + "entbudget_4.0": { + "validity": 0.28466796875, + "nfe": 3.0, + "unique": 0.05660377358490566 + } + }, + "reached_ceiling": false +} \ No newline at end of file diff --git a/results/gr1-2regular-mdlm-s1.json b/results/gr1-2regular-mdlm-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..29aa7732ac26eca79a8ace4b9d6fdeed074d2b18 100644 --- a/results/gr1-2regular-mdlm-s1.json +++ b/results/gr1-2regular-mdlm-s1.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "2regular", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-2regular-mdlm-s1" + }, + "exact": { + "family": "2regular", + "n": 6, + "n_edges": 15, + "size": 70, + "sum_marginal_entropy_bits": 14.56425891682003, + "joint_entropy_bits": 6.129283016944966, + "total_correlation_bits": 8.434975899875063, + "one_pass_ceiling": 0.002889476997120002, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.00390625, + "nfe": 1.0, + "unique": 0.9375 + }, + "fixedK_2": { + "validity": 0.242431640625, + "nfe": 2.0, + "unique": 0.06445115810674723 + }, + "fixedK_4": { + "validity": 0.681640625, + "nfe": 4.0, + "unique": 0.025071633237822348 + }, + "fixedK_8": { + "validity": 0.936279296875, + "nfe": 8.0, + "unique": 0.018252933507170794 + }, + "fixedK_15": { + "validity": 0.99951171875, + "nfe": 15.0, + "unique": 0.01709819247679531 + }, + "entbudget_0.05": { + "validity": 0.999755859375, + "nfe": 13.0, + "unique": 0.017094017094017096 + }, + "entbudget_0.25": { + "validity": 0.99951171875, + "nfe": 13.0, + "unique": 0.01709819247679531 + }, + "entbudget_1.0": { + "validity": 0.980712890625, + "nfe": 10.0, + "unique": 0.017425939756036842 + }, + "entbudget_4.0": { + "validity": 0.365234375, + "nfe": 3.0, + "unique": 0.046122994652406414 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr1-bipartite-anybudget-s0.json b/results/gr1-bipartite-anybudget-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..e6ac458e52d7444c84a9df97fa4a6325e14a4f3b 100644 --- a/results/gr1-bipartite-anybudget-s0.json +++ b/results/gr1-bipartite-anybudget-s0.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "bipartite", + "n": 6, + "mode": "anybudget", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-bipartite-anybudget-s0" + }, + "exact": { + "family": "bipartite", + "n": 6, + "n_edges": 15, + "size": 466, + "sum_marginal_entropy_bits": 8.976547296981545, + "joint_entropy_bits": 8.86418614465428, + "total_correlation_bits": 0.1123611523272654, + "one_pass_ceiling": 0.9363238709405111, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.059814453125, + "nfe": 1.0, + "unique": 0.6653061224489796 + }, + "fixedK_2": { + "validity": 0.14697265625, + "nfe": 2.0, + "unique": 0.49667774086378735 + }, + "fixedK_4": { + "validity": 0.44970703125, + "nfe": 4.0, + "unique": 0.23832790445168295 + }, + "fixedK_8": { + "validity": 0.994384765625, + "nfe": 8.0, + "unique": 0.11342990424748342 + }, + "fixedK_15": { + "validity": 0.999267578125, + "nfe": 15.0, + "unique": 0.11360860004886392 + }, + "entbudget_0.05": { + "validity": 1.0, + "nfe": 10.0, + "unique": 0.113525390625 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 10.0, + "unique": 0.11328125 + }, + "entbudget_1.0": { + "validity": 0.999267578125, + "nfe": 9.0, + "unique": 0.1131199609088688 + }, + "entbudget_4.0": { + "validity": 0.423583984375, + "nfe": 2.0, + "unique": 0.2270893371757925 + } + }, + "reached_ceiling": false +} \ No newline at end of file diff --git a/results/gr1-bipartite-anybudget-s1.json b/results/gr1-bipartite-anybudget-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..4b9801ce1d1d0faaceca149385f1f6780f70419c 100644 --- a/results/gr1-bipartite-anybudget-s1.json +++ b/results/gr1-bipartite-anybudget-s1.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "bipartite", + "n": 6, + "mode": "anybudget", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-bipartite-anybudget-s1" + }, + "exact": { + "family": "bipartite", + "n": 6, + "n_edges": 15, + "size": 466, + "sum_marginal_entropy_bits": 8.976547296981545, + "joint_entropy_bits": 8.86418614465428, + "total_correlation_bits": 0.1123611523272654, + "one_pass_ceiling": 0.9363238709405111, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.059326171875, + "nfe": 1.0, + "unique": 0.6872427983539094 + }, + "fixedK_2": { + "validity": 0.198486328125, + "nfe": 2.0, + "unique": 0.38253382533825336 + }, + "fixedK_4": { + "validity": 0.94775390625, + "nfe": 4.0, + "unique": 0.11540443070582174 + }, + "fixedK_8": { + "validity": 0.6884765625, + "nfe": 8.0, + "unique": 0.16453900709219857 + }, + "fixedK_15": { + "validity": 0.99951171875, + "nfe": 15.0, + "unique": 0.11309233023937469 + }, + "entbudget_0.05": { + "validity": 0.999267578125, + "nfe": 10.0, + "unique": 0.11360860004886392 + }, + "entbudget_0.25": { + "validity": 0.999267578125, + "nfe": 10.0, + "unique": 0.11336428047886636 + }, + "entbudget_1.0": { + "validity": 0.999267578125, + "nfe": 9.0, + "unique": 0.11336428047886636 + }, + "entbudget_4.0": { + "validity": 0.422607421875, + "nfe": 2.0, + "unique": 0.22992489890236859 + } + }, + "reached_ceiling": false +} \ No newline at end of file diff --git a/results/gr1-bipartite-mdlm-s0.json b/results/gr1-bipartite-mdlm-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ba2c113a87d7a7a3777c4128f86930d094fbfb2b 100644 --- a/results/gr1-bipartite-mdlm-s0.json +++ b/results/gr1-bipartite-mdlm-s0.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "bipartite", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-bipartite-mdlm-s0" + }, + "exact": { + "family": "bipartite", + "n": 6, + "n_edges": 15, + "size": 466, + "sum_marginal_entropy_bits": 8.976547296981545, + "joint_entropy_bits": 8.86418614465428, + "total_correlation_bits": 0.1123611523272654, + "one_pass_ceiling": 0.9363238709405111, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.059326171875, + "nfe": 1.0, + "unique": 0.6954732510288066 + }, + "fixedK_2": { + "validity": 0.284423828125, + "nfe": 2.0, + "unique": 0.30300429184549355 + }, + "fixedK_4": { + "validity": 0.646484375, + "nfe": 4.0, + "unique": 0.16805135951661632 + }, + "fixedK_8": { + "validity": 0.994384765625, + "nfe": 8.0, + "unique": 0.11367542352074637 + }, + "fixedK_15": { + "validity": 1.0, + "nfe": 15.0, + "unique": 0.113525390625 + }, + "entbudget_0.05": { + "validity": 0.999755859375, + "nfe": 10.0, + "unique": 0.11306471306471307 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 10.0, + "unique": 0.113525390625 + }, + "entbudget_1.0": { + "validity": 0.999267578125, + "nfe": 9.0, + "unique": 0.11360860004886392 + }, + "entbudget_4.0": { + "validity": 0.299072265625, + "nfe": 3.0, + "unique": 0.3093877551020408 + } + }, + "reached_ceiling": false +} \ No newline at end of file diff --git a/results/gr1-bipartite-mdlm-s1.json b/results/gr1-bipartite-mdlm-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..87a682bbe8d1f26ac3f7ce895b64fb8e258b490f 100644 --- a/results/gr1-bipartite-mdlm-s1.json +++ b/results/gr1-bipartite-mdlm-s1.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "bipartite", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-bipartite-mdlm-s1" + }, + "exact": { + "family": "bipartite", + "n": 6, + "n_edges": 15, + "size": 466, + "sum_marginal_entropy_bits": 8.976547296981545, + "joint_entropy_bits": 8.86418614465428, + "total_correlation_bits": 0.1123611523272654, + "one_pass_ceiling": 0.9363238709405111, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.058349609375, + "nfe": 1.0, + "unique": 0.7071129707112971 + }, + "fixedK_2": { + "validity": 0.2841796875, + "nfe": 2.0, + "unique": 0.3024054982817869 + }, + "fixedK_4": { + "validity": 0.6435546875, + "nfe": 4.0, + "unique": 0.1646433990895296 + }, + "fixedK_8": { + "validity": 0.995361328125, + "nfe": 8.0, + "unique": 0.11258278145695365 + }, + "fixedK_15": { + "validity": 1.0, + "nfe": 15.0, + "unique": 0.113525390625 + }, + "entbudget_0.05": { + "validity": 0.999755859375, + "nfe": 10.0, + "unique": 0.11330891330891331 + }, + "entbudget_0.25": { + "validity": 0.999755859375, + "nfe": 10.0, + "unique": 0.11355311355311355 + }, + "entbudget_1.0": { + "validity": 0.999755859375, + "nfe": 9.0, + "unique": 0.11355311355311355 + }, + "entbudget_4.0": { + "validity": 0.427001953125, + "nfe": 3.0, + "unique": 0.2338479130931961 + } + }, + "reached_ceiling": false +} \ No newline at end of file diff --git a/results/gr1-matching-anybudget-s0.json b/results/gr1-matching-anybudget-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..49fb38af838350db22eab08573b1d8fa2a0fdde8 100644 --- a/results/gr1-matching-anybudget-s0.json +++ b/results/gr1-matching-anybudget-s0.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "matching", + "n": 6, + "mode": "anybudget", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-matching-anybudget-s0" + }, + "exact": { + "family": "matching", + "n": 6, + "n_edges": 15, + "size": 15, + "sum_marginal_entropy_bits": 10.828921423310433, + "joint_entropy_bits": 3.9068905956085187, + "total_correlation_bits": 6.9220308277019145, + "one_pass_ceiling": 0.008246337208320003, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.00927734375, + "nfe": 1.0, + "unique": 0.34210526315789475 + }, + "fixedK_2": { + "validity": 0.415283203125, + "nfe": 2.0, + "unique": 0.008818342151675485 + }, + "fixedK_4": { + "validity": 0.6923828125, + "nfe": 4.0, + "unique": 0.005289139633286318 + }, + "fixedK_8": { + "validity": 0.947265625, + "nfe": 8.0, + "unique": 0.003865979381443299 + }, + "fixedK_15": { + "validity": 0.994140625, + "nfe": 15.0, + "unique": 0.0036836935166994107 + }, + "entbudget_0.05": { + "validity": 1.0, + "nfe": 11.0, + "unique": 0.003662109375 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 11.0, + "unique": 0.003662109375 + }, + "entbudget_1.0": { + "validity": 0.965087890625, + "nfe": 7.0, + "unique": 0.003794586390083481 + }, + "entbudget_4.0": { + "validity": 0.177978515625, + "nfe": 3.0, + "unique": 0.0205761316872428 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr1-matching-anybudget-s1.json b/results/gr1-matching-anybudget-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..f138e4c3706a0db5dfbf019bcd42e4cb95654757 100644 --- a/results/gr1-matching-anybudget-s1.json +++ b/results/gr1-matching-anybudget-s1.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "matching", + "n": 6, + "mode": "anybudget", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-matching-anybudget-s1" + }, + "exact": { + "family": "matching", + "n": 6, + "n_edges": 15, + "size": 15, + "sum_marginal_entropy_bits": 10.828921423310433, + "joint_entropy_bits": 3.9068905956085187, + "total_correlation_bits": 6.9220308277019145, + "one_pass_ceiling": 0.008246337208320003, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.0078125, + "nfe": 1.0, + "unique": 0.375 + }, + "fixedK_2": { + "validity": 0.254150390625, + "nfe": 2.0, + "unique": 0.01440922190201729 + }, + "fixedK_4": { + "validity": 0.6904296875, + "nfe": 4.0, + "unique": 0.005304101838755304 + }, + "fixedK_8": { + "validity": 0.781005859375, + "nfe": 8.0, + "unique": 0.004688965301656768 + }, + "fixedK_15": { + "validity": 0.972900390625, + "nfe": 15.0, + "unique": 0.0037641154328732747 + }, + "entbudget_0.05": { + "validity": 0.999755859375, + "nfe": 12.125, + "unique": 0.003663003663003663 + }, + "entbudget_0.25": { + "validity": 0.999755859375, + "nfe": 12.0, + "unique": 0.003663003663003663 + }, + "entbudget_1.0": { + "validity": 0.96826171875, + "nfe": 8.0, + "unique": 0.0037821482602118004 + }, + "entbudget_4.0": { + "validity": 0.486328125, + "nfe": 2.0, + "unique": 0.007530120481927711 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr1-matching-mdlm-s0.json b/results/gr1-matching-mdlm-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..368f6a3cf633326852173f04160d8cc5faa0ba3d 100644 --- a/results/gr1-matching-mdlm-s0.json +++ b/results/gr1-matching-mdlm-s0.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "matching", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-matching-mdlm-s0" + }, + "exact": { + "family": "matching", + "n": 6, + "n_edges": 15, + "size": 15, + "sum_marginal_entropy_bits": 10.828921423310433, + "joint_entropy_bits": 3.9068905956085187, + "total_correlation_bits": 6.9220308277019145, + "one_pass_ceiling": 0.008246337208320003, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.0078125, + "nfe": 1.0, + "unique": 0.40625 + }, + "fixedK_2": { + "validity": 0.426513671875, + "nfe": 2.0, + "unique": 0.008586147681740127 + }, + "fixedK_4": { + "validity": 0.827880859375, + "nfe": 4.0, + "unique": 0.0044234739015039815 + }, + "fixedK_8": { + "validity": 0.794921875, + "nfe": 8.0, + "unique": 0.004606879606879607 + }, + "fixedK_15": { + "validity": 0.999755859375, + "nfe": 15.0, + "unique": 0.003663003663003663 + }, + "entbudget_0.05": { + "validity": 1.0, + "nfe": 12.0, + "unique": 0.003662109375 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 12.0, + "unique": 0.003662109375 + }, + "entbudget_1.0": { + "validity": 0.81640625, + "nfe": 8.0, + "unique": 0.004485645933014354 + }, + "entbudget_4.0": { + "validity": 0.4267578125, + "nfe": 2.0, + "unique": 0.008581235697940504 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr1-matching-mdlm-s1.json b/results/gr1-matching-mdlm-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..2c215ea120f064b5b27e2fd5c4cb29ee77e0b54c 100644 --- a/results/gr1-matching-mdlm-s1.json +++ b/results/gr1-matching-mdlm-s1.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "matching", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-matching-mdlm-s1" + }, + "exact": { + "family": "matching", + "n": 6, + "n_edges": 15, + "size": 15, + "sum_marginal_entropy_bits": 10.828921423310433, + "joint_entropy_bits": 3.9068905956085187, + "total_correlation_bits": 6.9220308277019145, + "one_pass_ceiling": 0.008246337208320003, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.007568359375, + "nfe": 1.0, + "unique": 0.3870967741935484 + }, + "fixedK_2": { + "validity": 0.273681640625, + "nfe": 2.0, + "unique": 0.013380909901873328 + }, + "fixedK_4": { + "validity": 0.843505859375, + "nfe": 4.0, + "unique": 0.004341534008683068 + }, + "fixedK_8": { + "validity": 0.73876953125, + "nfe": 8.0, + "unique": 0.00495703899537343 + }, + "fixedK_15": { + "validity": 1.0, + "nfe": 15.0, + "unique": 0.003662109375 + }, + "entbudget_0.05": { + "validity": 1.0, + "nfe": 11.0, + "unique": 0.003662109375 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 11.0, + "unique": 0.003662109375 + }, + "entbudget_1.0": { + "validity": 0.8095703125, + "nfe": 8.0, + "unique": 0.004523522316043426 + }, + "entbudget_4.0": { + "validity": 0.260498046875, + "nfe": 2.0, + "unique": 0.014058106841611996 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr1-tree-anybudget-s0.json b/results/gr1-tree-anybudget-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5c49da3c0ea628dc07fce06b05d92b9b90c4622d 100644 --- a/results/gr1-tree-anybudget-s0.json +++ b/results/gr1-tree-anybudget-s0.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "tree", + "n": 6, + "mode": "anybudget", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-tree-anybudget-s0" + }, + "exact": { + "family": "tree", + "n": 6, + "n_edges": 15, + "size": 1296, + "sum_marginal_entropy_bits": 13.774437510817346, + "joint_entropy_bits": 10.339850002884624, + "total_correlation_bits": 3.434587507932722, + "one_pass_ceiling": 0.09248815955110604, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.09912109375, + "nfe": 1.0, + "unique": 0.8522167487684729 + }, + "fixedK_2": { + "validity": 0.267822265625, + "nfe": 2.0, + "unique": 0.5432999088422972 + }, + "fixedK_4": { + "validity": 0.380859375, + "nfe": 4.0, + "unique": 0.47628205128205126 + }, + "fixedK_8": { + "validity": 0.766357421875, + "nfe": 8.0, + "unique": 0.3395985982797069 + }, + "fixedK_15": { + "validity": 0.36572265625, + "nfe": 15.0, + "unique": 0.3324432576769025 + }, + "entbudget_0.05": { + "validity": 0.99365234375, + "nfe": 15.0, + "unique": 0.3034398034398034 + }, + "entbudget_0.25": { + "validity": 0.993896484375, + "nfe": 15.0, + "unique": 0.3026283468435274 + }, + "entbudget_1.0": { + "validity": 0.994140625, + "nfe": 15.0, + "unique": 0.3037819253438114 + }, + "entbudget_4.0": { + "validity": 0.3916015625, + "nfe": 3.0, + "unique": 0.4869077306733167 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr1-tree-anybudget-s1.json b/results/gr1-tree-anybudget-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..8068ef8684cab8a67ee6e9bf9a1e9773ba96f1f2 --- /dev/null +++ b/results/gr1-tree-anybudget-s1.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "tree", + "n": 6, + "mode": "anybudget", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-tree-anybudget-s1" + }, + "exact": { + "family": "tree", + "n": 6, + "n_edges": 15, + "size": 1296, + "sum_marginal_entropy_bits": 13.774437510817346, + "joint_entropy_bits": 10.339850002884624, + "total_correlation_bits": 3.434587507932722, + "one_pass_ceiling": 0.09248815955110604, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.10205078125, + "nfe": 1.0, + "unique": 0.8325358851674641 + }, + "fixedK_2": { + "validity": 0.2578125, + "nfe": 2.0, + "unique": 0.5625 + }, + "fixedK_4": { + "validity": 0.391357421875, + "nfe": 4.0, + "unique": 0.48034934497816595 + }, + "fixedK_8": { + "validity": 0.723876953125, + "nfe": 8.0, + "unique": 0.36458684654300166 + }, + "fixedK_15": { + "validity": 0.0, + "nfe": 15.0, + "unique": 0.0 + }, + "entbudget_0.05": { + "validity": 0.99560546875, + "nfe": 15.0, + "unique": 0.3006375674350172 + }, + "entbudget_0.25": { + "validity": 0.99462890625, + "nfe": 15.0, + "unique": 0.3021600392734413 + }, + "entbudget_1.0": { + "validity": 0.995849609375, + "nfe": 15.0, + "unique": 0.30178965432704097 + }, + "entbudget_4.0": { + "validity": 0.38916015625, + "nfe": 3.0, + "unique": 0.4504391468005019 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr1-tree-mdlm-s0.json b/results/gr1-tree-mdlm-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..f456eb62a22ae15a45a98e7fd77ac3c6e6486f9e --- /dev/null +++ b/results/gr1-tree-mdlm-s0.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "tree", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-tree-mdlm-s0" + }, + "exact": { + "family": "tree", + "n": 6, + "n_edges": 15, + "size": 1296, + "sum_marginal_entropy_bits": 13.774437510817346, + "joint_entropy_bits": 10.339850002884624, + "total_correlation_bits": 3.434587507932722, + "one_pass_ceiling": 0.09248815955110604, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.0966796875, + "nfe": 1.0, + "unique": 0.851010101010101 + }, + "fixedK_2": { + "validity": 0.260009765625, + "nfe": 2.0, + "unique": 0.5690140845070423 + }, + "fixedK_4": { + "validity": 0.372802734375, + "nfe": 4.0, + "unique": 0.4715127701375246 + }, + "fixedK_8": { + "validity": 0.77734375, + "nfe": 8.0, + "unique": 0.3476758793969849 + }, + "fixedK_15": { + "validity": 1.0, + "nfe": 15.0, + "unique": 0.29833984375 + }, + "entbudget_0.05": { + "validity": 0.999755859375, + "nfe": 15.0, + "unique": 0.30256410256410254 + }, + "entbudget_0.25": { + "validity": 0.99951171875, + "nfe": 15.0, + "unique": 0.3006839276990718 + }, + "entbudget_1.0": { + "validity": 0.999267578125, + "nfe": 15.0, + "unique": 0.30442218421695577 + }, + "entbudget_4.0": { + "validity": 0.3876953125, + "nfe": 3.0, + "unique": 0.4534005037783375 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr1-tree-mdlm-s1.json b/results/gr1-tree-mdlm-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b6ddc03f96bfb952107198737fed13be53c41b7e 100644 --- a/results/gr1-tree-mdlm-s1.json +++ b/results/gr1-tree-mdlm-s1.json @@ -0,0 +1,78 @@ +{ + "args": { + "family": "tree", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr1-tree-mdlm-s1" + }, + "exact": { + "family": "tree", + "n": 6, + "n_edges": 15, + "size": 1296, + "sum_marginal_entropy_bits": 13.774437510817346, + "joint_entropy_bits": 10.339850002884624, + "total_correlation_bits": 3.434587507932722, + "one_pass_ceiling": 0.09248815955110604, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.08740234375, + "nfe": 1.0, + "unique": 0.888268156424581 + }, + "fixedK_2": { + "validity": 0.2509765625, + "nfe": 2.0, + "unique": 0.5875486381322957 + }, + "fixedK_4": { + "validity": 0.385009765625, + "nfe": 4.0, + "unique": 0.476854787571338 + }, + "fixedK_8": { + "validity": 0.783203125, + "nfe": 8.0, + "unique": 0.33884039900249374 + }, + "fixedK_15": { + "validity": 0.997314453125, + "nfe": 15.0, + "unique": 0.3025703794369645 + }, + "entbudget_0.05": { + "validity": 0.999755859375, + "nfe": 15.0, + "unique": 0.30378510378510376 + }, + "entbudget_0.25": { + "validity": 0.9990234375, + "nfe": 15.0, + "unique": 0.3010752688172043 + }, + "entbudget_1.0": { + "validity": 0.998291015625, + "nfe": 15.0, + "unique": 0.3037417461482025 + }, + "entbudget_4.0": { + "validity": 0.40185546875, + "nfe": 3.0, + "unique": 0.4513973268529769 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr2-bipartite-ape-s0.json b/results/gr2-bipartite-ape-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..051f5936383476ac2fe8f5d83c73b90f74446e7a --- /dev/null +++ b/results/gr2-bipartite-ape-s0.json @@ -0,0 +1,79 @@ +{ + "args": { + "family": "bipartite", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "pe": "ape", + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr2-bipartite-ape-s0" + }, + "exact": { + "family": "bipartite", + "n": 6, + "n_edges": 15, + "size": 466, + "sum_marginal_entropy_bits": 8.976547296981545, + "joint_entropy_bits": 8.86418614465428, + "total_correlation_bits": 0.1123611523272654, + "one_pass_ceiling": 0.9363238709405111, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.94189453125, + "nfe": 1.0, + "unique": 0.12078797304302748 + }, + "fixedK_2": { + "validity": 0.949462890625, + "nfe": 2.0, + "unique": 0.1195680123425045 + }, + "fixedK_4": { + "validity": 0.97412109375, + "nfe": 4.0, + "unique": 0.11654135338345864 + }, + "fixedK_8": { + "validity": 0.994873046875, + "nfe": 8.0, + "unique": 0.1143558282208589 + }, + "fixedK_15": { + "validity": 1.0, + "nfe": 15.0, + "unique": 0.11376953125 + }, + "entbudget_0.05": { + "validity": 0.99951171875, + "nfe": 10.0, + "unique": 0.11382510991695163 + }, + "entbudget_0.25": { + "validity": 0.999755859375, + "nfe": 10.0, + "unique": 0.1137973137973138 + }, + "entbudget_1.0": { + "validity": 0.999755859375, + "nfe": 9.0, + "unique": 0.1137973137973138 + }, + "entbudget_4.0": { + "validity": 0.96484375, + "nfe": 2.0, + "unique": 0.11791497975708502 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr2-bipartite-ape-s1.json b/results/gr2-bipartite-ape-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..fe169a634f63de38e5cdce17a8fb7310ecf513d2 --- /dev/null +++ b/results/gr2-bipartite-ape-s1.json @@ -0,0 +1,79 @@ +{ + "args": { + "family": "bipartite", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "pe": "ape", + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr2-bipartite-ape-s1" + }, + "exact": { + "family": "bipartite", + "n": 6, + "n_edges": 15, + "size": 466, + "sum_marginal_entropy_bits": 8.976547296981545, + "joint_entropy_bits": 8.86418614465428, + "total_correlation_bits": 0.1123611523272654, + "one_pass_ceiling": 0.9363238709405111, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.936279296875, + "nfe": 1.0, + "unique": 0.12151238591916558 + }, + "fixedK_2": { + "validity": 0.943115234375, + "nfe": 2.0, + "unique": 0.12063163344550867 + }, + "fixedK_4": { + "validity": 0.973876953125, + "nfe": 4.0, + "unique": 0.11682125846076712 + }, + "fixedK_8": { + "validity": 0.99560546875, + "nfe": 8.0, + "unique": 0.114271701814615 + }, + "fixedK_15": { + "validity": 1.0, + "nfe": 15.0, + "unique": 0.11376953125 + }, + "entbudget_0.05": { + "validity": 1.0, + "nfe": 10.0, + "unique": 0.11376953125 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 10.0, + "unique": 0.11376953125 + }, + "entbudget_1.0": { + "validity": 0.999267578125, + "nfe": 9.0, + "unique": 0.11385291961886147 + }, + "entbudget_4.0": { + "validity": 0.957275390625, + "nfe": 2.0, + "unique": 0.11884723284876307 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr2-bipartite-rope-s0.json b/results/gr2-bipartite-rope-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..c8c76fc80e1b9d400d317f23898ab39cc09685af --- /dev/null +++ b/results/gr2-bipartite-rope-s0.json @@ -0,0 +1,79 @@ +{ + "args": { + "family": "bipartite", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "pe": "rope", + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr2-bipartite-rope-s0" + }, + "exact": { + "family": "bipartite", + "n": 6, + "n_edges": 15, + "size": 466, + "sum_marginal_entropy_bits": 8.976547296981545, + "joint_entropy_bits": 8.86418614465428, + "total_correlation_bits": 0.1123611523272654, + "one_pass_ceiling": 0.9363238709405111, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.059326171875, + "nfe": 1.0, + "unique": 0.691358024691358 + }, + "fixedK_2": { + "validity": 0.130859375, + "nfe": 2.0, + "unique": 0.5149253731343284 + }, + "fixedK_4": { + "validity": 0.309326171875, + "nfe": 4.0, + "unique": 0.324388318863457 + }, + "fixedK_8": { + "validity": 0.463134765625, + "nfe": 8.0, + "unique": 0.2430152872957301 + }, + "fixedK_15": { + "validity": 0.6787109375, + "nfe": 15.0, + "unique": 0.16618705035971224 + }, + "entbudget_0.05": { + "validity": 0.6787109375, + "nfe": 11.0, + "unique": 0.1669064748201439 + }, + "entbudget_0.25": { + "validity": 0.68701171875, + "nfe": 11.0, + "unique": 0.16453447050461975 + }, + "entbudget_1.0": { + "validity": 0.68310546875, + "nfe": 10.0, + "unique": 0.16654753395282346 + }, + "entbudget_4.0": { + "validity": 0.139892578125, + "nfe": 3.0, + "unique": 0.5392670157068062 + } + }, + "reached_ceiling": false +} \ No newline at end of file diff --git a/results/gr2-bipartite-rope-s1.json b/results/gr2-bipartite-rope-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..ee50242ea04b99ea28047964633e77455c67a97f --- /dev/null +++ b/results/gr2-bipartite-rope-s1.json @@ -0,0 +1,79 @@ +{ + "args": { + "family": "bipartite", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "pe": "rope", + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr2-bipartite-rope-s1" + }, + "exact": { + "family": "bipartite", + "n": 6, + "n_edges": 15, + "size": 466, + "sum_marginal_entropy_bits": 8.976547296981545, + "joint_entropy_bits": 8.86418614465428, + "total_correlation_bits": 0.1123611523272654, + "one_pass_ceiling": 0.9363238709405111, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.05810546875, + "nfe": 1.0, + "unique": 0.7142857142857143 + }, + "fixedK_2": { + "validity": 0.135498046875, + "nfe": 2.0, + "unique": 0.4900900900900901 + }, + "fixedK_4": { + "validity": 0.648681640625, + "nfe": 4.0, + "unique": 0.16409484380880693 + }, + "fixedK_8": { + "validity": 0.994384765625, + "nfe": 8.0, + "unique": 0.11244782715443162 + }, + "fixedK_15": { + "validity": 1.0, + "nfe": 15.0, + "unique": 0.11328125 + }, + "entbudget_0.05": { + "validity": 0.999755859375, + "nfe": 10.0, + "unique": 0.1137973137973138 + }, + "entbudget_0.25": { + "validity": 0.999755859375, + "nfe": 10.0, + "unique": 0.1137973137973138 + }, + "entbudget_1.0": { + "validity": 0.999755859375, + "nfe": 9.0, + "unique": 0.1137973137973138 + }, + "entbudget_4.0": { + "validity": 0.30859375, + "nfe": 3.0, + "unique": 0.29193037974683544 + } + }, + "reached_ceiling": false +} \ No newline at end of file diff --git a/results/gr2-bipartite-sin-s0.json b/results/gr2-bipartite-sin-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..335ea9f719ab1524755faf1a25e93f2819c8fdfe --- /dev/null +++ b/results/gr2-bipartite-sin-s0.json @@ -0,0 +1,79 @@ +{ + "args": { + "family": "bipartite", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "pe": "sin", + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr2-bipartite-sin-s0" + }, + "exact": { + "family": "bipartite", + "n": 6, + "n_edges": 15, + "size": 466, + "sum_marginal_entropy_bits": 8.976547296981545, + "joint_entropy_bits": 8.86418614465428, + "total_correlation_bits": 0.1123611523272654, + "one_pass_ceiling": 0.9363238709405111, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.94140625, + "nfe": 1.0, + "unique": 0.120850622406639 + }, + "fixedK_2": { + "validity": 0.947998046875, + "nfe": 2.0, + "unique": 0.11975276847798094 + }, + "fixedK_4": { + "validity": 0.972900390625, + "nfe": 4.0, + "unique": 0.11668757841907151 + }, + "fixedK_8": { + "validity": 0.995849609375, + "nfe": 8.0, + "unique": 0.11424368717822996 + }, + "fixedK_15": { + "validity": 1.0, + "nfe": 15.0, + "unique": 0.11376953125 + }, + "entbudget_0.05": { + "validity": 0.99951171875, + "nfe": 10.0, + "unique": 0.11382510991695163 + }, + "entbudget_0.25": { + "validity": 0.99951171875, + "nfe": 10.0, + "unique": 0.11382510991695163 + }, + "entbudget_1.0": { + "validity": 0.999267578125, + "nfe": 9.0, + "unique": 0.11385291961886147 + }, + "entbudget_4.0": { + "validity": 0.961181640625, + "nfe": 2.0, + "unique": 0.11811023622047244 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr2-bipartite-sin-s1.json b/results/gr2-bipartite-sin-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..838ac28d3a52430987fac40738772faa7815859d --- /dev/null +++ b/results/gr2-bipartite-sin-s1.json @@ -0,0 +1,79 @@ +{ + "args": { + "family": "bipartite", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "pe": "sin", + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr2-bipartite-sin-s1" + }, + "exact": { + "family": "bipartite", + "n": 6, + "n_edges": 15, + "size": 466, + "sum_marginal_entropy_bits": 8.976547296981545, + "joint_entropy_bits": 8.86418614465428, + "total_correlation_bits": 0.1123611523272654, + "one_pass_ceiling": 0.9363238709405111, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.9375, + "nfe": 1.0, + "unique": 0.12135416666666667 + }, + "fixedK_2": { + "validity": 0.94384765625, + "nfe": 2.0, + "unique": 0.12053802379720642 + }, + "fixedK_4": { + "validity": 0.976318359375, + "nfe": 4.0, + "unique": 0.11652913228307077 + }, + "fixedK_8": { + "validity": 0.996337890625, + "nfe": 8.0, + "unique": 0.11418769909335948 + }, + "fixedK_15": { + "validity": 0.999755859375, + "nfe": 15.0, + "unique": 0.1137973137973138 + }, + "entbudget_0.05": { + "validity": 1.0, + "nfe": 10.0, + "unique": 0.11376953125 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 10.0, + "unique": 0.11376953125 + }, + "entbudget_1.0": { + "validity": 0.999267578125, + "nfe": 9.0, + "unique": 0.11385291961886147 + }, + "entbudget_4.0": { + "validity": 0.965087890625, + "nfe": 2.0, + "unique": 0.11788515051859347 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr2-matching-ape-s0.json b/results/gr2-matching-ape-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..8ead54cb50f606ff0f2987fdb8e082883d956ece --- /dev/null +++ b/results/gr2-matching-ape-s0.json @@ -0,0 +1,79 @@ +{ + "args": { + "family": "matching", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "pe": "ape", + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr2-matching-ape-s0" + }, + "exact": { + "family": "matching", + "n": 6, + "n_edges": 15, + "size": 15, + "sum_marginal_entropy_bits": 10.828921423310433, + "joint_entropy_bits": 3.9068905956085187, + "total_correlation_bits": 6.9220308277019145, + "one_pass_ceiling": 0.008246337208320003, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.007568359375, + "nfe": 1.0, + "unique": 0.45161290322580644 + }, + "fixedK_2": { + "validity": 0.272216796875, + "nfe": 2.0, + "unique": 0.013452914798206279 + }, + "fixedK_4": { + "validity": 0.732666015625, + "nfe": 4.0, + "unique": 0.004998333888703766 + }, + "fixedK_8": { + "validity": 0.7783203125, + "nfe": 8.0, + "unique": 0.004705144291091593 + }, + "fixedK_15": { + "validity": 1.0, + "nfe": 15.0, + "unique": 0.003662109375 + }, + "entbudget_0.05": { + "validity": 1.0, + "nfe": 11.0, + "unique": 0.003662109375 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 11.0, + "unique": 0.003662109375 + }, + "entbudget_1.0": { + "validity": 0.823486328125, + "nfe": 8.0, + "unique": 0.004447079750963534 + }, + "entbudget_4.0": { + "validity": 0.269287109375, + "nfe": 2.0, + "unique": 0.013599274705349048 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr2-matching-ape-s1.json b/results/gr2-matching-ape-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..cfe6bbc75f89cbb76432fecb91b8d5a3921395c4 --- /dev/null +++ b/results/gr2-matching-ape-s1.json @@ -0,0 +1,79 @@ +{ + "args": { + "family": "matching", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "pe": "ape", + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr2-matching-ape-s1" + }, + "exact": { + "family": "matching", + "n": 6, + "n_edges": 15, + "size": 15, + "sum_marginal_entropy_bits": 10.828921423310433, + "joint_entropy_bits": 3.9068905956085187, + "total_correlation_bits": 6.9220308277019145, + "one_pass_ceiling": 0.008246337208320003, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.007568359375, + "nfe": 1.0, + "unique": 0.3548387096774194 + }, + "fixedK_2": { + "validity": 0.281494140625, + "nfe": 2.0, + "unique": 0.013009540329575022 + }, + "fixedK_4": { + "validity": 0.79296875, + "nfe": 4.0, + "unique": 0.004618226600985222 + }, + "fixedK_8": { + "validity": 0.816650390625, + "nfe": 8.0, + "unique": 0.004484304932735426 + }, + "fixedK_15": { + "validity": 1.0, + "nfe": 15.0, + "unique": 0.003662109375 + }, + "entbudget_0.05": { + "validity": 1.0, + "nfe": 12.0, + "unique": 0.003662109375 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 12.0, + "unique": 0.003662109375 + }, + "entbudget_1.0": { + "validity": 0.822998046875, + "nfe": 8.0, + "unique": 0.004449718184514981 + }, + "entbudget_4.0": { + "validity": 0.283203125, + "nfe": 3.0, + "unique": 0.01293103448275862 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr2-matching-rope-s0.json b/results/gr2-matching-rope-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..5bb8ee8580e34f41389d9f5bb49f91740e11c8e8 --- /dev/null +++ b/results/gr2-matching-rope-s0.json @@ -0,0 +1,79 @@ +{ + "args": { + "family": "matching", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "pe": "rope", + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr2-matching-rope-s0" + }, + "exact": { + "family": "matching", + "n": 6, + "n_edges": 15, + "size": 15, + "sum_marginal_entropy_bits": 10.828921423310433, + "joint_entropy_bits": 3.9068905956085187, + "total_correlation_bits": 6.9220308277019145, + "one_pass_ceiling": 0.008246337208320003, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.0078125, + "nfe": 1.0, + "unique": 0.40625 + }, + "fixedK_2": { + "validity": 0.42626953125, + "nfe": 2.0, + "unique": 0.00859106529209622 + }, + "fixedK_4": { + "validity": 0.65625, + "nfe": 4.0, + "unique": 0.005580357142857143 + }, + "fixedK_8": { + "validity": 0.738525390625, + "nfe": 8.0, + "unique": 0.0049586776859504135 + }, + "fixedK_15": { + "validity": 0.999755859375, + "nfe": 15.0, + "unique": 0.003663003663003663 + }, + "entbudget_0.05": { + "validity": 1.0, + "nfe": 12.0, + "unique": 0.003662109375 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 12.0, + "unique": 0.003662109375 + }, + "entbudget_1.0": { + "validity": 0.82080078125, + "nfe": 8.0, + "unique": 0.00446162998215348 + }, + "entbudget_4.0": { + "validity": 0.4287109375, + "nfe": 2.0, + "unique": 0.008542141230068337 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr2-matching-rope-s1.json b/results/gr2-matching-rope-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..42b73f6f47f57a81423b824a6bb1137a6dee0f7b --- /dev/null +++ b/results/gr2-matching-rope-s1.json @@ -0,0 +1,79 @@ +{ + "args": { + "family": "matching", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "pe": "rope", + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr2-matching-rope-s1" + }, + "exact": { + "family": "matching", + "n": 6, + "n_edges": 15, + "size": 15, + "sum_marginal_entropy_bits": 10.828921423310433, + "joint_entropy_bits": 3.9068905956085187, + "total_correlation_bits": 6.9220308277019145, + "one_pass_ceiling": 0.008246337208320003, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.0078125, + "nfe": 1.0, + "unique": 0.375 + }, + "fixedK_2": { + "validity": 0.421875, + "nfe": 2.0, + "unique": 0.008680555555555556 + }, + "fixedK_4": { + "validity": 0.73095703125, + "nfe": 4.0, + "unique": 0.00501002004008016 + }, + "fixedK_8": { + "validity": 0.79443359375, + "nfe": 8.0, + "unique": 0.004609711124769515 + }, + "fixedK_15": { + "validity": 0.999755859375, + "nfe": 15.0, + "unique": 0.003663003663003663 + }, + "entbudget_0.05": { + "validity": 1.0, + "nfe": 11.0, + "unique": 0.003662109375 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 11.0, + "unique": 0.003662109375 + }, + "entbudget_1.0": { + "validity": 0.822998046875, + "nfe": 8.0, + "unique": 0.004449718184514981 + }, + "entbudget_4.0": { + "validity": 0.425537109375, + "nfe": 2.0, + "unique": 0.008605851979345954 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr2-matching-sin-s0.json b/results/gr2-matching-sin-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..360a004ed72f88083cec624d9dda9cb7c10fbbc5 --- /dev/null +++ b/results/gr2-matching-sin-s0.json @@ -0,0 +1,79 @@ +{ + "args": { + "family": "matching", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "pe": "sin", + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr2-matching-sin-s0" + }, + "exact": { + "family": "matching", + "n": 6, + "n_edges": 15, + "size": 15, + "sum_marginal_entropy_bits": 10.828921423310433, + "joint_entropy_bits": 3.9068905956085187, + "total_correlation_bits": 6.9220308277019145, + "one_pass_ceiling": 0.008246337208320003, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.008056640625, + "nfe": 1.0, + "unique": 0.42424242424242425 + }, + "fixedK_2": { + "validity": 0.4375, + "nfe": 2.0, + "unique": 0.008370535714285714 + }, + "fixedK_4": { + "validity": 0.79833984375, + "nfe": 4.0, + "unique": 0.0045871559633027525 + }, + "fixedK_8": { + "validity": 0.94873046875, + "nfe": 8.0, + "unique": 0.0038600102933607824 + }, + "fixedK_15": { + "validity": 1.0, + "nfe": 15.0, + "unique": 0.003662109375 + }, + "entbudget_0.05": { + "validity": 0.999755859375, + "nfe": 11.0, + "unique": 0.003663003663003663 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 11.0, + "unique": 0.003662109375 + }, + "entbudget_1.0": { + "validity": 0.96630859375, + "nfe": 7.0, + "unique": 0.0037897928246589186 + }, + "entbudget_4.0": { + "validity": 0.43359375, + "nfe": 2.0, + "unique": 0.008445945945945946 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/gr2-matching-sin-s1.json b/results/gr2-matching-sin-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..b5a25a8f8bb4623c256919a2054114d58cda6c49 --- /dev/null +++ b/results/gr2-matching-sin-s1.json @@ -0,0 +1,79 @@ +{ + "args": { + "family": "matching", + "n": 6, + "mode": "mdlm", + "budget_bins": 8, + "pe": "sin", + "d": 256, + "layers": 6, + "heads": 8, + "bs": 256, + "steps": 30000, + "lr": 0.0001, + "warmup": 1000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/gr2-matching-sin-s1" + }, + "exact": { + "family": "matching", + "n": 6, + "n_edges": 15, + "size": 15, + "sum_marginal_entropy_bits": 10.828921423310433, + "joint_entropy_bits": 3.9068905956085187, + "total_correlation_bits": 6.9220308277019145, + "one_pass_ceiling": 0.008246337208320003, + "bound_holds": true + }, + "decode": { + "fixedK_1": { + "validity": 0.007568359375, + "nfe": 1.0, + "unique": 0.3548387096774194 + }, + "fixedK_2": { + "validity": 0.43603515625, + "nfe": 2.0, + "unique": 0.0083986562150056 + }, + "fixedK_4": { + "validity": 0.835205078125, + "nfe": 4.0, + "unique": 0.0043846828412744815 + }, + "fixedK_8": { + "validity": 0.84130859375, + "nfe": 8.0, + "unique": 0.004352872896111433 + }, + "fixedK_15": { + "validity": 0.999755859375, + "nfe": 15.0, + "unique": 0.003663003663003663 + }, + "entbudget_0.05": { + "validity": 1.0, + "nfe": 12.0, + "unique": 0.003662109375 + }, + "entbudget_0.25": { + "validity": 1.0, + "nfe": 12.0, + "unique": 0.003662109375 + }, + "entbudget_1.0": { + "validity": 0.9677734375, + "nfe": 7.0, + "unique": 0.0037840565085771947 + }, + "entbudget_4.0": { + "validity": 0.424072265625, + "nfe": 2.0, + "unique": 0.008635578583765112 + } + }, + "reached_ceiling": true +} \ No newline at end of file diff --git a/results/mol1-matched-ape-s0.json b/results/mol1-matched-ape-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..3d516a0e5e1b0c136c78b4d927420f47896818e7 --- /dev/null +++ b/results/mol1-matched-ape-s0.json @@ -0,0 +1,107 @@ +{ + "args": { + "mode": "matched", + "K": 8, + "budget_bins": 8, + "pe": "ape", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/qm9", + "limit": 0, + "d": 384, + "layers": 8, + "heads": 8, + "bs": 256, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/mol1-matched-ape-s0" + }, + "decode": { + "fixedK_1": { + "validity": 0.30078125, + "nfe": 1.0, + "unique": 0.9991883116883117, + "novel": 0.9439480097481722 + }, + "fixedK_2": { + "validity": 0.43896484375, + "nfe": 2.0, + "unique": 0.9983314794215795, + "novel": 0.8451253481894151 + }, + "fixedK_4": { + "validity": 0.578857421875, + "nfe": 4.0, + "unique": 0.9974694221847322, + "novel": 0.7167019027484144 + }, + "fixedK_8": { + "validity": 0.732177734375, + "nfe": 8.0, + "unique": 0.9896632210736912, + "novel": 0.5215633423180593 + }, + "fixedK_16": { + "validity": 0.873779296875, + "nfe": 16.0, + "unique": 0.9865884325230512, + "novel": 0.3806287170773152 + }, + "fixedK_32": { + "validity": 0.968017578125, + "nfe": 32.0, + "unique": 0.980327868852459, + "novel": 0.23411371237458195 + }, + "fixedK_45": { + "validity": 0.995849609375, + "nfe": 45.0, + "unique": 0.9835744054915421, + "novel": 0.1557826520438684 + }, + "entbudget_0.05": { + "validity": 0.998046875, + "nfe": 45.0, + "unique": 0.9806751467710372, + "novel": 0.15764529807932154 + }, + "entbudget_0.25": { + "validity": 0.995361328125, + "nfe": 42.0, + "unique": 0.9847927397596272, + "novel": 0.1651307596513076 + }, + "entbudget_1.0": { + "validity": 0.95849609375, + "nfe": 30.875, + "unique": 0.9834437086092715, + "novel": 0.21082621082621084 + }, + "entbudget_4.0": { + "validity": 0.694091796875, + "nfe": 6.875, + "unique": 0.9897995075624341, + "novel": 0.5536602700781805 + }, + "conf_0.7": { + "validity": 0.489990234375, + "nfe": 7.0, + "unique": 0.98903836571998, + "novel": 0.7561712846347607 + }, + "conf_0.9": { + "validity": 0.974853515625, + "nfe": 32.0, + "unique": 0.9852241422489356, + "novel": 0.19801728520589731 + }, + "conf_0.99": { + "validity": 0.99658203125, + "nfe": 45.0, + "unique": 0.9806467417932386, + "novel": 0.1671246565076193 + } + } +} \ No newline at end of file diff --git a/results/mol1-matched-ape-s1.json b/results/mol1-matched-ape-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..11d1701fef6d26f2bd5b9e2e5c16eb10aa0a18c0 --- /dev/null +++ b/results/mol1-matched-ape-s1.json @@ -0,0 +1,107 @@ +{ + "args": { + "mode": "matched", + "K": 8, + "budget_bins": 8, + "pe": "ape", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/qm9", + "limit": 0, + "d": 384, + "layers": 8, + "heads": 8, + "bs": 256, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/mol1-matched-ape-s1" + }, + "decode": { + "fixedK_1": { + "validity": 0.302490234375, + "nfe": 1.0, + "unique": 0.9951573849878934, + "novel": 0.9334955393349554 + }, + "fixedK_2": { + "validity": 0.44140625, + "nfe": 2.0, + "unique": 0.9988938053097345, + "novel": 0.832779623477298 + }, + "fixedK_4": { + "validity": 0.573486328125, + "nfe": 4.0, + "unique": 0.9931885908897403, + "novel": 0.7201028718388341 + }, + "fixedK_8": { + "validity": 0.736083984375, + "nfe": 8.0, + "unique": 0.9920398009950249, + "novel": 0.5242393848211301 + }, + "fixedK_16": { + "validity": 0.875732421875, + "nfe": 16.0, + "unique": 0.9849456370225815, + "novel": 0.3656948768751769 + }, + "fixedK_32": { + "validity": 0.961181640625, + "nfe": 32.0, + "unique": 0.9832359664719329, + "novel": 0.23017308189098423 + }, + "fixedK_45": { + "validity": 0.9970703125, + "nfe": 45.0, + "unique": 0.9857982370225269, + "novel": 0.15424739195231 + }, + "entbudget_0.05": { + "validity": 0.9970703125, + "nfe": 45.0, + "unique": 0.9799216454456415, + "novel": 0.1584207896051974 + }, + "entbudget_0.25": { + "validity": 0.99462890625, + "nfe": 42.0, + "unique": 0.9847815414825725, + "novel": 0.1625124626121635 + }, + "entbudget_1.0": { + "validity": 0.962158203125, + "nfe": 31.0, + "unique": 0.987059122050241, + "novel": 0.20899742930591259 + }, + "entbudget_4.0": { + "validity": 0.6923828125, + "nfe": 6.875, + "unique": 0.9908321579689704, + "novel": 0.5398576512455516 + }, + "conf_0.7": { + "validity": 0.476318359375, + "nfe": 7.0, + "unique": 0.9861609431060995, + "novel": 0.7401247401247402 + }, + "conf_0.9": { + "validity": 0.96533203125, + "nfe": 31.0, + "unique": 0.9830551340414769, + "novel": 0.20066889632107024 + }, + "conf_0.99": { + "validity": 0.996337890625, + "nfe": 45.0, + "unique": 0.9818671894143592, + "novel": 0.1594709258797105 + } + } +} \ No newline at end of file diff --git a/results/mol1-matched-rope-s0.json b/results/mol1-matched-rope-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..17a3bdf0e6708eab50b0e20ffde225e8ec65ba46 --- /dev/null +++ b/results/mol1-matched-rope-s0.json @@ -0,0 +1,107 @@ +{ + "args": { + "mode": "matched", + "K": 8, + "budget_bins": 8, + "pe": "rope", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/qm9", + "limit": 0, + "d": 384, + "layers": 8, + "heads": 8, + "bs": 256, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/mol1-matched-rope-s0" + }, + "decode": { + "fixedK_1": { + "validity": 0.000244140625, + "nfe": 1.0, + "unique": 1.0, + "novel": 1.0 + }, + "fixedK_2": { + "validity": 0.011474609375, + "nfe": 2.0, + "unique": 0.5531914893617021, + "novel": 0.8076923076923077 + }, + "fixedK_4": { + "validity": 0.299560546875, + "nfe": 4.0, + "unique": 0.07416462917685411, + "novel": 0.8461538461538461 + }, + "fixedK_8": { + "validity": 0.65771484375, + "nfe": 8.0, + "unique": 0.10690423162583519, + "novel": 0.8576388888888888 + }, + "fixedK_16": { + "validity": 0.599853515625, + "nfe": 16.0, + "unique": 0.10337810337810338, + "novel": 0.6023622047244095 + }, + "fixedK_32": { + "validity": 0.54443359375, + "nfe": 32.0, + "unique": 0.13273542600896862, + "novel": 0.34121621621621623 + }, + "fixedK_45": { + "validity": 0.5498046875, + "nfe": 45.0, + "unique": 0.44582593250444047, + "novel": 0.2340637450199203 + }, + "entbudget_0.05": { + "validity": 0.9794921875, + "nfe": 45.0, + "unique": 0.9793120638085743, + "novel": 0.2349198269279715 + }, + "entbudget_0.25": { + "validity": 0.97900390625, + "nfe": 42.0, + "unique": 0.9805486284289276, + "novel": 0.2347405900305188 + }, + "entbudget_1.0": { + "validity": 0.94677734375, + "nfe": 31.625, + "unique": 0.983754512635379, + "novel": 0.2851900393184797 + }, + "entbudget_4.0": { + "validity": 0.640380859375, + "nfe": 6.875, + "unique": 0.9416698436904308, + "novel": 0.7453441295546559 + }, + "conf_0.7": { + "validity": 0.0, + "nfe": 1.0, + "unique": 0.0, + "novel": 0.0 + }, + "conf_0.9": { + "validity": 0.4755859375, + "nfe": 32.0, + "unique": 0.5241273100616016, + "novel": 0.29774730656219395 + }, + "conf_0.99": { + "validity": 0.547607421875, + "nfe": 45.0, + "unique": 0.4592064199732501, + "novel": 0.24174757281553397 + } + } +} \ No newline at end of file diff --git a/results/mol1-matched-rope-s1.json b/results/mol1-matched-rope-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..c579db73653fdf0898efd742982431f2720e09fb --- /dev/null +++ b/results/mol1-matched-rope-s1.json @@ -0,0 +1,107 @@ +{ + "args": { + "mode": "matched", + "K": 8, + "budget_bins": 8, + "pe": "rope", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/qm9", + "limit": 0, + "d": 384, + "layers": 8, + "heads": 8, + "bs": 256, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/mol1-matched-rope-s1" + }, + "decode": { + "fixedK_1": { + "validity": 0.0, + "nfe": 1.0, + "unique": 0.0, + "novel": 0.0 + }, + "fixedK_2": { + "validity": 0.017333984375, + "nfe": 2.0, + "unique": 0.5633802816901409, + "novel": 0.85 + }, + "fixedK_4": { + "validity": 0.327880859375, + "nfe": 4.0, + "unique": 0.0766939687267312, + "novel": 0.883495145631068 + }, + "fixedK_8": { + "validity": 0.39111328125, + "nfe": 8.0, + "unique": 0.07303370786516854, + "novel": 0.8290598290598291 + }, + "fixedK_16": { + "validity": 0.276123046875, + "nfe": 16.0, + "unique": 0.1962864721485411, + "novel": 0.4864864864864865 + }, + "fixedK_32": { + "validity": 0.20654296875, + "nfe": 32.0, + "unique": 0.32505910165484636, + "novel": 0.4254545454545455 + }, + "fixedK_45": { + "validity": 0.34765625, + "nfe": 45.0, + "unique": 0.7394662921348315, + "novel": 0.25925925925925924 + }, + "entbudget_0.05": { + "validity": 0.962646484375, + "nfe": 45.0, + "unique": 0.9703271620593457, + "novel": 0.22895974908520647 + }, + "entbudget_0.25": { + "validity": 0.959228515625, + "nfe": 42.0, + "unique": 0.961567828964113, + "novel": 0.22419269454737956 + }, + "entbudget_1.0": { + "validity": 0.924072265625, + "nfe": 31.625, + "unique": 0.9706737120211361, + "novel": 0.2822536744692433 + }, + "entbudget_4.0": { + "validity": 0.6279296875, + "nfe": 7.0, + "unique": 0.9731726283048211, + "novel": 0.6272473032361167 + }, + "conf_0.7": { + "validity": 0.0, + "nfe": 1.0, + "unique": 0.0, + "novel": 0.0 + }, + "conf_0.9": { + "validity": 0.3349609375, + "nfe": 34.0, + "unique": 0.8002915451895044, + "novel": 0.30327868852459017 + }, + "conf_0.99": { + "validity": 0.3017578125, + "nfe": 45.0, + "unique": 0.8325242718446602, + "novel": 0.24101068999028183 + } + } +} \ No newline at end of file diff --git a/results/mol1-mdlm-ape-s0.json b/results/mol1-mdlm-ape-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..33ec54189af7696c0145e16619fe8b5085b84125 --- /dev/null +++ b/results/mol1-mdlm-ape-s0.json @@ -0,0 +1,107 @@ +{ + "args": { + "mode": "mdlm", + "K": 8, + "budget_bins": 8, + "pe": "ape", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/qm9", + "limit": 0, + "d": 384, + "layers": 8, + "heads": 8, + "bs": 256, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/mol1-mdlm-ape-s0" + }, + "decode": { + "fixedK_1": { + "validity": 0.29833984375, + "nfe": 1.0, + "unique": 0.9991816693944353, + "novel": 0.9426699426699426 + }, + "fixedK_2": { + "validity": 0.427734375, + "nfe": 2.0, + "unique": 0.9982876712328768, + "novel": 0.8524871355060034 + }, + "fixedK_4": { + "validity": 0.545654296875, + "nfe": 4.0, + "unique": 0.9941834451901566, + "novel": 0.7407740774077408 + }, + "fixedK_8": { + "validity": 0.711669921875, + "nfe": 8.0, + "unique": 0.9897084048027445, + "novel": 0.5580589254766031 + }, + "fixedK_16": { + "validity": 0.8564453125, + "nfe": 16.0, + "unique": 0.9851767388825542, + "novel": 0.4088541666666667 + }, + "fixedK_32": { + "validity": 0.953369140625, + "nfe": 32.0, + "unique": 0.982842509603073, + "novel": 0.24648254299114122 + }, + "fixedK_45": { + "validity": 0.995849609375, + "nfe": 45.0, + "unique": 0.9821034567295905, + "novel": 0.1767348976535197 + }, + "entbudget_0.05": { + "validity": 0.99560546875, + "nfe": 45.0, + "unique": 0.9816086316821971, + "novel": 0.17836622533100174 + }, + "entbudget_0.25": { + "validity": 0.99560546875, + "nfe": 42.0, + "unique": 0.9820990681706719, + "novel": 0.18551810237203495 + }, + "entbudget_1.0": { + "validity": 0.9609375, + "nfe": 30.75, + "unique": 0.9824695121951219, + "novel": 0.23506594259115593 + }, + "entbudget_4.0": { + "validity": 0.686279296875, + "nfe": 7.0, + "unique": 0.991462113127001, + "novel": 0.5479009687836384 + }, + "conf_0.7": { + "validity": 0.491455078125, + "nfe": 7.0, + "unique": 0.9865871833084948, + "novel": 0.7512588116817724 + }, + "conf_0.9": { + "validity": 0.968994140625, + "nfe": 31.0, + "unique": 0.9858906525573192, + "novel": 0.22616917965755176 + }, + "conf_0.99": { + "validity": 0.9970703125, + "nfe": 45.0, + "unique": 0.9821253672869735, + "novel": 0.18274744452754924 + } + } +} \ No newline at end of file diff --git a/results/mol1-mdlm-ape-s1.json b/results/mol1-mdlm-ape-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..b32dc509f50aadb4ba0cce581b161ad7c7a7bbd6 --- /dev/null +++ b/results/mol1-mdlm-ape-s1.json @@ -0,0 +1,107 @@ +{ + "args": { + "mode": "mdlm", + "K": 8, + "budget_bins": 8, + "pe": "ape", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/qm9", + "limit": 0, + "d": 384, + "layers": 8, + "heads": 8, + "bs": 256, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/mol1-mdlm-ape-s1" + }, + "decode": { + "fixedK_1": { + "validity": 0.300048828125, + "nfe": 1.0, + "unique": 0.9934906427990235, + "novel": 0.9352989352989353 + }, + "fixedK_2": { + "validity": 0.432373046875, + "nfe": 2.0, + "unique": 0.9983060417843026, + "novel": 0.8501131221719457 + }, + "fixedK_4": { + "validity": 0.5703125, + "nfe": 4.0, + "unique": 0.9965753424657534, + "novel": 0.7469931271477663 + }, + "fixedK_8": { + "validity": 0.73046875, + "nfe": 8.0, + "unique": 0.9936497326203209, + "novel": 0.5452404978136562 + }, + "fixedK_16": { + "validity": 0.86474609375, + "nfe": 16.0, + "unique": 0.9884246188594015, + "novel": 0.38131962296486716 + }, + "fixedK_32": { + "validity": 0.96142578125, + "nfe": 32.0, + "unique": 0.9824784154393092, + "novel": 0.24838459550271388 + }, + "fixedK_45": { + "validity": 0.998046875, + "nfe": 45.0, + "unique": 0.9794520547945206, + "novel": 0.18731268731268733 + }, + "entbudget_0.05": { + "validity": 0.995361328125, + "nfe": 45.0, + "unique": 0.9820946774589159, + "novel": 0.17757242757242758 + }, + "entbudget_0.25": { + "validity": 0.994384765625, + "nfe": 42.0, + "unique": 0.9835502086913823, + "novel": 0.17748377433849227 + }, + "entbudget_1.0": { + "validity": 0.958984375, + "nfe": 30.75, + "unique": 0.9814154786150713, + "novel": 0.2409857328145266 + }, + "entbudget_4.0": { + "validity": 0.686279296875, + "nfe": 6.75, + "unique": 0.9875489149768766, + "novel": 0.5554755043227666 + }, + "conf_0.7": { + "validity": 0.503173828125, + "nfe": 7.0, + "unique": 0.9815623483745755, + "novel": 0.7399901136925359 + }, + "conf_0.9": { + "validity": 0.968505859375, + "nfe": 31.0, + "unique": 0.982606503655155, + "novel": 0.22396100564391996 + }, + "conf_0.99": { + "validity": 0.99462890625, + "nfe": 45.0, + "unique": 0.9808541973490427, + "novel": 0.1841841841841842 + } + } +} \ No newline at end of file diff --git a/results/mol1-mdlm-rope-s0.json b/results/mol1-mdlm-rope-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..83ada3fbd48acf9e183ad1ae5b3317bb9371dbdc --- /dev/null +++ b/results/mol1-mdlm-rope-s0.json @@ -0,0 +1,107 @@ +{ + "args": { + "mode": "mdlm", + "K": 8, + "budget_bins": 8, + "pe": "rope", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/qm9", + "limit": 0, + "d": 384, + "layers": 8, + "heads": 8, + "bs": 256, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 0, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/mol1-mdlm-rope-s0" + }, + "decode": { + "fixedK_1": { + "validity": 0.000244140625, + "nfe": 1.0, + "unique": 1.0, + "novel": 1.0 + }, + "fixedK_2": { + "validity": 0.019775390625, + "nfe": 2.0, + "unique": 0.4444444444444444, + "novel": 0.8888888888888888 + }, + "fixedK_4": { + "validity": 0.265625, + "nfe": 4.0, + "unique": 0.0818014705882353, + "novel": 0.8539325842696629 + }, + "fixedK_8": { + "validity": 0.749267578125, + "nfe": 8.0, + "unique": 0.05408927989573151, + "novel": 0.8493975903614458 + }, + "fixedK_16": { + "validity": 0.71875, + "nfe": 16.0, + "unique": 0.1480978260869565, + "novel": 0.6995412844036697 + }, + "fixedK_32": { + "validity": 0.87451171875, + "nfe": 32.0, + "unique": 0.20993858179787828, + "novel": 0.586436170212766 + }, + "fixedK_45": { + "validity": 0.960693359375, + "nfe": 45.0, + "unique": 0.3593392630241423, + "novel": 0.36916548797736914 + }, + "entbudget_0.05": { + "validity": 0.984375, + "nfe": 45.0, + "unique": 0.9739583333333334, + "novel": 0.24013241660300483 + }, + "entbudget_0.25": { + "validity": 0.98095703125, + "nfe": 42.0, + "unique": 0.9813339970134395, + "novel": 0.2505706314988587 + }, + "entbudget_1.0": { + "validity": 0.945556640625, + "nfe": 30.0, + "unique": 0.9783113865220759, + "novel": 0.30509369226708893 + }, + "entbudget_4.0": { + "validity": 0.6259765625, + "nfe": 7.0, + "unique": 0.984399375975039, + "novel": 0.6537242472266244 + }, + "conf_0.7": { + "validity": 0.0, + "nfe": 1.0, + "unique": 0.0, + "novel": 0.0 + }, + "conf_0.9": { + "validity": 0.891357421875, + "nfe": 33.0, + "unique": 0.3886606409202958, + "novel": 0.3671599718111346 + }, + "conf_0.99": { + "validity": 0.9404296875, + "nfe": 45.0, + "unique": 0.3694184839044652, + "novel": 0.34293745607870696 + } + } +} \ No newline at end of file diff --git a/results/mol1-mdlm-rope-s1.json b/results/mol1-mdlm-rope-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..c6fd4277413eaf5b6a41311ba1ce61748ebd4a47 --- /dev/null +++ b/results/mol1-mdlm-rope-s1.json @@ -0,0 +1,107 @@ +{ + "args": { + "mode": "mdlm", + "K": 8, + "budget_bins": 8, + "pe": "rope", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/qm9", + "limit": 0, + "d": 384, + "layers": 8, + "heads": 8, + "bs": 256, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 1, + "n_gen": 4096, + "gen_bs": 512, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/mol1-mdlm-rope-s1" + }, + "decode": { + "fixedK_1": { + "validity": 0.0, + "nfe": 1.0, + "unique": 0.0, + "novel": 0.0 + }, + "fixedK_2": { + "validity": 0.017822265625, + "nfe": 2.0, + "unique": 0.5753424657534246, + "novel": 0.8095238095238095 + }, + "fixedK_4": { + "validity": 0.321044921875, + "nfe": 4.0, + "unique": 0.08973384030418251, + "novel": 0.864406779661017 + }, + "fixedK_8": { + "validity": 0.69482421875, + "nfe": 8.0, + "unique": 0.04321855235418131, + "novel": 0.8536585365853658 + }, + "fixedK_16": { + "validity": 0.667236328125, + "nfe": 16.0, + "unique": 0.045737285034760336, + "novel": 0.704 + }, + "fixedK_32": { + "validity": 0.47998046875, + "nfe": 32.0, + "unique": 0.15615462868769076, + "novel": 0.4592833876221498 + }, + "fixedK_45": { + "validity": 0.5888671875, + "nfe": 45.0, + "unique": 0.4635157545605307, + "novel": 0.30500894454382826 + }, + "entbudget_0.05": { + "validity": 0.97607421875, + "nfe": 45.0, + "unique": 0.9697348674337168, + "novel": 0.2636058808356977 + }, + "entbudget_0.25": { + "validity": 0.97607421875, + "nfe": 42.0, + "unique": 0.9677338669334667, + "novel": 0.27345567330059445 + }, + "entbudget_1.0": { + "validity": 0.937255859375, + "nfe": 31.75, + "unique": 0.9747330033862985, + "novel": 0.31373597006948156 + }, + "entbudget_4.0": { + "validity": 0.64208984375, + "nfe": 6.125, + "unique": 0.9452471482889734, + "novel": 0.6444086886564763 + }, + "conf_0.7": { + "validity": 0.0, + "nfe": 1.0, + "unique": 0.0, + "novel": 0.0 + }, + "conf_0.9": { + "validity": 0.36865234375, + "nfe": 29.875, + "unique": 0.8152317880794702, + "novel": 0.3931762794476036 + }, + "conf_0.99": { + "validity": 0.566162109375, + "nfe": 45.0, + "unique": 0.47477360931435963, + "novel": 0.3060853769300636 + } + } +} \ No newline at end of file diff --git a/results/sud1-full-aug-s0.json b/results/sud1-full-aug-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..076a16e22d0b7606961e5478487a4a17dfa00468 --- /dev/null +++ b/results/sud1-full-aug-s0.json @@ -0,0 +1,135 @@ +{ + "args": { + "mode": "full", + "K": 8, + "lam": 1.0, + "pe": "ape", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sudoku", + "augment": true, + "d": 512, + "layers": 10, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 0, + "eval_bs": 200, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/sud1-full-aug-s0" + }, + "decode": { + "fixedK_1": { + "board": 0.98, + "cell": 0.9991513121161363, + "valid_sudoku": 0.98, + "clues_kept": 1.0, + "nfe": 1.0 + }, + "fixedK_2": { + "board": 0.998, + "cell": 0.9999553322166388, + "valid_sudoku": 0.998, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "fixedK_4": { + "board": 0.997, + "cell": 0.9999106644332775, + "valid_sudoku": 0.997, + "clues_kept": 1.0, + "nfe": 4.0 + }, + "fixedK_8": { + "board": 0.998, + "cell": 0.9998659966499163, + "valid_sudoku": 0.998, + "clues_kept": 1.0, + "nfe": 8.0 + }, + "fixedK_16": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 16.0 + }, + "fixedK_32": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 32.0 + }, + "fixedK_45": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 45.0 + }, + "entbudget_0.01": { + "board": 0.999, + "cell": 0.9999329983249581, + "valid_sudoku": 0.999, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "entbudget_0.05": { + "board": 0.999, + "cell": 0.9999329983249581, + "valid_sudoku": 0.999, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "entbudget_0.2": { + "board": 0.993, + "cell": 0.99964265773311, + "valid_sudoku": 0.993, + "clues_kept": 1.0, + "nfe": 2.4 + }, + "entbudget_0.5": { + "board": 0.988, + "cell": 0.999463986599665, + "valid_sudoku": 0.988, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "entbudget_1.0": { + "board": 0.98, + "cell": 0.9992183137911781, + "valid_sudoku": 0.98, + "clues_kept": 1.0, + "nfe": 1.8 + }, + "entbudget_2.0": { + "board": 0.982, + "cell": 0.9991513121161363, + "valid_sudoku": 0.982, + "clues_kept": 1.0, + "nfe": 1.8 + }, + "conf_0.9": { + "board": 0.99, + "cell": 0.9995309882747069, + "valid_sudoku": 0.99, + "clues_kept": 1.0, + "nfe": 1.8 + }, + "conf_0.99": { + "board": 0.995, + "cell": 0.9997096594081518, + "valid_sudoku": 0.995, + "clues_kept": 1.0, + "nfe": 2.2 + }, + "conf_0.999": { + "board": 0.999, + "cell": 0.9999329983249581, + "valid_sudoku": 0.999, + "clues_kept": 1.0, + "nfe": 2.0 + } + }, + "best_board": 1.0 +} \ No newline at end of file diff --git a/results/sud1-full-aug-s1.json b/results/sud1-full-aug-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..bedc0dbd77fcba96477d16fd1de8102fbebc9244 --- /dev/null +++ b/results/sud1-full-aug-s1.json @@ -0,0 +1,135 @@ +{ + "args": { + "mode": "full", + "K": 8, + "lam": 1.0, + "pe": "ape", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sudoku", + "augment": true, + "d": 512, + "layers": 10, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 1, + "eval_bs": 200, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/sud1-full-aug-s1" + }, + "decode": { + "fixedK_1": { + "board": 0.984, + "cell": 0.9990619765494138, + "valid_sudoku": 0.984, + "clues_kept": 1.0, + "nfe": 1.0 + }, + "fixedK_2": { + "board": 0.998, + "cell": 0.9999553322166388, + "valid_sudoku": 0.998, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "fixedK_4": { + "board": 0.998, + "cell": 0.9999553322166388, + "valid_sudoku": 0.998, + "clues_kept": 1.0, + "nfe": 4.0 + }, + "fixedK_8": { + "board": 0.999, + "cell": 0.9999776661083194, + "valid_sudoku": 0.999, + "clues_kept": 1.0, + "nfe": 8.0 + }, + "fixedK_16": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 16.0 + }, + "fixedK_32": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 32.0 + }, + "fixedK_45": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 45.0 + }, + "entbudget_0.01": { + "board": 0.994, + "cell": 0.9995979899497487, + "valid_sudoku": 0.994, + "clues_kept": 1.0, + "nfe": 3.2 + }, + "entbudget_0.05": { + "board": 0.989, + "cell": 0.9993299832495812, + "valid_sudoku": 0.989, + "clues_kept": 1.0, + "nfe": 3.0 + }, + "entbudget_0.2": { + "board": 0.988, + "cell": 0.9992406476828587, + "valid_sudoku": 0.988, + "clues_kept": 1.0, + "nfe": 2.6 + }, + "entbudget_0.5": { + "board": 0.987, + "cell": 0.9992629815745394, + "valid_sudoku": 0.987, + "clues_kept": 1.0, + "nfe": 2.2 + }, + "entbudget_1.0": { + "board": 0.985, + "cell": 0.9991513121161363, + "valid_sudoku": 0.985, + "clues_kept": 1.0, + "nfe": 1.8 + }, + "entbudget_2.0": { + "board": 0.984, + "cell": 0.9991289782244556, + "valid_sudoku": 0.984, + "clues_kept": 1.0, + "nfe": 1.2 + }, + "conf_0.9": { + "board": 0.988, + "cell": 0.9992183137911781, + "valid_sudoku": 0.988, + "clues_kept": 1.0, + "nfe": 1.8 + }, + "conf_0.99": { + "board": 0.989, + "cell": 0.99928531546622, + "valid_sudoku": 0.989, + "clues_kept": 1.0, + "nfe": 2.8 + }, + "conf_0.999": { + "board": 0.995, + "cell": 0.9995979899497487, + "valid_sudoku": 0.995, + "clues_kept": 1.0, + "nfe": 2.8 + } + }, + "best_board": 1.0 +} \ No newline at end of file diff --git a/results/sud1-full-plain-s0.json b/results/sud1-full-plain-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..04fa8b8a8d2538023a3ae934c68841a4e17b295f --- /dev/null +++ b/results/sud1-full-plain-s0.json @@ -0,0 +1,135 @@ +{ + "args": { + "mode": "full", + "K": 8, + "lam": 1.0, + "pe": "ape", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sudoku", + "augment": false, + "d": 512, + "layers": 10, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 0, + "eval_bs": 200, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/sud1-full-plain-s0" + }, + "decode": { + "fixedK_1": { + "board": 0.956, + "cell": 0.998034617532105, + "valid_sudoku": 0.956, + "clues_kept": 1.0, + "nfe": 1.0 + }, + "fixedK_2": { + "board": 0.998, + "cell": 0.9999553322166388, + "valid_sudoku": 0.998, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "fixedK_4": { + "board": 0.999, + "cell": 0.9999776661083194, + "valid_sudoku": 0.999, + "clues_kept": 1.0, + "nfe": 4.0 + }, + "fixedK_8": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 8.0 + }, + "fixedK_16": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 16.0 + }, + "fixedK_32": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 32.0 + }, + "fixedK_45": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 45.0 + }, + "entbudget_0.01": { + "board": 0.988, + "cell": 0.9994193188163037, + "valid_sudoku": 0.988, + "clues_kept": 1.0, + "nfe": 2.8 + }, + "entbudget_0.05": { + "board": 0.982, + "cell": 0.99892797319933, + "valid_sudoku": 0.982, + "clues_kept": 1.0, + "nfe": 3.0 + }, + "entbudget_0.2": { + "board": 0.974, + "cell": 0.9987046342825238, + "valid_sudoku": 0.974, + "clues_kept": 1.0, + "nfe": 2.6 + }, + "entbudget_0.5": { + "board": 0.967, + "cell": 0.9984142936906756, + "valid_sudoku": 0.967, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "entbudget_1.0": { + "board": 0.955, + "cell": 0.9981016192071468, + "valid_sudoku": 0.955, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "entbudget_2.0": { + "board": 0.953, + "cell": 0.9979676158570631, + "valid_sudoku": 0.953, + "clues_kept": 1.0, + "nfe": 1.8 + }, + "conf_0.9": { + "board": 0.965, + "cell": 0.9984589614740369, + "valid_sudoku": 0.965, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "conf_0.99": { + "board": 0.982, + "cell": 0.9989726409826912, + "valid_sudoku": 0.982, + "clues_kept": 1.0, + "nfe": 2.8 + }, + "conf_0.999": { + "board": 0.988, + "cell": 0.9994193188163037, + "valid_sudoku": 0.988, + "clues_kept": 1.0, + "nfe": 2.8 + } + }, + "best_board": 1.0 +} \ No newline at end of file diff --git a/results/sud1-full-plain-s1.json b/results/sud1-full-plain-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..52ba9541cd2d18f9f0e8a2ccfac637091e6d5c60 --- /dev/null +++ b/results/sud1-full-plain-s1.json @@ -0,0 +1,135 @@ +{ + "args": { + "mode": "full", + "K": 8, + "lam": 1.0, + "pe": "ape", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sudoku", + "augment": false, + "d": 512, + "layers": 10, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 1, + "eval_bs": 200, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/sud1-full-plain-s1" + }, + "decode": { + "fixedK_1": { + "board": 0.955, + "cell": 0.9975432719151313, + "valid_sudoku": 0.955, + "clues_kept": 1.0, + "nfe": 1.0 + }, + "fixedK_2": { + "board": 0.997, + "cell": 0.9997989949748743, + "valid_sudoku": 0.997, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "fixedK_4": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 4.0 + }, + "fixedK_8": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 8.0 + }, + "fixedK_16": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 16.0 + }, + "fixedK_32": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 32.0 + }, + "fixedK_45": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 45.0 + }, + "entbudget_0.01": { + "board": 0.984, + "cell": 0.9990173087660524, + "valid_sudoku": 0.984, + "clues_kept": 1.0, + "nfe": 3.0 + }, + "entbudget_0.05": { + "board": 0.979, + "cell": 0.9986376326074818, + "valid_sudoku": 0.979, + "clues_kept": 1.0, + "nfe": 3.0 + }, + "entbudget_0.2": { + "board": 0.969, + "cell": 0.9979452819653825, + "valid_sudoku": 0.969, + "clues_kept": 1.0, + "nfe": 2.8 + }, + "entbudget_0.5": { + "board": 0.962, + "cell": 0.9977219430485762, + "valid_sudoku": 0.962, + "clues_kept": 1.0, + "nfe": 2.2 + }, + "entbudget_1.0": { + "board": 0.959, + "cell": 0.9975656058068119, + "valid_sudoku": 0.959, + "clues_kept": 1.0, + "nfe": 2.2 + }, + "entbudget_2.0": { + "board": 0.956, + "cell": 0.9974762702400893, + "valid_sudoku": 0.956, + "clues_kept": 1.0, + "nfe": 1.4 + }, + "conf_0.9": { + "board": 0.966, + "cell": 0.9977889447236181, + "valid_sudoku": 0.966, + "clues_kept": 1.0, + "nfe": 2.2 + }, + "conf_0.99": { + "board": 0.978, + "cell": 0.99857063093244, + "valid_sudoku": 0.978, + "clues_kept": 1.0, + "nfe": 3.0 + }, + "conf_0.999": { + "board": 0.983, + "cell": 0.9988386376326075, + "valid_sudoku": 0.983, + "clues_kept": 1.0, + "nfe": 3.0 + } + }, + "best_board": 1.0 +} \ No newline at end of file diff --git a/results/sud1-mdlm-aug-s0.json b/results/sud1-mdlm-aug-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..3e75483cbff6836b6ddd657ea30beda97d045a2e --- /dev/null +++ b/results/sud1-mdlm-aug-s0.json @@ -0,0 +1,135 @@ +{ + "args": { + "mode": "mdlm", + "K": 8, + "lam": 1.0, + "pe": "ape", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sudoku", + "augment": true, + "d": 512, + "layers": 10, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 0, + "eval_bs": 200, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/sud1-mdlm-aug-s0" + }, + "decode": { + "fixedK_1": { + "board": 0.945, + "cell": 0.9977219430485762, + "valid_sudoku": 0.945, + "clues_kept": 1.0, + "nfe": 1.0 + }, + "fixedK_2": { + "board": 0.991, + "cell": 0.9996873255164712, + "valid_sudoku": 0.991, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "fixedK_4": { + "board": 0.992, + "cell": 0.9997543271915131, + "valid_sudoku": 0.992, + "clues_kept": 1.0, + "nfe": 4.0 + }, + "fixedK_8": { + "board": 0.999, + "cell": 0.9999553322166388, + "valid_sudoku": 0.999, + "clues_kept": 1.0, + "nfe": 8.0 + }, + "fixedK_16": { + "board": 0.999, + "cell": 0.9999329983249581, + "valid_sudoku": 0.999, + "clues_kept": 1.0, + "nfe": 16.0 + }, + "fixedK_32": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 32.0 + }, + "fixedK_45": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 45.0 + }, + "entbudget_0.01": { + "board": 0.996, + "cell": 0.9996649916247906, + "valid_sudoku": 0.996, + "clues_kept": 1.0, + "nfe": 3.8 + }, + "entbudget_0.05": { + "board": 0.991, + "cell": 0.9993746510329425, + "valid_sudoku": 0.991, + "clues_kept": 1.0, + "nfe": 2.6 + }, + "entbudget_0.2": { + "board": 0.983, + "cell": 0.9991513121161363, + "valid_sudoku": 0.983, + "clues_kept": 1.0, + "nfe": 2.6 + }, + "entbudget_0.5": { + "board": 0.969, + "cell": 0.9986376326074818, + "valid_sudoku": 0.969, + "clues_kept": 1.0, + "nfe": 2.6 + }, + "entbudget_1.0": { + "board": 0.956, + "cell": 0.9980569514237856, + "valid_sudoku": 0.956, + "clues_kept": 1.0, + "nfe": 2.4 + }, + "entbudget_2.0": { + "board": 0.943, + "cell": 0.9975432719151313, + "valid_sudoku": 0.943, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "conf_0.9": { + "board": 0.967, + "cell": 0.9982802903405918, + "valid_sudoku": 0.967, + "clues_kept": 1.0, + "nfe": 2.8 + }, + "conf_0.99": { + "board": 0.987, + "cell": 0.9991736460078169, + "valid_sudoku": 0.987, + "clues_kept": 1.0, + "nfe": 3.0 + }, + "conf_0.999": { + "board": 0.994, + "cell": 0.9994416527079844, + "valid_sudoku": 0.994, + "clues_kept": 1.0, + "nfe": 3.2 + } + }, + "best_board": 1.0 +} \ No newline at end of file diff --git a/results/sud1-mdlm-aug-s1.json b/results/sud1-mdlm-aug-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..e77e3591366d9fe476b0fe6958aa0b66c8e134d3 --- /dev/null +++ b/results/sud1-mdlm-aug-s1.json @@ -0,0 +1,135 @@ +{ + "args": { + "mode": "mdlm", + "K": 8, + "lam": 1.0, + "pe": "ape", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sudoku", + "augment": true, + "d": 512, + "layers": 10, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 1, + "eval_bs": 200, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/sud1-mdlm-aug-s1" + }, + "decode": { + "fixedK_1": { + "board": 0.941, + "cell": 0.9981016192071468, + "valid_sudoku": 0.941, + "clues_kept": 1.0, + "nfe": 1.0 + }, + "fixedK_2": { + "board": 0.994, + "cell": 0.9997989949748743, + "valid_sudoku": 0.994, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "fixedK_4": { + "board": 0.995, + "cell": 0.9998436627582357, + "valid_sudoku": 0.995, + "clues_kept": 1.0, + "nfe": 4.0 + }, + "fixedK_8": { + "board": 0.999, + "cell": 0.9999776661083194, + "valid_sudoku": 0.999, + "clues_kept": 1.0, + "nfe": 8.0 + }, + "fixedK_16": { + "board": 0.999, + "cell": 0.9999553322166388, + "valid_sudoku": 0.999, + "clues_kept": 1.0, + "nfe": 16.0 + }, + "fixedK_32": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 32.0 + }, + "fixedK_45": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 45.0 + }, + "entbudget_0.01": { + "board": 0.998, + "cell": 0.9997319932998325, + "valid_sudoku": 0.998, + "clues_kept": 1.0, + "nfe": 2.6 + }, + "entbudget_0.05": { + "board": 0.994, + "cell": 0.9997096594081518, + "valid_sudoku": 0.994, + "clues_kept": 1.0, + "nfe": 2.2 + }, + "entbudget_0.2": { + "board": 0.982, + "cell": 0.9992183137911781, + "valid_sudoku": 0.982, + "clues_kept": 1.0, + "nfe": 2.2 + }, + "entbudget_0.5": { + "board": 0.971, + "cell": 0.99892797319933, + "valid_sudoku": 0.971, + "clues_kept": 1.0, + "nfe": 2.2 + }, + "entbudget_1.0": { + "board": 0.959, + "cell": 0.9986152987158012, + "valid_sudoku": 0.959, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "entbudget_2.0": { + "board": 0.952, + "cell": 0.9982802903405918, + "valid_sudoku": 0.952, + "clues_kept": 1.0, + "nfe": 1.8 + }, + "conf_0.9": { + "board": 0.97, + "cell": 0.9989056393076493, + "valid_sudoku": 0.97, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "conf_0.99": { + "board": 0.99, + "cell": 0.9994863204913457, + "valid_sudoku": 0.99, + "clues_kept": 1.0, + "nfe": 2.2 + }, + "conf_0.999": { + "board": 0.996, + "cell": 0.9997543271915131, + "valid_sudoku": 0.996, + "clues_kept": 1.0, + "nfe": 2.4 + } + }, + "best_board": 1.0 +} \ No newline at end of file diff --git a/results/sud1-mdlm-plain-s0.json b/results/sud1-mdlm-plain-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..337791fd8bb4e78cd842bf2ec15f840160e3221c --- /dev/null +++ b/results/sud1-mdlm-plain-s0.json @@ -0,0 +1,135 @@ +{ + "args": { + "mode": "mdlm", + "K": 8, + "lam": 1.0, + "pe": "ape", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sudoku", + "augment": false, + "d": 512, + "layers": 10, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 0, + "eval_bs": 200, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/sud1-mdlm-plain-s0" + }, + "decode": { + "fixedK_1": { + "board": 0.875, + "cell": 0.9943271915131212, + "valid_sudoku": 0.875, + "clues_kept": 1.0, + "nfe": 1.0 + }, + "fixedK_2": { + "board": 0.981, + "cell": 0.9991289782244556, + "valid_sudoku": 0.981, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "fixedK_4": { + "board": 0.986, + "cell": 0.9995086543830263, + "valid_sudoku": 0.986, + "clues_kept": 1.0, + "nfe": 4.0 + }, + "fixedK_8": { + "board": 0.996, + "cell": 0.9998436627582357, + "valid_sudoku": 0.996, + "clues_kept": 1.0, + "nfe": 8.0 + }, + "fixedK_16": { + "board": 0.998, + "cell": 0.9999329983249581, + "valid_sudoku": 0.998, + "clues_kept": 1.0, + "nfe": 16.0 + }, + "fixedK_32": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 32.0 + }, + "fixedK_45": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 45.0 + }, + "entbudget_0.01": { + "board": 0.99, + "cell": 0.9992406476828587, + "valid_sudoku": 0.99, + "clues_kept": 1.0, + "nfe": 3.8 + }, + "entbudget_0.05": { + "board": 0.973, + "cell": 0.9980792853154662, + "valid_sudoku": 0.973, + "clues_kept": 1.0, + "nfe": 3.8 + }, + "entbudget_0.2": { + "board": 0.949, + "cell": 0.9972975991066443, + "valid_sudoku": 0.949, + "clues_kept": 1.0, + "nfe": 2.8 + }, + "entbudget_0.5": { + "board": 0.927, + "cell": 0.9964712451144612, + "valid_sudoku": 0.927, + "clues_kept": 1.0, + "nfe": 3.0 + }, + "entbudget_1.0": { + "board": 0.892, + "cell": 0.9952875488553881, + "valid_sudoku": 0.892, + "clues_kept": 1.0, + "nfe": 2.6 + }, + "entbudget_2.0": { + "board": 0.874, + "cell": 0.9945281965382468, + "valid_sudoku": 0.874, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "conf_0.9": { + "board": 0.926, + "cell": 0.9963372417643774, + "valid_sudoku": 0.926, + "clues_kept": 1.0, + "nfe": 3.0 + }, + "conf_0.99": { + "board": 0.958, + "cell": 0.9976549413735344, + "valid_sudoku": 0.958, + "clues_kept": 1.0, + "nfe": 3.4 + }, + "conf_0.999": { + "board": 0.98, + "cell": 0.9985929648241206, + "valid_sudoku": 0.98, + "clues_kept": 1.0, + "nfe": 4.4 + } + }, + "best_board": 1.0 +} \ No newline at end of file diff --git a/results/sud1-mdlm-plain-s1.json b/results/sud1-mdlm-plain-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..7c02474c848de55dd06ef786f5125bf8db2aa9dd --- /dev/null +++ b/results/sud1-mdlm-plain-s1.json @@ -0,0 +1,135 @@ +{ + "args": { + "mode": "mdlm", + "K": 8, + "lam": 1.0, + "pe": "ape", + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sudoku", + "augment": false, + "d": 512, + "layers": 10, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 2000, + "seed": 1, + "eval_bs": 200, + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/sud1-mdlm-plain-s1" + }, + "decode": { + "fixedK_1": { + "board": 0.882, + "cell": 0.9950865438302624, + "valid_sudoku": 0.882, + "clues_kept": 1.0, + "nfe": 1.0 + }, + "fixedK_2": { + "board": 0.992, + "cell": 0.9997766610831937, + "valid_sudoku": 0.992, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "fixedK_4": { + "board": 0.991, + "cell": 0.9995533221663875, + "valid_sudoku": 0.991, + "clues_kept": 1.0, + "nfe": 4.0 + }, + "fixedK_8": { + "board": 0.998, + "cell": 0.9998883305415969, + "valid_sudoku": 0.998, + "clues_kept": 1.0, + "nfe": 8.0 + }, + "fixedK_16": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 16.0 + }, + "fixedK_32": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 32.0 + }, + "fixedK_45": { + "board": 1.0, + "cell": 1.0, + "valid_sudoku": 1.0, + "clues_kept": 1.0, + "nfe": 45.0 + }, + "entbudget_0.01": { + "board": 0.991, + "cell": 0.999463986599665, + "valid_sudoku": 0.991, + "clues_kept": 1.0, + "nfe": 3.4 + }, + "entbudget_0.05": { + "board": 0.977, + "cell": 0.9986152987158012, + "valid_sudoku": 0.977, + "clues_kept": 1.0, + "nfe": 3.0 + }, + "entbudget_0.2": { + "board": 0.954, + "cell": 0.997677275265215, + "valid_sudoku": 0.954, + "clues_kept": 1.0, + "nfe": 3.2 + }, + "entbudget_0.5": { + "board": 0.939, + "cell": 0.99714126186488, + "valid_sudoku": 0.939, + "clues_kept": 1.0, + "nfe": 3.2 + }, + "entbudget_1.0": { + "board": 0.902, + "cell": 0.9959575656058068, + "valid_sudoku": 0.902, + "clues_kept": 1.0, + "nfe": 2.4 + }, + "entbudget_2.0": { + "board": 0.883, + "cell": 0.9951982132886655, + "valid_sudoku": 0.883, + "clues_kept": 1.0, + "nfe": 2.0 + }, + "conf_0.9": { + "board": 0.921, + "cell": 0.9965382467895031, + "valid_sudoku": 0.921, + "clues_kept": 1.0, + "nfe": 2.4 + }, + "conf_0.99": { + "board": 0.971, + "cell": 0.9984366275823562, + "valid_sudoku": 0.971, + "clues_kept": 1.0, + "nfe": 3.4 + }, + "conf_0.999": { + "board": 0.989, + "cell": 0.9992629815745394, + "valid_sudoku": 0.989, + "clues_kept": 1.0, + "nfe": 3.4 + } + }, + "best_board": 1.0 +} \ No newline at end of file diff --git a/results/t8-anybudget-s0.json b/results/t8-anybudget-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..4bd7e3c036699dda66845513eb4c1a30f4246e41 100644 --- a/results/t8-anybudget-s0.json +++ b/results/t8-anybudget-s0.json @@ -0,0 +1,184 @@ +{ + "args": { + "mode": "anybudget", + "budget_bins": 10, + "K": 8, + "lam": 1.0, + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/text8", + "seq_len": 256, + "d": 512, + "layers": 12, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 0, + "n_gen": 256, + "gen_bs": 64, + "evallm": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/evallm", + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/t8-anybudget-s0" + }, + "valid_bpc": 2.6432830130237086, + "reference": { + "gen_ppl": 2.8611169809244217, + "entropy": 2.8548588407315023, + "distinct4": 0.17659955533596838 + }, + "decode": { + "fixedK_1": { + "nfe": 1.0, + "gen_ppl": 647.1523971491781, + "entropy": 2.8519201096932343, + "distinct4": 0.6255712697628458, + "sample": " saifmjhtletm xciusfeatetessio ie nctouh rveeeosvoewre hoh e ini ectni natheoen nier tu grrlsoc r doi f natreuox " + }, + "fixedK_2": { + "nfe": 2.0, + "gen_ppl": 284.71145051118185, + "entropy": 2.8714188294814846, + "distinct4": 0.6150259387351779, + "sample": "cviedteees nina souline soue storn su iaau sterh spa io aau regr eafror aerodlaeuro hsidlesio neum sdaredeiaeg tmo repb " + }, + "fixedK_4": { + "nfe": 4.0, + "gen_ppl": 141.59876005616246, + "entropy": 2.8714743018728703, + "distinct4": 0.5442039278656127, + "sample": "eltoka endeties insehundius spouier rocse xperies alatysts upoleaa cotcortichippeiorw lit salancayncc natglun ezusodchip" + }, + "fixedK_8": { + "nfe": 8.0, + "gen_ppl": 131.3458219008855, + "entropy": 2.863585890380679, + "distinct4": 0.5383368330039525, + "sample": "roe tra scouai csuno shiwaenry vinca mosea cifflen ikao h ticaretai aventaurf zhn h tevna h nt unctress of hhm paetci dp" + }, + "fixedK_16": { + "nfe": 16.0, + "gen_ppl": 94.35387684273293, + "entropy": 2.8635864724516917, + "distinct4": 0.5053884634387352, + "sample": "mar mehmar or miy oyx five paxla igd ahrud yad sumer belove this is first seen atienly of three eveen or ch ahis a chh " + }, + "fixedK_32": { + "nfe": 32.0, + "gen_ppl": 98.25667381964884, + "entropy": 2.8618309548447387, + "distinct4": 0.5039834486166008, + "sample": "ukerthur ds hell but aht de ks nesum ds telhe buits bct b tastid flat balat bullwcar baeaben lujmyebairs carshtigm of th" + }, + "fixedK_64": { + "nfe": 64.0, + "gen_ppl": 73.03552547506037, + "entropy": 2.874860615633551, + "distinct4": 0.4676537796442688, + "sample": "youists were bunbrielry s ahe df p additional sundisdoes since used to pardeaux eryptians also fause herocdem iea the on" + }, + "fixedK_128": { + "nfe": 128.0, + "gen_ppl": 40.850602236968165, + "entropy": 2.87052370612463, + "distinct4": 0.40087697628458496, + "sample": "st struam fiid protected jaluccututgun banli shows bn panys inverko kiniat cutsilo eusimbn beils roctrd outgurype was th" + }, + "fixedK_256": { + "nfe": 256.0, + "gen_ppl": 8.72295816502438, + "entropy": 2.850434203450186, + "distinct4": 0.23295454545454544, + "sample": "s in pleborous flan together benrits and phato yeg witner plusenes this is the most plague friendly that see also little" + }, + "conf_0.5": { + "nfe": 236.5, + "gen_ppl": 33.13419900213926, + "entropy": 2.8688767244016877, + "distinct4": 0.35678112648221344, + "sample": "rs lionum flores zerosparna p ztto ch flore one v six zere foprcaster m four zero a hcov three zero two zepo two ones ma" + }, + "conf_0.7": { + "nfe": 247.75, + "gen_ppl": 15.753179224495673, + "entropy": 2.857834114877501, + "distinct4": 0.2855576828063241, + "sample": "abap providion later chapter bach forged a healing structures at it control of reach recording of thehrim he has the pla" + }, + "conf_0.9": { + "nfe": 249.75, + "gen_ppl": 9.759803453625768, + "entropy": 2.8594265127853107, + "distinct4": 0.2432528409090909, + "sample": "ports he is a member of the same vest l under new opera bust two we eight we make for best weight varies that the advent" + }, + "conf_0.95": { + "nfe": 253.75, + "gen_ppl": 9.203248887287545, + "entropy": 2.852782799374823, + "distinct4": 0.2413383152173913, + "sample": "ulture to the geography at the expense or duchief faroe length but he would somewhat her well because of its paralyze to" + }, + "conf_0.99": { + "nfe": 255.75, + "gen_ppl": 8.886583497471843, + "entropy": 2.8482387413247445, + "distinct4": 0.23454483695652173, + "sample": " decided to that in one nine eight two jeanstr derman character the bar has also off d to gain economics nine nine two o" + }, + "entbudget_0.05": { + "nfe": 256.0, + "gen_ppl": 8.31621137140298, + "entropy": 2.856814624760518, + "distinct4": 0.23494626976284586, + "sample": "open empire s discussion which expected beria would hold on the construction of beria as his emperor despite the opposit" + }, + "entbudget_0.1": { + "nfe": 255.75, + "gen_ppl": 8.651806126641523, + "entropy": 2.859121773870919, + "distinct4": 0.23740118577075098, + "sample": "ware class queted with an other four facilities both trip ettes such as a guars trip of the decko that would leak howeve" + }, + "entbudget_0.25": { + "nfe": 254.75, + "gen_ppl": 8.37114810664664, + "entropy": 2.857402221087463, + "distinct4": 0.23594985177865613, + "sample": "ntoni baiefried the current forum tanking compare johnson battle the first beginning the close part of electronyclism th" + }, + "entbudget_0.5": { + "nfe": 253.25, + "gen_ppl": 8.916751814989125, + "entropy": 2.852354949481287, + "distinct4": 0.23885251976284586, + "sample": "barly overweed or usly dry named diffies with wall blows for leased trained the must called vinland sharap of vinland s " + }, + "entbudget_1.0": { + "nfe": 246.5, + "gen_ppl": 9.663616835171089, + "entropy": 2.854885589659516, + "distinct4": 0.24261981225296442, + "sample": "attern ben lebanon dog metal and stathoof at as friend of his time hu was prompted william argestan third law st the eur" + }, + "entbudget_2.0": { + "nfe": 245.0, + "gen_ppl": 15.050124866134697, + "entropy": 2.865182671245583, + "distinct4": 0.2816514328063241, + "sample": "l phone kom nyl neber elizabeth cher libo kojito wab sya manage polit costata aner obilise jad reference reservie dku an" + }, + "entbudget_4.0": { + "nfe": 222.25, + "gen_ppl": 49.30991338095145, + "entropy": 2.8821868194813383, + "distinct4": 0.4022202322134387, + "sample": "pointtail of general as minerial chericter since che fen red ethay focrenand and the spury are ficemire tw kingmin thems" + }, + "entbudget_8.0": { + "nfe": 94.0, + "gen_ppl": 92.85964451588463, + "entropy": 2.8779042137228212, + "distinct4": 0.4966495800395257, + "sample": "t press jeaf rov in thn gauss iy ghrrented by jean by guitor of biphicol exiltng ine etc sv co ipapess laad sv praxedus " + } + } +} \ No newline at end of file diff --git a/results/t8-anybudget-s1.json b/results/t8-anybudget-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..644bdfc0e94c3d65b0c80a4d5f68f64cf53ae86b 100644 --- a/results/t8-anybudget-s1.json +++ b/results/t8-anybudget-s1.json @@ -0,0 +1,184 @@ +{ + "args": { + "mode": "anybudget", + "budget_bins": 10, + "K": 8, + "lam": 1.0, + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/text8", + "seq_len": 256, + "d": 512, + "layers": 12, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 1, + "n_gen": 256, + "gen_bs": 64, + "evallm": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/evallm", + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/t8-anybudget-s1" + }, + "valid_bpc": 2.5007115896367123, + "reference": { + "gen_ppl": 2.8611169809244217, + "entropy": 2.8548588407315023, + "distinct4": 0.17659955533596838 + }, + "decode": { + "fixedK_1": { + "nfe": 1.0, + "gen_ppl": 652.743296840149, + "entropy": 2.861510732218865, + "distinct4": 0.6352828557312253, + "sample": "e si bgr c ivasse ege gbk pi er chtimuescu aodnnvtmsfar seaearmwt ohnairu efo li k ornor hnaewnehigstnas p trct mwyglr" + }, + "fixedK_2": { + "nfe": 2.0, + "gen_ppl": 224.1773073399576, + "entropy": 2.8730419067273756, + "distinct4": 0.6006669960474308, + "sample": "nlneywre trobwvuseobn t scrorsbes p egga g neow funl ndoshtchits carfpedr ske e eid gnyex ret abucinsce penga ti arceen" + }, + "fixedK_4": { + "nfe": 4.0, + "gen_ppl": 142.3650435517088, + "entropy": 2.8747493329343174, + "distinct4": 0.5628859930830039, + "sample": "c as r itevalinipent sone giteau vu lyhjs so a rcec msin stut m ul ptris hoxme m t empanl newanclus jpg steepecdes thn" + }, + "fixedK_8": { + "nfe": 8.0, + "gen_ppl": 117.63621211982583, + "entropy": 2.8667358565245555, + "distinct4": 0.5222795207509882, + "sample": "ithtt w thtforwend raithta o tdf fcanta orn s cecholce k tg d xmarlcr g t s n w lft vcigmae vigve pa eimegvg s ba ix" + }, + "fixedK_16": { + "nfe": 16.0, + "gen_ppl": 95.51740285297666, + "entropy": 2.867975191567498, + "distinct4": 0.5044157608695652, + "sample": "d concri e s n in c or pconsormplly it a no ledge where the experimnnt r tnrmfly long biratect is a key the designatio " + }, + "fixedK_32": { + "nfe": 32.0, + "gen_ppl": 96.25850237642565, + "entropy": 2.8751065946676357, + "distinct4": 0.5102674160079052, + "sample": "ng minc and ensur en icgeivi vn and norau historyogonwf s ndoves an rahag of hlstall hnstofswski hirstall ioks edia an" + }, + "fixedK_64": { + "nfe": 64.0, + "gen_ppl": 82.64514220397459, + "entropy": 2.882195706954416, + "distinct4": 0.494827692687747, + "sample": "httc moath weachrist i one zero zero films j howanopkelinbalh hah acherh shah etc ahthmush ah hurn with ps brecht hnh an" + }, + "fixedK_128": { + "nfe": 128.0, + "gen_ppl": 41.806746152471064, + "entropy": 2.878154089411176, + "distinct4": 0.40465970849802374, + "sample": "le sirquitobus throses engried in ruburde greaty north grebads and keiya thit sisguates the morality be alerted to incre" + }, + "fixedK_256": { + "nfe": 256.0, + "gen_ppl": 8.455623698204066, + "entropy": 2.85002400174842, + "distinct4": 0.23309350296442688, + "sample": " magazine total part of austria by the department of the parties which have died of aisbrizi native party inspiring the " + }, + "conf_0.5": { + "nfe": 232.0, + "gen_ppl": 32.42533684319982, + "entropy": 2.871793462947959, + "distinct4": 0.3610270503952569, + "sample": "ratification newh tne modd the that did n t d horce tne darriion of karen forced noise on the songs of oppc aids of abib" + }, + "conf_0.7": { + "nfe": 244.75, + "gen_ppl": 16.41828926133972, + "entropy": 2.8608218840357984, + "distinct4": 0.2927834733201581, + "sample": " southern eagine dr anghes betzerland is given angreh cs challenge up prime monsters of meaning that its piloty of natch" + }, + "conf_0.9": { + "nfe": 249.0, + "gen_ppl": 9.529916913887313, + "entropy": 2.853858101802156, + "distinct4": 0.2445497776679842, + "sample": " poland in pres minister of encyclopederothe secretary of support the foreign featured by the two round regional histori" + }, + "conf_0.95": { + "nfe": 253.0, + "gen_ppl": 9.04114167430555, + "entropy": 2.860625875208359, + "distinct4": 0.23928483201581027, + "sample": "nd larger curses deserts so far throughout their were ald only were given the essential norms of the international natio" + }, + "conf_0.99": { + "nfe": 255.0, + "gen_ppl": 8.681781580058319, + "entropy": 2.852837609745673, + "distinct4": 0.23573369565217392, + "sample": " maxwell an echoemium lighting deaths video award one nine four five a body of orchie frequent at laesar vi la lal rocka" + }, + "entbudget_0.05": { + "nfe": 256.0, + "gen_ppl": 8.539377785066206, + "entropy": 2.858159617168711, + "distinct4": 0.23701519268774704, + "sample": " second a bone highbounded they was also it wonder assembles that gave asteroin in allow heads of still attending one ei" + }, + "entbudget_0.1": { + "nfe": 255.5, + "gen_ppl": 8.346377463287311, + "entropy": 2.854984494849832, + "distinct4": 0.23428236166007904, + "sample": "inity one nine five two country books winter levels accord with real one nine nine nine everymore race roger jpg rso one" + }, + "entbudget_0.25": { + "nfe": 252.5, + "gen_ppl": 8.161859687568855, + "entropy": 2.8500343549609264, + "distinct4": 0.23133337450592886, + "sample": " an ovorwild role these are recently able to derive a variety a local local proper and borde note in english years also " + }, + "entbudget_0.5": { + "nfe": 252.25, + "gen_ppl": 8.741198582091938, + "entropy": 2.8529416214484513, + "distinct4": 0.2367372776679842, + "sample": "t the second st has been mix to us with the ferst thange it was born generally coming later fromapart showed into a psyc" + }, + "entbudget_1.0": { + "nfe": 249.0, + "gen_ppl": 9.572941389458466, + "entropy": 2.856457684565826, + "distinct4": 0.24485857213438736, + "sample": "ous astrological subspecies cuu d thombes and three seven k scapinch he remains sca ila marcaku jacques masquer other pe" + }, + "entbudget_2.0": { + "nfe": 240.0, + "gen_ppl": 14.143091754404434, + "entropy": 2.862448555521914, + "distinct4": 0.2763401679841897, + "sample": "ven the troth of it thoen tne thoen one three three thrie treth thern one two three three tv thorn one six benontin fi y" + }, + "entbudget_4.0": { + "nfe": 224.5, + "gen_ppl": 48.00720553282236, + "entropy": 2.882350486005125, + "distinct4": 0.4010468132411067, + "sample": "tag thp tri meal stamww utso known as phaldaw atriolwhers tnis has been enevusible however suviyapadoo the stth six rese" + }, + "entbudget_8.0": { + "nfe": 95.5, + "gen_ppl": 87.2341998984942, + "entropy": 2.8880133093866447, + "distinct4": 0.4974987648221344, + "sample": "gll sk colors we thcoming utuat stachhewd pd what l sue llfrig von v lohiing wicjiog subtwh se wycliffe soueridn seakh" + } + } +} \ No newline at end of file diff --git a/results/t8-matched-s0.json b/results/t8-matched-s0.json new file mode 100644 index 0000000000000000000000000000000000000000..d9161393a6cc1549a90da59802f9f13b16d58ede --- /dev/null +++ b/results/t8-matched-s0.json @@ -0,0 +1,184 @@ +{ + "args": { + "mode": "matched", + "budget_bins": 10, + "K": 8, + "lam": 1.0, + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/text8", + "seq_len": 256, + "d": 512, + "layers": 12, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 0, + "n_gen": 256, + "gen_bs": 64, + "evallm": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/evallm", + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/t8-matched-s0" + }, + "valid_bpc": 1.6945707777936863, + "reference": { + "gen_ppl": 2.8611169809244217, + "entropy": 2.8548588407315023, + "distinct4": 0.17659955533596838 + }, + "decode": { + "fixedK_1": { + "nfe": 1.0, + "gen_ppl": 651.670990558339, + "entropy": 2.8596867892202513, + "distinct4": 0.6306818181818182, + "sample": "iyiee icn rte i a hahesd en at a tuncnh wlrrghtomisa hnplaeh oatareohas vl ihmsroseoec wl na ces olgsyhseu tetao e" + }, + "fixedK_2": { + "nfe": 2.0, + "gen_ppl": 295.7939325505372, + "entropy": 2.881068367608071, + "distinct4": 0.6202754446640316, + "sample": "ils lingr oc mapge yva ateduen pa en ih yies yoen po sspioeia pswa rharoenamme margsia affry org jr spitnt merdga cesw" + }, + "fixedK_4": { + "nfe": 4.0, + "gen_ppl": 146.64344242219326, + "entropy": 2.87963229856915, + "distinct4": 0.5575747282608695, + "sample": "jofastinol thrae h troidalrmoutc buobc efo hf sqrtopmel po frjw fa e est r ebrley iu gf equ ned hdac dat nerm gu ixal i" + }, + "fixedK_8": { + "nfe": 8.0, + "gen_ppl": 123.33127507188556, + "entropy": 2.8759862541635304, + "distinct4": 0.5395411314229249, + "sample": "a liago be dety biofferducks cappel detydoferid dr ce tian ie antanaa nicuae icrc laagw r irc ir ded noea rsdoets va " + }, + "fixedK_16": { + "nfe": 16.0, + "gen_ppl": 94.52461890205767, + "entropy": 2.8679006574918358, + "distinct4": 0.5024085968379447, + "sample": "r sshb hamlof arejonabsb jatwa set no anythetehadu jach ar phi cares nn an anset withfuzs cerche iu and e seu dilease no" + }, + "fixedK_32": { + "nfe": 32.0, + "gen_ppl": 93.32694606529006, + "entropy": 2.881981396902236, + "distinct4": 0.5099895009881423, + "sample": "ig calegendes and begin iig ethertise egnecurles fluenns cillitoses aad vig romate the romantic arjer denlzilt i ml te" + }, + "fixedK_64": { + "nfe": 64.0, + "gen_ppl": 76.88697586376091, + "entropy": 2.8837927884620282, + "distinct4": 0.48372653162055335, + "sample": " guyvarem and gebro frossingeritis gordon onisodod egor dor strem dor ond luke it can he s m s selsusee awgans seancing" + }, + "fixedK_128": { + "nfe": 128.0, + "gen_ppl": 42.38286162666533, + "entropy": 2.8688634954594003, + "distinct4": 0.4073770998023715, + "sample": "etaed cwb kn wn extirnal plaies would claim that the storms of the muream educational tentrance couple hurst freechp dit" + }, + "fixedK_256": { + "nfe": 256.0, + "gen_ppl": 8.190038162989037, + "entropy": 2.8523496773263997, + "distinct4": 0.23462203557312253, + "sample": " high afertider is a latestelectric sutra annumber of salvador aafl local pacific upper five four eight five four eight " + }, + "conf_0.5": { + "nfe": 237.5, + "gen_ppl": 32.44207025275455, + "entropy": 2.8729227647277003, + "distinct4": 0.3659060029644269, + "sample": "dutick when to survive is said to st is it also high sweeping elecstick against usuaety series with gold basins andlock " + }, + "conf_0.7": { + "nfe": 242.0, + "gen_ppl": 14.857149274501849, + "entropy": 2.8541855195015415, + "distinct4": 0.28189846837944665, + "sample": "one nine one six jich one two one one li ewalias de lila of the ire called lionl la of faudel one two twoemila bead one " + }, + "conf_0.9": { + "nfe": 249.25, + "gen_ppl": 9.66527136508687, + "entropy": 2.856233740945464, + "distinct4": 0.247560523715415, + "sample": "large gallentimarium callfornia and vulcania there are inside low low california during the south to sentura except supe" + }, + "conf_0.95": { + "nfe": 251.25, + "gen_ppl": 8.64833402393934, + "entropy": 2.8547744792145946, + "distinct4": 0.2407824851778656, + "sample": "ne longest day day pemple golden religious former day external links canada dominance day intrust day couture covers of " + }, + "conf_0.99": { + "nfe": 254.75, + "gen_ppl": 8.337975724958945, + "entropy": 2.852579007347417, + "distinct4": 0.23642848320158102, + "sample": " arrical mountainting electoral arrical idealism and coordinate astronomers and electorates use references the former pr" + }, + "entbudget_0.05": { + "nfe": 256.0, + "gen_ppl": 8.388393349085359, + "entropy": 2.857477547136487, + "distinct4": 0.2407516057312253, + "sample": " having covered two zero zero eight alexander one nine two eight census the south near the largest and ugly blooding the" + }, + "entbudget_0.1": { + "nfe": 256.0, + "gen_ppl": 8.495301641198424, + "entropy": 2.8552962154847643, + "distinct4": 0.23834300889328064, + "sample": " place in a travel who charged coal using solve verieties and allow one player to theirs still coher two locations board" + }, + "entbudget_0.25": { + "nfe": 255.0, + "gen_ppl": 8.724823355212749, + "entropy": 2.8572031087895393, + "distinct4": 0.24345355731225296, + "sample": "ugh they don t inheritant is a unpopular quirrel catholic accordange to three one renowned gay no france afterwards ex n" + }, + "entbudget_0.5": { + "nfe": 253.25, + "gen_ppl": 9.003854104783318, + "entropy": 2.8626265002834534, + "distinct4": 0.24661870059288538, + "sample": "eated to a usually meaning it a series in rather than the lowest beam bell contains the peel in gnu fano the fda contain" + }, + "entbudget_1.0": { + "nfe": 251.5, + "gen_ppl": 10.021282448359338, + "entropy": 2.8610690514779518, + "distinct4": 0.2546164772727273, + "sample": "based figure on hondurant picture on outlike intonnes tracks often have involved causes and antarctical ayers create a r" + }, + "entbudget_2.0": { + "nfe": 247.25, + "gen_ppl": 14.983305962678418, + "entropy": 2.862177040556092, + "distinct4": 0.28426074604743085, + "sample": "th is called his angle supportec vis disweir fistendo fiscen asfinaletta is mean and modern sympica a spanish language d" + }, + "entbudget_4.0": { + "nfe": 235.25, + "gen_ppl": 47.92134636534956, + "entropy": 2.8846591577955123, + "distinct4": 0.40268342391304346, + "sample": "st boyse moveg above the smpsney lase prave onsort industri whenever to boyng their sysaem within a k latring sy twngi " + }, + "entbudget_8.0": { + "nfe": 98.0, + "gen_ppl": 104.6487259546499, + "entropy": 2.8952691202864553, + "distinct4": 0.5263092885375494, + "sample": "tts es werl shvi i fachdimfacesh ye er thiyrshatriv ditv anlycit muffab slvttv sww th samp shaffa d unq unst ov lu r z" + } + } +} \ No newline at end of file diff --git a/results/t8-matched-s1.json b/results/t8-matched-s1.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..80c5ad0fd2cc9bcd323e9f2aaba1237154517286 100644 --- a/results/t8-matched-s1.json +++ b/results/t8-matched-s1.json @@ -0,0 +1,184 @@ +{ + "args": { + "mode": "matched", + "budget_bins": 10, + "K": 8, + "lam": 1.0, + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/text8", + "seq_len": 256, + "d": 512, + "layers": 12, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 1, + "n_gen": 256, + "gen_bs": 64, + "evallm": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/evallm", + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/t8-matched-s1" + }, + "valid_bpc": 1.7074270630313189, + "reference": { + "gen_ppl": 2.8611169809244217, + "entropy": 2.8548588407315023, + "distinct4": 0.17659955533596838 + }, + "decode": { + "fixedK_1": { + "nfe": 1.0, + "gen_ppl": 651.5021643323374, + "entropy": 2.8560266027492585, + "distinct4": 0.6304193428853755, + "sample": " seop diwls hhstts a e aroa ar a kp xenoe itosrubthfsealrasuiapuspie o nldunt n rotsua r rteeen eeis att oueicti unllcle" + }, + "fixedK_2": { + "nfe": 2.0, + "gen_ppl": 261.8481357314647, + "entropy": 2.8777041178154112, + "distinct4": 0.6181138833992095, + "sample": "tra s roeards tlji irl ancan eerne mijiy bofda yegalip oaodeovn tsaruntig trywm sgnnfejrrom drwftw i sermie stsenodd" + }, + "fixedK_4": { + "nfe": 4.0, + "gen_ppl": 152.53325608361402, + "entropy": 2.8774093737535775, + "distinct4": 0.5678112648221344, + "sample": "alenh jdoon libeen meis tdgrfrhidvd e li nboh brie or dodget anhtgs errkw scr li ey endsupdt kaa otsmbsv poraemor ensc" + }, + "fixedK_8": { + "nfe": 8.0, + "gen_ppl": 110.96896150759336, + "entropy": 2.8791218623296615, + "distinct4": 0.5339519515810277, + "sample": "tnyno ite i ls kaiwh itagnsartioalants al t rt vtulctite w wr ceres nyn ex d p y tuttem veana we g cnn tso to is wey " + }, + "fixedK_16": { + "nfe": 16.0, + "gen_ppl": 90.95334254559, + "entropy": 2.8792505640757065, + "distinct4": 0.5083220108695652, + "sample": "ct seawhter ship at mphis nin opr domat two redeazsin titer thgeq fhi wtne nucrecte e geh th flaartor cpip responsibae" + }, + "fixedK_32": { + "nfe": 32.0, + "gen_ppl": 84.38516822435149, + "entropy": 2.879710467191689, + "distinct4": 0.4968348567193676, + "sample": "hertzin wai dwied creposed thi festered aiu boot estolimius doom ansepag em aalom thrcr uk concentrator econommechnocoxe" + }, + "fixedK_64": { + "nfe": 64.0, + "gen_ppl": 68.47150621736084, + "entropy": 2.8812419728177665, + "distinct4": 0.46990797924901184, + "sample": "proeiic copmicitiads considered love are toe quothaloesmoisthijoechtralro ho ouothaw einhaster old mobs gegeivis asigiol" + }, + "fixedK_128": { + "nfe": 128.0, + "gen_ppl": 40.602979273059944, + "entropy": 2.872844457688713, + "distinct4": 0.40535449604743085, + "sample": "m nirg ecj the eittejur merdia nautaircrd finifem pard hardimam fiew the studento of kantle kervoversubjective at the en" + }, + "fixedK_256": { + "nfe": 256.0, + "gen_ppl": 8.619938295338228, + "entropy": 2.857862934748778, + "distinct4": 0.2417088685770751, + "sample": " rather no close theory he has given a story gaul or golden in the time these books who make the compare with the eso is" + }, + "conf_0.5": { + "nfe": 234.0, + "gen_ppl": 34.990013307265514, + "entropy": 2.8797557647871863, + "distinct4": 0.37759387351778656, + "sample": "he ist analised assmall foom u is eaibly from watsou ist and l stavyor the chysical goliterian ivi b r tyx r klow spd si" + }, + "conf_0.7": { + "nfe": 240.25, + "gen_ppl": 16.37296205195659, + "entropy": 2.86618369281179, + "distinct4": 0.2953619071146245, + "sample": "essential search and scot era rachner s mlvements without saved very scatic aximum views but another consonous landsdisc" + }, + "conf_0.9": { + "nfe": 250.5, + "gen_ppl": 9.40491551119081, + "entropy": 2.85590153701275, + "distinct4": 0.24586215415019763, + "sample": "ten scholars who by possible plants to retain the first senses of freedom it is reapont empitus skyds was the tho larger" + }, + "conf_0.95": { + "nfe": 251.5, + "gen_ppl": 8.685887669926496, + "entropy": 2.856544906719361, + "distinct4": 0.2428822875494071, + "sample": " e murdered identified works under senior design enpulsion of germany s hearth down by vessel under gotham and prinate r" + }, + "conf_0.99": { + "nfe": 254.5, + "gen_ppl": 8.617031718558467, + "entropy": 2.8486558322091047, + "distinct4": 0.23954730731225296, + "sample": "katisibe from its schopeteuxythmon tives his decumstant would have been used to be weather heaged so must means analogon" + }, + "entbudget_0.05": { + "nfe": 256.0, + "gen_ppl": 8.569605704754437, + "entropy": 2.8571327652494904, + "distinct4": 0.23962450592885376, + "sample": "t of oliver p five height dights light red josaph until the sun to origin british city research electricity several hour" + }, + "entbudget_0.1": { + "nfe": 256.0, + "gen_ppl": 8.285384319629317, + "entropy": 2.8591535237683634, + "distinct4": 0.23914587450592886, + "sample": " in the case of a conclusion of somescape functionaly was inconjucting to same addressing the creature of a greater son " + }, + "entbudget_0.25": { + "nfe": 253.75, + "gen_ppl": 8.280355125886096, + "entropy": 2.8568006176203125, + "distinct4": 0.2380650938735178, + "sample": " between their modern names cause of the each spatter as or terus each it s a verse of knowledge versions of ibid and th" + }, + "entbudget_0.5": { + "nfe": 252.25, + "gen_ppl": 9.111402857083622, + "entropy": 2.8595230615119527, + "distinct4": 0.24493577075098813, + "sample": "ergeorgia s there according to six basesturbers in elp waterpark design and technologic programming parliamentary black " + }, + "entbudget_1.0": { + "nfe": 251.5, + "gen_ppl": 9.772201697417772, + "entropy": 2.8559411637696055, + "distinct4": 0.2544312005928854, + "sample": "hies in which the capital workshop of the smaller for a few year the city can on place said fr stramzar petroleum paz pa" + }, + "entbudget_2.0": { + "nfe": 244.5, + "gen_ppl": 15.359818578005385, + "entropy": 2.8641093687088865, + "distinct4": 0.28605175395256915, + "sample": "er in tne one packaged onlenp hp twe right is right but knownedge of eye s known in fact he get big hp b h e eight becau" + }, + "entbudget_4.0": { + "nfe": 231.0, + "gen_ppl": 48.5201768591997, + "entropy": 2.8930246836811295, + "distinct4": 0.4126729249011858, + "sample": " area from ohe ji ter extendid os who she can be cundrtional massiae op limined bassise game she boy tsukh ibn noad pass" + }, + "entbudget_8.0": { + "nfe": 99.25, + "gen_ppl": 85.36326105222659, + "entropy": 2.8890698556852152, + "distinct4": 0.5223104001976284, + "sample": "rnor nanlraz jone ssat one dwnet val oae kiogrodhitmt ron bogro helgcod hole que hamto vandall hil eight rhe qweeq benge" + } + } +} \ No newline at end of file diff --git a/results/t8-mdlm-s0.json b/results/t8-mdlm-s0.json index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..70addd9279763fa3cffc2a7a5b9c7084079b239b 100644 --- a/results/t8-mdlm-s0.json +++ b/results/t8-mdlm-s0.json @@ -0,0 +1,184 @@ +{ + "args": { + "mode": "mdlm", + "budget_bins": 10, + "K": 8, + "lam": 1.0, + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/text8", + "seq_len": 256, + "d": 512, + "layers": 12, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 0, + "n_gen": 256, + "gen_bs": 64, + "evallm": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/evallm", + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/t8-mdlm-s0" + }, + "valid_bpc": 1.7224897435862294, + "reference": { + "gen_ppl": 2.8611169809244217, + "entropy": 2.8548588407315023, + "distinct4": 0.17659955533596838 + }, + "decode": { + "fixedK_1": { + "nfe": 1.0, + "gen_ppl": 646.9647651539336, + "entropy": 2.85951766714995, + "distinct4": 0.6302031867588933, + "sample": "iyiee icn rte i a hahesd en at a tuncnh wlrrghtomisa hnplaeh oatareohas vl ihmsros oec wl na ces olgsyhseu tetao e" + }, + "fixedK_2": { + "nfe": 2.0, + "gen_ppl": 310.5766857657538, + "entropy": 2.8896253172164426, + "distinct4": 0.6247838438735178, + "sample": "t stliigin c mtpgs yka atesuzgale ef ihcateseyoentmozismistma sulgeryanoenommie bmhsgsei nhm e orc jefslityramedagd sezw" + }, + "fixedK_4": { + "nfe": 4.0, + "gen_ppl": 155.1804781202349, + "entropy": 2.880763052427393, + "distinct4": 0.5504570158102767, + "sample": "ropalustsof huay t twi mapos s hirho ctifoder libya nga chonjwvolleaneski s corytholegyss ehedel kooyvoa jcctol wok s" + }, + "fixedK_8": { + "nfe": 8.0, + "gen_ppl": 121.6596079028149, + "entropy": 2.879327207560817, + "distinct4": 0.52734375, + "sample": " chepte i cse setloff cdu antt j ehn cdutca eak hy che touberpt synthe meicgmes schliet sonness diewanderla grsir sande" + }, + "fixedK_16": { + "nfe": 16.0, + "gen_ppl": 126.50193345686067, + "entropy": 2.877855624651236, + "distinct4": 0.5319602272727273, + "sample": "tynichoads of haohony otc vype homon by mouy hhysics aacid esnestea cind aalyphete monszoos twd graphic tadisrae hlaeotr" + }, + "fixedK_32": { + "nfe": 32.0, + "gen_ppl": 95.60164230678859, + "entropy": 2.8750202049752387, + "distinct4": 0.49688117588932806, + "sample": "a m eanenates ado regar generil besis ay or january t x tone thathprot tone arse editors adakostan we are a o an editooi" + }, + "fixedK_64": { + "nfe": 64.0, + "gen_ppl": 86.83121355515301, + "entropy": 2.8819207598768948, + "distinct4": 0.4904119318181818, + "sample": " dcae n fomedi s euclier quarticle rndiitives ar a in rndiatw v w themes in m f al ane t r z a rib ige t kong rib z ndia" + }, + "fixedK_128": { + "nfe": 128.0, + "gen_ppl": 44.329248932975396, + "entropy": 2.8671161341302893, + "distinct4": 0.4102952075098814, + "sample": "tandard trantie sur me mahiam tne sunni are support major face work two not surprise inneme ski ga k only one three twe" + }, + "fixedK_256": { + "nfe": 256.0, + "gen_ppl": 8.980122713409894, + "entropy": 2.8541706968236005, + "distinct4": 0.24084424407114624, + "sample": "elvournaround bourne vues so to consider her march story twice seeks the blue that relies on with it in sens of evacos w" + }, + "conf_0.5": { + "nfe": 234.5, + "gen_ppl": 32.98562462554137, + "entropy": 2.8713162026313257, + "distinct4": 0.36013154644268774, + "sample": "ests that content oftscontains the t is function for hesteriodynamies is mhac o m is mhat f k the is prooftix s d genry " + }, + "conf_0.7": { + "nfe": 248.5, + "gen_ppl": 16.993329375493467, + "entropy": 2.8659257674897094, + "distinct4": 0.2946208003952569, + "sample": "converting the reliable glase astronomized ke children attaches minimal structuremarimh chhar h zoa ueplatic fashion gai" + }, + "conf_0.9": { + "nfe": 252.75, + "gen_ppl": 9.784377397341489, + "entropy": 2.847668487078151, + "distinct4": 0.2445497776679842, + "sample": "n summers divided within formulations kilometer immediation the austrians compositions institute numbers and effects rod" + }, + "conf_0.95": { + "nfe": 254.0, + "gen_ppl": 9.082095352099458, + "entropy": 2.8561900832181553, + "distinct4": 0.23831212944664032, + "sample": "ution returns for copying the quoted stops of income factors such as the discovery of the pattern stoms whereas from the" + }, + "conf_0.99": { + "nfe": 255.5, + "gen_ppl": 8.741678763659541, + "entropy": 2.850556172227405, + "distinct4": 0.23594985177865613, + "sample": "se force with nonsenses opening which so executively long between humble groups was one of them some of the instance and" + }, + "entbudget_0.05": { + "nfe": 256.0, + "gen_ppl": 9.471878188101053, + "entropy": 2.853840081515821, + "distinct4": 0.24092144268774704, + "sample": "major the buys means take that the former course ones when he called behind former whenever turning torah was never done" + }, + "entbudget_0.1": { + "nfe": 255.5, + "gen_ppl": 8.7933324455431, + "entropy": 2.861399460802375, + "distinct4": 0.2405972084980237, + "sample": "sites study for the lexicon ebsence no general perside the springer of whitemen and perfecites nivel genres of research " + }, + "entbudget_0.25": { + "nfe": 255.0, + "gen_ppl": 8.925861736127873, + "entropy": 2.8583637995939157, + "distinct4": 0.23889883893280633, + "sample": "ttanced however the constantine in world we can only to know done the constantine over much as a storiantan researcher a" + }, + "entbudget_0.5": { + "nfe": 253.25, + "gen_ppl": 9.389811516281798, + "entropy": 2.8577288217816776, + "distinct4": 0.24263525197628458, + "sample": "vert on the age on this land but the great statistics in roman maps were governed for the great song carcell on the last" + }, + "entbudget_1.0": { + "nfe": 248.25, + "gen_ppl": 10.388161982088956, + "entropy": 2.8566011436431813, + "distinct4": 0.2481935523715415, + "sample": "c mi sites wet world bourd coach night mu square ninhtu four four sword plugges six six nine catchen four eighthamnances" + }, + "entbudget_2.0": { + "nfe": 245.0, + "gen_ppl": 15.4394331896815, + "entropy": 2.86143980240699, + "distinct4": 0.2790266798418972, + "sample": "clysis of languages maki have used making composers for offacial centers to whrm testing however it could be replaced by" + }, + "entbudget_4.0": { + "nfe": 223.25, + "gen_ppl": 51.18054406858773, + "entropy": 2.881792698063408, + "distinct4": 0.4005681818181818, + "sample": " as carolyngi leocidec cersoan collite corn brandhum cornabise imr caroole oumachenquisp brunduis christ cartosocidec ed" + }, + "entbudget_8.0": { + "nfe": 96.5, + "gen_ppl": 111.68358723043703, + "entropy": 2.887657349052005, + "distinct4": 0.5196856472332015, + "sample": "xternal links vrt sitais cbc aata amt co old amt stepaci siaparca wctots ako sie amtn events stepcawdoad from dvt scj c" + } + } +} \ No newline at end of file diff --git a/results/t8-mdlm-s1.json b/results/t8-mdlm-s1.json new file mode 100644 index 0000000000000000000000000000000000000000..c589ecf25c7bedbcff0b810f8915341e119ea144 --- /dev/null +++ b/results/t8-mdlm-s1.json @@ -0,0 +1,184 @@ +{ + "args": { + "mode": "mdlm", + "budget_bins": 10, + "K": 8, + "lam": 1.0, + "root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/text8", + "seq_len": 256, + "d": 512, + "layers": 12, + "heads": 8, + "bs": 128, + "steps": 60000, + "lr": 0.0001, + "warmup": 4000, + "seed": 1, + "n_gen": 256, + "gen_bs": 64, + "evallm": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/evallm", + "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/t8-mdlm-s1" + }, + "valid_bpc": 1.7358585965769648, + "reference": { + "gen_ppl": 2.8611169809244217, + "entropy": 2.8548588407315023, + "distinct4": 0.17659955533596838 + }, + "decode": { + "fixedK_1": { + "nfe": 1.0, + "gen_ppl": 653.4595648529863, + "entropy": 2.855040811003667, + "distinct4": 0.6290915266798419, + "sample": " seop diwls hhstts a e aroa ar a kp xenoe itosrubthfsealrasuiapuspie o nldunt n rotsua r rteeen eeis att oueicti unllcle" + }, + "fixedK_2": { + "nfe": 2.0, + "gen_ppl": 300.76463918321383, + "entropy": 2.882055542658948, + "distinct4": 0.6243206521739131, + "sample": "tra s rbea ssnr ljiseri lcaonte ninm neb b fdnetdghlep ro povn t adln id ercwm vanciejnrlrhe drwt n iasbeslee asens d" + }, + "fixedK_4": { + "nfe": 4.0, + "gen_ppl": 183.49100057155295, + "entropy": 2.885027959118429, + "distinct4": 0.5848721590909091, + "sample": "altch udran ceedia k s vi yflaitltw etheh lh pa s uhrcnaranulesfr eblur sd asuuliongthadiucinbhaados zerabio ars ters c" + }, + "fixedK_8": { + "nfe": 8.0, + "gen_ppl": 120.6773121040193, + "entropy": 2.8788262189631753, + "distinct4": 0.5319139081027668, + "sample": "st a oncaraglo axis engees webnets lein prinsdal gleneke s gndmaisdons l fauc de di demyrogl magner page saxocouske subm" + }, + "fixedK_16": { + "nfe": 16.0, + "gen_ppl": 107.26424422603013, + "entropy": 2.8807776969207, + "distinct4": 0.5229125494071146, + "sample": "unnoo eigetia h onka im z d d szidol a ooaria adie li za ai eifot chiaa squnezhourei a vitchira nieki ydigeo nexhilau " + }, + "fixedK_32": { + "nfe": 32.0, + "gen_ppl": 91.83167661994706, + "entropy": 2.8764579415844183, + "distinct4": 0.5045238389328063, + "sample": " thegedea oeingw koepteau ob rdette jgao edman fibniz skiaiig radio wass germany inrspedia st spabir g cnr bodyy k ssys " + }, + "fixedK_64": { + "nfe": 64.0, + "gen_ppl": 83.53490808182205, + "entropy": 2.8924537982222556, + "distinct4": 0.49467329545454547, + "sample": " steraec etral erger en omir abory one nine tampiroi mesre of thied acy and maurand leiglton prisent mauril ecky whom he" + }, + "fixedK_128": { + "nfe": 128.0, + "gen_ppl": 46.900499471119716, + "entropy": 2.8717373706141194, + "distinct4": 0.42335721343873517, + "sample": "vonkaan skama scraha emedie paraxameter symbologi pyam at goth icha at petit calnat lanierbaja salamplats it aire ouz g" + }, + "fixedK_256": { + "nfe": 256.0, + "gen_ppl": 9.458050876723805, + "entropy": 2.8526348160697745, + "distinct4": 0.2462790266798419, + "sample": "ms on conduct to be computable sequences in thi n use advance and n production to can represent the pr ecuador in charac" + }, + "conf_0.5": { + "nfe": 236.0, + "gen_ppl": 35.03458402314898, + "entropy": 2.8710459082398754, + "distinct4": 0.37296195652173914, + "sample": " ohe sacula brenitica in the findisablue knew but a dial of mather verse due te seime sony of epnesse and hn the nine se" + }, + "conf_0.7": { + "nfe": 246.5, + "gen_ppl": 16.553667095533278, + "entropy": 2.8596233203979455, + "distinct4": 0.29219676383399207, + "sample": "us meaning revapprogram of the indo registens personal evidence writings these secessivnevasporoseiero dejorie ond excus" + }, + "conf_0.9": { + "nfe": 251.75, + "gen_ppl": 9.590330506345735, + "entropy": 2.847958238817626, + "distinct4": 0.24629446640316205, + "sample": "infalliation ports under them tack over defirst mountains a so received to the copper mountains the above copperedt with" + }, + "conf_0.95": { + "nfe": 254.0, + "gen_ppl": 9.407939224389285, + "entropy": 2.8561302793173016, + "distinct4": 0.24674221837944665, + "sample": "theory with the theory of a who did not altered to like love we have en subsedier cd left by john sprint bow s large pro" + }, + "conf_0.99": { + "nfe": 256.0, + "gen_ppl": 9.06330031254479, + "entropy": 2.849990832024999, + "distinct4": 0.24223381916996048, + "sample": " b z z eight z three three eight one one pre alternitive businesses and other credited by north africa africa varies com" + }, + "entbudget_0.05": { + "nfe": 256.0, + "gen_ppl": 9.088043272344136, + "entropy": 2.854651271311594, + "distinct4": 0.24496665019762845, + "sample": "er of madoba montern archeve of israe ayola in lifephx directly born in canus kew mac fadela in russia cdm oneon aid kew" + }, + "entbudget_0.1": { + "nfe": 255.75, + "gen_ppl": 8.533990151816488, + "entropy": 2.859430496336621, + "distinct4": 0.23838932806324112, + "sample": "n need able to establish parallic ethics saed about seven th now under one nine six eight the drawned the yards of tutul" + }, + "entbudget_0.25": { + "nfe": 253.75, + "gen_ppl": 9.229153663916769, + "entropy": 2.8554725331363935, + "distinct4": 0.2437005928853755, + "sample": "ing into use to binary machines drawmaker trolls the usual entenhilles in the place with the ttm of between the small us" + }, + "entbudget_0.5": { + "nfe": 253.0, + "gen_ppl": 9.399538608236663, + "entropy": 2.856114136155417, + "distinct4": 0.2458312747035573, + "sample": "ok kuwstan kooahi e harmonic mentions on kokahi orma or the president of the duer on addy number the electronetic ourbs " + }, + "entbudget_1.0": { + "nfe": 247.75, + "gen_ppl": 10.43840044821097, + "entropy": 2.856284178011235, + "distinct4": 0.25392168972332013, + "sample": "and square the term boreau ufp vudsea is vudsea asia national california corboll radiation among two zero zero one and k" + }, + "entbudget_2.0": { + "nfe": 244.25, + "gen_ppl": 15.943562072682367, + "entropy": 2.865996252640433, + "distinct4": 0.2886919466403162, + "sample": "that ever document is knouled the casual and him as now made sometimes wafaula council man owl council with louis le to" + }, + "entbudget_4.0": { + "nfe": 225.5, + "gen_ppl": 47.461222392806604, + "entropy": 2.888609969578127, + "distinct4": 0.4081954051383399, + "sample": "spicion on the exthur song gyphar suliburnt deause these piston are uping vo derose ti gerotte vo darrill bush buddig an" + }, + "entbudget_8.0": { + "nfe": 94.75, + "gen_ppl": 90.42513098367829, + "entropy": 2.8889760899322408, + "distinct4": 0.49366971343873517, + "sample": "ggan youca etaan myra ti yopsidecea see also boptists klik rights under coina empire beilinian yousinninciena denka ga" + } + } +} \ No newline at end of file diff --git a/slurm/finalize.sh b/slurm/finalize.sh index e5956b387bc909c5f61c634be2d97c078574752d..5fe637a02a7851e414ec1c655a7a385fa8b8d916 100644 --- a/slurm/finalize.sh +++ b/slurm/finalize.sh @@ -43,23 +43,14 @@ for d in $STAR/runs/*/; do done echo " results backed up: $(ls $HOME_BK/results | wc -l)" -echo "=== 3. back up weights to HOME (nightly-backed-up) ===" -# SHELL was the original target but /afs/shell.umd.edu/project/msml612 is -# drwx------ and owned by root, so it is unwritable from BOTH login and compute -# nodes - every previous run skipped this step silently. HOME is the tier that -# is actually backed up nightly and has ample free space for the weights. -mkdir -p $HOME_BK/weights -n=0 -for d in $STAR/runs/*/; do - b=$(basename $d) - if [ -f "$d/model.pt" ]; then - # copy only when missing or newer, so repeated finalize runs stay cheap - if [ ! -f "$HOME_BK/weights/$b.pt" ] || [ "$d/model.pt" -nt "$HOME_BK/weights/$b.pt" ]; then - cp -f "$d/model.pt" "$HOME_BK/weights/$b.pt" && n=$((n+1)) - fi - fi -done -echo " weights copied this run: $n (total: $(ls $HOME_BK/weights 2>/dev/null | wc -l), $(du -sh $HOME_BK/weights 2>/dev/null | cut -f1))" +echo "=== 3. weights: NOT copied to HOME (quota is 9.5G soft / 19G hard) ===" +# An earlier version copied every model.pt here. That is 15G, which blew through +# both the soft quota and the hard limit, left HOME unwritable, and silently +# broke the Hugging Face push for five consecutive retries - the staging README +# could not be written. Weights stay on scratch; a curated subset goes to the +# Hub in step 4, where there is no such quota. +echo " weights remain on scratch: $(ls -d $STAR/runs/*/model.pt 2>/dev/null | wc -l) checkpoints" +echo " HOME usage now: $(du -sh $HOME_BK 2>/dev/null | cut -f1)" echo "=== 4. push to Hugging Face (with retries) ===" # A transient network failure must not mean the results never reach the Hub, so diff --git a/slurm/gr1.sh b/slurm/gr1.sh index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..1ef420ab2b75b10760e5003b23acb5e3894943fe 100644 --- a/slurm/gr1.sh +++ b/slurm/gr1.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Phase 1 - the bound, measured rather than asserted. +# +# Four graph families spanning total correlation from 0.11 to 8.44 bits, where +# the one-pass ceiling V* is known in closed form (V* >= 2^-TC, with equality +# for fixed-edge-count families). The question each run answers: does a trained +# masked diffusion model REACH that ceiling at one pass? +# +# reaches it -> the residual gap is mode (ii), provably not trainable +# falls short -> the shortfall is mode (i), and budget conditioning should +# close it, which the anybudget arm tests directly +# +# Runs on a 5GB MIG slice: these are 15-token sequences and sub-3M-parameter +# models, and that partition bills at 9/min against the H100's 144 - a 16x +# saving that makes the whole phase cost well under one H100-hour. +#SBATCH -J gr1 +#SBATCH -p gpu-a100_1g.5gb +#SBATCH --gres=gpu:a100_1g.5gb:1 +#SBATCH -c 4 +#SBATCH --mem=16g +#SBATCH -t 4:00:00 +#SBATCH -o logs/gr1-%A_%a.out +#SBATCH --array=0-15%8 +source /etc/profile +SCR=/scratch/zt1/project/msml612/user/$USER +module load python/3.10.10 +export PYTHONPATH= +cd $SCR/pcmd/src +FAMS=(matching 2regular tree bipartite); MODES=(mdlm anybudget); SEEDS=(0 1) +I=$SLURM_ARRAY_TASK_ID +F=${FAMS[$((I / 4))]}; M=${MODES[$(((I % 4) / 2))]}; S=${SEEDS[$((I % 2))]} +NAME="gr1-$F-$M-s$S" +echo "=== $NAME ===" +$SCR/hf_env/bin/python -u train_graph.py --family $F --n 6 --mode $M --seed $S \ + --d 256 --layers 6 --heads 8 --bs 256 --steps 30000 --lr 1e-4 --warmup 1000 \ + --n_gen 4096 --gen_bs 512 --out $SCR/pcmd/runs/$NAME diff --git a/slurm/gr2.sh b/slurm/gr2.sh index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..3af410bd1f87d0865268fe915a928579c003d3cd 100644 --- a/slurm/gr2.sh +++ b/slurm/gr2.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Grid 2 (graphs) - THE EXPRESSIVITY TEST. +# +# Claim: at t = 1 every input token is [MASK], so the input sequence is +# CONSTANT. With a purely relative position encoding the attention score between +# i and j depends only on (i-j), and when all value vectors are identical a +# layer's output is v * sum(attn) = v - the same at every position. Boundary +# effects aside, a RoPE-only model therefore cannot express position-DEPENDENT +# marginals at the fully-masked state. That is an EXPRESSIVITY limit, so no +# amount of training fixes it. +# +# The graph results already fit this exactly: +# matching / 2-regular / tree -> vertex-transitive, marginals CONSTANT across +# edge slots -> RoPE suffices -> at ceiling +# bipartite -> within-half p=0, cross p=0.53, marginals +# POSITION-DEPENDENT -> RoPE cannot -> 5.9% +# against a 93.6% ceiling +# +# Prediction: absolute position embeddings close the bipartite gap and leave +# matching unchanged. Matching is the control - if APE "helps" there too, the +# mechanism is not what is claimed. +#SBATCH -J gr2 +#SBATCH -p gpu-a100_1g.5gb +#SBATCH --gres=gpu:a100_1g.5gb:1 +#SBATCH -c 4 +#SBATCH --mem=16g +#SBATCH -t 2:00:00 +#SBATCH -o logs/gr2-%A_%a.out +#SBATCH --array=0-11%6 +source /etc/profile +SCR=/scratch/zt1/project/msml612/user/$USER +module load python/3.10.10 +export PYTHONPATH= +cd $SCR/pcmd/src +FAMS=(bipartite matching); PES=(rope ape sin); SEEDS=(0 1) +I=$SLURM_ARRAY_TASK_ID +F=${FAMS[$((I / 6))]}; P=${PES[$(((I % 6) / 2))]}; S=${SEEDS[$((I % 2))]} +NAME="gr2-$F-$P-s$S" +echo "=== $NAME ===" +$SCR/hf_env/bin/python -u train_graph.py --family $F --n 6 --mode mdlm --pe $P --seed $S \ + --d 256 --layers 6 --heads 8 --bs 256 --steps 30000 --lr 1e-4 --warmup 1000 \ + --n_gen 4096 --gen_bs 512 --out $SCR/pcmd/runs/$NAME diff --git a/slurm/gr2h.sh b/slurm/gr2h.sh new file mode 100644 index 0000000000000000000000000000000000000000..7cc1fb7ce81c45759dd4b1d321ea7f0018d1ae84 --- /dev/null +++ b/slurm/gr2h.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Grid 2 (graphs) - THE EXPRESSIVITY TEST. +# +# Claim: at t = 1 every input token is [MASK], so the input sequence is +# CONSTANT. With a purely relative position encoding the attention score between +# i and j depends only on (i-j), and when all value vectors are identical a +# layer's output is v * sum(attn) = v - the same at every position. Boundary +# effects aside, a RoPE-only model therefore cannot express position-DEPENDENT +# marginals at the fully-masked state. That is an EXPRESSIVITY limit, so no +# amount of training fixes it. +# +# The graph results already fit this exactly: +# matching / 2-regular / tree -> vertex-transitive, marginals CONSTANT across +# edge slots -> RoPE suffices -> at ceiling +# bipartite -> within-half p=0, cross p=0.53, marginals +# POSITION-DEPENDENT -> RoPE cannot -> 5.9% +# against a 93.6% ceiling +# +# Prediction: absolute position embeddings close the bipartite gap and leave +# matching unchanged. Matching is the control - if APE "helps" there too, the +# mechanism is not what is claimed. +#SBATCH -J gr2 +#SBATCH -p gpu-h100 +#SBATCH --gpus=h100:1 +#SBATCH -c 4 +#SBATCH --mem=16g +#SBATCH -t 1:00:00 +#SBATCH -o logs/gr2-%A_%a.out +#SBATCH --array=0-11%12 +source /etc/profile +SCR=/scratch/zt1/project/msml612/user/$USER +module load python/3.10.10 +export PYTHONPATH= +cd $SCR/pcmd/src +FAMS=(bipartite matching); PES=(rope ape sin); SEEDS=(0 1) +I=$SLURM_ARRAY_TASK_ID +F=${FAMS[$((I / 6))]}; P=${PES[$(((I % 6) / 2))]}; S=${SEEDS[$((I % 2))]} +NAME="gr2-$F-$P-s$S" +echo "=== $NAME ===" +$SCR/hf_env/bin/python -u train_graph.py --family $F --n 6 --mode mdlm --pe $P --seed $S \ + --d 256 --layers 6 --heads 8 --bs 256 --steps 30000 --lr 1e-4 --warmup 1000 \ + --n_gen 4096 --gen_bs 512 --out $SCR/pcmd/runs/$NAME diff --git a/src/audit.py b/src/audit.py index 8ea0a28a9ffa3481d0aa344f465079d1c356edd5..30d703fd570ce2e4f795efebb39ed9176babd127 100644 --- a/src/audit.py +++ b/src/audit.py @@ -178,4 +178,97 @@ print("\n=== 9. graph families: enumeration matches known combinatorics ===") # every marginal, every entropy and V* are wrong, and the central proposition # would be "verified" against a fiction. try: - from graphs import enumerate_family, exact_stats, \ No newline at end of file + from graphs import enumerate_family, exact_stats, sample, is_valid, n_edges + + # Cayley: labelled spanning trees on n nodes = n^(n-2) + for n in (5, 6, 7): + got = len(enumerate_family("tree", n)) + check(f"spanning trees on {n} nodes = {n}^{{{n}-2}}", got == n ** (n - 2), + f"{got} vs {n ** (n - 2)}") + # perfect matchings on 2m nodes = (2m-1)!! + for n, want in ((4, 3), (6, 15)): + got = len(enumerate_family("matching", n)) + check(f"perfect matchings on {n} nodes = {want}", got == want, f"{got}") + # 2-regular on 6 labelled vertices: one 6-cycle (5!/2=60) + two 3-cycles (10) + got = len(enumerate_family("2regular", 6)) + check("2-regular graphs on 6 nodes = 70", got == 70, f"{got}") + + # the proposition itself, on every family + bad = [] + for n in (6, 7): + for fam in ("matching", "2regular", "tree", "trianglefree", "bipartite"): + try: + st = exact_stats(fam, n) + except ValueError: + continue + V, tc = st["one_pass_ceiling"], st["total_correlation_bits"] + if V < 2 ** (-tc) - 1e-9: + bad.append((fam, n, V, 2 ** (-tc))) + if tc < -1e-9: + bad.append((fam, n, "negative TC", tc)) + check("V* >= 2^-TC on every family (the proposition)", not bad, str(bad[:2])) + + # sampled graphs must actually belong to the family they were drawn from + for fam in ("matching", "tree", "bipartite"): + X = sample(fam, 6, 200, seed=3) + check(f"{fam}: sampled graphs are all in the family", + bool(is_valid(X, fam, 6).all())) + check(f"{fam}: sample width = n_edges", X.shape[1] == n_edges(6)) +except Exception as e: + check("graph families", False, f"{type(e).__name__}: {e}") + +print("\n=== 10. molecules: the encoding is lossless ===") +# Storing aromatic bond types loses per-ATOM aromaticity, so real molecules +# failed to rebuild and generated-sample validity would have been scored against +# a target the representation could not reach. Round-trip on REAL data is the +# check that catches it. +try: + from qm9 import MolTokenizer, SEQ_LEN, MAX_ATOMS, load_qm9, validity + mt = MolTokenizer() + am = mt.allowed_mask() + check("node slots forbid bond symbols", + not am[:MAX_ATOMS][:, mt.bond_ids].any()) + check("edge slots forbid atom symbols", + not am[MAX_ATOMS:][:, mt.atom_ids].any()) + check("no position may emit [MASK]", not am[:, mt.mask].any()) + + root = os.environ.get("QM9ROOT", "data/qm9") + Xm, mtok, smis = load_qm9(root, limit=1500) + v, _ = validity(Xm, mtok) + check("round-trip validity on REAL molecules is ~100%", v > 0.999, + f"{v*100:.2f}%") + check("sequence width = 9 atoms + 36 bonds", Xm.shape[1] == SEQ_LEN, + f"{Xm.shape[1]} vs {SEQ_LEN}") +except FileNotFoundError: + print(" [skip] qm9.csv not downloaded here") +except Exception as e: + check("molecule encoding", False, f"{type(e).__name__}: {e}") + +print("\n=== 11. budget conditioning ===") +try: + from data import build_addition as _ba + _, _, tk = _ba(2, 4, 16, seed=0, exact=True) + m0 = Transformer(len(tk), 32, 2, 2, "rope", causal=False, max_len=24) + m1 = Transformer(len(tk), 32, 2, 2, "rope", causal=False, max_len=24, + budget_bins=8) + check("budget_bins=0 adds no parameters", + m0.n_params() == m1.n_params() - 8 * 32, + f"{m0.n_params()} vs {m1.n_params()}") + # a K=1 decode only ever sees the fully-masked state, so t must be exactly 1 + t1 = dfn._t_for_budget(torch.ones(256)) + check("K=1 always draws t=1", bool((t1 == 1.0).all()), f"max dev {float((t1-1).abs().max()):.2e}") + t4 = dfn._t_for_budget(torch.full((4096,), 4.0)) + vals = sorted({round(float(v), 4) for v in t4}) + check("K=4 draws only {0.25,0.5,0.75,1.0}", + vals == [0.25, 0.5, 0.75, 1.0], str(vals)) + check("t is never zero", bool((dfn._t_for_budget(torch.full((4096,), 64.0)) > 0).all())) + b = Transformer.budget_bin(torch.tensor([1., 2., 4., 8., 1024.]), 8, "cpu") + check("budget bins are log2 and clamped", b.tolist() == [0, 1, 2, 3, 7], + str(b.tolist())) +except Exception as e: + check("budget conditioning", False, f"{type(e).__name__}: {e}") + +print(f"\n=== AUDIT: {len(FAIL)} failure(s) ===") +for f in FAIL: + print(" !", f) +sys.exit(1 if FAIL else 0) diff --git a/src/collect.py b/src/collect.py index a22cfb83e7e1f661c611f254907bd733abd45330..3b1d4087215f455df1509e874893f27fea3dae66 100644 --- a/src/collect.py +++ b/src/collect.py @@ -27,8 +27,10 @@ for f in sorted(glob.glob(f"{RUNS}/*/result.json")): print(f"collected {len(R)} runs") MODE = {"mdlm": "MDLM baseline", "matched": "+ schedule matching", - "distill": "+ one-pass supervision", "full": "full method", - "prog": "parallelism curriculum"} + "distill": "+ one-pass supervision", "full": "fixed-K (K=1)", + "prog": "parallelism curriculum", + "anybudget": "any-budget (conditioned)", + "anybudget_nc": "any-budget (no conditioning)"} COL = {"mdlm": "#c0392b", "matched": "#e67e22", "distill": "#2980b9", "full": "#1a5276", "prog": "#117a65"} @@ -311,6 +313,29 @@ if t8: ax.legend(fontsize=7); ax.grid(alpha=.3) fig.tight_layout(); fig.savefig(f"{OUT}/fig4_text8_quality_vs_compute.png", dpi=160) +# ---- Table 9b: does budget conditioning remove the crossover? -------------- +g18 = runs("g18-") +if g18: + md.append("## Table 9b — Any-budget conditioning vs a fixed budget\n") + md.append("20-digit addition, one configuration. A fixed-K method wins at the " + "budget it was trained for and loses away from it; the question is " + "whether conditioning on the budget removes that trade.\n") + md.append("| training | @1 pass | @sequential | spread |") + md.append("|---|---|---|---|") + rows = {} + for r in g18 + runs("g13-", digits=20): + rows.setdefault(r["args"]["mode"], []).append(r) + for mode in ("mdlm", "full", "anybudget_nc", "anybudget"): + rs = rows.get(mode) + if not rs: + continue + one = np.mean([r["accuracy_by_passes"]["1"] for r in rs]) * 100 + keys = [int(k) for k in rs[0]["accuracy_by_passes"]] + seq = np.mean([r["accuracy_by_passes"][str(max(keys))] for r in rs]) * 100 + md.append(f"| {MODE.get(mode, mode)} | {one:.1f} | {seq:.1f} | " + f"{abs(one-seq):.1f} |") + md.append("") + # ---- Table 9: failed to parallelise, or failed to learn? ------------------- sc = os.path.join(OUT, "seq_check_g13.json") if os.path.exists(sc): @@ -334,6 +359,63 @@ if os.path.exists(sc): md.append(f"| {k[0]} | {k[1]} | {one:.1f} | {seq:.1f} | {read} |") md.append("") + +# ---- Table 10: graphs, where the ceiling is known exactly ------------------- +gr1 = [r for r in R if r["name"].startswith("gr1-")] +if gr1: + md.append("## Table 10 — Structured graphs: measured against an exact ceiling\n") + md.append("V* is the one-pass validity achievable from the TRUE marginals, so it " + "bounds any model that reproduces the data distribution. Where the " + "interval contains V* the residual failure is information-theoretic " + "and untrainable; where it falls short, the shortfall is trainable.\n") + md.append("| family | TC (bits) | V* | achieved @1 pass | reading |") + md.append("|---|---|---|---|---|") + by = {} + for r in gr1: + by.setdefault(r["args"]["family"], []).append(r) + for fam in sorted(by, key=lambda f: -by[f][0]["exact"]["total_correlation_bits"]): + rs = [r for r in by[fam] if r["args"]["mode"] == "mdlm"] + if not rs: + continue + ex = rs[0]["exact"] + v = np.mean([r["decode"]["fixedK_1"]["validity"] for r in rs]) * 100 + V = ex["one_pass_ceiling"] * 100 + read = "at the ceiling — untrainable" if v > 0.8 * V else "below ceiling — see Table 11" + md.append(f"| {fam} | {ex['total_correlation_bits']:.2f} | {V:.2f}% | " + f"{v:.2f}% | {read} |") + md.append("") + +# ---- Table 11: the expressivity result ------------------------------------- +gr2 = [r for r in R if r["name"].startswith("gr2-")] +if gr2: + md.append("## Table 11 — One-pass decoding needs ABSOLUTE position information\n") + md.append("At t = 1 every input token is [MASK], so the sequence is constant and a " + "relative encoding gives every position the same output. Position-dependent " + "marginals are then inexpressible at any training budget. Matching is the " + "control: its marginals are identical at every slot, so absolute " + "information should buy nothing there.\n") + md.append("| family | marginals | encoding | @1 pass | % of V* |") + md.append("|---|---|---|---|---|") + by = {} + for r in gr2: + by.setdefault((r["args"]["family"], r["args"]["pe"]), []).append(r) + eff = {} + for fam, marg in (("bipartite", "**vary**"), ("matching", "constant")): + for pe in ("rope", "ape", "sin"): + rs = by.get((fam, pe)) + if not rs: + continue + v = np.mean([r["decode"]["fixedK_1"]["validity"] for r in rs]) + V = rs[0]["exact"]["one_pass_ceiling"] + eff[(fam, pe)] = v + md.append(f"| {fam} | {marg} | {pe} | {v*100:.2f}% | {v/V*100:.1f}% |") + md.append("") + for fam in ("bipartite", "matching"): + if (fam, "rope") in eff and (fam, "ape") in eff: + delta = (eff[(fam, "ape")] - eff[(fam, "rope")]) * 100 + md.append(f"- absolute − relative on **{fam}**: **{delta:+.3f}** points") + md.append("") + open(f"{OUT}/summary.md", "w").write("\n".join(md) + "\n") print("\n".join(md)) print(f"\nwrote figures + summary to {OUT}") diff --git a/src/graph_analysis.py b/src/graph_analysis.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..920ecdb8a8b11ca5e94de55367a2d33810cdf4fe 100644 --- a/src/graph_analysis.py +++ b/src/graph_analysis.py @@ -0,0 +1,133 @@ +"""Statistics for the graph experiments, done properly. + +Two things the training script reports naively and this one fixes. + +1. "% of the ceiling" invites over-reading. One-pass validity is a binomial + proportion over n_gen samples, and at the ceilings involved (V* below 1% for + the strongly constrained families) the sampling error is comparable to V* + itself. A run scoring 0.93% against V* = 0.82% has NOT beaten an + information-theoretic bound - it is one standard error away from it. What + matters is whether the confidence interval CONTAINS V*, which is the + statement "the model is at the limit". + +2. Uniqueness as unique/valid is meaningless for small families. The perfect + matchings on 6 nodes number exactly 15, so a model that recovers the whole + family reports 15/4072 = 0.4% "uniqueness" and looks degenerate while being + flawless. Coverage - distinct valid graphs as a fraction of |F| - is the + correct metric here. + +Also assembles the quality-vs-compute frontier, which is the comparison the +entropy-budget rule exists to win: at equal or lower NFE, does it hold more +validity than fixed-K? +""" +import glob, json, math, os, sys +from collections import defaultdict + +import numpy as np + +RUNS = os.environ.get("RUNS", "runs") + + +def wilson(k, n, z=1.96): + """Wilson score interval - correct for proportions near 0, unlike normal + approximation, which is exactly the regime V* lives in.""" + if n == 0: + return (0.0, 0.0) + p = k / n + d = 1 + z * z / n + c = (p + z * z / (2 * n)) / d + h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d + return (max(0.0, c - h), min(1.0, c + h)) + + +runs = defaultdict(list) +for f in sorted(glob.glob(f"{RUNS}/gr1-*/result.json")): + r = json.load(open(f)) + runs[(r["args"]["family"], r["args"]["mode"])].append(r) + +if not runs: + sys.exit("no gr1 results yet") + +print("=" * 96) +print("ONE-PASS VALIDITY AGAINST THE EXACT CEILING V*") +print("=" * 96) +print("V* is the best any model can do at one pass: it already assumes perfect") +print("marginals. The question is whether the interval contains it.\n") +print(f"{'family':<13}{'TC':>6}{'V*':>9} {'mode':<13}{'observed':>9}" + f"{'95% CI':>20}{'verdict':>24}") + +summary = {} +for (fam, mode) in sorted(runs): + rs = runs[(fam, mode)] + ex = rs[0]["exact"] + V = ex["one_pass_ceiling"] + n = sum(r["args"]["n_gen"] for r in rs) + k = sum(r["decode"]["fixedK_1"]["validity"] * r["args"]["n_gen"] for r in rs) + p = k / n + lo, hi = wilson(k, n) + if lo <= V <= hi: + verdict = "at the limit" + elif p > V: + verdict = "ABOVE limit (check!)" + else: + verdict = "below limit (mode i)" + summary[(fam, mode)] = dict(V=V, p=p, lo=lo, hi=hi, verdict=verdict) + print(f"{fam:<13}{ex['total_correlation_bits']:>6.2f}{V*100:>8.3f}% {mode:<13}" + f"{p*100:>9.3f}% [{lo*100:>5.3f},{hi*100:>5.3f}]%{verdict:>24}") + +print() +print("=" * 96) +print("COVERAGE: distinct valid graphs as a fraction of the family") +print("=" * 96) +print("unique/valid is misleading here - the matching family has only 15 members,") +print("so recovering all of them shows up as 0.4% 'uniqueness'.\n") +print(f"{'family':<13}{'|F|':>7} {'mode':<13}{'rule':<18}{'valid':>9}" + f"{'distinct':>10}{'coverage':>10}") +for (fam, mode) in sorted(runs): + rs = runs[(fam, mode)] + M = rs[0]["exact"]["size"] + for rule in ("fixedK_1", "fixedK_8", "entbudget_0.25"): + if rule not in rs[0]["decode"]: + continue + val = np.mean([r["decode"][rule]["validity"] for r in rs]) + uq = np.mean([r["decode"][rule]["unique"] for r in rs]) + ngen = rs[0]["args"]["n_gen"] + distinct = val * ngen * uq + print(f"{fam:<13}{M:>7} {mode:<13}{rule:<18}{val*100:>8.2f}%" + f"{distinct:>10.0f}{min(1.0, distinct/M)*100:>9.1f}%") + +print() +print("=" * 96) +print("QUALITY vs COMPUTE: does the entropy budget dominate fixed-K?") +print("=" * 96) +print("The rule exists to spend passes where the dependence actually is. It wins") +print("only if it holds more validity at the SAME or LOWER number of passes.\n") +for (fam, mode) in sorted(runs): + rs = runs[(fam, mode)] + pts = defaultdict(list) + for r in rs: + for k_, v in r["decode"].items(): + fam_kind = "entropy-budget" if k_.startswith("entbudget") else "fixed-K" + pts[fam_kind].append((v["nfe"], v["validity"])) + def curve(kind): + # average validity across seeds at each distinct NFE, ordered by NFE + out = [] + for n in sorted({p[0] for p in pts[kind]}): + vs = [v for m, v in pts[kind] if m == n] + out.append((float(n), float(np.mean(vs)))) + return out + + fk, eb = curve("fixed-K"), curve("entropy-budget") + print(f"-- {fam} / {mode}") + print(f" {'fixed-K':<16}" + " ".join(f"{n:.0f}->{v*100:.1f}%" for n, v in fk)) + print(f" {'entropy-budget':<16}" + " ".join(f"{n:.0f}->{v*100:.1f}%" for n, v in eb)) + # a dominating point: at most the NFE and strictly more validity + wins = [(n, v) for n, v in eb + if any(n <= fn and v > fv + 1e-9 for fn, fv in fk)] + print(f" entropy-budget points dominating some fixed-K point: " + f"{[(int(n), round(v*100,1)) for n, v in wins] or 'none'}") + +json.dump({f"{k[0]}/{k[1]}": v for k, v in summary.items()}, + open(os.path.join(os.environ.get("OUT", "figures"), + "graph_ceiling.json"), "w"), indent=2, default=float) +print("\nwrote graph_ceiling.json") diff --git a/src/graphs.py b/src/graphs.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..43b6605bfb824765616aa7113e09c794dbe2ab64 100644 --- a/src/graphs.py +++ b/src/graphs.py @@ -0,0 +1,243 @@ +"""Structured graph families where the information-theoretic quantities are EXACT. + +The bound this project rests on, + + TC(S) = sum_i H(x_i | c) - H(S | c) <= sum_i H(x_i | c), + +has so far been asserted rather than measured. On addition TC is zero by +construction, and on natural text H(S | c) is not computable, so the entropy +budget is motivated by a quantity we never verify. That is the soft spot. + +Graphs close it. A graph on n nodes is a binary sequence of n(n-1)/2 edge +indicators, so the same masked-diffusion machinery applies unchanged - but for +small n the ENTIRE family can be enumerated, which makes every quantity exact: + + * the true edge marginals p(e_i = 1), + * the true joint entropy H(S) for any subset of edges, + * therefore the true total correlation TC(S), + * and - the decisive one - the BEST POSSIBLE one-pass validity, + + V* = sum over valid graphs g of prod_i p_i(g_i), + + which is the probability that independent draws from the true marginals + happen to land on a valid graph. No model, however well trained, can beat + V* at one pass, because V* already assumes perfect marginals. + +That last quantity turns a qualitative claim into an airtight one. If a trained +model reaches V*, the remaining gap to 100% is *provably* information-theoretic +rather than an optimisation failure - which is exactly what mode (ii) asserts +and what no previous experiment here could demonstrate. + +Enumeration is 2^(n(n-1)/2) graphs, so n <= 7 is the practical range: n=6 gives +32,768 candidates and n=7 gives 2,097,152. +""" +import math +from functools import lru_cache + +import numpy as np + + +def edge_list(n): + """Upper-triangular edge slots in a fixed canonical order.""" + return [(i, j) for i in range(n) for j in range(i + 1, n)] + + +def n_edges(n): + return n * (n - 1) // 2 + + +def mask_to_adj(mask, n): + a = np.zeros((n, n), dtype=np.int8) + for k, (i, j) in enumerate(edge_list(n)): + if (mask >> k) & 1: + a[i, j] = a[j, i] = 1 + return a + + +def degrees(mask, n): + d = [0] * n + for k, (i, j) in enumerate(edge_list(n)): + if (mask >> k) & 1: + d[i] += 1 + d[j] += 1 + return d + + +def _components(mask, n): + parent = list(range(n)) + + def find(x): + while parent[x] != x: + parent[x] = parent[parent[x]] + x = parent[x] + return x + + cnt = 0 + for k, (i, j) in enumerate(edge_list(n)): + if (mask >> k) & 1: + a, b = find(i), find(j) + if a != b: + parent[a] = b + cnt += 1 + return len({find(x) for x in range(n)}), cnt + + +def has_triangle(mask, n): + adj = mask_to_adj(mask, n) + for i in range(n): + for j in range(i + 1, n): + if not adj[i, j]: + continue + for k in range(j + 1, n): + if adj[i, k] and adj[j, k]: + return True + return False + + +# ---------------------------------------------------------------- families --- +# Each predicate defines a set of graphs; the distribution is uniform over it, +# which is the maximum-entropy choice and keeps H(S) = log2(|family|) exact. + +def _is_perfect_matching(mask, n): + return all(d == 1 for d in degrees(mask, n)) + + +def _is_two_regular(mask, n): + return all(d == 2 for d in degrees(mask, n)) + + +def _is_tree(mask, n): + comps, m = _components(mask, n) + return comps == 1 and m == n - 1 + + +def _is_triangle_free_dense(mask, n): + # enough edges that the constraint actually bites, and no triangle + return bin(mask).count("1") >= n and not has_triangle(mask, n) + + +def _is_bipartite_fixed(mask, n): + half = n // 2 + for k, (i, j) in enumerate(edge_list(n)): + if (mask >> k) & 1 and ((i < half) == (j < half)): + return False + return bin(mask).count("1") >= half + + +FAMILIES = { + "matching": _is_perfect_matching, # every degree exactly 1 + "2regular": _is_two_regular, # every degree exactly 2 + "tree": _is_tree, # connected and acyclic + "trianglefree": _is_triangle_free_dense, + "bipartite": _is_bipartite_fixed, +} + + +@lru_cache(maxsize=None) +def enumerate_family(name, n): + """Every graph in the family, as a tuple of bit masks. Exhaustive.""" + pred = FAMILIES[name] + E = n_edges(n) + if E > 22: + raise ValueError(f"n={n} needs 2^{E} enumeration; use n <= 7") + return tuple(m for m in range(1 << E) if pred(m, n)) + + +def exact_stats(name, n): + """Every information-theoretic quantity, computed exactly by enumeration. + + Returns a dict with the true marginals, the summed marginal entropy (the + BOUND), the true joint entropy, the true total correlation, and V* - the + highest one-pass validity any model can achieve. + """ + fam = enumerate_family(name, n) + M = len(fam) + if M == 0: + raise ValueError(f"family {name} is empty at n={n}") + E = n_edges(n) + + bits = np.zeros((M, E), dtype=np.float64) + for r, m in enumerate(fam): + for k in range(E): + bits[r, k] = (m >> k) & 1 + + p = bits.mean(0) # true edge marginals + with np.errstate(divide="ignore", invalid="ignore"): + h = -(np.where(p > 0, p * np.log2(p), 0.0) + + np.where(p < 1, (1 - p) * np.log2(1 - p), 0.0)) + sum_h = float(h.sum()) # the BOUND + h_joint = math.log2(M) # uniform over the family + tc = sum_h - h_joint # the TRUE dependence + + # V*: probability that independent draws from the true marginals land in + # the family. This is the ceiling on one-pass validity for ANY model. + logp1 = np.where(p > 0, np.log(np.clip(p, 1e-300, 1)), -np.inf) + logp0 = np.where(p < 1, np.log(np.clip(1 - p, 1e-300, 1)), -np.inf) + v = 0.0 + for r in range(M): + lp = float(np.where(bits[r] > 0, logp1, logp0).sum()) + v += math.exp(lp) + + return {"family": name, "n": n, "n_edges": E, "size": M, + "marginals": p.tolist(), + "sum_marginal_entropy_bits": sum_h, + "joint_entropy_bits": h_joint, + "total_correlation_bits": tc, + "one_pass_ceiling": v, + "bound_holds": tc <= sum_h + 1e-9} + + +def sample(name, n, k, seed=0): + """Draw k graphs uniformly from the family, as (k, E) arrays of 0/1.""" + fam = enumerate_family(name, n) + rng = np.random.default_rng(seed) + idx = rng.integers(0, len(fam), size=k) + E = n_edges(n) + out = np.zeros((k, E), dtype=np.int64) + for r, t in enumerate(idx): + m = fam[t] + for b in range(E): + out[r, b] = (m >> b) & 1 + return out + + +def is_valid(rows, name, n): + """Validity of generated edge sequences, checked against the family.""" + pred = FAMILIES[name] + ok = np.zeros(len(rows), dtype=bool) + for r, row in enumerate(rows): + m = 0 + for b, v in enumerate(row): + if v == 1: + m |= (1 << b) + ok[r] = pred(m, n) + return ok + + +class EdgeTokenizer: + """Three symbols: absent, present, and the absorbing state.""" + + def __init__(self): + self.itos = ["0", "1", "[MASK]"] + self.stoi = {c: i for i, c in enumerate(self.itos)} + self.mask = 2 + self.pad = 2 + + def __len__(self): + return 3 + + +if __name__ == "__main__": + print(f"{'family':<14}{'n':>3}{'size':>8}{'sum H_i':>10}{'H(S)':>8}" + f"{'TC':>8}{'V* (1-pass ceiling)':>22}") + for n in (6, 7): + for name in FAMILIES: + try: + s = exact_stats(name, n) + except ValueError: + continue + print(f"{name:<14}{n:>3}{s['size']:>8}" + f"{s['sum_marginal_entropy_bits']:>10.3f}" + f"{s['joint_entropy_bits']:>8.3f}" + f"{s['total_correlation_bits']:>8.3f}" + f"{s['one_pass_ceiling']*100:>21.4f}%") diff --git a/src/marginal_check.py b/src/marginal_check.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..eb7fadb051e1705ee6185ee817a4851a290b8d3f 100644 --- a/src/marginal_check.py +++ b/src/marginal_check.py @@ -0,0 +1,30 @@ +"""Are the failing family's marginals position-DEPENDENT, and the succeeding ones not? + +At t = 1 every input token is [MASK], so the input sequence is constant. With a +purely RELATIVE position encoding the attention score between positions i and j +depends only on (i - j); when every value vector is identical the layer output is +v * sum(attn) = v, the same at every position. Boundary effects aside, a +RoPE-only model therefore cannot express position-DEPENDENT marginals at the +fully-masked state. This is an expressivity limit, not an optimisation one, so +more training cannot fix it. + +That predicts exactly which families succeed: those whose true marginals are +constant across edge slots. +""" +import sys, os +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import numpy as np +from graphs import exact_stats, FAMILIES + +print(f"{'family':<14}{'n':>3}{'V*':>10}{'marginal spread':>18}{'position-dependent?':>22}") +for n in (6,): + for name in FAMILIES: + try: + s = exact_stats(name, n) + except ValueError: + continue + p = np.array(s["marginals"]) + spread = float(p.max() - p.min()) + dep = "YES - RoPE cannot" if spread > 1e-6 else "no - RoPE suffices" + print(f"{name:<14}{n:>3}{s['one_pass_ceiling']*100:>9.2f}%{spread:>18.4f}{dep:>22}") + print(f"{'':>17}min={p.min():.3f} max={p.max():.3f} distinct={len(set(np.round(p,4)))}") diff --git a/src/mol_analysis.py b/src/mol_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..574b2700be61fd2e20ee871abd7a3d7dc24b3dae --- /dev/null +++ b/src/mol_analysis.py @@ -0,0 +1,113 @@ +"""QM9 results as a quality-vs-compute frontier. + +The number that matters on this benchmark is not validity at some fixed step +count but **how few denoising steps are needed to reach a given validity**. That +is the axis the discrete graph-diffusion literature competes on, and it is the +one determined by the training schedule and the decoding rule rather than by +model scale. + +Reported here as a speed-up over the standard formulation. The `mdlm` + `rope` +arm IS the conventional setup - uniform-t discrete diffusion with the rotary +encoding that modern implementations default to - so the comparison is +controlled: same data, same model size, same training budget, same decoder, one +factor changed at a time. + +A note on external comparison. Published DiGress-style numbers come from larger +models trained longer and with auxiliary spectral features that neither arm here +uses, so quoting them beside these would not be like-for-like. The claim +supported by this script is the controlled one: at matched scale and budget, how +many steps does each formulation need. Any external number should be taken from +the paper itself rather than from memory. +""" +import glob, json, os, sys +from collections import defaultdict + +import numpy as np + +RUNS = os.environ.get("RUNS", "runs") + +runs = defaultdict(list) +for f in sorted(glob.glob(f"{RUNS}/mol1-*/result.json")): + r = json.load(open(f)) + a = r["args"] + runs[(a["mode"], a["pe"])].append(r) + +if not runs: + sys.exit("no QM9 results yet") + +LABEL = {("mdlm", "rope"): "uniform-t + relative (standard)", + ("mdlm", "ape"): "uniform-t + absolute", + ("matched", "rope"): "matched + relative", + ("matched", "ape"): "matched + absolute (ours)"} + + +def curve(rs): + """(steps, validity, unique, novel) for every fixed-K setting, by step count.""" + pts = defaultdict(list) + for r in rs: + for k, v in r["decode"].items(): + if k.startswith("fixedK_"): + pts[int(k.split("_")[1])].append(v) + out = [] + for k in sorted(pts): + vs = pts[k] + out.append((k, + float(np.mean([x["validity"] for x in vs])), + float(np.mean([x["unique"] for x in vs])), + float(np.mean([x["novel"] for x in vs])))) + return out + + +def steps_for(c, target): + """Fewest denoising steps reaching `target` validity, or None.""" + for k, v, _, _ in c: + if v >= target: + return k + return None + + +print("=" * 84) +print("QM9 — validity against number of denoising steps") +print("=" * 84) +for key in [("mdlm", "rope"), ("mdlm", "ape"), ("matched", "rope"), ("matched", "ape")]: + rs = runs.get(key) + if not rs: + continue + c = curve(rs) + print(f"\n{LABEL[key]} ({len(rs)} seeds)") + print(f" {'steps':>7}{'validity':>11}{'unique':>9}{'novel':>9}") + for k, v, u, n in c: + print(f" {k:>7}{v*100:>10.2f}%{u*100:>8.1f}%{n*100:>8.1f}%") + +print() +print("=" * 84) +print("STEPS NEEDED TO REACH A VALIDITY TARGET (fewer is better)") +print("=" * 84) +hdr = "".join(f"{str(int(t*100)) + '%':>9}" for t in (0.5, 0.8, 0.9, 0.95)) +print(f"{'formulation':<36}" + hdr) +base = runs.get(("mdlm", "rope")) +base_c = curve(base) if base else None +for key in [("mdlm", "rope"), ("mdlm", "ape"), ("matched", "rope"), ("matched", "ape")]: + rs = runs.get(key) + if not rs: + continue + c = curve(rs) + cells = [] + for t in (0.5, 0.8, 0.9, 0.95): + s = steps_for(c, t) + cells.append(f"{s:>9}" if s else f"{'—':>9}") + print(f"{LABEL[key]:<36}" + "".join(cells)) + +if base_c: + print() + print("SPEED-UP over the standard formulation, at equal validity:") + for key in [("mdlm", "ape"), ("matched", "rope"), ("matched", "ape")]: + rs = runs.get(key) + if not rs: + continue + c = curve(rs) + for t in (0.8, 0.9, 0.95): + b, o = steps_for(base_c, t), steps_for(c, t) + if b and o: + print(f" {LABEL[key]:<34} at {int(t*100)}% validity: " + f"{b} steps -> {o} steps ({b/o:.1f}x fewer)") diff --git a/src/mol_fair.py b/src/mol_fair.py new file mode 100644 index 0000000000000000000000000000000000000000..8a52dc40f40bdd6df6793dc37143eaf88d7c0e3a --- /dev/null +++ b/src/mol_fair.py @@ -0,0 +1,127 @@ +"""Is the QM9 result real, or an artifact of our own representation? + +We encode a molecule as 9 fixed node slots plus 36 fixed edge slots. That forces +the model to learn slot-specific facts - slot 8 is never occupied, slot 7 is +occupied 80% of the time - which is precisely the position-dependence a relative +encoding cannot represent at the fully-masked state. Our RoPE baseline caps below +80% validity, and we attributed that to the encoding. + +But DiGress and the graph-diffusion line do NOT generate this way. They sample +the number of atoms n from the training distribution FIRST, then generate an n-node +graph with a permutation-equivariant model. "Slot 8 is empty" never arises for +them. If that is the whole of our baseline's disadvantage, then our comparison is +against a strawman of our own construction and the QM9 claim does not stand. + +This script removes exactly that disadvantage, without retraining. For each +sample it draws n from the empirical training distribution, fixes node slots +n..8 and every edge touching them to "none" as OBSERVED context, and decodes only +the remaining positions. That is the DiGress setup expressed in our +representation, and it is strictly generous to the baseline. + + RoPE recovers -> the finding was representation-specific and the QM9 claim + must be withdrawn + RoPE still fails -> the disadvantage is not the padding slots, and the finding + survives a fair baseline + +Either way the answer goes in the paper. +""" +import argparse, glob, json, os, sys +import numpy as np +import torch + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from model import Transformer +import diffusion as dfn +from qm9 import EDGES, MAX_ATOMS, SEQ_LEN, MolTokenizer, load_qm9, validity + +p = argparse.ArgumentParser() +p.add_argument("--runs", default="runs") +p.add_argument("--root", default="data/qm9") +p.add_argument("--n_gen", type=int, default=2048) +p.add_argument("--gen_bs", type=int, default=512) +p.add_argument("--out", default="figures/mol_fair.json") +a = p.parse_args() + +dev = "cuda" if torch.cuda.is_available() else "cpu" +X, tok, train_smiles = load_qm9(a.root, 40000) +ALLOWED = torch.from_numpy(tok.allowed_mask()).to(dev) + +# empirical distribution over heavy-atom counts, which is what DiGress samples +counts = np.array([(row[:MAX_ATOMS] != tok.none).sum() for row in X]) +sizes, freqs = np.unique(counts, return_counts=True) +probs = freqs / freqs.sum() +print(f"atom-count distribution from training data: " + f"{dict(zip(sizes.tolist(), np.round(probs, 3).tolist()))}", flush=True) + + +def scaffold(bs, rng): + """A canvas with n sampled first: slots beyond n are OBSERVED as 'none'. + + This is the advantage DiGress has and our flat encoding withholds: the model + never has to discover which slots exist, because the unused ones are given. + """ + x = torch.full((bs, SEQ_LEN), tok.mask, dtype=torch.long) + fill = torch.ones((bs, SEQ_LEN), dtype=torch.bool) + ns = rng.choice(sizes, size=bs, p=probs) + for b, n in enumerate(ns): + for j in range(int(n), MAX_ATOMS): + x[b, j] = tok.none + fill[b, j] = False + for k, (i, j) in enumerate(EDGES): + if i >= n or j >= n: + x[b, MAX_ATOMS + k] = tok.none + fill[b, MAX_ATOMS + k] = False + return x, fill + + +def load(run_dir): + cfg = json.load(open(os.path.join(run_dir, "result.json")))["args"] + m = Transformer(len(tok), cfg["d"], cfg["layers"], cfg["heads"], cfg["pe"], + causal=False, max_len=SEQ_LEN + 8).to(dev) + m.load_state_dict(torch.load(os.path.join(run_dir, "model.pt"), map_location=dev)) + m.eval() + return m, cfg + + +@torch.no_grad() +def run(model, steps, seed): + rng = np.random.default_rng(seed) + outs = [] + for _ in range(max(1, a.n_gen // a.gen_bs)): + x, fill = scaffold(a.gen_bs, rng) + y, _ = dfn.fixed_k_decode(model, x.to(dev), tok, dev, steps, + fillable=fill.to(dev), allowed=ALLOWED) + outs.append(y.cpu()) + y = torch.cat(outs).numpy() + v, smis = validity(y, tok) + uniq = len(set(smis)) / max(1, len(smis)) + return v, uniq + + +res = {} +print(f"\n{'arm':<28}{'steps':>7}{'validity':>11}{'unique':>9}", flush=True) +for d in sorted(glob.glob(os.path.join(a.runs, "mol1-*"))): + if not os.path.exists(os.path.join(d, "model.pt")): + continue + m, cfg = load(d) + name = os.path.basename(d) + key = f"{cfg['mode']}-{cfg['pe']}" + for steps in (1, 4, 8, 16, 32, 45): + v, u = run(m, steps, cfg["seed"]) + res.setdefault(key, {}).setdefault(str(steps), []).append(v) + print(f"{name:<28}{steps:>7}{v*100:>10.2f}%{u*100:>8.1f}%", flush=True) + del m + torch.cuda.empty_cache() + +print(f"\n{'='*70}") +print("FAIR BASELINE: atom count given, unused slots observed (the DiGress setup)") +print(f"{'='*70}") +print(f"{'arm':<22}" + "".join(f"{s:>9}" for s in (1, 4, 8, 16, 32, 45))) +for key in sorted(res): + print(f"{key:<22}" + "".join(f"{np.mean(res[key][str(s)])*100:>8.1f}%" + for s in (1, 4, 8, 16, 32, 45))) +os.makedirs(os.path.dirname(a.out) or ".", exist_ok=True) +json.dump(res, open(a.out, "w"), indent=2) +print(f"\nwrote {a.out}") +print("\nRead: if the relative-encoding arms now track the absolute ones, the QM9") +print("claim was an artifact of the flat fixed-slot encoding and must be withdrawn.") diff --git a/src/qm9.py b/src/qm9.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5012e91e21750d8a03abf615704c5517e83818b5 100644 --- a/src/qm9.py +++ b/src/qm9.py @@ -0,0 +1,213 @@ +"""QM9 molecular graphs as flat token sequences. + +QM9 is the standard small-molecule generation benchmark and the one DiGress and +the discrete graph-diffusion line report on. Molecules have at most 9 heavy +atoms drawn from {C, N, O, F}, with bonds in {single, double, triple, aromatic}, +so a molecule is fully described by + + 9 node slots (atom type, or absent) + + 36 edge slots (bond type, or absent) = 45 tokens. + +That is small enough to train from scratch cheaply, which is the point: the +claim under test is about the training schedule and the decoding rule, not about +model scale. + +Why molecules are the right Phase 2. Validity here is a genuine JOINT +constraint - valency must balance at every atom simultaneously - so mode (ii) is +both dominant and, unlike natural text, *checkable*: RDKit says yes or no. The +metric is therefore unambiguous in a way generative perplexity is not, and the +field's live axis (validity at few denoising steps) is exactly the +quality-vs-compute frontier this project is about. + +Node and edge slots take disjoint symbol sets, which is a property of the +representation rather than something a model should spend capacity +rediscovering; `allowed_mask` expresses it and the decoders enforce it. +""" +import csv +import os + +import numpy as np + +MAX_ATOMS = 9 +ATOMS = ["C", "N", "O", "F"] +# Kekulised: aromatic rings are stored as alternating single/double bonds, so +# no aromaticity perception is needed to rebuild a molecule. Storing AROMATIC +# directly loses the per-ATOM aromatic flags, and reconstruction then fails with +# "Can't kekulize mol" on every aromatic species - 7% of QM9, and 100% of the +# round-trip failures measured before this fix. +BONDS = ["SINGLE", "DOUBLE", "TRIPLE"] +N_EDGE = MAX_ATOMS * (MAX_ATOMS - 1) // 2 # 36 +SEQ_LEN = MAX_ATOMS + N_EDGE # 45 + +EDGES = [(i, j) for i in range(MAX_ATOMS) for j in range(i + 1, MAX_ATOMS)] + + +class MolTokenizer: + """One vocabulary, but node and edge positions draw from disjoint parts. + + 0 no atom / no bond (shared "absent" symbol) + 1..4 C N O F (node slots only) + 5..8 single double triple aromatic (edge slots only) + 9 [MASK] + """ + + def __init__(self): + self.itos = ([""] + ATOMS + BONDS + ["[MASK]"]) + self.stoi = {s: i for i, s in enumerate(self.itos)} + self.mask = self.stoi["[MASK]"] + self.pad = self.mask + self.none = 0 + self.atom_ids = [self.stoi[a] for a in ATOMS] + self.bond_ids = [self.stoi[b] for b in BONDS] + + def __len__(self): + return len(self.itos) + + def allowed_mask(self): + """(SEQ_LEN, V) boolean: which symbols each position may take.""" + m = np.zeros((SEQ_LEN, len(self)), dtype=bool) + m[:MAX_ATOMS, self.none] = True + for i in self.atom_ids: + m[:MAX_ATOMS, i] = True + m[MAX_ATOMS:, self.none] = True + for i in self.bond_ids: + m[MAX_ATOMS:, i] = True + return m + + +def mol_to_seq(mol, tok): + """RDKit molecule -> (SEQ_LEN,) token array, or None if it cannot be encoded. + + Rejects molecules this representation cannot express exactly, rather than + encoding them lossily: anything over MAX_ATOMS, anything carrying a formal + charge (the atom vocabulary is uncharged), and anything that will not + kekulise. Encoding must be lossless or generated-sample validity is measured + against a target the encoding itself cannot hit. + """ + from rdkit import Chem + n = mol.GetNumAtoms() + if n > MAX_ATOMS: + return None + if any(a.GetFormalCharge() != 0 for a in mol.GetAtoms()): + return None + try: + mol = Chem.Mol(mol) + Chem.Kekulize(mol, clearAromaticFlags=True) + except Exception: + return None + seq = np.zeros(SEQ_LEN, dtype=np.int64) + for a in mol.GetAtoms(): + s = a.GetSymbol() + if s not in tok.stoi: + return None + seq[a.GetIdx()] = tok.stoi[s] + pos = {e: k for k, e in enumerate(EDGES)} + for b in mol.GetBonds(): + i, j = sorted((b.GetBeginAtomIdx(), b.GetEndAtomIdx())) + bt = str(b.GetBondType()) + if bt not in tok.stoi: + return None + seq[MAX_ATOMS + pos[(i, j)]] = tok.stoi[bt] + return seq + + +def seq_to_mol(seq, tok): + """Token array -> RDKit molecule (unsanitised), or None if incoherent.""" + from rdkit import Chem + bond_map = {tok.stoi["SINGLE"]: Chem.BondType.SINGLE, + tok.stoi["DOUBLE"]: Chem.BondType.DOUBLE, + tok.stoi["TRIPLE"]: Chem.BondType.TRIPLE} + m = Chem.RWMol() + idx = {} + for i in range(MAX_ATOMS): + t = int(seq[i]) + if t in tok.atom_ids: + idx[i] = m.AddAtom(Chem.Atom(tok.itos[t])) + if not idx: + return None + for k, (i, j) in enumerate(EDGES): + t = int(seq[MAX_ATOMS + k]) + if t in bond_map: + # a bond to an absent atom is incoherent, not merely invalid + if i not in idx or j not in idx: + return None + m.AddBond(idx[i], idx[j], bond_map[t]) + return m + + +def validity(seqs, tok): + """Fraction that sanitise, plus canonical SMILES of those that do. + + Sanitisation is the standard validity criterion for this benchmark: it + enforces valency at every atom at once, which is precisely the joint + constraint parallel decoding risks breaking. + """ + from rdkit import Chem + from rdkit import RDLogger + RDLogger.DisableLog("rdApp.*") + smiles = [] + for s in seqs: + m = seq_to_mol(s, tok) + if m is None: + continue + try: + mm = m.GetMol() + Chem.SanitizeMol(mm) + smi = Chem.MolToSmiles(mm) + if smi: + smiles.append(smi) + except Exception: + continue + return len(smiles) / max(1, len(seqs)), smiles + + +def load_qm9(root="data/qm9", limit=None): + """Return (sequences, tokenizer, train_smiles_set). + + The SMILES set is kept so novelty can be measured against the training data + rather than only uniqueness within a sample. + """ + from rdkit import Chem + from rdkit import RDLogger + RDLogger.DisableLog("rdApp.*") + path = os.path.join(root, "qm9.csv") + if not os.path.exists(path): + raise FileNotFoundError( + f"{path} not found. Fetch once with:\n mkdir -p {root} && curl -sL -o {path} " + "https://deepchemdata.s3.us-west-1.amazonaws.com/datasets/qm9.csv") + tok = MolTokenizer() + seqs, smis = [], set() + n_kept, n_seen = [0], 0 + with open(path) as f: + for r in csv.DictReader(f): + n_seen += 1 + smi = r.get("smiles") + if not smi: + continue + m = Chem.MolFromSmiles(smi) + if m is None: + continue + s = mol_to_seq(m, tok) + if s is None: + continue + seqs.append(s) + smis.add(Chem.MolToSmiles(m)) + n_kept[0] += 1 + if limit and len(seqs) >= limit: + break + print(f"qm9: kept {n_kept[0]}/{n_seen} molecules " + f"({100*n_kept[0]/max(1,n_seen):.1f}%); the remainder are charged, " + f"over {MAX_ATOMS} heavy atoms, or will not kekulise") + return np.stack(seqs), tok, smis + + +if __name__ == "__main__": + X, tok, smis = load_qm9(os.environ.get("QM9ROOT", "data/qm9"), limit=5000) + print(f"sequences {X.shape} vocab {len(tok)} unique SMILES {len(smis)}") + am = tok.allowed_mask() + print(f"allowed mask {am.shape}: node slots allow {am[0].sum()} symbols, " + f"edge slots allow {am[MAX_ATOMS].sum()}") + # round-trip: every training molecule must re-sanitise from its own tokens + v, sm = validity(X[:2000], tok) + print(f"round-trip validity on real data: {v*100:.2f}% " + f"(must be ~100% or the encoding is lossy)") diff --git a/src/sudoku.py b/src/sudoku.py new file mode 100644 index 0000000000000000000000000000000000000000..18cdfccde8a3f1715b28bf94c2f3bb5d707bc3cc --- /dev/null +++ b/src/sudoku.py @@ -0,0 +1,144 @@ +"""Sudoku as a constraint-satisfaction benchmark for masked diffusion. + +Uses the SATNet dataset (Wang et al., ICML 2019), which is the standard 9x9 +Sudoku benchmark for neural solvers: 10,000 puzzles with the conventional +9,000 / 1,000 train/test split, scored by BOARD-level accuracy - every one of the +81 cells correct, nothing partial. That metric is unambiguous and it is what the +published numbers report, so a comparison means something. + +Why this task. Sudoku is the cleanest possible case of the regime this project +is about: the answer is UNIQUE, so there is no information-theoretic obstacle at +all, and the dependence between cells is enormous, since each cell is +constrained by twenty others through its row, column and box. Our framework says +the entire difficulty is therefore mode (i) - and, more sharply, that a decoder +which commits cells in order of certainty is performing classical constraint +propagation. Filling the forced cells first, recomputing, and repeating IS the +textbook algorithm, and it is exactly what an entropy-budgeted commit rule does +without being told to. + +The prediction, registered before any run: fixed-K parallel decoding should fail +badly on this task while entropy-budgeted decoding should succeed, and the gap +should be far larger than anything seen on text, where dependence is diffuse. +""" +import os + +import numpy as np + +GRID = 9 +CELLS = GRID * GRID # 81 +N_TRAIN, N_TEST = 9000, 1000 # the conventional SATNet split + + +class SudokuTokenizer: + """Ten symbols: the digits 1-9 and the absorbing state. + + There is no separate 'blank' symbol. A blank cell is simply one that starts + masked, which is what makes this a conditional masked-diffusion problem + rather than a classification one: clues are observed context, blanks are the + positions to fill. + """ + + def __init__(self): + self.itos = [str(d) for d in range(1, 10)] + ["[MASK]"] + self.stoi = {s: i for i, s in enumerate(self.itos)} + self.mask = self.stoi["[MASK]"] + self.pad = self.mask + + def __len__(self): + return len(self.itos) + + +def _onehot_to_int(t): + """(N, 9, 9, 9) one-hot -> (N, 81) with 0 for an empty cell, 1-9 otherwise.""" + filled = t.sum(-1) > 0 + digits = t.argmax(-1) + 1 + return np.where(filled, digits, 0).reshape(len(t), CELLS).astype(np.int64) + + +def is_valid_solution(grid): + """Does a completed 81-cell grid satisfy every Sudoku constraint? + + Checked rather than assumed: if the dataset's labels were not valid Sudoku + the entire benchmark would be meaningless, and the failure would be silent. + """ + g = np.asarray(grid).reshape(GRID, GRID) + target = set(range(1, 10)) + for i in range(GRID): + if set(g[i]) != target or set(g[:, i]) != target: + return False + for bi in range(0, GRID, 3): + for bj in range(0, GRID, 3): + if set(g[bi:bi + 3, bj:bj + 3].ravel()) != target: + return False + return True + + +def consistent_with_clues(pred, puzzle): + """Does a prediction keep every given clue? (It should: clues are observed.)""" + p, q = np.asarray(pred), np.asarray(puzzle) + given = q > 0 + return bool((p[given] == q[given]).all()) + + +def load_satnet(root="data/sudoku"): + """Return (puzzles, solutions, tokenizer) as (N, 81) integer arrays. + + puzzles: 0 where the cell is blank, 1-9 where a clue is given + solutions: 1-9 everywhere + """ + import torch + base = os.path.join(root, "sudoku") + if not os.path.isdir(base): + base = root + fx = os.path.join(base, "features.pt") + fy = os.path.join(base, "labels.pt") + if not (os.path.exists(fx) and os.path.exists(fy)): + raise FileNotFoundError( + f"SATNet Sudoku not found under {root}. Fetch once with:\n" + f" mkdir -p {root} && cd {root} && curl -sLO https://powei.tw/sudoku.zip " + f"&& unzip -q sudoku.zip") + X = _onehot_to_int(torch.load(fx, map_location="cpu").numpy()) + Y = _onehot_to_int(torch.load(fy, map_location="cpu").numpy()) + return X, Y, SudokuTokenizer() + + +def split(X, Y): + """The conventional 9,000 / 1,000 split, in dataset order.""" + return (X[:N_TRAIN], Y[:N_TRAIN]), (X[N_TRAIN:N_TRAIN + N_TEST], + Y[N_TRAIN:N_TRAIN + N_TEST]) + + +def board_accuracy(pred, gold): + """Fraction of boards with ALL 81 cells correct - the published metric.""" + return float((np.asarray(pred) == np.asarray(gold)).all(axis=1).mean()) + + +def cell_accuracy(pred, gold, blanks): + """Per-cell accuracy over blanks only, reported alongside as context.""" + p, g, b = np.asarray(pred), np.asarray(gold), np.asarray(blanks) + return float((p[b] == g[b]).mean()) + + +if __name__ == "__main__": + root = os.environ.get("SUDOKUROOT", "data/sudoku") + X, Y, tok = load_satnet(root) + (Xtr, Ytr), (Xte, Yte) = split(X, Y) + print(f"puzzles {X.shape} solutions {Y.shape} vocab {len(tok)}") + print(f"train {Xtr.shape[0]} test {Xte.shape[0]} (conventional split)") + + clues = (X > 0).sum(1) + print(f"clues per puzzle: min={clues.min()} max={clues.max()} " + f"mean={clues.mean():.1f}") + + # every label must be a valid Sudoku solution, or the benchmark is meaningless + bad = sum(not is_valid_solution(Y[i]) for i in range(0, len(Y), 7)) + print(f"labels that are NOT valid Sudoku (sampled): {bad} (must be 0)") + + # every clue must agree with its solution + dis = sum(not consistent_with_clues(Y[i], X[i]) for i in range(0, len(X), 7)) + print(f"puzzles whose clues disagree with the label (sampled): {dis} (must be 0)") + + # train and test must not overlap + tr = {bytes(r) for r in Xtr[::3]} + ov = sum(bytes(r) in tr for r in Xte) + print(f"test puzzles also seen in the training sample: {ov} (must be 0)") diff --git a/src/sudoku_analysis.py b/src/sudoku_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..251bac1af825bcdaa99003623792d8212270c34c --- /dev/null +++ b/src/sudoku_analysis.py @@ -0,0 +1,104 @@ +"""Sudoku results: the decoding comparison, and the benchmark comparison. + +Two separate questions, kept separate because they have different standards of +evidence. + +1. THE INTERNAL CLAIM. Does committing by certainty beat committing a fixed + share per pass? This is a controlled comparison - same trained model, same + test split, only the decoder changed - so it is clean regardless of how the + absolute numbers land, and it is the claim the framework actually makes. + +2. THE BENCHMARK COMPARISON. SATNet (Wang et al., ICML 2019) reports 98.3% + board-level accuracy on this split. Our number is comparable only for the + un-augmented arm, since SATNet did not use digit-relabelling augmentation. + The augmented arm is reported separately and must not be quoted against them. + +Board accuracy means all 81 cells correct. Cell accuracy over blanks and the +fraction of outputs that are valid Sudoku are reported alongside, because a +solver can score well per-cell while almost never completing a board. +""" +import glob, json, os, sys +from collections import defaultdict + +import numpy as np + +RUNS = os.environ.get("RUNS", "runs") +SATNET = 0.983 + +runs = defaultdict(list) +for f in sorted(glob.glob(f"{RUNS}/sud1-*/result.json")): + r = json.load(open(f)) + a = r["args"] + runs[(a["mode"], "aug" if a.get("augment") else "plain")].append(r) + +if not runs: + sys.exit("no sudoku results yet") + + +def rows(rs, prefix): + out = {} + for r in rs: + for k, v in r["decode"].items(): + if k.startswith(prefix): + out.setdefault(k, []).append(v) + return out + + +print("=" * 88) +print("SUDOKU (SATNet benchmark, 1000-puzzle test split) — BOARD accuracy = all 81 cells") +print("=" * 88) + +best_overall = {} +for key in sorted(runs): + rs = runs[key] + print(f"\n--- {key[0]} / {key[1]} ({len(rs)} seeds)") + print(f" {'rule':<24}{'NFE':>7}{'BOARD':>10}{'cell':>9}{'valid':>9}") + for prefix, label in (("fixedK_", "fixed-K"), + ("entbudget_", "entropy-budget"), + ("conf_", "confidence")): + got = rows(rs, prefix) + for k in sorted(got, key=lambda s: float(s.split("_")[1])): + v = got[k] + b = np.mean([x["board"] for x in v]) + print(f" {k:<24}{np.mean([x['nfe'] for x in v]):>7.1f}" + f"{b*100:>9.2f}%{np.mean([x['cell'] for x in v])*100:>8.2f}%" + f"{np.mean([x['valid_sudoku'] for x in v])*100:>8.2f}%") + fam = best_overall.setdefault((key, label), {"board": -1}) + if b > fam["board"]: + best_overall[(key, label)] = { + "board": b, "rule": k, + "nfe": float(np.mean([x["nfe"] for x in v]))} + +print() +print("=" * 88) +print("1. THE CONTROLLED CLAIM — same model, same data, only the decoder changes") +print("=" * 88) +print(f"{'arm':<22}{'fixed-K best':>16}{'entropy-budget':>17}{'gap':>10}{'conf best':>12}") +for key in sorted(runs): + fk = best_overall.get((key, "fixed-K")) + eb = best_overall.get((key, "entropy-budget")) + cf = best_overall.get((key, "confidence")) + if not (fk and eb): + continue + print(f"{key[0] + '/' + key[1]:<22}{fk['board']*100:>15.2f}%{eb['board']*100:>16.2f}%" + f"{(eb['board']-fk['board'])*100:>+9.2f}" + f"{(cf['board']*100 if cf else float('nan')):>11.2f}%") + +print() +print("=" * 88) +print("2. THE BENCHMARK COMPARISON — like-for-like is the UN-AUGMENTED arm only") +print("=" * 88) +print(f" SATNet (Wang et al., ICML 2019): {SATNET*100:.1f}% board accuracy\n") +for key in sorted(runs): + eb = best_overall.get((key, "entropy-budget")) + if not eb: + continue + tag = "comparable" if key[1] == "plain" else "NOT comparable (augmented)" + delta = (eb["board"] - SATNET) * 100 + verdict = f"{delta:+.2f} pts vs SATNet" if key[1] == "plain" else "—" + print(f" {key[0] + '/' + key[1]:<20}{eb['board']*100:>7.2f}% at {eb['nfe']:>5.1f} passes" + f" [{tag}] {verdict}") + +print() +print("Note: a decoder is only as good as the model it decodes. The controlled") +print("claim in section 1 holds regardless of whether section 2 clears 98.3%.") diff --git a/src/train_graph.py b/src/train_graph.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..13ba55dcd60348df87177efdf2ea821a30aceb88 100644 --- a/src/train_graph.py +++ b/src/train_graph.py @@ -0,0 +1,163 @@ +"""Masked diffusion on structured graphs, where the ceiling is known exactly. + +This is the experiment the project could not run before. On every earlier task +the best achievable parallel-decoding accuracy was unknown, so a gap could +always be blamed on undertraining. Here it is computable in closed form: + + V* = sum over valid graphs of prod_i p_i(g_i) and V* >= 2^(-TC), + +with equality when the family has a fixed edge count. V* is the probability +that independent draws from the TRUE marginals land on a valid graph, so it is +the one-pass ceiling for any model whatsoever - it already assumes the marginals +are perfect. + +That makes the logic airtight in a way it was not on addition or text: + + * if a trained model reaches V* at one pass, the remaining gap to 100% is + PROVABLY information-theoretic - mode (ii) - and no training objective can + recover it; + * if it falls short of V*, the shortfall is exactly mode (i), the trainable + part, and budget conditioning should close it. + +The two failure modes are therefore separated numerically rather than argued +about, on the same task, at the same time. + +A graph on n nodes is a binary sequence of n(n-1)/2 edge indicators, so the +masked-diffusion machinery is used unchanged - only the data and the validity +check are new. +""" +import argparse, json, math, os, time +import numpy as np +import torch + +from model import Transformer +import diffusion as dfn +from graphs import (EdgeTokenizer, exact_stats, is_valid, n_edges, sample) + + +def get_args(): + p = argparse.ArgumentParser() + p.add_argument("--family", default="matching", + choices=["matching", "2regular", "tree", "trianglefree", + "bipartite"]) + p.add_argument("--n", type=int, default=6) + p.add_argument("--mode", choices=["mdlm", "anybudget", "anybudget_nc"], + default="mdlm") + p.add_argument("--budget_bins", type=int, default=8) + p.add_argument("--pe", default="rope", choices=["rope", "ape", "sin", "alibi", "nope"], + help="position encoding; decisive at the fully-masked state") + p.add_argument("--d", type=int, default=256) + p.add_argument("--layers", type=int, default=6) + p.add_argument("--heads", type=int, default=8) + p.add_argument("--bs", type=int, default=256) + p.add_argument("--steps", type=int, default=30000) + p.add_argument("--lr", type=float, default=1e-4) + p.add_argument("--warmup", type=int, default=1000) + p.add_argument("--seed", type=int, default=0) + p.add_argument("--n_gen", type=int, default=4096) + p.add_argument("--gen_bs", type=int, default=512) + p.add_argument("--out", default="runs/gdev") + return p.parse_args() + + +a = get_args() +torch.manual_seed(a.seed) +np.random.seed(a.seed) +dev = "cuda" if torch.cuda.is_available() else "cpu" +os.makedirs(a.out, exist_ok=True) + +E = n_edges(a.n) +tok = EdgeTokenizer() +stats = exact_stats(a.family, a.n) +print(f"family={a.family} n={a.n} edges={E} |family|={stats['size']}", flush=True) +print(f" sum_i H(e_i) = {stats['sum_marginal_entropy_bits']:.3f} bits (the bound)", flush=True) +print(f" H(S) = {stats['joint_entropy_bits']:.3f} bits", flush=True) +print(f" TC = {stats['total_correlation_bits']:.3f} bits", flush=True) +print(f" V* ceiling = {stats['one_pass_ceiling']*100:.4f}% " + f"(2^-TC = {2**-stats['total_correlation_bits']*100:.4f}%)", flush=True) + +X = torch.from_numpy(sample(a.family, a.n, 200_000, seed=a.seed)) + +COND = a.mode == "anybudget" +model = Transformer(len(tok), a.d, a.layers, a.heads, a.pe, causal=False, + max_len=E + 8, budget_bins=a.budget_bins if COND else 0).to(dev) +print(f"params {model.n_params()/1e6:.2f}M mode={a.mode} pe={a.pe}", flush=True) + +opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=0.01) +sched = torch.optim.lr_scheduler.LambdaLR( + opt, lambda s: min(1.0, (s + 1) / a.warmup) * + 0.5 * (1 + math.cos(math.pi * min(1.0, s / a.steps)))) + +t0 = time.time() +for step in range(a.steps): + idx = torch.randint(0, len(X), (a.bs,)) + xb = X[idx].to(dev) + opt.zero_grad(set_to_none=True) + with torch.autocast("cuda", dtype=torch.bfloat16, enabled=(dev == "cuda")): + if a.mode in ("anybudget", "anybudget_nc"): + loss = dfn.uncond_any_budget_loss( + model, xb, tok, choices=tuple(2 ** i for i in range(int(math.log2(E)) + 1)), + condition=COND) + else: + loss = dfn.uncond_mdlm_loss(model, xb, tok, "uniform") + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) + opt.step(); sched.step() + if step % 5000 == 0: + print(f"step {step:6d} loss {loss.item():.4f} ({time.time()-t0:.0f}s)", flush=True) + +model.eval() +res = {"args": vars(a), "exact": {k: v for k, v in stats.items() if k != "marginals"}} + + +@torch.no_grad() +def gen(fn): + outs, nfes = [], [] + for _ in range(max(1, a.n_gen // a.gen_bs)): + blank = torch.full((a.gen_bs, E), tok.mask, dtype=torch.long) + x, nfe = fn(blank) + outs.append(x.cpu()); nfes.append(nfe) + x = torch.cat(outs).numpy() + ok = is_valid(x, a.family, a.n) + uniq = len({tuple(r) for r in x[ok]}) / max(1, int(ok.sum())) + return float(ok.mean()), float(np.mean(nfes)), float(uniq) + + +def bud(K): + return torch.full((a.gen_bs,), float(K), device=dev) if COND else None + + +print("\n=== validity against decoding budget ===", flush=True) +print(f"{'rule':<22}{'NFE':>7}{'validity':>11}{'vs V*':>10}{'uniq':>8}", flush=True) +res["decode"] = {} +V = stats["one_pass_ceiling"] + +for K in sorted({1, 2, 4, 8, E}): + v, nfe, u = gen(lambda b, K=K: dfn.fixed_k_decode(model, b, tok, dev, K, + budget_cond=bud(K))) + res["decode"][f"fixedK_{K}"] = {"validity": v, "nfe": nfe, "unique": u} + rel = f"{v/V:8.2f}x" if K == 1 else f"{'—':>9}" + print(f"{'fixed-K K=' + str(K):<22}{nfe:>7.1f}{v*100:>10.2f}%{rel}{u*100:>7.1f}%", flush=True) + +for B in (0.05, 0.25, 1.0, 4.0): + v, nfe, u = gen(lambda b, B=B: dfn.entropy_budget_decode(model, b, tok, dev, B, + max_steps=E + 2, + budget_cond=bud(8))) + res["decode"][f"entbudget_{B}"] = {"validity": v, "nfe": nfe, "unique": u} + print(f"{'entropy-budget B=' + str(B):<22}{nfe:>7.1f}{v*100:>10.2f}%{'—':>9}" + f"{u*100:>7.1f}%", flush=True) + +one = res["decode"]["fixedK_1"]["validity"] +res["reached_ceiling"] = one >= 0.9 * V +print(f"\nV* (exact one-pass ceiling) : {V*100:.4f}%", flush=True) +print(f"achieved at one pass : {one*100:.4f}% " + f"({one/V*100:.1f}% of the ceiling)", flush=True) +print("reading: " + ("the model is at the information-theoretic limit, so the " + "residual gap is mode (ii) and NOT trainable" + if res["reached_ceiling"] else + "the model falls short of the ceiling, so part of the gap " + "is mode (i) and IS trainable"), flush=True) + +torch.save(model.state_dict(), os.path.join(a.out, "model.pt")) +json.dump(res, open(os.path.join(a.out, "result.json"), "w"), indent=2) +print(f"\nsaved to {a.out}", flush=True) diff --git a/src/train_mol.py b/src/train_mol.py new file mode 100644 index 0000000000000000000000000000000000000000..05a7e2d8059e5caf23b08a5fe5faae1902aec390 --- /dev/null +++ b/src/train_mol.py @@ -0,0 +1,149 @@ +"""QM9 molecule generation: validity against denoising budget. + +Phase 2. Where the synthetic graph families give an exactly computable ceiling +but an artificial task, QM9 gives a real benchmark whose metric is unambiguous: +RDKit sanitisation either succeeds or it does not, and it enforces valency at +every atom simultaneously. That makes it a joint constraint of exactly the kind +parallel decoding breaks - mode (ii) - on data nobody designed for this project. + +The axis of comparison is the one the graph-diffusion literature competes on: +how much validity survives as the number of denoising steps falls. Discrete +graph diffusion conventionally uses hundreds of steps; the claim under test is +that budget-conditioned training plus an entropy-budgeted commit rule holds +validity at far fewer. + +All arms share the model, the data and the training budget, so the comparison +isolates the schedule and the decoding rule rather than scale. +""" +import argparse, json, math, os, time +import numpy as np +import torch + +from model import Transformer +import diffusion as dfn +from qm9 import MolTokenizer, SEQ_LEN, load_qm9, validity + + +def get_args(): + p = argparse.ArgumentParser() + p.add_argument("--mode", + choices=["mdlm", "matched", "anybudget", "anybudget_nc"], + default="mdlm") + p.add_argument("--K", type=int, default=8, + help="budget the schedule is matched to (mode=matched)") + p.add_argument("--budget_bins", type=int, default=8) + p.add_argument("--pe", default="rope", + choices=["rope", "ape", "sin", "alibi", "nope"], + help="QM9 marginals are strongly position-dependent (slot 8 is " + "never occupied, some edge slots never bond), so this is " + "predicted to matter a great deal at low step counts") + p.add_argument("--root", default="data/qm9") + p.add_argument("--limit", type=int, default=0, help="0 = all molecules") + p.add_argument("--d", type=int, default=384) + p.add_argument("--layers", type=int, default=8) + p.add_argument("--heads", type=int, default=8) + p.add_argument("--bs", type=int, default=256) + p.add_argument("--steps", type=int, default=60000) + p.add_argument("--lr", type=float, default=1e-4) + p.add_argument("--warmup", type=int, default=2000) + p.add_argument("--seed", type=int, default=0) + p.add_argument("--n_gen", type=int, default=4096) + p.add_argument("--gen_bs", type=int, default=512) + p.add_argument("--out", default="runs/moldev") + return p.parse_args() + + +a = get_args() +torch.manual_seed(a.seed); np.random.seed(a.seed) +dev = "cuda" if torch.cuda.is_available() else "cpu" +os.makedirs(a.out, exist_ok=True) + +X, tok, train_smiles = load_qm9(a.root, a.limit or None) +print(f"qm9 sequences {X.shape} vocab {len(tok)} train SMILES {len(train_smiles)}", + flush=True) +Xt = torch.from_numpy(X) +ALLOWED = torch.from_numpy(tok.allowed_mask()).to(dev) + +COND = a.mode == "anybudget" +model = Transformer(len(tok), a.d, a.layers, a.heads, a.pe, causal=False, + max_len=SEQ_LEN + 8, + budget_bins=a.budget_bins if COND else 0).to(dev) +print(f"params {model.n_params()/1e6:.2f}M mode={a.mode} pe={a.pe}", flush=True) + +opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=0.01) +sched = torch.optim.lr_scheduler.LambdaLR( + opt, lambda s: min(1.0, (s + 1) / a.warmup) * + 0.5 * (1 + math.cos(math.pi * min(1.0, s / a.steps)))) +BUDGETS = tuple(2 ** i for i in range(int(math.log2(SEQ_LEN)) + 2)) + +t0 = time.time() +for step in range(a.steps): + xb = Xt[torch.randint(0, len(Xt), (a.bs,))].to(dev) + opt.zero_grad(set_to_none=True) + with torch.autocast("cuda", dtype=torch.bfloat16, enabled=(dev == "cuda")): + if a.mode in ("anybudget", "anybudget_nc"): + loss = dfn.uncond_any_budget_loss(model, xb, tok, choices=BUDGETS, + condition=COND) + elif a.mode == "matched": + loss = dfn.uncond_mdlm_loss(model, xb, tok, "matched", a.K) + else: + loss = dfn.uncond_mdlm_loss(model, xb, tok, "uniform") + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) + opt.step(); sched.step() + if step % 5000 == 0: + print(f"step {step:6d} loss {loss.item():.4f} ({time.time()-t0:.0f}s)", flush=True) + +model.eval() +res = {"args": vars(a), "decode": {}} + + +@torch.no_grad() +def gen(fn): + outs, nfes = [], [] + for _ in range(max(1, a.n_gen // a.gen_bs)): + blank = torch.full((a.gen_bs, SEQ_LEN), tok.mask, dtype=torch.long) + x, nfe = fn(blank) + outs.append(x.cpu()); nfes.append(nfe) + x = torch.cat(outs).numpy() + v, smis = validity(x, tok) + uniq = len(set(smis)) / max(1, len(smis)) + novel = sum(s not in train_smiles for s in set(smis)) / max(1, len(set(smis))) + return {"validity": v, "nfe": float(np.mean(nfes)), "unique": uniq, + "novel": novel} + + +def bud(K): + return torch.full((a.gen_bs,), float(K), device=dev) if COND else None + + +print("\n=== validity / uniqueness / novelty against denoising budget ===", flush=True) +print(f"{'rule':<24}{'NFE':>7}{'valid':>9}{'unique':>9}{'novel':>9}", flush=True) + + +def report(tag, key, r): + res["decode"][key] = r + print(f"{tag:<24}{r['nfe']:>7.1f}{r['validity']*100:>8.2f}%" + f"{r['unique']*100:>8.1f}%{r['novel']*100:>8.1f}%", flush=True) + + +for K in (1, 2, 4, 8, 16, 32, SEQ_LEN): + report(f"fixed-K K={K}", f"fixedK_{K}", + gen(lambda b, K=K: dfn.fixed_k_decode(model, b, tok, dev, K, + budget_cond=bud(K), allowed=ALLOWED))) + +for B in (0.05, 0.25, 1.0, 4.0): + report(f"entropy-budget B={B}", f"entbudget_{B}", + gen(lambda b, B=B: dfn.entropy_budget_decode( + model, b, tok, dev, B, max_steps=SEQ_LEN + 2, + budget_cond=bud(8), allowed=ALLOWED))) + +for tau in (0.7, 0.9, 0.99): + report(f"confidence tau={tau}", f"conf_{tau}", + gen(lambda b, tau=tau: dfn.confidence_threshold_decode( + model, b, tok, dev, tau, max_steps=SEQ_LEN + 2, + budget_cond=bud(8), allowed=ALLOWED))) + +torch.save(model.state_dict(), os.path.join(a.out, "model.pt")) +json.dump(res, open(os.path.join(a.out, "result.json"), "w"), indent=2) +print(f"\nsaved to {a.out}", flush=True) diff --git a/src/train_sudoku.py b/src/train_sudoku.py new file mode 100644 index 0000000000000000000000000000000000000000..041a910e55696bfbc510b4bcac009c108c2363c1 --- /dev/null +++ b/src/train_sudoku.py @@ -0,0 +1,157 @@ +"""Masked diffusion on Sudoku: does committing by certainty solve constraints? + +The benchmark is SATNet's 9x9 Sudoku (Wang et al., ICML 2019) with its +conventional 9,000 / 1,000 split, scored by BOARD-level accuracy - all 81 cells +correct - which is the metric the published numbers use. + +The claim under test is specific. Sudoku has a unique answer, so there is no +information-theoretic obstacle whatsoever: every cell's true conditional is a +point mass and TC is zero. The entire difficulty is therefore mode (i), plus the +ORDER in which cells are committed. A decoder that commits the lowest-entropy +cells first, recomputes, and repeats is doing classical constraint propagation - +fill the forced cells, propagate, repeat - without being told to. A fixed-K +decoder that commits a fixed fraction per pass regardless of certainty is not. + +Prediction, registered before the runs: entropy-budgeted decoding should beat +fixed-K decoding by a very large margin here, larger than anywhere else in this +project, because Sudoku is the case where commit ORDER carries almost all of the +information. Confidence-thresholding is included as the published competitor. +""" +import argparse, json, math, os, time +import numpy as np +import torch + +from model import Transformer +import diffusion as dfn +from sudoku import (CELLS, SudokuTokenizer, board_accuracy, cell_accuracy, + consistent_with_clues, is_valid_solution, load_satnet, split) + + +def get_args(): + p = argparse.ArgumentParser() + p.add_argument("--mode", choices=["mdlm", "matched", "full"], default="mdlm") + p.add_argument("--K", type=int, default=8) + p.add_argument("--lam", type=float, default=1.0) + p.add_argument("--pe", default="ape", choices=["ape", "rope", "sin"]) + p.add_argument("--root", default="data/sudoku") + p.add_argument("--augment", action="store_true", + help="relabel digits by a random permutation (a Sudoku symmetry)") + p.add_argument("--d", type=int, default=512) + p.add_argument("--layers", type=int, default=10) + p.add_argument("--heads", type=int, default=8) + p.add_argument("--bs", type=int, default=128) + p.add_argument("--steps", type=int, default=80000) + p.add_argument("--lr", type=float, default=1e-4) + p.add_argument("--warmup", type=int, default=2000) + p.add_argument("--seed", type=int, default=0) + p.add_argument("--eval_bs", type=int, default=200) + p.add_argument("--out", default="runs/sud") + return p.parse_args() + + +a = get_args() +torch.manual_seed(a.seed); np.random.seed(a.seed) +dev = "cuda" if torch.cuda.is_available() else "cpu" +os.makedirs(a.out, exist_ok=True) + +X, Y, tok = load_satnet(a.root) +(Xtr, Ytr), (Xte, Yte) = split(X, Y) +print(f"sudoku: train {Xtr.shape} test {Xte.shape} vocab {len(tok)} " + f"clues/puzzle mean {(X>0).sum(1).mean():.1f}", flush=True) + +# tokens are the SOLUTION digits shifted to 0-8; blanks are what gets masked +sol_tr = torch.from_numpy(Ytr - 1) +blank_tr = torch.from_numpy(Xtr == 0) +sol_te = torch.from_numpy(Yte - 1) +blank_te = torch.from_numpy(Xte == 0) + +model = Transformer(len(tok), a.d, a.layers, a.heads, a.pe, causal=False, + max_len=CELLS + 8).to(dev) +print(f"params {model.n_params()/1e6:.2f}M mode={a.mode} pe={a.pe}", flush=True) + +opt = torch.optim.AdamW(model.parameters(), lr=a.lr, weight_decay=0.01) +sched = torch.optim.lr_scheduler.LambdaLR( + opt, lambda s: min(1.0, (s + 1) / a.warmup) * + 0.5 * (1 + math.cos(math.pi * min(1.0, s / a.steps)))) + +rng = np.random.default_rng(a.seed) +t0 = time.time() +for step in range(a.steps): + idx = torch.randint(0, len(sol_tr), (a.bs,)) + xb, ab = sol_tr[idx].clone(), blank_tr[idx] + if a.augment: + # relabelling the nine digits maps a valid grid to a valid grid, and + # leaves the constraint structure - the thing being learned - untouched + perm = torch.from_numpy(rng.permutation(9)) + xb = perm[xb] + xb, ab = xb.to(dev), ab.to(dev) + opt.zero_grad(set_to_none=True) + with torch.autocast("cuda", dtype=torch.bfloat16, enabled=(dev == "cuda")): + loss = dfn.pmd_loss(model, xb, ab, tok, a.mode, a.K, a.lam) + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) + opt.step(); sched.step() + if step % 5000 == 0: + print(f"step {step:6d} loss {loss.item():.4f} ({time.time()-t0:.0f}s)", flush=True) + +model.eval() +res = {"args": vars(a), "decode": {}} + + +@torch.no_grad() +def evaluate(fn): + """Board accuracy over the full test split, plus the diagnostics.""" + preds, nfes = [], [] + for i in range(0, len(sol_te), a.eval_bs): + gold = sol_te[i:i + a.eval_bs].to(dev) + bl = blank_te[i:i + a.eval_bs].to(dev) + x = torch.where(bl, torch.full_like(gold, tok.mask), gold) + y, nfe = fn(x, bl) + preds.append(y.cpu()); nfes.append(nfe) + P = torch.cat(preds).numpy() + 1 + G = Yte + board = board_accuracy(P, G) + cell = cell_accuracy(P, G, Xte == 0) + valid = float(np.mean([is_valid_solution(P[i]) for i in range(len(P))])) + kept = float(np.mean([consistent_with_clues(P[i], Xte[i]) for i in range(len(P))])) + return {"board": board, "cell": cell, "valid_sudoku": valid, + "clues_kept": kept, "nfe": float(np.mean(nfes))} + + +def report(tag, key, r): + res["decode"][key] = r + print(f" {tag:<26}{r['nfe']:>7.1f}{r['board']*100:>10.2f}%{r['cell']*100:>9.2f}%" + f"{r['valid_sudoku']*100:>9.2f}%", flush=True) + + +print("\n=== board accuracy against decoding budget ===", flush=True) +print(f" {'rule':<26}{'NFE':>7}{'BOARD':>10}{'cell':>9}{'valid':>9}", flush=True) + +print("fixed-K (commits a fixed share per pass, ignoring certainty):", flush=True) +for K in (1, 2, 4, 8, 16, 32, 45): + report(f"fixed-K K={K}", f"fixedK_{K}", + evaluate(lambda x, bl, K=K: dfn.fixed_k_decode( + model, x, tok, dev, K, fillable=bl))) + +print("entropy-budget (= constraint propagation):", flush=True) +for B in (0.01, 0.05, 0.2, 0.5, 1.0, 2.0): + report(f"entropy-budget B={B}", f"entbudget_{B}", + evaluate(lambda x, bl, B=B: dfn.entropy_budget_decode( + model, x, tok, dev, B, max_steps=CELLS + 2, fillable=bl))) + +print("confidence-threshold (published inference-time method):", flush=True) +for tau in (0.9, 0.99, 0.999): + report(f"confidence tau={tau}", f"conf_{tau}", + evaluate(lambda x, bl, tau=tau: dfn.confidence_threshold_decode( + model, x, tok, dev, tau, max_steps=CELLS + 2, fillable=bl))) + +best = max(res["decode"].values(), key=lambda r: r["board"]) +res["best_board"] = best["board"] +print(f"\nbest board accuracy: {best['board']*100:.2f}% at {best['nfe']:.1f} passes", + flush=True) +print(f"SATNet (Wang et al. ICML 2019) reports 98.3% board accuracy on this split.", + flush=True) + +torch.save(model.state_dict(), os.path.join(a.out, "model.pt")) +json.dump(res, open(os.path.join(a.out, "result.json"), "w"), indent=2) +print(f"saved to {a.out}", flush=True)