Automated push from Zaratan

README.md

@@ -40,19 +40,43 @@ Answers are unique but only ~10 characters with shallow, local dependencies. One
 | english | 95.3 | 95.2 | -0.1 |
 | finnish | 50.9 | 55.8 | +4.9 |
 | finnish | 48.6 | 56.6 | +8.0 |
+| finnish | 50.9 | 55.8 | +4.9 |
+| finnish | 52.9 | 57.9 | +4.9 |
+| finnish | 48.6 | 56.6 | +8.0 |
+| finnish | 54.3 | 61.7 | +7.4 |
 | georgian | 94.3 | 94.1 | -0.2 |
 | georgian | 95.4 | 95.3 | -0.1 |
+| georgian | 94.3 | 94.1 | -0.2 |
+| georgian | 93.8 | 94.0 | +0.1 |
+| georgian | 95.4 | 95.3 | -0.1 |
+| georgian | 93.9 | 93.9 | +0.0 |
 | german | 78.9 | 79.3 | +0.4 |
 | german | 75.6 | 79.5 | +3.9 |
+| german | 78.9 | 79.3 | +0.4 |
+| german | 77.9 | 78.0 | +0.1 |
+| german | 75.6 | 79.5 | +3.9 |
+| german | 76.4 | 78.9 | +2.5 |
 | german | 77.2 | 78.3 | +1.0 |
 | navajo | 29.5 | 33.8 | +4.3 |
 | navajo | 31.4 | 40.2 | +8.9 |
+| navajo | 29.5 | 33.8 | +4.3 |
+| navajo | 24.3 | 29.0 | +4.8 |
+| navajo | 31.4 | 40.2 | +8.9 |
+| navajo | 24.6 | 31.7 | +7.1 |
 | russian | 72.7 | 72.0 | -0.7 |
 | russian | 69.1 | 71.7 | +2.6 |
+| russian | 72.7 | 72.0 | -0.7 |
+| russian | 71.9 | 72.0 | +0.1 |
+| russian | 69.1 | 71.7 | +2.6 |
+| russian | 74.1 | 75.9 | +1.8 |
 | spanish | 84.9 | 84.7 | -0.2 |
 | spanish | 83.9 | 84.1 | +0.1 |
 | turkish | 80.2 | 81.2 | +1.0 |
 | turkish | 77.5 | 82.2 | +4.7 |
+| turkish | 80.2 | 81.2 | +1.0 |
+| turkish | 84.5 | 85.1 | +0.7 |
+| turkish | 77.5 | 82.2 | +4.7 |
+| turkish | 79.9 | 82.2 | +2.3 |
 | turkish | 76.4 | 81.8 | +5.4 |
 
 ## Table 3 — Non-unique answers: the limit no training removes
@@ -78,4 +102,24 @@ Every apparent failure at 10+ digits was a learning-rate artifact, not a limit o
 | 10 | 0.0003 | 500 | 47.3 | 1/3 |
 | 10 | 0.0003 | 4000 | 100.0 | 3/3 |
 
+## Table 6 — Minimum depth for one-pass addition
+
+The account in THEORY.md predicts minimum depth growing as log2(n), since the carry is a prefix scan. A ripple-carry account predicts linear growth; an impossibility account predicts no depth suffices.
+
+| digits | 2L | 3L | 4L | 6L | 8L | log2(n) | min depth >90% |
+|---|---|---|---|---|---|---|---|
+| **4** | 100 | 100 | 100 | 100 | 100 | 2.0 | 2 |
+| **8** | 99 | 100 | – | – | – | 3.0 | 2 |
+
+## Table 7 — Which component carries the gain (inflection)
+
+| language | mdlm | matched | distill | full | (gap 1→32) |
+|---|---|---|---|---|---|
+| finnish | 51.4 | – | – | 51.9 | +7.7 / – / – / +4.9 |
+| georgian | 94.6 | – | – | 94.1 | -0.0 / – / – / -0.0 |
+| german | 76.0 | – | – | 78.4 | +3.2 / – / – / +0.2 |
+| navajo | 28.0 | – | – | 26.9 | +8.0 / – / – / +4.5 |
+| russian | 71.6 | – | – | 72.3 | +2.2 / – / – / -0.3 |
+| turkish | 78.7 | – | – | 82.3 | +3.5 / – / – / +0.8 |
+
 

figures/summary.md

@@ -23,19 +23,43 @@ Answers are unique but only ~10 characters with shallow, local dependencies. One
 | english | 95.3 | 95.2 | -0.1 |
 | finnish | 50.9 | 55.8 | +4.9 |
 | finnish | 48.6 | 56.6 | +8.0 |
+| finnish | 50.9 | 55.8 | +4.9 |
+| finnish | 52.9 | 57.9 | +4.9 |
+| finnish | 48.6 | 56.6 | +8.0 |
+| finnish | 54.3 | 61.7 | +7.4 |
 | georgian | 94.3 | 94.1 | -0.2 |
 | georgian | 95.4 | 95.3 | -0.1 |
+| georgian | 94.3 | 94.1 | -0.2 |
+| georgian | 93.8 | 94.0 | +0.1 |
+| georgian | 95.4 | 95.3 | -0.1 |
+| georgian | 93.9 | 93.9 | +0.0 |
 | german | 78.9 | 79.3 | +0.4 |
 | german | 75.6 | 79.5 | +3.9 |
+| german | 78.9 | 79.3 | +0.4 |
+| german | 77.9 | 78.0 | +0.1 |
+| german | 75.6 | 79.5 | +3.9 |
+| german | 76.4 | 78.9 | +2.5 |
 | german | 77.2 | 78.3 | +1.0 |
 | navajo | 29.5 | 33.8 | +4.3 |
 | navajo | 31.4 | 40.2 | +8.9 |
+| navajo | 29.5 | 33.8 | +4.3 |
+| navajo | 24.3 | 29.0 | +4.8 |
+| navajo | 31.4 | 40.2 | +8.9 |
+| navajo | 24.6 | 31.7 | +7.1 |
 | russian | 72.7 | 72.0 | -0.7 |
 | russian | 69.1 | 71.7 | +2.6 |
+| russian | 72.7 | 72.0 | -0.7 |
+| russian | 71.9 | 72.0 | +0.1 |
+| russian | 69.1 | 71.7 | +2.6 |
+| russian | 74.1 | 75.9 | +1.8 |
 | spanish | 84.9 | 84.7 | -0.2 |
 | spanish | 83.9 | 84.1 | +0.1 |
 | turkish | 80.2 | 81.2 | +1.0 |
 | turkish | 77.5 | 82.2 | +4.7 |
+| turkish | 80.2 | 81.2 | +1.0 |
+| turkish | 84.5 | 85.1 | +0.7 |
+| turkish | 77.5 | 82.2 | +4.7 |
+| turkish | 79.9 | 82.2 | +2.3 |
 | turkish | 76.4 | 81.8 | +5.4 |
 
 ## Table 3 — Non-unique answers: the limit no training removes
@@ -61,3 +85,23 @@ Every apparent failure at 10+ digits was a learning-rate artifact, not a limit o
 | 10 | 0.0003 | 500 | 47.3 | 1/3 |
 | 10 | 0.0003 | 4000 | 100.0 | 3/3 |
 
+## Table 6 — Minimum depth for one-pass addition
+
+The account in THEORY.md predicts minimum depth growing as log2(n), since the carry is a prefix scan. A ripple-carry account predicts linear growth; an impossibility account predicts no depth suffices.
+
+| digits | 2L | 3L | 4L | 6L | 8L | log2(n) | min depth >90% |
+|---|---|---|---|---|---|---|---|
+| **4** | 100 | 100 | 100 | 100 | 100 | 2.0 | 2 |
+| **8** | 99 | 100 | – | – | – | 3.0 | 2 |
+
+## Table 7 — Which component carries the gain (inflection)
+
+| language | mdlm | matched | distill | full | (gap 1→32) |
+|---|---|---|---|---|---|
+| finnish | 51.4 | – | – | 51.9 | +7.7 / – / – / +4.9 |
+| georgian | 94.6 | – | – | 94.1 | -0.0 / – / – / -0.0 |
+| german | 76.0 | – | – | 78.4 | +3.2 / – / – / +0.2 |
+| navajo | 28.0 | – | – | 26.9 | +8.0 / – / – / +4.5 |
+| russian | 71.6 | – | – | 72.3 | +2.2 / – / – / -0.3 |
+| turkish | 78.7 | – | – | 82.3 | +3.5 / – / – / +0.8 |
+

results/g14-d4-L2-s0.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 4,
+    "seq_len": 16,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 2,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 0,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d4-L2-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 6.882920638720194,
+  "compute": {
+    "params": 3557376,
+    "train_tokens": 122880000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d4-L2-s1.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 4,
+    "seq_len": 16,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 2,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 1,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d4-L2-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 6.816111731529236,
+  "compute": {
+    "params": 3557376,
+    "train_tokens": 122880000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d4-L3-s0.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 4,
+    "seq_len": 16,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 3,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 0,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d4-L3-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 9.519309445222218,
+  "compute": {
+    "params": 5330304,
+    "train_tokens": 122880000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d4-L3-s1.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 4,
+    "seq_len": 16,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 3,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 1,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d4-L3-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 9.63311020930608,
+  "compute": {
+    "params": 5330304,
+    "train_tokens": 122880000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d4-L4-s0.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 4,
+    "seq_len": 16,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 4,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 0,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d4-L4-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 12.398685284455617,
+  "compute": {
+    "params": 7103232,
+    "train_tokens": 122880000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d4-L4-s1.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 4,
+    "seq_len": 16,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 4,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 1,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d4-L4-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 12.252778033415476,
+  "compute": {
+    "params": 7103232,
+    "train_tokens": 122880000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d4-L6-s0.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 4,
+    "seq_len": 16,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 6,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 0,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d4-L6-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 16.965274214744568,
+  "compute": {
+    "params": 10649088,
+    "train_tokens": 122880000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d4-L6-s1.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 4,
+    "seq_len": 16,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 6,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 1,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d4-L6-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 17.125181821982064,
+  "compute": {
+    "params": 10649088,
+    "train_tokens": 122880000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d4-L8-s0.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 4,
+    "seq_len": 16,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 8,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 0,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d4-L8-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 21.700168172518413,
+  "compute": {
+    "params": 14194944,
+    "train_tokens": 122880000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d4-L8-s1.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 4,
+    "seq_len": 16,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 8,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 1,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d4-L8-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 21.729793294270834,
+  "compute": {
+    "params": 14194944,
+    "train_tokens": 122880000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d8-L2-s0.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 8,
+    "seq_len": 28,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 2,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 0,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d8-L2-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.99,
+    "2": 0.9975,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 7.338654283682505,
+  "compute": {
+    "params": 3557376,
+    "train_tokens": 215040000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d8-L2-s1.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 8,
+    "seq_len": 28,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 2,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 1,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d8-L2-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.9975,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 7.30421595176061,
+  "compute": {
+    "params": 3557376,
+    "train_tokens": 215040000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d8-L3-s0.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 8,
+    "seq_len": 28,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 3,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 0,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d8-L3-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 9.942399954795837,
+  "compute": {
+    "params": 5330304,
+    "train_tokens": 215040000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d8-L3-s1.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 8,
+    "seq_len": 28,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 3,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 1,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d8-L3-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.9975,
+    "2": 1.0,
+    "3": 0.9975,
+    "4": 0.9975,
+    "6": 0.9975,
+    "8": 0.9975,
+    "12": 0.9975,
+    "16": 0.9975
+  },
+  "minutes": 9.788207534948985,
+  "compute": {
+    "params": 5330304,
+    "train_tokens": 215040000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d8-L4-s0.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 8,
+    "seq_len": 28,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 4,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 0,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d8-L4-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 12.345258478323618,
+  "compute": {
+    "params": 7103232,
+    "train_tokens": 215040000,
+    "fwd_per_step": 2
+  }
+}

results/g14-d8-L4-s1.json

@@ -0,0 +1,39 @@
+{
+  "args": {
+    "mode": "full",
+    "K_max": 16,
+    "K": 4,
+    "lam": 1.0,
+    "digits": 8,
+    "seq_len": 28,
+    "n_train": 800000,
+    "n_eval": 400,
+    "eval_bs": 100,
+    "d": 384,
+    "layers": 4,
+    "heads": 6,
+    "pe": "rope",
+    "bs": 128,
+    "steps": 60000,
+    "lr": 0.0001,
+    "warmup": 4000,
+    "seed": 1,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g14-d8-L4-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 1.0,
+    "2": 1.0,
+    "3": 1.0,
+    "4": 1.0,
+    "6": 1.0,
+    "8": 1.0,
+    "12": 1.0,
+    "16": 1.0
+  },
+  "minutes": 12.423506192366283,
+  "compute": {
+    "params": 7103232,
+    "train_tokens": 215040000,
+    "fwd_per_step": 2
+  }
+}

results/g15-finnish-full-s0.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "finnish",
+    "size": "medium",
+    "mode": "full",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 0,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-finnish-full-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.5093256353021978,
+    "2": 0.511461195054945,
+    "4": 0.5276978880494505,
+    "8": 0.5495793269230769,
+    "16": 0.5583683894230769,
+    "32": 0.5583683894230769,
+    "ar": 0.05099587912087912
+  },
+  "minutes": 4.275957576433817,
+  "compute": {
+    "params": 4769792,
+    "answer_len_mean": 15.12968635559082
+  }
+}

results/g15-finnish-full-s1.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "finnish",
+    "size": "medium",
+    "mode": "full",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 1,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-finnish-full-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.5292217548076923,
+    "2": 0.5335250686813187,
+    "4": 0.549547132554945,
+    "8": 0.5708490728021978,
+    "16": 0.5776850103021978,
+    "32": 0.5786615728021978,
+    "ar": 0.05548162774725275
+  },
+  "minutes": 4.345337744553884,
+  "compute": {
+    "params": 4769792,
+    "answer_len_mean": 15.12968635559082
+  }
+}

results/g15-finnish-mdlm-s0.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "finnish",
+    "size": "medium",
+    "mode": "mdlm",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 0,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-finnish-mdlm-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.4858237465659341,
+    "2": 0.5041959993131868,
+    "4": 0.5329133756868132,
+    "8": 0.545640882554945,
+    "16": 0.5636160714285714,
+    "32": 0.5655691964285714,
+    "ar": 0.05392556662087912
+  },
+  "minutes": 2.4385203957557677,
+  "compute": {
+    "params": 4769792,
+    "answer_len_mean": 15.12968635559082
+  }
+}

results/g15-finnish-mdlm-s1.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "finnish",
+    "size": "medium",
+    "mode": "mdlm",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 1,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-finnish-mdlm-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.5428936298076923,
+    "2": 0.5407902644230769,
+    "4": 0.5704842032967032,
+    "8": 0.5990191449175823,
+    "16": 0.6165972699175823,
+    "32": 0.6165972699175823,
+    "ar": 0.06173806662087912
+  },
+  "minutes": 2.4639089981714886,
+  "compute": {
+    "params": 4769792,
+    "answer_len_mean": 15.12968635559082
+  }
+}

results/g15-georgian-full-s0.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "georgian",
+    "size": "medium",
+    "mode": "full",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 0,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-georgian-full-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.9432091346153846,
+    "2": 0.9434344951923077,
+    "4": 0.9402794471153846,
+    "8": 0.9424579326923077,
+    "16": 0.9412560096153846,
+    "32": 0.9412560096153846,
+    "ar": 0.1258263221153846
+  },
+  "minutes": 4.286119385560354,
+  "compute": {
+    "params": 4766208,
+    "answer_len_mean": 9.932000160217285
+  }
+}

results/g15-georgian-full-s1.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "georgian",
+    "size": "medium",
+    "mode": "full",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 1,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-georgian-full-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.9383263221153846,
+    "2": 0.9385516826923077,
+    "4": 0.9385516826923077,
+    "8": 0.9395282451923077,
+    "16": 0.9395282451923077,
+    "32": 0.9395282451923077,
+    "ar": 0.12387319711538461
+  },
+  "minutes": 4.304093313217163,
+  "compute": {
+    "params": 4766208,
+    "answer_len_mean": 9.932000160217285
+  }
+}

results/g15-georgian-mdlm-s0.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "georgian",
+    "size": "medium",
+    "mode": "mdlm",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 0,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-georgian-mdlm-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.9539513221153846,
+    "2": 0.9522235576923077,
+    "4": 0.9502704326923077,
+    "8": 0.9532001201923077,
+    "16": 0.9532001201923077,
+    "32": 0.9532001201923077,
+    "ar": 0.1258263221153846
+  },
+  "minutes": 2.4665067235628766,
+  "compute": {
+    "params": 4766208,
+    "answer_len_mean": 9.932000160217285
+  }
+}

results/g15-georgian-mdlm-s1.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "georgian",
+    "size": "medium",
+    "mode": "mdlm",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 1,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-georgian-mdlm-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.9385516826923077,
+    "2": 0.9375751201923077,
+    "4": 0.9375751201923077,
+    "8": 0.9395282451923077,
+    "16": 0.9385516826923077,
+    "32": 0.9385516826923077,
+    "ar": 0.12387319711538461
+  },
+  "minutes": 2.4444581945737203,
+  "compute": {
+    "params": 4766208,
+    "answer_len_mean": 9.932000160217285
+  }
+}

results/g15-german-full-s0.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "german",
+    "size": "medium",
+    "mode": "full",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 0,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-german-full-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.7893359375,
+    "2": 0.792265625,
+    "4": 0.7903125,
+    "8": 0.79296875,
+    "16": 0.79296875,
+    "32": 0.79296875,
+    "ar": 0.0776171875
+  },
+  "minutes": 4.36189177831014,
+  "compute": {
+    "params": 4776448,
+    "answer_len_mean": 10.743975639343262
+  }
+}

results/g15-german-full-s1.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "german",
+    "size": "medium",
+    "mode": "full",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 1,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-german-full-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.7788671875,
+    "2": 0.7705078125,
+    "4": 0.7734375,
+    "8": 0.7795703125,
+    "16": 0.7795703125,
+    "32": 0.7795703125,
+    "ar": 0.0724609375
+  },
+  "minutes": 4.429418742656708,
+  "compute": {
+    "params": 4776448,
+    "answer_len_mean": 10.743975639343262
+  }
+}

results/g15-german-mdlm-s0.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "german",
+    "size": "medium",
+    "mode": "mdlm",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 0,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-german-mdlm-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.7555859375,
+    "2": 0.7710546875,
+    "4": 0.7808203125,
+    "8": 0.7911328125,
+    "16": 0.7950390625,
+    "32": 0.7950390625,
+    "ar": 0.076640625
+  },
+  "minutes": 2.526710093021393,
+  "compute": {
+    "params": 4776448,
+    "answer_len_mean": 10.743975639343262
+  }
+}

results/g15-german-mdlm-s1.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "german",
+    "size": "medium",
+    "mode": "mdlm",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 1,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-german-mdlm-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.7641015625,
+    "2": 0.76421875,
+    "4": 0.7815234375,
+    "8": 0.786953125,
+    "16": 0.7886328125,
+    "32": 0.7886328125,
+    "ar": 0.0769140625
+  },
+  "minutes": 2.5262390971183777,
+  "compute": {
+    "params": 4776448,
+    "answer_len_mean": 10.743975639343262
+  }
+}

results/g15-navajo-full-s0.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "navajo",
+    "size": "medium",
+    "mode": "full",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 0,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-navajo-full-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.29529747596153844,
+    "2": 0.30814302884615385,
+    "4": 0.3108473557692308,
+    "8": 0.33743990384615385,
+    "16": 0.33841646634615385,
+    "32": 0.33841646634615385,
+    "ar": 0.06332632211538461
+  },
+  "minutes": 4.317290306091309,
+  "compute": {
+    "params": 4765696,
+    "answer_len_mean": 10.005999565124512
+  }
+}

results/g15-navajo-full-s1.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "navajo",
+    "size": "medium",
+    "mode": "full",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 1,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-navajo-full-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.24263822115384615,
+    "2": 0.25045072115384615,
+    "4": 0.265625,
+    "8": 0.2872596153846154,
+    "16": 0.2901893028846154,
+    "32": 0.2901893028846154,
+    "ar": 0.0625751201923077
+  },
+  "minutes": 4.282497262954712,
+  "compute": {
+    "params": 4765696,
+    "answer_len_mean": 10.005999565124512
+  }
+}

results/g15-navajo-mdlm-s0.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "navajo",
+    "size": "medium",
+    "mode": "mdlm",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 0,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-navajo-mdlm-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.3137019230769231,
+    "2": 0.3273737980769231,
+    "4": 0.37169471153846156,
+    "8": 0.4004657451923077,
+    "16": 0.4024188701923077,
+    "32": 0.4024188701923077,
+    "ar": 0.08473557692307693
+  },
+  "minutes": 2.4744708220163982,
+  "compute": {
+    "params": 4765696,
+    "answer_len_mean": 10.005999565124512
+  }
+}

results/g15-navajo-mdlm-s1.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "navajo",
+    "size": "medium",
+    "mode": "mdlm",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 1,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-navajo-mdlm-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.24579326923076922,
+    "2": 0.2675030048076923,
+    "4": 0.29725060096153844,
+    "8": 0.3133263221153846,
+    "16": 0.3172325721153846,
+    "32": 0.3172325721153846,
+    "ar": 0.06475360576923077
+  },
+  "minutes": 2.43903698126475,
+  "compute": {
+    "params": 4765696,
+    "answer_len_mean": 10.005999565124512
+  }
+}

results/g15-russian-full-s0.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "russian",
+    "size": "medium",
+    "mode": "full",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 0,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-russian-full-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.7271740845959596,
+    "2": 0.7232678345959596,
+    "4": 0.7213147095959596,
+    "8": 0.7218868371212122,
+    "16": 0.7199337121212122,
+    "32": 0.7199337121212122,
+    "ar": 0.03911182133838384
+  },
+  "minutes": 4.331714260578155,
+  "compute": {
+    "params": 4770304,
+    "answer_len_mean": 11.276381492614746
+  }
+}

results/g15-russian-full-s1.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "russian",
+    "size": "medium",
+    "mode": "full",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 1,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-russian-full-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.7187894570707071,
+    "2": 0.7197660195707071,
+    "4": 0.7158597695707071,
+    "8": 0.7190755208333333,
+    "16": 0.7200520833333333,
+    "32": 0.7200520833333333,
+    "ar": 0.03813525883838384
+  },
+  "minutes": 4.2800087610880535,
+  "compute": {
+    "params": 4770304,
+    "answer_len_mean": 11.276381492614746
+  }
+}

results/g15-russian-mdlm-s0.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "russian",
+    "size": "medium",
+    "mode": "mdlm",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 0,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-russian-mdlm-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.6910412720959596,
+    "2": 0.7068339646464646,
+    "4": 0.7132161458333333,
+    "8": 0.7106908933080809,
+    "16": 0.7165502683080809,
+    "32": 0.7165502683080809,
+    "ar": 0.035896070075757576
+  },
+  "minutes": 2.446616017818451,
+  "compute": {
+    "params": 4770304,
+    "answer_len_mean": 11.276381492614746
+  }
+}

results/g15-russian-mdlm-s1.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "russian",
+    "size": "medium",
+    "mode": "mdlm",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 1,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-russian-mdlm-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.7412997159090909,
+    "2": 0.7453243371212122,
+    "4": 0.7542810921717171,
+    "8": 0.7633562184343434,
+    "16": 0.7591639046717171,
+    "32": 0.7591639046717171,
+    "ar": 0.03618213383838384
+  },
+  "minutes": 2.493031601111094,
+  "compute": {
+    "params": 4770304,
+    "answer_len_mean": 11.276381492614746
+  }
+}

results/g15-turkish-full-s0.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "turkish",
+    "size": "medium",
+    "mode": "full",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 0,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-turkish-full-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.8020938340053764,
+    "2": 0.7859753024193549,
+    "4": 0.7945228494623656,
+    "8": 0.8103998655913979,
+    "16": 0.8119854670698925,
+    "32": 0.8119854670698925,
+    "ar": 0.10095346102150538
+  },
+  "minutes": 4.366493237018585,
+  "compute": {
+    "params": 4772864,
+    "answer_len_mean": 13.684530258178711
+  }
+}

results/g15-turkish-full-s1.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "turkish",
+    "size": "medium",
+    "mode": "full",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 1,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-turkish-full-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.844705561155914,
+    "2": 0.8401902721774194,
+    "4": 0.841775873655914,
+    "8": 0.8499558971774194,
+    "16": 0.8512999831989247,
+    "32": 0.8512999831989247,
+    "ar": 0.1005859375
+  },
+  "minutes": 4.334676361083984,
+  "compute": {
+    "params": 4772864,
+    "answer_len_mean": 13.684530258178711
+  }
+}

results/g15-turkish-mdlm-s0.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "turkish",
+    "size": "medium",
+    "mode": "mdlm",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 0,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-turkish-mdlm-s0"
+  },
+  "accuracy_by_passes": {
+    "1": 0.7749915994623656,
+    "2": 0.7828040994623656,
+    "4": 0.8090557795698925,
+    "8": 0.8204070060483871,
+    "16": 0.8217510920698925,
+    "32": 0.8217510920698925,
+    "ar": 0.09607064852150538
+  },
+  "minutes": 2.545037277539571,
+  "compute": {
+    "params": 4772864,
+    "answer_len_mean": 13.684530258178711
+  }
+}

results/g15-turkish-mdlm-s1.json

@@ -0,0 +1,36 @@
+{
+  "args": {
+    "lang": "turkish",
+    "size": "medium",
+    "mode": "mdlm",
+    "seq_len": 48,
+    "d": 256,
+    "layers": 6,
+    "heads": 8,
+    "pe": "rope",
+    "bs": 64,
+    "steps": 15000,
+    "lr": 0.0001,
+    "warmup": 1000,
+    "K": 4,
+    "lam": 1.0,
+    "seed": 1,
+    "data_root": "/scratch/zt1/project/msml612/user/govind02/pcmd/data/sig",
+    "eval_bs": 128,
+    "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g15-turkish-mdlm-s1"
+  },
+  "accuracy_by_passes": {
+    "1": 0.7992901545698925,
+    "2": 0.8051495295698925,
+    "4": 0.8152826780913979,
+    "8": 0.8168682795698925,
+    "16": 0.8221186155913979,
+    "32": 0.8221186155913979,
+    "ar": 0.10193002352150538
+  },
+  "minutes": 2.530856684843699,
+  "compute": {
+    "params": 4772864,
+    "answer_len_mean": 13.684530258178711
+  }
+}

slurm/finalize.sh

@@ -72,16 +72,30 @@ table = summary.read_text() if summary.exists() else "_(pending)_"
 
 (stage / "README.md").write_text(f"""---
 license: apache-2.0
-tags: [masked-diffusion, discrete-diffusion, length-generalization, positional-encoding, from-scratch]
+tags: [masked-diffusion, discrete-diffusion, parallel-decoding, text8, from-scratch]
 ---
 
-# Length Generalization in Masked Diffusion Language Models
+# Two Failure Modes of Parallel Decoding in Masked Diffusion
 
-From-scratch masked diffusion language models (MDLM-style absorbing state) and
-matched autoregressive baselines, trained on arithmetic and algorithmic tasks and
-evaluated far beyond the lengths seen in training.
+Parallel decoding in masked diffusion loses accuracy for two different reasons
+that need opposite fixes:
 
-**No pretrained weights or tokenizers are used anywhere.**
+1. **The marginals are untrained at high mask ratio.** A K-pass decode only ever
+   evaluates the model at mask ratios {{1, (K-1)/K, ..., 1/K}}, while uniform-t
+   training spends most of its capacity on nearly-complete states that
+   high-parallelism decoding never visits. **Fixable only at training time.**
+2. **The committed positions carry real mutual information.** Parallel decoding
+   samples the product of marginals; the truth is the joint. The discarded
+   dependence is the total correlation of the committed set, upper-bounded by
+   its summed conditional entropy. **Fixable only at inference, by committing
+   less.**
+
+Prior work applies inference-time fixes to both, which is why it flatlines on
+arithmetic - where mode 2 is identically zero and all the loss is mode 1.
+
+Everything is trained from scratch: no pretrained weights, no pretrained
+tokenizer, and the autoregressive evaluator used to score generated text is
+itself trained from scratch on the same corpus.
 
 Code: https://github.com/GIND123/MSML612
 

slurm/g14.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Grid 14 - THE THEORY TEST, now at a learning rate that is not broken.
+#
+# THEORY.md predicts the minimum depth for one-pass decoding grows as log2(n),
+# because addition's carry is a prefix scan computable by carry lookahead in
+# O(log n) depth, and constant-depth transformers sit in TC0 which contains
+# addition.
+#
+#   log-depth (ours)   : min depth ~ 2, 3, 4, 4  for 4, 8, 12, 16 digits
+#   ripple-carry       : min depth ~ 4, 8, 12, 16
+#   impossibility      : no depth suffices
+#
+# The first attempt at this sweep was cancelled on discovering it would have run
+# at lr 3e-4 with 500 warmup, where deep models fail for optimisation reasons and
+# would have faked a clean "depth does not help" curve.
+#SBATCH -J g14
+#SBATCH -p gpu-h100
+#SBATCH --gpus=h100:1
+#SBATCH -c 8
+#SBATCH --mem=48g
+#SBATCH -t 08:00:00
+#SBATCH -o logs/g14-%A_%a.out
+#SBATCH --array=0-47%12
+source /etc/profile
+SCR=/scratch/zt1/project/msml612/user/$USER
+module load python/3.10.10
+export PYTHONPATH=
+cd $SCR/pcmd/src
+DIG=(4 8 12 16); LAYERS=(2 3 4 6 8 12); SEEDS=(0 1)
+I=$SLURM_ARRAY_TASK_ID
+D=${DIG[$((I / 12))]}; L=${LAYERS[$(((I % 12) / 2))]}; S=${SEEDS[$((I % 2))]}
+SL=$(( 3 * D + 4 ))
+NAME="g14-d$D-L$L-s$S"
+echo "=== $NAME ==="
+$SCR/hf_env/bin/python train_add.py --mode full --digits $D --seq_len $SL --seed $S \
+  --n_train 800000 --n_eval 400 --eval_bs 100 \
+  --d 384 --layers $L --heads 6 --bs 128 --steps 60000 --lr 1e-4 --warmup 4000 \
+  --K 4 --lam 1.0 --out $SCR/pcmd/runs/$NAME

slurm/g15.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Grid 15 - component ablation on inflection. Which part carries the gain, and
+# does the answer hold across typologically diverse languages rather than just
+# on addition?
+#SBATCH -J g15
+#SBATCH -p gpu-h100
+#SBATCH --gpus=h100:1
+#SBATCH -c 8
+#SBATCH --mem=32g
+#SBATCH -t 06:00:00
+#SBATCH -o logs/g15-%A_%a.out
+#SBATCH --array=0-47%12
+source /etc/profile
+SCR=/scratch/zt1/project/msml612/user/$USER
+module load python/3.10.10
+export PYTHONPATH=
+cd $SCR/pcmd/src
+LANGS=(german turkish finnish russian navajo georgian)
+MODES=(mdlm matched distill full); SEEDS=(0 1)
+I=$SLURM_ARRAY_TASK_ID
+L=${LANGS[$((I / 8))]}; M=${MODES[$(((I % 8) / 2))]}; S=${SEEDS[$((I % 2))]}
+NAME="g15-$L-$M-s$S"
+echo "=== $NAME ==="
+$SCR/hf_env/bin/python train_sig.py --lang $L --mode $M --size medium --seed $S \
+  --steps 15000 --d 256 --layers 6 --lr 1e-4 --warmup 1000 \
+  --data_root $SCR/pcmd/data/sig --out $SCR/pcmd/runs/$NAME

slurm/g16.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Grid 16 - THE DEFINITIVE FRONTIER, one consistent configuration throughout.
+#
+# Grid 11 (10 layers, 60k steps) understated the method: it gave 75% at sixteen
+# digits, while grid 13 (12 layers, 150k steps) reached 99.9% at TWENTY digits
+# with the baseline at exactly zero. Mixing configurations across digit counts
+# makes the curve meaningless, so this reruns every length at one setting.
+#SBATCH -J g16
+#SBATCH -p gpu-h100
+#SBATCH --gpus=h100:1
+#SBATCH -c 8
+#SBATCH --mem=64g
+#SBATCH -t 16:00:00
+#SBATCH -o logs/g16-%A_%a.out
+#SBATCH --array=0-29%12
+source /etc/profile
+SCR=/scratch/zt1/project/msml612/user/$USER
+module load python/3.10.10
+export PYTHONPATH=
+cd $SCR/pcmd/src
+DIG=(12 16 20 24 28); MODES=(mdlm full); SEEDS=(0 1 2)
+I=$SLURM_ARRAY_TASK_ID
+D=${DIG[$((I / 6))]}; M=${MODES[$(((I % 6) / 3))]}; S=${SEEDS[$((I % 3))]}
+SL=$(( 3 * D + 4 ))
+NAME="g16-d$D-$M-s$S"
+echo "=== $NAME ==="
+$SCR/hf_env/bin/python train_add.py --mode $M --digits $D --seq_len $SL --seed $S \
+  --n_train 1500000 --n_eval 400 --eval_bs 100 \
+  --d 512 --layers 12 --heads 8 --bs 128 --steps 150000 --lr 1e-4 --warmup 6000 \
+  --K 4 --lam 1.0 --out $SCR/pcmd/runs/$NAME

slurm/g16b.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Grid 16 - THE DEFINITIVE FRONTIER, one consistent configuration throughout.
+#
+# Grid 11 (10 layers, 60k steps) understated the method: it gave 75% at sixteen
+# digits, while grid 13 (12 layers, 150k steps) reached 99.9% at TWENTY digits
+# with the baseline at exactly zero. Mixing configurations across digit counts
+# makes the curve meaningless, so this reruns every length at one setting.
+#SBATCH -J g16
+#SBATCH -p gpu-h100
+#SBATCH --gpus=h100:1
+#SBATCH -c 8
+#SBATCH --mem=64g
+#SBATCH -t 16:00:00
+#SBATCH -o logs/g16-%A_%a.out
+#SBATCH --array=0-11%6
+source /etc/profile
+SCR=/scratch/zt1/project/msml612/user/$USER
+module load python/3.10.10
+export PYTHONPATH=
+cd $SCR/pcmd/src
+DIG=(12 16); MODES=(mdlm full); SEEDS=(0 1 2)
+I=$SLURM_ARRAY_TASK_ID
+D=${DIG[$((I / 6))]}; M=${MODES[$(((I % 6) / 3))]}; S=${SEEDS[$((I % 3))]}
+SL=$(( 3 * D + 4 ))
+NAME="g16-d$D-$M-s$S"
+echo "=== $NAME ==="
+$SCR/hf_env/bin/python train_add.py --mode $M --digits $D --seq_len $SL --seed $S \
+  --n_train 1500000 --n_eval 400 --eval_bs 100 \
+  --d 512 --layers 12 --heads 8 --bs 128 --steps 150000 --lr 1e-4 --warmup 6000 \
+  --K 4 --lam 1.0 --out $SCR/pcmd/runs/$NAME

slurm/g17.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Grid 17 - COMPUTE-MATCHED CONTROL.
+#
+# `full` evaluates the model twice per training step (matched-schedule term plus
+# high-ratio term), so at equal step counts it spends roughly twice the training
+# FLOPs of the baseline. Any reviewer will ask whether the gap is the method or
+# simply the extra compute. This gives the BASELINE the larger budget: 300k
+# steps of plain MDLM against 150k steps of the method, which is if anything
+# generous to the baseline since the method's second pass is on a shared batch.
+#SBATCH -J g17
+#SBATCH -p gpu-h100
+#SBATCH --gpus=h100:1
+#SBATCH -c 8
+#SBATCH --mem=64g
+#SBATCH -t 24:00:00
+#SBATCH -o logs/g17-%A_%a.out
+#SBATCH --array=0-8%6
+source /etc/profile
+SCR=/scratch/zt1/project/msml612/user/$USER
+module load python/3.10.10
+export PYTHONPATH=
+cd $SCR/pcmd/src
+DIG=(12 16 20); SEEDS=(0 1 2)
+I=$SLURM_ARRAY_TASK_ID
+D=${DIG[$((I / 3))]}; S=${SEEDS[$((I % 3))]}
+SL=$(( 3 * D + 4 ))
+NAME="g17-d$D-mdlm2x-s$S"
+echo "=== $NAME : baseline with DOUBLE the training steps ==="
+$SCR/hf_env/bin/python -u train_add.py --mode mdlm --digits $D --seq_len $SL --seed $S \
+  --n_train 1500000 --n_eval 400 --eval_bs 100 \
+  --d 512 --layers 12 --heads 8 --bs 128 --steps 300000 --lr 1e-4 --warmup 6000 \
+  --out $SCR/pcmd/runs/$NAME

slurm/g17b.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Grid 17 - COMPUTE-MATCHED CONTROL.
+#
+# `full` evaluates the model twice per training step (matched-schedule term plus
+# high-ratio term), so at equal step counts it spends roughly twice the training
+# FLOPs of the baseline. Any reviewer will ask whether the gap is the method or
+# simply the extra compute. This gives the BASELINE the larger budget: 300k
+# steps of plain MDLM against 150k steps of the method, which is if anything
+# generous to the baseline since the method's second pass is on a shared batch.
+#SBATCH -J g17
+#SBATCH -p gpu-h100
+#SBATCH --gpus=h100:1
+#SBATCH -c 8
+#SBATCH --mem=64g
+#SBATCH -t 24:00:00
+#SBATCH -o logs/g17-%A_%a.out
+#SBATCH --array=0-3%4
+source /etc/profile
+SCR=/scratch/zt1/project/msml612/user/$USER
+module load python/3.10.10
+export PYTHONPATH=
+cd $SCR/pcmd/src
+DIG=(16 20); SEEDS=(0 1)
+I=$SLURM_ARRAY_TASK_ID
+D=${DIG[$((I / 2))]}; S=${SEEDS[$((I % 2))]}
+SL=$(( 3 * D + 4 ))
+NAME="g17-d$D-mdlm2x-s$S"
+echo "=== $NAME : baseline with DOUBLE the training steps ==="
+$SCR/hf_env/bin/python -u train_add.py --mode mdlm --digits $D --seq_len $SL --seed $S \
+  --n_train 1500000 --n_eval 400 --eval_bs 100 \
+  --d 512 --layers 12 --heads 8 --bs 128 --steps 300000 --lr 1e-4 --warmup 6000 \
+  --out $SCR/pcmd/runs/$NAME