Hugging Face's logo

GOVINDFROM
/
parallel-consistent-masked-diffusion

masked-diffusion
discrete-diffusion
constraint-satisfaction
sudoku
recurrent-transformer
from-scratch
Model card Files
xet
 Community
GOVINDFROM commited on
Commit
38a786f
·
verified ·
1 Parent(s): 7fa1b76

Automated push from Zaratan

Browse files
Files changed (18) hide show
  1. results/g2-agree-cmd-s0.json +46 -0
  2. results/g2-agree-cmd-s1.json +46 -0
  3. results/g2-agree-coord-s0.json +46 -0
  4. results/g2-agree-coord-s1.json +46 -0
  5. results/g2-agree-selfcorrupt-s0.json +46 -0
  6. results/g2-agree-selfcorrupt-s1.json +46 -0
  7. results/g2-copy-cmd-s0.json +46 -0
  8. results/g2-copy-cmd-s1.json +46 -0
  9. results/g2-copy-selfcorrupt-s0.json +46 -0
  10. results/g2-copy-selfcorrupt-s1.json +46 -0
  11. results/g2-parity-cmd-s0.json +46 -0
  12. results/g2-parity-cmd-s1.json +46 -0
  13. results/g2-parity-coord-s0.json +46 -0
  14. results/g2-parity-coord-s1.json +46 -0
  15. results/g2-parity-mdlm-s0.json +46 -0
  16. results/g2-parity-mdlm-s1.json +46 -0
  17. results/g2-parity-selfcorrupt-s0.json +46 -0
  18. results/g2-parity-selfcorrupt-s1.json +46 -0
results/g2-agree-cmd-s0.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 0.0,
10
+ "128": 0.0
11
+ },
12
+ "args": {
13
+ "objective": "cmd",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "agree",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 0,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-agree-cmd-s0"
38
+ },
39
+ "minutes": 9.635117228825887,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 1
45
+ }
46
+ }
results/g2-agree-cmd-s1.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 0.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "cmd",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "agree",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 1,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-agree-cmd-s1"
38
+ },
39
+ "minutes": 9.589203838507334,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 1
45
+ }
46
+ }
results/g2-agree-coord-s0.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 0.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "coord",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "agree",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 0,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-agree-coord-s0"
38
+ },
39
+ "minutes": 5.399196537335714,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 1
45
+ }
46
+ }
results/g2-agree-coord-s1.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 1.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "coord",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "agree",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 1,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-agree-coord-s1"
38
+ },
39
+ "minutes": 5.436067247390747,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 1
45
+ }
46
+ }
results/g2-agree-selfcorrupt-s0.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 0.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "selfcorrupt",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "agree",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 0,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-agree-selfcorrupt-s0"
38
+ },
39
+ "minutes": 7.810237646102905,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 2
45
+ }
46
+ }
results/g2-agree-selfcorrupt-s1.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 0.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "selfcorrupt",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "agree",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 1,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-agree-selfcorrupt-s1"
38
+ },
39
+ "minutes": 7.882202510039011,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 2
45
+ }
46
+ }
results/g2-copy-cmd-s0.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 1.0,
4
+ "2": 1.0,
5
+ "4": 1.0,
6
+ "8": 1.0,
7
+ "16": 1.0,
8
+ "32": 1.0,
9
+ "64": 1.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "cmd",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "copy",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 0,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-copy-cmd-s0"
38
+ },
39
+ "minutes": 9.477424037456512,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 1
45
+ }
46
+ }
results/g2-copy-cmd-s1.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 1.0,
4
+ "2": 1.0,
5
+ "4": 1.0,
6
+ "8": 1.0,
7
+ "16": 1.0,
8
+ "32": 1.0,
9
+ "64": 1.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "cmd",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "copy",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 1,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-copy-cmd-s1"
38
+ },
39
+ "minutes": 9.575367267926534,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 1
45
+ }
46
+ }
results/g2-copy-selfcorrupt-s0.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 1.0,
4
+ "2": 1.0,
5
+ "4": 1.0,
6
+ "8": 1.0,
7
+ "16": 1.0,
8
+ "32": 1.0,
9
+ "64": 1.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "selfcorrupt",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "copy",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 0,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-copy-selfcorrupt-s0"
38
+ },
39
+ "minutes": 7.878946936130523,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 2
45
+ }
46
+ }
results/g2-copy-selfcorrupt-s1.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 1.0,
4
+ "2": 1.0,
5
+ "4": 1.0,
6
+ "8": 1.0,
7
+ "16": 1.0,
8
+ "32": 1.0,
9
+ "64": 1.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "selfcorrupt",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "copy",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 1,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-copy-selfcorrupt-s1"
38
+ },
39
+ "minutes": 7.865754318237305,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 2
45
+ }
46
+ }
results/g2-parity-cmd-s0.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 0.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "cmd",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "parity",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 0,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-parity-cmd-s0"
38
+ },
39
+ "minutes": 9.53271158138911,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 1
45
+ }
46
+ }
results/g2-parity-cmd-s1.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 1.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "cmd",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "parity",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 1,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-parity-cmd-s1"
38
+ },
39
+ "minutes": 9.619432377815247,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 1
45
+ }
46
+ }
results/g2-parity-coord-s0.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 0.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "coord",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "parity",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 0,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-parity-coord-s0"
38
+ },
39
+ "minutes": 5.548012403647105,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 1
45
+ }
46
+ }
results/g2-parity-coord-s1.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 0.0,
10
+ "128": 0.0
11
+ },
12
+ "args": {
13
+ "objective": "coord",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "parity",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 1,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-parity-coord-s1"
38
+ },
39
+ "minutes": 5.450428613026937,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 1
45
+ }
46
+ }
results/g2-parity-mdlm-s0.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 0.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "mdlm",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "parity",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 0,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-parity-mdlm-s0"
38
+ },
39
+ "minutes": 3.7591087341308596,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 1
45
+ }
46
+ }
results/g2-parity-mdlm-s1.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 0.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "mdlm",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "parity",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 1,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-parity-mdlm-s1"
38
+ },
39
+ "minutes": 3.6820839484532675,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 1
45
+ }
46
+ }
results/g2-parity-selfcorrupt-s0.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 1.0,
9
+ "64": 1.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "selfcorrupt",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "parity",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 0,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-parity-selfcorrupt-s0"
38
+ },
39
+ "minutes": 7.905708706378936,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 2
45
+ }
46
+ }
results/g2-parity-selfcorrupt-s1.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "accuracy_by_passes": {
3
+ "1": 0.0,
4
+ "2": 0.0,
5
+ "4": 0.0,
6
+ "8": 0.0,
7
+ "16": 0.0,
8
+ "32": 0.0,
9
+ "64": 0.0,
10
+ "128": 1.0
11
+ },
12
+ "args": {
13
+ "objective": "selfcorrupt",
14
+ "alpha": 0.5,
15
+ "beta": 1.0,
16
+ "group": 4,
17
+ "coord_every": 4,
18
+ "k_frac": 0.25,
19
+ "correct_weight": 1.0,
20
+ "ramp_frac": 0.3,
21
+ "task": "parity",
22
+ "coupling": 4,
23
+ "seq_len": 128,
24
+ "n_train": 200000,
25
+ "n_eval": 500,
26
+ "d": 384,
27
+ "layers": 6,
28
+ "heads": 6,
29
+ "pe": "rope",
30
+ "bs": 64,
31
+ "steps": 20000,
32
+ "lr": 0.0003,
33
+ "warmup": 500,
34
+ "seed": 1,
35
+ "data_root": "data",
36
+ "eval_bs": 100,
37
+ "out": "/scratch/zt1/project/msml612/user/govind02/pcmd/runs/g2-parity-selfcorrupt-s1"
38
+ },
39
+ "minutes": 7.818812612692515,
40
+ "compute": {
41
+ "params": 10660608,
42
+ "train_tokens": 163840000,
43
+ "train_flops_approx": 10479804088320000,
44
+ "fwd_per_step": 2
45
+ }
46
+ }