DavidSunok commited on
Commit
8da2be4
·
verified ·
1 Parent(s): 7f214fa

Unify audited E1 and E5 result sources

Browse files
INTACT-unified/PAPER_E5_GOAL_MANIFEST.json CHANGED
@@ -22,17 +22,19 @@
22
  },
23
  "evaluation": {
24
  "interface": "Direct",
 
 
25
  "candidate_sequences": 0,
26
  "terminal_cost_calls": 0,
27
  "evaluation_seeds": [0, 1, 42],
28
  "episodes_per_evaluation_seed": 100,
29
- "evaluator_fingerprint_sha256": "1e475a069338bdad6c9e6a38b8ea4e2d5557ecc4cff88844e5efee79121a9fc7",
30
  "aggregate_official_sr_percent": {
31
- "pusht": {"mean": 80.2222222222, "sample_std": 1.2619796324},
32
- "cube": {"mean": 99.5555555556, "sample_std": 0.7698003589},
33
- "reacher": {"mean": 95.6666666667, "sample_std": 1.7638342074},
34
- "tworoom": {"mean": 82.1111111111, "sample_std": 4.1141130181},
35
- "macro": {"mean": 89.3888888889, "sample_std": 0.7742977703}
36
  }
37
  },
38
  "training_seeds": [
@@ -40,7 +42,7 @@
40
  "seed": 0,
41
  "asset": "intact-goal-e5-seed0.tar.gz",
42
  "shared_state_sha256": "05c6b2f030c11698038134789e02ad00bc6d01e50aebb56910de05dfaf2d5178",
43
- "official_direct_sr_percent": {"pusht": 79.3333333333, "cube": 100.0, "reacher": 93.6666666667, "tworoom": 86.6666666667, "macro": 89.9166666667},
44
  "shards": [
45
  {"task": "pusht", "path": "checkpoints/recovery_delta_full_pusht_s0/weights_epoch_5.pt", "bytes": 84734848, "sha256": "80312771c15e50ba4b3eb3eb9a910fe6dc0a3bd140f7df1396eea306bf89d012"},
46
  {"task": "cube", "path": "checkpoints/recovery_delta_full_cube_s0/weights_epoch_5.pt", "bytes": 84858432, "sha256": "96a4d525e996c2b78fa384873496396024364850ad4af0003274b8c1426ea270"},
@@ -52,7 +54,7 @@
52
  "seed": 42,
53
  "asset": "intact-goal-e5-seed42.tar.gz",
54
  "shared_state_sha256": "e8adcbe1c27a564b3be8193c1ce0737a50247686e647e544f9985856d635661d",
55
- "official_direct_sr_percent": {"pusht": 81.6666666667, "cube": 100.0, "reacher": 96.3333333333, "tworoom": 81.0, "macro": 89.75},
56
  "shards": [
57
  {"task": "pusht", "path": "checkpoints/recovery_delta_full_pusht_s42/weights_epoch_5.pt", "bytes": 84734848, "sha256": "fbcf4dd23eb1ffefdde3ec133609ff00eba4774bd8a697ef3e1586b0fdca917d"},
58
  {"task": "cube", "path": "checkpoints/recovery_delta_full_cube_s42/weights_epoch_5.pt", "bytes": 84858432, "sha256": "d155d9d0c71cc48614d822b14456f341748a25524c7a899b959a54312755e94a"},
@@ -64,7 +66,7 @@
64
  "seed": 3072,
65
  "asset": "intact-goal-e5-seed3072.tar.gz",
66
  "shared_state_sha256": "7cf4b0ca65011c7f54bcabdfe6664e9c82fe650e13656fbb4fe0351c582e65b8",
67
- "official_direct_sr_percent": {"pusht": 79.6666666667, "cube": 98.6666666667, "reacher": 97.0, "tworoom": 78.6666666667, "macro": 88.5},
68
  "shards": [
69
  {"task": "pusht", "path": "checkpoints/recovery_delta_full_pusht_s3072/weights_epoch_5.pt", "bytes": 84734848, "sha256": "24a598acac5faa1e6e548e84c4d17bd79d9e9e48aae38eee384a92ce3269b590"},
70
  {"task": "cube", "path": "checkpoints/recovery_delta_full_cube_s3072/weights_epoch_5.pt", "bytes": 84858432, "sha256": "d2abfd5d907491cf770cc9d3f5bd003f53d0d16f01ac4732111b9dc459895f07"},
 
22
  },
23
  "evaluation": {
24
  "interface": "Direct",
25
+ "result_revision": "2026-09-14",
26
+ "evaluation_protocol": "official_expert_continuation_v1",
27
  "candidate_sequences": 0,
28
  "terminal_cost_calls": 0,
29
  "evaluation_seeds": [0, 1, 42],
30
  "episodes_per_evaluation_seed": 100,
31
+ "history_boundary": "strictly pre-start actions only; append each policy-executed action exactly once",
32
  "aggregate_official_sr_percent": {
33
+ "pusht": {"mean": 86.1111111111, "sample_std": 0.96},
34
+ "cube": {"mean": 100.0, "sample_std": 0.0},
35
+ "reacher": {"mean": 97.2222222222, "sample_std": 0.84},
36
+ "tworoom": {"mean": 81.5555555556, "sample_std": 3.67},
37
+ "macro": {"mean": 91.2222222222, "sample_std": 0.51}
38
  }
39
  },
40
  "training_seeds": [
 
42
  "seed": 0,
43
  "asset": "intact-goal-e5-seed0.tar.gz",
44
  "shared_state_sha256": "05c6b2f030c11698038134789e02ad00bc6d01e50aebb56910de05dfaf2d5178",
45
+ "official_direct_sr_percent": {"pusht": 85.0, "cube": 100.0, "reacher": 96.3333333333, "tworoom": 85.3333333333, "macro": 91.6666666667},
46
  "shards": [
47
  {"task": "pusht", "path": "checkpoints/recovery_delta_full_pusht_s0/weights_epoch_5.pt", "bytes": 84734848, "sha256": "80312771c15e50ba4b3eb3eb9a910fe6dc0a3bd140f7df1396eea306bf89d012"},
48
  {"task": "cube", "path": "checkpoints/recovery_delta_full_cube_s0/weights_epoch_5.pt", "bytes": 84858432, "sha256": "96a4d525e996c2b78fa384873496396024364850ad4af0003274b8c1426ea270"},
 
54
  "seed": 42,
55
  "asset": "intact-goal-e5-seed42.tar.gz",
56
  "shared_state_sha256": "e8adcbe1c27a564b3be8193c1ce0737a50247686e647e544f9985856d635661d",
57
+ "official_direct_sr_percent": {"pusht": 86.6666666667, "cube": 100.0, "reacher": 97.3333333333, "tworoom": 81.3333333333, "macro": 91.3333333333},
58
  "shards": [
59
  {"task": "pusht", "path": "checkpoints/recovery_delta_full_pusht_s42/weights_epoch_5.pt", "bytes": 84734848, "sha256": "fbcf4dd23eb1ffefdde3ec133609ff00eba4774bd8a697ef3e1586b0fdca917d"},
60
  {"task": "cube", "path": "checkpoints/recovery_delta_full_cube_s42/weights_epoch_5.pt", "bytes": 84858432, "sha256": "d155d9d0c71cc48614d822b14456f341748a25524c7a899b959a54312755e94a"},
 
66
  "seed": 3072,
67
  "asset": "intact-goal-e5-seed3072.tar.gz",
68
  "shared_state_sha256": "7cf4b0ca65011c7f54bcabdfe6664e9c82fe650e13656fbb4fe0351c582e65b8",
69
+ "official_direct_sr_percent": {"pusht": 86.6666666667, "cube": 100.0, "reacher": 98.0, "tworoom": 78.0, "macro": 90.6666666667},
70
  "shards": [
71
  {"task": "pusht", "path": "checkpoints/recovery_delta_full_pusht_s3072/weights_epoch_5.pt", "bytes": 84734848, "sha256": "24a598acac5faa1e6e548e84c4d17bd79d9e9e48aae38eee384a92ce3269b590"},
72
  {"task": "cube", "path": "checkpoints/recovery_delta_full_cube_s3072/weights_epoch_5.pt", "bytes": 84858432, "sha256": "d2abfd5d907491cf770cc9d3f5bd003f53d0d16f01ac4732111b9dc459895f07"},
INTACT-unified/PAPER_E5_MATRIX_MANIFEST.json CHANGED
@@ -7,6 +7,8 @@
7
  },
8
  "protocol": {
9
  "name": "Controlled Math-SDPA shared-encoder E5 matrix",
 
 
10
  "epoch": 5,
11
  "training_seeds": [
12
  0,
@@ -25,7 +27,7 @@
25
  "reacher",
26
  "tworoom"
27
  ],
28
- "evaluator_fingerprint_sha256": "1e475a069338bdad6c9e6a38b8ea4e2d5557ecc4cff88844e5efee79121a9fc7"
29
  },
30
  "cells": [
31
  {
@@ -168,24 +170,24 @@
168
  "native_interface": "Direct",
169
  "aggregate_official_sr_percent": {
170
  "pusht": {
171
- "mean": 35.0,
172
- "sample_std": 1.45
173
  },
174
  "cube": {
175
- "mean": 61.89,
176
- "sample_std": 1.26
177
  },
178
  "reacher": {
179
- "mean": 61.0,
180
- "sample_std": 0.58
181
  },
182
  "tworoom": {
183
- "mean": 68.78,
184
- "sample_std": 3.83
185
  },
186
  "macro": {
187
- "mean": 56.67,
188
- "sample_std": 0.22
189
  }
190
  },
191
  "training_seeds": [
@@ -436,24 +438,24 @@
436
  "native_interface": "Direct",
437
  "aggregate_official_sr_percent": {
438
  "pusht": {
439
- "mean": 70.67,
440
- "sample_std": 1.76
441
  },
442
  "cube": {
443
  "mean": 100.0,
444
  "sample_std": 0.0
445
  },
446
  "reacher": {
447
- "mean": 82.33,
448
- "sample_std": 3.18
449
  },
450
  "tworoom": {
451
- "mean": 69.44,
452
- "sample_std": 6.75
453
  },
454
  "macro": {
455
- "mean": 80.61,
456
- "sample_std": 2.11
457
  }
458
  },
459
  "training_seeds": [
@@ -704,24 +706,24 @@
704
  "native_interface": "Direct",
705
  "aggregate_official_sr_percent": {
706
  "pusht": {
707
- "mean": 80.22,
708
- "sample_std": 1.26
709
  },
710
  "cube": {
711
- "mean": 99.56,
712
- "sample_std": 0.77
713
  },
714
  "reacher": {
715
- "mean": 95.67,
716
- "sample_std": 1.76
717
  },
718
  "tworoom": {
719
- "mean": 82.11,
720
- "sample_std": 4.11
721
  },
722
  "macro": {
723
- "mean": 89.39,
724
- "sample_std": 0.77
725
  }
726
  },
727
  "training_seeds": [
 
7
  },
8
  "protocol": {
9
  "name": "Controlled Math-SDPA shared-encoder E5 matrix",
10
+ "result_revision": "2026-09-14",
11
+ "evaluation_protocol": "official_expert_continuation_v1",
12
  "epoch": 5,
13
  "training_seeds": [
14
  0,
 
27
  "reacher",
28
  "tworoom"
29
  ],
30
+ "history_boundary": "strictly pre-start actions only; append each policy-executed action exactly once"
31
  },
32
  "cells": [
33
  {
 
170
  "native_interface": "Direct",
171
  "aggregate_official_sr_percent": {
172
  "pusht": {
173
+ "mean": 36.11,
174
+ "sample_std": 0.19
175
  },
176
  "cube": {
177
+ "mean": 67.56,
178
+ "sample_std": 4.03
179
  },
180
  "reacher": {
181
+ "mean": 90.56,
182
+ "sample_std": 1.35
183
  },
184
  "tworoom": {
185
+ "mean": 78.56,
186
+ "sample_std": 6.68
187
  },
188
  "macro": {
189
+ "mean": 68.19,
190
+ "sample_std": 0.67
191
  }
192
  },
193
  "training_seeds": [
 
438
  "native_interface": "Direct",
439
  "aggregate_official_sr_percent": {
440
  "pusht": {
441
+ "mean": 81.78,
442
+ "sample_std": 0.96
443
  },
444
  "cube": {
445
  "mean": 100.0,
446
  "sample_std": 0.0
447
  },
448
  "reacher": {
449
+ "mean": 88.67,
450
+ "sample_std": 0.33
451
  },
452
  "tworoom": {
453
+ "mean": 69.22,
454
+ "sample_std": 7.04
455
  },
456
  "macro": {
457
+ "mean": 84.92,
458
+ "sample_std": 1.86
459
  }
460
  },
461
  "training_seeds": [
 
706
  "native_interface": "Direct",
707
  "aggregate_official_sr_percent": {
708
  "pusht": {
709
+ "mean": 86.11,
710
+ "sample_std": 0.96
711
  },
712
  "cube": {
713
+ "mean": 100.0,
714
+ "sample_std": 0.0
715
  },
716
  "reacher": {
717
+ "mean": 97.22,
718
+ "sample_std": 0.84
719
  },
720
  "tworoom": {
721
+ "mean": 81.56,
722
+ "sample_std": 3.67
723
  },
724
  "macro": {
725
+ "mean": 91.22,
726
+ "sample_std": 0.51
727
  }
728
  },
729
  "training_seeds": [
INTACT/AUDITED_RESULTS_20260914.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "schema_version": 1,
3
+ "result_revision": "2026-09-14",
4
+ "authoritative_for": "project_headline_success_rates",
5
+ "protocol": {
6
+ "benchmark": "Official LeWM",
7
+ "training_scope": "task-specific E1",
8
+ "training_seeds": [3072, 3073, 3074],
9
+ "evaluation_seeds": [0, 1, 42],
10
+ "episodes_per_evaluation_seed": 100,
11
+ "history_boundary": "strictly pre-start actions only; append each policy-executed action exactly once"
12
+ },
13
+ "results": {
14
+ "direct": {
15
+ "interface": "Direct",
16
+ "candidate_sequences": 0,
17
+ "official_sr_percent": {
18
+ "pusht": {"mean": 87.44, "sample_std": 1.26},
19
+ "cube": {"mean": 100.00, "sample_std": 0.00},
20
+ "reacher": {"mean": 97.33, "sample_std": 0.00},
21
+ "tworoom": {"mean": 97.67, "sample_std": 1.20},
22
+ "macro": {"mean": 95.61, "sample_std": 0.59}
23
+ }
24
+ },
25
+ "guarded_a": {
26
+ "interface": "Guarded A 128x3",
27
+ "candidate_sequences": 384,
28
+ "official_sr_percent": {
29
+ "pusht": {"mean": 91.56, "sample_std": 0.69},
30
+ "cube": {"mean": 99.67, "sample_std": 0.33},
31
+ "reacher": {"mean": 97.56, "sample_std": 0.51},
32
+ "tworoom": {"mean": 97.56, "sample_std": 1.26},
33
+ "macro": {"mean": 96.58, "sample_std": 0.44}
34
+ }
35
+ }
36
+ },
37
+ "manifest_note": "MANIFEST.json preserves the identity, hashes, and package-specific validation of the released reproduction archives; this file is authoritative for project headline success rates."
38
+ }
INTACT/MANIFEST.json CHANGED
@@ -7,6 +7,11 @@
7
  "path": "INTACT",
8
  "source_revision": "intact-goal-e1-single-p77-v1"
9
  },
 
 
 
 
 
10
  "model": {
11
  "name": "Goal-displacement INTACT single-task reproduction",
12
  "epoch": 1,
 
7
  "path": "INTACT",
8
  "source_revision": "intact-goal-e1-single-p77-v1"
9
  },
10
+ "result_scope": {
11
+ "role": "archive-specific reproduction validation",
12
+ "project_headline_results": "AUDITED_RESULTS_20260914.json",
13
+ "use_for_project_headline": false
14
+ },
15
  "model": {
16
  "name": "Goal-displacement INTACT single-task reproduction",
17
  "epoch": 1,
INTACT/README.md CHANGED
@@ -15,23 +15,24 @@ checkpoints: four tasks for each training seed (`0`, `42`, and `3072`).
15
  - No current or future expert action is visible
16
  - Official Direct evaluation, `H=5`, `RH=5`, zero search
17
 
18
- ## Official Direct results
19
 
20
- | Task | Mean SR (%) | Train-seed sample std (pp) |
21
- |---|---:|---:|
22
- | PushT | 86.44 | 1.07 |
23
- | Cube | 100.00 | 0.00 |
24
- | Reacher | 97.78 | 0.51 |
25
- | TwoRoom | 98.78 | 0.69 |
26
- | **Macro** | **95.75** | **0.52** |
27
 
28
- The matrix contains 36 evaluations: training seeds `0/42/3072`, evaluation
29
- seeds `0/1/42`, four tasks, and 100 episodes per cell.
 
 
 
30
 
31
  ## Files
32
 
33
  - `intact-goal-e1-single-seed<seed>.tar.gz`: four task shards for one seed
34
- - `MANIFEST.json`: release contract, aggregate results, and shard hashes
 
35
  - `RESULTS.csv`: complete 36-cell continuation evaluation
36
  - `SHA256SUMS`: archive checksums
37
 
 
15
  - No current or future expert action is visible
16
  - Official Direct evaluation, `H=5`, `RH=5`, zero search
17
 
18
+ ## Audited project results
19
 
20
+ | Inference | PushT | Cube | Reacher | TwoRoom | Macro SR |
21
+ |---|---:|---:|---:|---:|---:|
22
+ | **Direct, zero search** | 87.44 +/- 1.26 | 100.00 +/- 0.00 | 97.33 +/- 0.00 | 97.67 +/- 1.20 | **95.61 +/- 0.59** |
23
+ | **Guarded A 128x3** | 91.56 +/- 0.69 | 99.67 +/- 0.33 | 97.56 +/- 0.51 | 97.56 +/- 1.26 | **96.58 +/- 0.44** |
 
 
 
24
 
25
+ [`AUDITED_RESULTS_20260914.json`](AUDITED_RESULTS_20260914.json) is the
26
+ machine-readable source of truth for project headline SR. `MANIFEST.json` is
27
+ retained for the identity, hashes, and package-specific validation of the
28
+ released `0/42/3072` reproduction archives; its validation aggregate is not
29
+ the project headline.
30
 
31
  ## Files
32
 
33
  - `intact-goal-e1-single-seed<seed>.tar.gz`: four task shards for one seed
34
+ - `AUDITED_RESULTS_20260914.json`: authoritative project headline SR
35
+ - `MANIFEST.json`: archive identity, package validation, and shard hashes
36
  - `RESULTS.csv`: complete 36-cell continuation evaluation
37
  - `SHA256SUMS`: archive checksums
38
 
README.md CHANGED
@@ -33,6 +33,12 @@ point and appends every policy-executed action exactly once. The history-free
33
  ablation removes the previous-action channel from both training and evaluation.
34
  Guarded A is an optional local verifier around the Direct plan; it evaluates
35
  384 sampled candidates rather than the 9,000 used by CEM 300x30.
 
 
 
 
 
 
36
 
37
  ## Download
38
 
 
33
  ablation removes the previous-action channel from both training and evaluation.
34
  Guarded A is an optional local verifier around the Direct plan; it evaluates
35
  384 sampled candidates rather than the 9,000 used by CEM 300x30.
36
+ Machine-readable headline values are frozen in
37
+ [`INTACT/AUDITED_RESULTS_20260914.json`](INTACT/AUDITED_RESULTS_20260914.json)
38
+ and
39
+ [`INTACT-unified/AUDITED_RESULTS_20260914.json`](INTACT-unified/AUDITED_RESULTS_20260914.json).
40
+ Model manifests retain archive identities and hashes; use the audited result
41
+ files for project-level reporting.
42
 
43
  ## Download
44