Unify audited E1 and E5 result sources
Browse files- INTACT-unified/PAPER_E5_GOAL_MANIFEST.json +11 -9
- INTACT-unified/PAPER_E5_MATRIX_MANIFEST.json +31 -29
- INTACT/AUDITED_RESULTS_20260914.json +38 -0
- INTACT/MANIFEST.json +5 -0
- INTACT/README.md +12 -11
- README.md +6 -0
INTACT-unified/PAPER_E5_GOAL_MANIFEST.json
CHANGED
|
@@ -22,17 +22,19 @@
|
|
| 22 |
},
|
| 23 |
"evaluation": {
|
| 24 |
"interface": "Direct",
|
|
|
|
|
|
|
| 25 |
"candidate_sequences": 0,
|
| 26 |
"terminal_cost_calls": 0,
|
| 27 |
"evaluation_seeds": [0, 1, 42],
|
| 28 |
"episodes_per_evaluation_seed": 100,
|
| 29 |
-
"
|
| 30 |
"aggregate_official_sr_percent": {
|
| 31 |
-
"pusht": {"mean":
|
| 32 |
-
"cube": {"mean":
|
| 33 |
-
"reacher": {"mean":
|
| 34 |
-
"tworoom": {"mean":
|
| 35 |
-
"macro": {"mean":
|
| 36 |
}
|
| 37 |
},
|
| 38 |
"training_seeds": [
|
|
@@ -40,7 +42,7 @@
|
|
| 40 |
"seed": 0,
|
| 41 |
"asset": "intact-goal-e5-seed0.tar.gz",
|
| 42 |
"shared_state_sha256": "05c6b2f030c11698038134789e02ad00bc6d01e50aebb56910de05dfaf2d5178",
|
| 43 |
-
"official_direct_sr_percent": {"pusht":
|
| 44 |
"shards": [
|
| 45 |
{"task": "pusht", "path": "checkpoints/recovery_delta_full_pusht_s0/weights_epoch_5.pt", "bytes": 84734848, "sha256": "80312771c15e50ba4b3eb3eb9a910fe6dc0a3bd140f7df1396eea306bf89d012"},
|
| 46 |
{"task": "cube", "path": "checkpoints/recovery_delta_full_cube_s0/weights_epoch_5.pt", "bytes": 84858432, "sha256": "96a4d525e996c2b78fa384873496396024364850ad4af0003274b8c1426ea270"},
|
|
@@ -52,7 +54,7 @@
|
|
| 52 |
"seed": 42,
|
| 53 |
"asset": "intact-goal-e5-seed42.tar.gz",
|
| 54 |
"shared_state_sha256": "e8adcbe1c27a564b3be8193c1ce0737a50247686e647e544f9985856d635661d",
|
| 55 |
-
"official_direct_sr_percent": {"pusht":
|
| 56 |
"shards": [
|
| 57 |
{"task": "pusht", "path": "checkpoints/recovery_delta_full_pusht_s42/weights_epoch_5.pt", "bytes": 84734848, "sha256": "fbcf4dd23eb1ffefdde3ec133609ff00eba4774bd8a697ef3e1586b0fdca917d"},
|
| 58 |
{"task": "cube", "path": "checkpoints/recovery_delta_full_cube_s42/weights_epoch_5.pt", "bytes": 84858432, "sha256": "d155d9d0c71cc48614d822b14456f341748a25524c7a899b959a54312755e94a"},
|
|
@@ -64,7 +66,7 @@
|
|
| 64 |
"seed": 3072,
|
| 65 |
"asset": "intact-goal-e5-seed3072.tar.gz",
|
| 66 |
"shared_state_sha256": "7cf4b0ca65011c7f54bcabdfe6664e9c82fe650e13656fbb4fe0351c582e65b8",
|
| 67 |
-
"official_direct_sr_percent": {"pusht":
|
| 68 |
"shards": [
|
| 69 |
{"task": "pusht", "path": "checkpoints/recovery_delta_full_pusht_s3072/weights_epoch_5.pt", "bytes": 84734848, "sha256": "24a598acac5faa1e6e548e84c4d17bd79d9e9e48aae38eee384a92ce3269b590"},
|
| 70 |
{"task": "cube", "path": "checkpoints/recovery_delta_full_cube_s3072/weights_epoch_5.pt", "bytes": 84858432, "sha256": "d2abfd5d907491cf770cc9d3f5bd003f53d0d16f01ac4732111b9dc459895f07"},
|
|
|
|
| 22 |
},
|
| 23 |
"evaluation": {
|
| 24 |
"interface": "Direct",
|
| 25 |
+
"result_revision": "2026-09-14",
|
| 26 |
+
"evaluation_protocol": "official_expert_continuation_v1",
|
| 27 |
"candidate_sequences": 0,
|
| 28 |
"terminal_cost_calls": 0,
|
| 29 |
"evaluation_seeds": [0, 1, 42],
|
| 30 |
"episodes_per_evaluation_seed": 100,
|
| 31 |
+
"history_boundary": "strictly pre-start actions only; append each policy-executed action exactly once",
|
| 32 |
"aggregate_official_sr_percent": {
|
| 33 |
+
"pusht": {"mean": 86.1111111111, "sample_std": 0.96},
|
| 34 |
+
"cube": {"mean": 100.0, "sample_std": 0.0},
|
| 35 |
+
"reacher": {"mean": 97.2222222222, "sample_std": 0.84},
|
| 36 |
+
"tworoom": {"mean": 81.5555555556, "sample_std": 3.67},
|
| 37 |
+
"macro": {"mean": 91.2222222222, "sample_std": 0.51}
|
| 38 |
}
|
| 39 |
},
|
| 40 |
"training_seeds": [
|
|
|
|
| 42 |
"seed": 0,
|
| 43 |
"asset": "intact-goal-e5-seed0.tar.gz",
|
| 44 |
"shared_state_sha256": "05c6b2f030c11698038134789e02ad00bc6d01e50aebb56910de05dfaf2d5178",
|
| 45 |
+
"official_direct_sr_percent": {"pusht": 85.0, "cube": 100.0, "reacher": 96.3333333333, "tworoom": 85.3333333333, "macro": 91.6666666667},
|
| 46 |
"shards": [
|
| 47 |
{"task": "pusht", "path": "checkpoints/recovery_delta_full_pusht_s0/weights_epoch_5.pt", "bytes": 84734848, "sha256": "80312771c15e50ba4b3eb3eb9a910fe6dc0a3bd140f7df1396eea306bf89d012"},
|
| 48 |
{"task": "cube", "path": "checkpoints/recovery_delta_full_cube_s0/weights_epoch_5.pt", "bytes": 84858432, "sha256": "96a4d525e996c2b78fa384873496396024364850ad4af0003274b8c1426ea270"},
|
|
|
|
| 54 |
"seed": 42,
|
| 55 |
"asset": "intact-goal-e5-seed42.tar.gz",
|
| 56 |
"shared_state_sha256": "e8adcbe1c27a564b3be8193c1ce0737a50247686e647e544f9985856d635661d",
|
| 57 |
+
"official_direct_sr_percent": {"pusht": 86.6666666667, "cube": 100.0, "reacher": 97.3333333333, "tworoom": 81.3333333333, "macro": 91.3333333333},
|
| 58 |
"shards": [
|
| 59 |
{"task": "pusht", "path": "checkpoints/recovery_delta_full_pusht_s42/weights_epoch_5.pt", "bytes": 84734848, "sha256": "fbcf4dd23eb1ffefdde3ec133609ff00eba4774bd8a697ef3e1586b0fdca917d"},
|
| 60 |
{"task": "cube", "path": "checkpoints/recovery_delta_full_cube_s42/weights_epoch_5.pt", "bytes": 84858432, "sha256": "d155d9d0c71cc48614d822b14456f341748a25524c7a899b959a54312755e94a"},
|
|
|
|
| 66 |
"seed": 3072,
|
| 67 |
"asset": "intact-goal-e5-seed3072.tar.gz",
|
| 68 |
"shared_state_sha256": "7cf4b0ca65011c7f54bcabdfe6664e9c82fe650e13656fbb4fe0351c582e65b8",
|
| 69 |
+
"official_direct_sr_percent": {"pusht": 86.6666666667, "cube": 100.0, "reacher": 98.0, "tworoom": 78.0, "macro": 90.6666666667},
|
| 70 |
"shards": [
|
| 71 |
{"task": "pusht", "path": "checkpoints/recovery_delta_full_pusht_s3072/weights_epoch_5.pt", "bytes": 84734848, "sha256": "24a598acac5faa1e6e548e84c4d17bd79d9e9e48aae38eee384a92ce3269b590"},
|
| 72 |
{"task": "cube", "path": "checkpoints/recovery_delta_full_cube_s3072/weights_epoch_5.pt", "bytes": 84858432, "sha256": "d2abfd5d907491cf770cc9d3f5bd003f53d0d16f01ac4732111b9dc459895f07"},
|
INTACT-unified/PAPER_E5_MATRIX_MANIFEST.json
CHANGED
|
@@ -7,6 +7,8 @@
|
|
| 7 |
},
|
| 8 |
"protocol": {
|
| 9 |
"name": "Controlled Math-SDPA shared-encoder E5 matrix",
|
|
|
|
|
|
|
| 10 |
"epoch": 5,
|
| 11 |
"training_seeds": [
|
| 12 |
0,
|
|
@@ -25,7 +27,7 @@
|
|
| 25 |
"reacher",
|
| 26 |
"tworoom"
|
| 27 |
],
|
| 28 |
-
"
|
| 29 |
},
|
| 30 |
"cells": [
|
| 31 |
{
|
|
@@ -168,24 +170,24 @@
|
|
| 168 |
"native_interface": "Direct",
|
| 169 |
"aggregate_official_sr_percent": {
|
| 170 |
"pusht": {
|
| 171 |
-
"mean":
|
| 172 |
-
"sample_std":
|
| 173 |
},
|
| 174 |
"cube": {
|
| 175 |
-
"mean":
|
| 176 |
-
"sample_std":
|
| 177 |
},
|
| 178 |
"reacher": {
|
| 179 |
-
"mean":
|
| 180 |
-
"sample_std":
|
| 181 |
},
|
| 182 |
"tworoom": {
|
| 183 |
-
"mean":
|
| 184 |
-
"sample_std":
|
| 185 |
},
|
| 186 |
"macro": {
|
| 187 |
-
"mean":
|
| 188 |
-
"sample_std": 0.
|
| 189 |
}
|
| 190 |
},
|
| 191 |
"training_seeds": [
|
|
@@ -436,24 +438,24 @@
|
|
| 436 |
"native_interface": "Direct",
|
| 437 |
"aggregate_official_sr_percent": {
|
| 438 |
"pusht": {
|
| 439 |
-
"mean":
|
| 440 |
-
"sample_std":
|
| 441 |
},
|
| 442 |
"cube": {
|
| 443 |
"mean": 100.0,
|
| 444 |
"sample_std": 0.0
|
| 445 |
},
|
| 446 |
"reacher": {
|
| 447 |
-
"mean":
|
| 448 |
-
"sample_std":
|
| 449 |
},
|
| 450 |
"tworoom": {
|
| 451 |
-
"mean": 69.
|
| 452 |
-
"sample_std":
|
| 453 |
},
|
| 454 |
"macro": {
|
| 455 |
-
"mean":
|
| 456 |
-
"sample_std":
|
| 457 |
}
|
| 458 |
},
|
| 459 |
"training_seeds": [
|
|
@@ -704,24 +706,24 @@
|
|
| 704 |
"native_interface": "Direct",
|
| 705 |
"aggregate_official_sr_percent": {
|
| 706 |
"pusht": {
|
| 707 |
-
"mean":
|
| 708 |
-
"sample_std":
|
| 709 |
},
|
| 710 |
"cube": {
|
| 711 |
-
"mean":
|
| 712 |
-
"sample_std": 0.
|
| 713 |
},
|
| 714 |
"reacher": {
|
| 715 |
-
"mean":
|
| 716 |
-
"sample_std":
|
| 717 |
},
|
| 718 |
"tworoom": {
|
| 719 |
-
"mean":
|
| 720 |
-
"sample_std":
|
| 721 |
},
|
| 722 |
"macro": {
|
| 723 |
-
"mean":
|
| 724 |
-
"sample_std": 0.
|
| 725 |
}
|
| 726 |
},
|
| 727 |
"training_seeds": [
|
|
|
|
| 7 |
},
|
| 8 |
"protocol": {
|
| 9 |
"name": "Controlled Math-SDPA shared-encoder E5 matrix",
|
| 10 |
+
"result_revision": "2026-09-14",
|
| 11 |
+
"evaluation_protocol": "official_expert_continuation_v1",
|
| 12 |
"epoch": 5,
|
| 13 |
"training_seeds": [
|
| 14 |
0,
|
|
|
|
| 27 |
"reacher",
|
| 28 |
"tworoom"
|
| 29 |
],
|
| 30 |
+
"history_boundary": "strictly pre-start actions only; append each policy-executed action exactly once"
|
| 31 |
},
|
| 32 |
"cells": [
|
| 33 |
{
|
|
|
|
| 170 |
"native_interface": "Direct",
|
| 171 |
"aggregate_official_sr_percent": {
|
| 172 |
"pusht": {
|
| 173 |
+
"mean": 36.11,
|
| 174 |
+
"sample_std": 0.19
|
| 175 |
},
|
| 176 |
"cube": {
|
| 177 |
+
"mean": 67.56,
|
| 178 |
+
"sample_std": 4.03
|
| 179 |
},
|
| 180 |
"reacher": {
|
| 181 |
+
"mean": 90.56,
|
| 182 |
+
"sample_std": 1.35
|
| 183 |
},
|
| 184 |
"tworoom": {
|
| 185 |
+
"mean": 78.56,
|
| 186 |
+
"sample_std": 6.68
|
| 187 |
},
|
| 188 |
"macro": {
|
| 189 |
+
"mean": 68.19,
|
| 190 |
+
"sample_std": 0.67
|
| 191 |
}
|
| 192 |
},
|
| 193 |
"training_seeds": [
|
|
|
|
| 438 |
"native_interface": "Direct",
|
| 439 |
"aggregate_official_sr_percent": {
|
| 440 |
"pusht": {
|
| 441 |
+
"mean": 81.78,
|
| 442 |
+
"sample_std": 0.96
|
| 443 |
},
|
| 444 |
"cube": {
|
| 445 |
"mean": 100.0,
|
| 446 |
"sample_std": 0.0
|
| 447 |
},
|
| 448 |
"reacher": {
|
| 449 |
+
"mean": 88.67,
|
| 450 |
+
"sample_std": 0.33
|
| 451 |
},
|
| 452 |
"tworoom": {
|
| 453 |
+
"mean": 69.22,
|
| 454 |
+
"sample_std": 7.04
|
| 455 |
},
|
| 456 |
"macro": {
|
| 457 |
+
"mean": 84.92,
|
| 458 |
+
"sample_std": 1.86
|
| 459 |
}
|
| 460 |
},
|
| 461 |
"training_seeds": [
|
|
|
|
| 706 |
"native_interface": "Direct",
|
| 707 |
"aggregate_official_sr_percent": {
|
| 708 |
"pusht": {
|
| 709 |
+
"mean": 86.11,
|
| 710 |
+
"sample_std": 0.96
|
| 711 |
},
|
| 712 |
"cube": {
|
| 713 |
+
"mean": 100.0,
|
| 714 |
+
"sample_std": 0.0
|
| 715 |
},
|
| 716 |
"reacher": {
|
| 717 |
+
"mean": 97.22,
|
| 718 |
+
"sample_std": 0.84
|
| 719 |
},
|
| 720 |
"tworoom": {
|
| 721 |
+
"mean": 81.56,
|
| 722 |
+
"sample_std": 3.67
|
| 723 |
},
|
| 724 |
"macro": {
|
| 725 |
+
"mean": 91.22,
|
| 726 |
+
"sample_std": 0.51
|
| 727 |
}
|
| 728 |
},
|
| 729 |
"training_seeds": [
|
INTACT/AUDITED_RESULTS_20260914.json
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 1,
|
| 3 |
+
"result_revision": "2026-09-14",
|
| 4 |
+
"authoritative_for": "project_headline_success_rates",
|
| 5 |
+
"protocol": {
|
| 6 |
+
"benchmark": "Official LeWM",
|
| 7 |
+
"training_scope": "task-specific E1",
|
| 8 |
+
"training_seeds": [3072, 3073, 3074],
|
| 9 |
+
"evaluation_seeds": [0, 1, 42],
|
| 10 |
+
"episodes_per_evaluation_seed": 100,
|
| 11 |
+
"history_boundary": "strictly pre-start actions only; append each policy-executed action exactly once"
|
| 12 |
+
},
|
| 13 |
+
"results": {
|
| 14 |
+
"direct": {
|
| 15 |
+
"interface": "Direct",
|
| 16 |
+
"candidate_sequences": 0,
|
| 17 |
+
"official_sr_percent": {
|
| 18 |
+
"pusht": {"mean": 87.44, "sample_std": 1.26},
|
| 19 |
+
"cube": {"mean": 100.00, "sample_std": 0.00},
|
| 20 |
+
"reacher": {"mean": 97.33, "sample_std": 0.00},
|
| 21 |
+
"tworoom": {"mean": 97.67, "sample_std": 1.20},
|
| 22 |
+
"macro": {"mean": 95.61, "sample_std": 0.59}
|
| 23 |
+
}
|
| 24 |
+
},
|
| 25 |
+
"guarded_a": {
|
| 26 |
+
"interface": "Guarded A 128x3",
|
| 27 |
+
"candidate_sequences": 384,
|
| 28 |
+
"official_sr_percent": {
|
| 29 |
+
"pusht": {"mean": 91.56, "sample_std": 0.69},
|
| 30 |
+
"cube": {"mean": 99.67, "sample_std": 0.33},
|
| 31 |
+
"reacher": {"mean": 97.56, "sample_std": 0.51},
|
| 32 |
+
"tworoom": {"mean": 97.56, "sample_std": 1.26},
|
| 33 |
+
"macro": {"mean": 96.58, "sample_std": 0.44}
|
| 34 |
+
}
|
| 35 |
+
}
|
| 36 |
+
},
|
| 37 |
+
"manifest_note": "MANIFEST.json preserves the identity, hashes, and package-specific validation of the released reproduction archives; this file is authoritative for project headline success rates."
|
| 38 |
+
}
|
INTACT/MANIFEST.json
CHANGED
|
@@ -7,6 +7,11 @@
|
|
| 7 |
"path": "INTACT",
|
| 8 |
"source_revision": "intact-goal-e1-single-p77-v1"
|
| 9 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
"model": {
|
| 11 |
"name": "Goal-displacement INTACT single-task reproduction",
|
| 12 |
"epoch": 1,
|
|
|
|
| 7 |
"path": "INTACT",
|
| 8 |
"source_revision": "intact-goal-e1-single-p77-v1"
|
| 9 |
},
|
| 10 |
+
"result_scope": {
|
| 11 |
+
"role": "archive-specific reproduction validation",
|
| 12 |
+
"project_headline_results": "AUDITED_RESULTS_20260914.json",
|
| 13 |
+
"use_for_project_headline": false
|
| 14 |
+
},
|
| 15 |
"model": {
|
| 16 |
"name": "Goal-displacement INTACT single-task reproduction",
|
| 17 |
"epoch": 1,
|
INTACT/README.md
CHANGED
|
@@ -15,23 +15,24 @@ checkpoints: four tasks for each training seed (`0`, `42`, and `3072`).
|
|
| 15 |
- No current or future expert action is visible
|
| 16 |
- Official Direct evaluation, `H=5`, `RH=5`, zero search
|
| 17 |
|
| 18 |
-
##
|
| 19 |
|
| 20 |
-
|
|
| 21 |
-
|---|---:|---:|
|
| 22 |
-
|
|
| 23 |
-
|
|
| 24 |
-
| Reacher | 97.78 | 0.51 |
|
| 25 |
-
| TwoRoom | 98.78 | 0.69 |
|
| 26 |
-
| **Macro** | **95.75** | **0.52** |
|
| 27 |
|
| 28 |
-
|
| 29 |
-
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
## Files
|
| 32 |
|
| 33 |
- `intact-goal-e1-single-seed<seed>.tar.gz`: four task shards for one seed
|
| 34 |
-
- `
|
|
|
|
| 35 |
- `RESULTS.csv`: complete 36-cell continuation evaluation
|
| 36 |
- `SHA256SUMS`: archive checksums
|
| 37 |
|
|
|
|
| 15 |
- No current or future expert action is visible
|
| 16 |
- Official Direct evaluation, `H=5`, `RH=5`, zero search
|
| 17 |
|
| 18 |
+
## Audited project results
|
| 19 |
|
| 20 |
+
| Inference | PushT | Cube | Reacher | TwoRoom | Macro SR |
|
| 21 |
+
|---|---:|---:|---:|---:|---:|
|
| 22 |
+
| **Direct, zero search** | 87.44 +/- 1.26 | 100.00 +/- 0.00 | 97.33 +/- 0.00 | 97.67 +/- 1.20 | **95.61 +/- 0.59** |
|
| 23 |
+
| **Guarded A 128x3** | 91.56 +/- 0.69 | 99.67 +/- 0.33 | 97.56 +/- 0.51 | 97.56 +/- 1.26 | **96.58 +/- 0.44** |
|
|
|
|
|
|
|
|
|
|
| 24 |
|
| 25 |
+
[`AUDITED_RESULTS_20260914.json`](AUDITED_RESULTS_20260914.json) is the
|
| 26 |
+
machine-readable source of truth for project headline SR. `MANIFEST.json` is
|
| 27 |
+
retained for the identity, hashes, and package-specific validation of the
|
| 28 |
+
released `0/42/3072` reproduction archives; its validation aggregate is not
|
| 29 |
+
the project headline.
|
| 30 |
|
| 31 |
## Files
|
| 32 |
|
| 33 |
- `intact-goal-e1-single-seed<seed>.tar.gz`: four task shards for one seed
|
| 34 |
+
- `AUDITED_RESULTS_20260914.json`: authoritative project headline SR
|
| 35 |
+
- `MANIFEST.json`: archive identity, package validation, and shard hashes
|
| 36 |
- `RESULTS.csv`: complete 36-cell continuation evaluation
|
| 37 |
- `SHA256SUMS`: archive checksums
|
| 38 |
|
README.md
CHANGED
|
@@ -33,6 +33,12 @@ point and appends every policy-executed action exactly once. The history-free
|
|
| 33 |
ablation removes the previous-action channel from both training and evaluation.
|
| 34 |
Guarded A is an optional local verifier around the Direct plan; it evaluates
|
| 35 |
384 sampled candidates rather than the 9,000 used by CEM 300x30.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
|
| 37 |
## Download
|
| 38 |
|
|
|
|
| 33 |
ablation removes the previous-action channel from both training and evaluation.
|
| 34 |
Guarded A is an optional local verifier around the Direct plan; it evaluates
|
| 35 |
384 sampled candidates rather than the 9,000 used by CEM 300x30.
|
| 36 |
+
Machine-readable headline values are frozen in
|
| 37 |
+
[`INTACT/AUDITED_RESULTS_20260914.json`](INTACT/AUDITED_RESULTS_20260914.json)
|
| 38 |
+
and
|
| 39 |
+
[`INTACT-unified/AUDITED_RESULTS_20260914.json`](INTACT-unified/AUDITED_RESULTS_20260914.json).
|
| 40 |
+
Model manifests retain archive identities and hashes; use the audited result
|
| 41 |
+
files for project-level reporting.
|
| 42 |
|
| 43 |
## Download
|
| 44 |
|