Publish codex SIMPLE seed-0 evaluation evidence

comparison.json

@@ -3175,7 +3175,92 @@
     "limitation": "One episode per harness; this is not a multi-seed success-rate estimate."
   },
   "simple": {
-    "tasks": [],
+    "tasks": [
+      {
+        "task_key": "task06/01",
+        "kinex_success": true,
+        "codex_success": true,
+        "same_instruction": true,
+        "same_roboenv_source": true,
+        "kinex_steps": 2001,
+        "codex_steps": 1647,
+        "kinex_usage": {
+          "input_tokens": 3121253,
+          "output_tokens": 9863,
+          "cached_input_tokens": 2984704,
+          "cache_write_input_tokens": 0,
+          "reasoning_output_tokens": 2815,
+          "cache_reported_input_tokens": 3121253,
+          "cache_write_reported_input_tokens": 3121253,
+          "reasoning_reported_output_tokens": 9863,
+          "schema": "rlebench/token-usage/1",
+          "source": "Provider terminal response usage",
+          "response_count": 84,
+          "reported_responses": {
+            "input_tokens": 84,
+            "output_tokens": 84,
+            "cached_input_tokens": 84,
+            "cache_write_input_tokens": 84,
+            "reasoning_output_tokens": 84,
+            "cache_reported_input_tokens": 84,
+            "cache_write_reported_input_tokens": 84,
+            "reasoning_reported_output_tokens": 84
+          },
+          "known_cached_input_tokens": 2984704,
+          "known_input_tokens": 3121253,
+          "known_output_tokens": 9863,
+          "known_cache_write_input_tokens": 0,
+          "known_reasoning_output_tokens": 2815,
+          "cache_hit_rate": 0.9562518642352927,
+          "uncached_input_tokens": 136549,
+          "cost_usd": null,
+          "response_ids_complete": true,
+          "accounting": "reported-responses",
+          "usage_source": "provider-raw",
+          "request_attempts": 84,
+          "rejected_attempts": 0,
+          "unresolved_attempts": [],
+          "audit_complete": true
+        },
+        "codex_usage": {
+          "accounting": "reported-responses",
+          "audit_complete": true,
+          "cache_hit_rate": 0.9760161493873662,
+          "cache_reported_input_tokens": 2168751,
+          "cache_write_input_tokens": 0,
+          "cache_write_reported_input_tokens": 2168751,
+          "cached_input_tokens": 2116736,
+          "completed_turns": 1,
+          "cost_usd": null,
+          "failed_turns": 0,
+          "input_tokens": 2168751,
+          "known_cache_write_input_tokens": 0,
+          "known_cached_input_tokens": 2116736,
+          "known_input_tokens": 2168751,
+          "known_output_tokens": 10358,
+          "known_reasoning_output_tokens": 2642,
+          "output_tokens": 10358,
+          "reasoning_output_tokens": 2642,
+          "reasoning_reported_output_tokens": 10358,
+          "reported_responses": {
+            "cache_reported_input_tokens": 51,
+            "cache_write_input_tokens": 51,
+            "cache_write_reported_input_tokens": 51,
+            "cached_input_tokens": 51,
+            "input_tokens": 51,
+            "output_tokens": 51,
+            "reasoning_output_tokens": 51,
+            "reasoning_reported_output_tokens": 51
+          },
+          "response_count": 51,
+          "response_ids_complete": true,
+          "schema": "rlebench/token-usage/1",
+          "source": "Codex token_usage_record per response",
+          "uncached_input_tokens": 52015,
+          "unidentified_usage_records": 0
+        }
+      }
+    ],
     "limits": [
       "One independent seed-0 attempt per agent and task; wall time is not a controlled speed comparison.",
       "Pending or interrupted attempts are not imputed as successes or failures."

data.json

@@ -6,7 +6,7 @@
   "benchmark_complete": false,
   "edition": "codex-simple-astra-high-seed0",
   "created_at": "2026-10-09T00:09:42.989838+00:00",
-  "updated_at": "2026-10-09T23:28:20.523018+00:00",
+  "updated_at": "2026-10-09T23:47:35.140643+00:00",
   "model": "gpt-6-astra",
   "effort": "high",
   "seed": 0,
@@ -129,8 +129,8 @@
     "new_evaluations": 2,
     "progress": {
       "finished": 74,
-      "running": 2,
-      "queued": 8,
+      "running": 3,
+      "queued": 7,
       "needs_review": 0,
       "interrupted": 0,
       "native_successes": 50,
@@ -6425,7 +6425,7 @@
       "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.",
       "status": "pending",
       "episode_id": null,
-      "run_status": "queued",
+      "run_status": "running",
       "status_note": "Authorized episode is queued, running, or awaiting evidence review.",
       "planned_protocol": {
         "episodes": 1,

manifest.json

@@ -18,8 +18,8 @@
   "catalog/task06/10-open-faucet/task.yaml": "6752776b0ebe55a237ee8cc08c4341cbb8561936afd1b63079e800f7add1f91c",
   "catalog/task06/11-push-office-chair/task.yaml": "ab19bf9b7cec8992e2dcc1fb30ad9f6ce0f8343063b44777b2b68afc39d7f426",
   "catalog/task06/12-open-trash-can/task.yaml": "8d8b4b0ec848f91283e73416d257f5b0f816ac4b872fccd10b213e74d953085c",
-  "comparison.json": "dd9f47f2432cb870f8b174e93e96d753c04b88535da99eefdb4d916c0a3f1c06",
-  "data.json": "c8a833de78ff76a2e61574b8914a447bd961de0becb3984bb595bd936460e4dd",
+  "comparison.json": "d33c23b55c4b646ceb29f63bd77435bdd7dd3fca2dbe890b4b810602919c36df",
+  "data.json": "d5cc695c69f7aebd19450638a773811b0ce6b4999f388d46e54c9278c0e13277",
   "episodes.csv": "47ca70e21d4885e51cbba42ef44fb13adb711f6cf3f38c61756e3b8b61f1fd86",
   "episodes.json": "1c831cb223f2f14009b47ef8d439c25a69745a0c64a2c26fc574744497ed0608",
   "failure-reviews.json": "c66a997c9ecd6f7f43f130a8a33cd64c7a5b2049033e9ccde653ee30e4796f23",
@@ -32,6 +32,6 @@
   "licenses/robosuite-LICENSE.txt": "177978cbece0a4c454c2aaec5b3f145b39270814874c43109da9e829c39d9cba",
   "protocol.json": "11cbed51e7428bce14397c11ad7cb1420e02eb4010bfe51b29aec88e235cac05",
   "style.css": "dd31b30951c220ebb82bc96224b5e640913945679e42b2e0a42e2c373a329e6c",
-  "summary.json": "f55eda1feb28e219b64f275dfb57fd14bbc2216cfd1513d0db3512aad2b53747",
-  "task-index.json": "bfe5aacbe86e63cf135b2c61e7080a907e9f38d92e09b2a68222f41b878f02d3"
+  "summary.json": "ee7d11651f64a4e8a7f93328d665b8cb416f56b590dd6c8e113e0313f482377f",
+  "task-index.json": "e4aa3cc87e1bbee12248f7b5b18151fa999686f4bec446765312f952c1918b4a"
 }

publication-manifest.json

@@ -78,12 +78,12 @@
       "bytes": 319
     },
     "comparison.json": {
-      "sha256": "dd9f47f2432cb870f8b174e93e96d753c04b88535da99eefdb4d916c0a3f1c06",
-      "bytes": 118378
+      "sha256": "d33c23b55c4b646ceb29f63bd77435bdd7dd3fca2dbe890b4b810602919c36df",
+      "bytes": 121628
     },
     "data.json": {
-      "sha256": "c8a833de78ff76a2e61574b8914a447bd961de0becb3984bb595bd936460e4dd",
-      "bytes": 1094645
+      "sha256": "d5cc695c69f7aebd19450638a773811b0ce6b4999f388d46e54c9278c0e13277",
+      "bytes": 1094646
     },
     "episodes.csv": {
       "sha256": "47ca70e21d4885e51cbba42ef44fb13adb711f6cf3f38c61756e3b8b61f1fd86",
@@ -134,15 +134,15 @@
       "bytes": 16560
     },
     "summary.json": {
-      "sha256": "f55eda1feb28e219b64f275dfb57fd14bbc2216cfd1513d0db3512aad2b53747",
+      "sha256": "ee7d11651f64a4e8a7f93328d665b8cb416f56b590dd6c8e113e0313f482377f",
       "bytes": 3205
     },
     "task-index.json": {
-      "sha256": "bfe5aacbe86e63cf135b2c61e7080a907e9f38d92e09b2a68222f41b878f02d3",
-      "bytes": 229773
+      "sha256": "e4aa3cc87e1bbee12248f7b5b18151fa999686f4bec446765312f952c1918b4a",
+      "bytes": 229774
     },
     "manifest.json": {
-      "sha256": "c5d5a5aeb3cec68e8da8dc6058beb0efa7c8f38a849e7a43d3b54b47b4b11abd",
+      "sha256": "ff1f8910b33e31eacfa104c812a12bfe23c6d576877204c48381fe4929d8ecfb",
       "bytes": 3481
     }
   }

summary.json

@@ -112,8 +112,8 @@
   "new_evaluations": 2,
   "progress": {
     "finished": 74,
-    "running": 2,
-    "queued": 8,
+    "running": 3,
+    "queued": 7,
     "needs_review": 0,
     "interrupted": 0,
     "native_successes": 50,

task-index.json

@@ -6174,7 +6174,7 @@
     "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.",
     "status": "pending",
     "episode_id": null,
-    "run_status": "queued",
+    "run_status": "running",
     "status_note": "Authorized episode is queued, running, or awaiting evidence review.",
     "planned_protocol": {
       "episodes": 1,