Publish codex SIMPLE seed-0 evaluation evidence

README.md

@@ -10,7 +10,7 @@ pinned: false
 
 # Codex Benchmark
 
-81 published results across 5 simulator families. This snapshot adds 9 of 12 authorized SIMPLE G1 results.
+82 published results across 5 simulator families. This snapshot adds 10 of 12 authorized SIMPLE G1 results.
 
 | Environment | Published / selected | Native successes / valid results |
 | --- | ---: | ---: |
@@ -18,7 +18,7 @@ pinned: false
 | LIBERO Long | 10 / 10 | 9 / 10 |
 | RoboTwin | 10 / 10 | 7 / 10 |
 | RoboDojo | 42 / 42 | 31 / 42 |
-| SIMPLE G1 | 9 / 12 | 3 / 9 |
+| SIMPLE G1 | 10 / 12 | 4 / 10 |
 
 SIMPLE uses GPT-6 Astra with high reasoning effort and default service tier, one fresh seed-0 episode per task, 10000 native control intervals for retained slots 01-08 and 15000 for restarted slots 09-12, at 50 Hz, stepped MuJoCo physics with Isaac rendering, domain randomization level 0, arm gravity compensation, stop-on-success, recording and an 8-hour wall-clock timeout. Request retries remain within the same session; whole episodes are never retried automatically.
 

REPORT.md

@@ -1,6 +1,6 @@
 # Codex Benchmark
 
-81 published results across 5 simulator families. This snapshot adds 9 of 12 authorized SIMPLE G1 results.
+82 published results across 5 simulator families. This snapshot adds 10 of 12 authorized SIMPLE G1 results.
 
 | Environment | Published / selected | Native successes / valid results |
 | --- | ---: | ---: |
@@ -8,7 +8,7 @@
 | LIBERO Long | 10 / 10 | 9 / 10 |
 | RoboTwin | 10 / 10 | 7 / 10 |
 | RoboDojo | 42 / 42 | 31 / 42 |
-| SIMPLE G1 | 9 / 12 | 3 / 9 |
+| SIMPLE G1 | 10 / 12 | 4 / 10 |
 
 SIMPLE uses GPT-6 Astra with high reasoning effort and default service tier, one fresh seed-0 episode per task, 10000 native control intervals for retained slots 01-08 and 15000 for restarted slots 09-12, at 50 Hz, stepped MuJoCo physics with Isaac rendering, domain randomization level 0, arm gravity compensation, stop-on-success, recording and an 8-hour wall-clock timeout. Request retries remain within the same session; whole episodes are never retried automatically.
 

comparison.json

@@ -3855,6 +3855,91 @@
           "uncached_input_tokens": 46590,
           "unidentified_usage_records": 0
         }
+      },
+      {
+        "task_key": "task06/11",
+        "kinex_success": true,
+        "codex_success": true,
+        "same_instruction": true,
+        "same_budget": true,
+        "same_roboenv_source": true,
+        "kinex_steps": 360,
+        "codex_steps": 581,
+        "kinex_usage": {
+          "input_tokens": 333016,
+          "output_tokens": 1394,
+          "cached_input_tokens": 271872,
+          "cache_write_input_tokens": 0,
+          "reasoning_output_tokens": 93,
+          "cache_reported_input_tokens": 333016,
+          "cache_write_reported_input_tokens": 333016,
+          "reasoning_reported_output_tokens": 1394,
+          "schema": "rlebench/token-usage/1",
+          "source": "Provider terminal response usage",
+          "response_count": 19,
+          "reported_responses": {
+            "input_tokens": 19,
+            "output_tokens": 19,
+            "cached_input_tokens": 19,
+            "cache_write_input_tokens": 19,
+            "reasoning_output_tokens": 19,
+            "cache_reported_input_tokens": 19,
+            "cache_write_reported_input_tokens": 19,
+            "reasoning_reported_output_tokens": 19
+          },
+          "known_cached_input_tokens": 271872,
+          "known_input_tokens": 333016,
+          "known_output_tokens": 1394,
+          "known_cache_write_input_tokens": 0,
+          "known_reasoning_output_tokens": 93,
+          "cache_hit_rate": 0.8163932063324285,
+          "uncached_input_tokens": 61144,
+          "cost_usd": null,
+          "response_ids_complete": true,
+          "accounting": "reported-responses",
+          "usage_source": "provider-raw",
+          "request_attempts": 19,
+          "rejected_attempts": 0,
+          "unresolved_attempts": [],
+          "audit_complete": true
+        },
+        "codex_usage": {
+          "accounting": "reported-responses",
+          "audit_complete": true,
+          "cache_hit_rate": 0.9631537448295527,
+          "cache_reported_input_tokens": 866221,
+          "cache_write_input_tokens": 0,
+          "cache_write_reported_input_tokens": 866221,
+          "cached_input_tokens": 834304,
+          "completed_turns": 1,
+          "cost_usd": null,
+          "failed_turns": 0,
+          "input_tokens": 866221,
+          "known_cache_write_input_tokens": 0,
+          "known_cached_input_tokens": 834304,
+          "known_input_tokens": 866221,
+          "known_output_tokens": 2765,
+          "known_reasoning_output_tokens": 543,
+          "output_tokens": 2765,
+          "reasoning_output_tokens": 543,
+          "reasoning_reported_output_tokens": 2765,
+          "reported_responses": {
+            "cache_reported_input_tokens": 28,
+            "cache_write_input_tokens": 28,
+            "cache_write_reported_input_tokens": 28,
+            "cached_input_tokens": 28,
+            "input_tokens": 28,
+            "output_tokens": 28,
+            "reasoning_output_tokens": 28,
+            "reasoning_reported_output_tokens": 28
+          },
+          "response_count": 28,
+          "response_ids_complete": true,
+          "schema": "rlebench/token-usage/1",
+          "source": "Codex token_usage_record per response",
+          "uncached_input_tokens": 31917,
+          "unidentified_usage_records": 0
+        }
       }
     ],
     "limits": [

data.json

@@ -6,7 +6,7 @@
   "benchmark_complete": false,
   "edition": "codex-simple-astra-high-seed0",
   "created_at": "2026-10-09T00:09:42.989838+00:00",
-  "updated_at": "2026-10-10T02:13:06.874108+00:00",
+  "updated_at": "2026-10-10T02:19:29.611615+00:00",
   "model": "gpt-6-astra",
   "effort": "high",
   "seed": 0,
@@ -17,8 +17,8 @@
   },
   "summary": {
     "planned_tasks": 84,
-    "published_results": 81,
-    "pending_tasks": 3,
+    "published_results": 82,
+    "pending_tasks": 2,
     "families": [
       {
         "id": "task01",
@@ -105,12 +105,12 @@
         "id": "task06",
         "name": "SIMPLE G1",
         "total": 12,
-        "completed": 9,
-        "pending": 3,
-        "successes": 3,
-        "valid_results": 9,
-        "success_rate": 0.3333333333333333,
-        "usage_complete": 9,
+        "completed": 10,
+        "pending": 2,
+        "successes": 4,
+        "valid_results": 10,
+        "success_rate": 0.4,
+        "usage_complete": 10,
         "control_frequency_hz": 50,
         "max_control_steps": null,
         "max_control_steps_by_task": {
@@ -128,31 +128,31 @@
           "task06/12": 15000
         },
         "preflight_results": 0,
-        "formal_results": 9,
+        "formal_results": 10,
         "modified_results": 0,
-        "original_results": 9,
-        "input_tokens": 109041603,
-        "cached_input_tokens": 107832320,
-        "output_tokens": 286623
+        "original_results": 10,
+        "input_tokens": 111703380,
+        "cached_input_tokens": 110436224,
+        "output_tokens": 298235
       }
     ],
     "interrupted_attempts": 4,
     "usage_incomplete_tasks": [],
     "execution_incomplete_tasks": [],
     "deferred_tasks": [],
-    "new_evaluations": 9,
+    "new_evaluations": 10,
     "progress": {
-      "finished": 81,
+      "finished": 82,
       "running": 2,
-      "queued": 1,
+      "queued": 0,
       "needs_review": 0,
       "interrupted": 0,
-      "native_successes": 52,
+      "native_successes": 53,
       "native_failures": 29
     },
     "simple_campaign": {
       "planned": 12,
-      "published": 9,
+      "published": 10,
       "agent": "codex",
       "model": "gpt-6-astra",
       "effort": "high",
@@ -246,12 +246,12 @@
       "id": "task06",
       "name": "SIMPLE G1",
       "total": 12,
-      "completed": 9,
-      "pending": 3,
-      "successes": 3,
-      "valid_results": 9,
-      "success_rate": 0.3333333333333333,
-      "usage_complete": 9,
+      "completed": 10,
+      "pending": 2,
+      "successes": 4,
+      "valid_results": 10,
+      "success_rate": 0.4,
+      "usage_complete": 10,
       "control_frequency_hz": 50,
       "max_control_steps": null,
       "max_control_steps_by_task": {
@@ -269,12 +269,12 @@
         "task06/12": 15000
       },
       "preflight_results": 0,
-      "formal_results": 9,
+      "formal_results": 10,
       "modified_results": 0,
-      "original_results": 9,
-      "input_tokens": 109041603,
-      "cached_input_tokens": 107832320,
-      "output_tokens": 286623
+      "original_results": 10,
+      "input_tokens": 111703380,
+      "cached_input_tokens": 110436224,
+      "output_tokens": 298235
     }
   ],
   "tasks": [
@@ -6727,10 +6727,10 @@
       "catalog_instruction": "turn the faucet",
       "native_instruction": "turn the faucet",
       "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.",
-      "status": "pending",
-      "episode_id": null,
-      "run_status": "running",
-      "status_note": "Authorized episode is queued, running, or awaiting evidence review.",
+      "status": "completed",
+      "episode_id": "task06-10-seed0-formal",
+      "run_status": "finished",
+      "status_note": "Native result, execution and usage complete.",
       "planned_protocol": {
         "episodes": 1,
         "seed": 0,
@@ -6817,7 +6817,7 @@
       "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.",
       "status": "pending",
       "episode_id": null,
-      "run_status": "queued",
+      "run_status": "running",
       "status_note": "Authorized episode is queued, running, or awaiting evidence review.",
       "planned_protocol": {
         "episodes": 1,
@@ -27473,6 +27473,277 @@
       "selected_for_formal_metrics": true,
       "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/08/seed-0/"
     },
+    {
+      "id": "task06-10-seed0-formal",
+      "task_key": "task06/10",
+      "family": "task06",
+      "slot": "10",
+      "seed": 0,
+      "episode": 1,
+      "phase": "formal",
+      "status": "completed",
+      "success": true,
+      "native_reward": 1.0,
+      "valid": true,
+      "execution": {
+        "reason": null,
+        "status": "finished"
+      },
+      "verdict": {
+        "evidence_valid": true,
+        "steps": 1710,
+        "success": true,
+        "termination": "success"
+      },
+      "steps": 1710,
+      "simulation_time_s": null,
+      "wall_time_s": 559.043041,
+      "model": "gpt-6-astra",
+      "effort": "high",
+      "harness": "codex",
+      "codex_version": "0.160.0",
+      "native_instruction": "turn the faucet",
+      "instruction": "turn the faucet",
+      "instruction_policy": "original_native",
+      "usage": {
+        "accounting": "reported-responses",
+        "audit_complete": true,
+        "cache_hit_rate": 0.9782577578812951,
+        "cache_reported_input_tokens": 2661777,
+        "cache_write_input_tokens": 0,
+        "cache_write_reported_input_tokens": 2661777,
+        "cached_input_tokens": 2603904,
+        "completed_turns": 1,
+        "cost_usd": null,
+        "failed_turns": 0,
+        "input_tokens": 2661777,
+        "known_cache_write_input_tokens": 0,
+        "known_cached_input_tokens": 2603904,
+        "known_input_tokens": 2661777,
+        "known_output_tokens": 11612,
+        "known_reasoning_output_tokens": 4313,
+        "output_tokens": 11612,
+        "reasoning_output_tokens": 4313,
+        "reasoning_reported_output_tokens": 11612,
+        "reported_responses": {
+          "cache_reported_input_tokens": 62,
+          "cache_write_input_tokens": 62,
+          "cache_write_reported_input_tokens": 62,
+          "cached_input_tokens": 62,
+          "input_tokens": 62,
+          "output_tokens": 62,
+          "reasoning_output_tokens": 62,
+          "reasoning_reported_output_tokens": 62
+        },
+        "response_count": 62,
+        "response_ids_complete": true,
+        "schema": "rlebench/token-usage/1",
+        "source": "Codex token_usage_record per response",
+        "uncached_input_tokens": 57873,
+        "unidentified_usage_records": 0
+      },
+      "call_activity": {
+        "model_tool_calls": 61,
+        "model_tool_calls_by_name": {
+          "exec": 61
+        },
+        "nested_python_tool_invocations": null,
+        "python_device_rpc_attempts": null,
+        "python_device_rpc_attempts_by_action": null,
+        "python_device_rpc_errors": null,
+        "python_instrumented_model_tool_calls": null,
+        "python_tool_invocations": null,
+        "python_tool_invocations_by_origin": null,
+        "schema": "rlebench/call-activity/1",
+        "source": "Codex native sessions"
+      },
+      "media": {
+        "passed": true,
+        "width": 1280,
+        "height": 360,
+        "duration_s": 8.55,
+        "speed": 4,
+        "source_fps": 10,
+        "output_fps": 20,
+        "recording": {
+          "accepted_samples": 343,
+          "captured_samples": 343,
+          "clock": "simulation",
+          "dropped_samples": 0,
+          "encoded_frames": 343,
+          "end_time_s": 34.19999999999975,
+          "error": null,
+          "experimental": true,
+          "fps": 10,
+          "received_samples": 343,
+          "schema": "roboenv/recording/1",
+          "state": "closed",
+          "status": "complete",
+          "views": [
+            {
+              "fov_y": 45.0,
+              "height": 360,
+              "name": "camera_head_left",
+              "pose": null,
+              "source": "camera_head_left",
+              "width": 640
+            },
+            {
+              "fov_y": 45.0,
+              "height": 360,
+              "name": "camera_head_right",
+              "pose": null,
+              "source": "camera_head_right",
+              "width": 640
+            }
+          ]
+        },
+        "view_names": [
+          "camera_head_left",
+          "camera_head_right"
+        ],
+        "sha256": "9ff6b51ac3df1d951d61e2c082370f22da0d92c477af7f6d7457011910bb9f4c",
+        "scope": "Native spectator recording postprocessed at 4x; no replay or rerender"
+      },
+      "analysis": {
+        "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002",
+        "duration": "\u4f7f\u7528 1,710 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002",
+        "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002"
+      },
+      "provenance": {
+        "sources": {
+          "RLE-Bench-inhouse": {
+            "build_inputs": [
+              "pyproject.toml",
+              "src",
+              "tasks",
+              "README.md",
+              "Makefile",
+              "tests",
+              "docs"
+            ],
+            "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc",
+            "dirty": true,
+            "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618",
+            "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse"
+          },
+          "RoboEnv": {
+            "build_inputs": [
+              "pyproject.toml",
+              "README.md",
+              "src",
+              "runtime/pyproject.toml",
+              "runtime/README.md",
+              "runtime/src",
+              "runtime/environments.json",
+              "runtime/locks",
+              "catalog",
+              "upstreams.lock.json",
+              "third_party/patches",
+              "docs/validation"
+            ],
+            "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2",
+            "dirty": true,
+            "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92",
+            "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv"
+          },
+          "kinex": {
+            "build_inputs": [
+              "package.json",
+              "package-lock.json",
+              ".nvmrc",
+              "tsconfig.json",
+              "VERSION",
+              "src",
+              "packages/core",
+              "packages/setup",
+              "script",
+              "assets",
+              "bin"
+            ],
+            "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38",
+            "dirty": false,
+            "revision": "caac19a8a36272972f762e0f74cfe381e8500048",
+            "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex"
+          }
+        },
+        "job": "task06-10-open-faucet-codex-seed0-attempt03",
+        "attempt": 3,
+        "harness": "stock Codex CLI",
+        "control_interface": "public RoboEnv SDK and CLI",
+        "kinex_agent_runtime_used": false,
+        "gpu_index": 6,
+        "gpu_model": "NVIDIA L40S",
+        "campaign_dispatch_concurrency_limit": 8,
+        "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.",
+        "codex_version": "0.160.0",
+        "model": "gpt-6-astra",
+        "effort": "high",
+        "service_tier": "default",
+        "fresh_session": true,
+        "source_jobs": [],
+        "resume_trajectory": false,
+        "imported_skills": [],
+        "automatic_harbor_retries": 0,
+        "request_policy": {
+          "max_request_retries": 50,
+          "configuration": "explicit retry50 SSE",
+          "usage_accounting": "reported-responses"
+        },
+        "classification": "formal",
+        "measured_images": {
+          "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217",
+          "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957"
+        },
+        "session_original_sha256": "b986ecabd7f40cc38364b5a3bbf7316dcc9e4f2aa2aa963bcedebacc501fbf8c",
+        "protocol_sha256": "3a6b2b718e41c395dbf3c891d8ed4164aa605473a8340f445d6ce59b9da0899a",
+        "deployment": "steps15000-20261010T012736Z",
+        "max_steps": 15000,
+        "budget_revision": "steps15000-20261010T012736Z",
+        "same_protocol_replacement": false,
+        "replaces_user_interrupted_attempt": "task06-10-open-faucet-codex-seed0-attempt01",
+        "replacement_reason": "User requested 300 seconds at 50 Hz with updated code."
+      },
+      "links": {
+        "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/agent/session.jsonl",
+        "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/agent/trajectory.json",
+        "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/agent/provider-usage.jsonl",
+        "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/transcript.json",
+        "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/verifier/episode.json",
+        "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/verifier/protocol.json",
+        "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/instructions.json",
+        "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/verifier/workspace.tar.gz",
+        "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/verifier/workspace.json",
+        "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl",
+        "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/evidence/episode/evidence/recording/manifest.json",
+        "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/usage.json",
+        "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/analysis.json",
+        "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/provenance.json",
+        "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/video.mp4",
+        "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/poster.jpg",
+        "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/media-validation.json",
+        "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/evidence/episode/evidence/final-observation.json"
+      },
+      "resources": [
+        {
+          "name": "tools/g1_control.py",
+          "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/resources/tools/g1_control.py",
+          "kind": "Created during this episode; final workspace snapshot."
+        },
+        {
+          "name": "memos/g1_faucet.md",
+          "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/resources/memos/g1_faucet.md",
+          "kind": "Created during this episode; final workspace snapshot."
+        }
+      ],
+      "session_counts": {
+        "visible_events": 138,
+        "observed_images": 30,
+        "tool_errors": 2
+      },
+      "selected_for_formal_metrics": true,
+      "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/"
+    },
     {
       "id": "task06-11-seed0-formal",
       "task_key": "task06/11",

episodes.csv

@@ -80,6 +80,6 @@ task06/06,simple/handover,completed,0,True,False,4040,1176.656816,5562825,547558
 task06/07,simple/pick-and-place-and-hug-container,completed,0,True,False,8645,2551.587007,16738844,16525440,38724
 task06/08,simple/close-door,completed,0,True,True,1277,369.21498,1340798,1294208,6729
 task06/09,simple/open-oven,pending,0,,,,,,,
-task06/10,simple/open-faucet,pending,0,,,,,,,
+task06/10,simple/open-faucet,completed,0,True,True,1710,559.043041,2661777,2603904,11612
 task06/11,simple/push-office-chair,completed,0,True,True,581,203.012415,866221,834304,2765
 task06/12,simple/open-trash-can,pending,0,,,,,,,

episodes.json

@@ -20621,6 +20621,277 @@
     "selected_for_formal_metrics": true,
     "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/08/seed-0/"
   },
+  {
+    "id": "task06-10-seed0-formal",
+    "task_key": "task06/10",
+    "family": "task06",
+    "slot": "10",
+    "seed": 0,
+    "episode": 1,
+    "phase": "formal",
+    "status": "completed",
+    "success": true,
+    "native_reward": 1.0,
+    "valid": true,
+    "execution": {
+      "reason": null,
+      "status": "finished"
+    },
+    "verdict": {
+      "evidence_valid": true,
+      "steps": 1710,
+      "success": true,
+      "termination": "success"
+    },
+    "steps": 1710,
+    "simulation_time_s": null,
+    "wall_time_s": 559.043041,
+    "model": "gpt-6-astra",
+    "effort": "high",
+    "harness": "codex",
+    "codex_version": "0.160.0",
+    "native_instruction": "turn the faucet",
+    "instruction": "turn the faucet",
+    "instruction_policy": "original_native",
+    "usage": {
+      "accounting": "reported-responses",
+      "audit_complete": true,
+      "cache_hit_rate": 0.9782577578812951,
+      "cache_reported_input_tokens": 2661777,
+      "cache_write_input_tokens": 0,
+      "cache_write_reported_input_tokens": 2661777,
+      "cached_input_tokens": 2603904,
+      "completed_turns": 1,
+      "cost_usd": null,
+      "failed_turns": 0,
+      "input_tokens": 2661777,
+      "known_cache_write_input_tokens": 0,
+      "known_cached_input_tokens": 2603904,
+      "known_input_tokens": 2661777,
+      "known_output_tokens": 11612,
+      "known_reasoning_output_tokens": 4313,
+      "output_tokens": 11612,
+      "reasoning_output_tokens": 4313,
+      "reasoning_reported_output_tokens": 11612,
+      "reported_responses": {
+        "cache_reported_input_tokens": 62,
+        "cache_write_input_tokens": 62,
+        "cache_write_reported_input_tokens": 62,
+        "cached_input_tokens": 62,
+        "input_tokens": 62,
+        "output_tokens": 62,
+        "reasoning_output_tokens": 62,
+        "reasoning_reported_output_tokens": 62
+      },
+      "response_count": 62,
+      "response_ids_complete": true,
+      "schema": "rlebench/token-usage/1",
+      "source": "Codex token_usage_record per response",
+      "uncached_input_tokens": 57873,
+      "unidentified_usage_records": 0
+    },
+    "call_activity": {
+      "model_tool_calls": 61,
+      "model_tool_calls_by_name": {
+        "exec": 61
+      },
+      "nested_python_tool_invocations": null,
+      "python_device_rpc_attempts": null,
+      "python_device_rpc_attempts_by_action": null,
+      "python_device_rpc_errors": null,
+      "python_instrumented_model_tool_calls": null,
+      "python_tool_invocations": null,
+      "python_tool_invocations_by_origin": null,
+      "schema": "rlebench/call-activity/1",
+      "source": "Codex native sessions"
+    },
+    "media": {
+      "passed": true,
+      "width": 1280,
+      "height": 360,
+      "duration_s": 8.55,
+      "speed": 4,
+      "source_fps": 10,
+      "output_fps": 20,
+      "recording": {
+        "accepted_samples": 343,
+        "captured_samples": 343,
+        "clock": "simulation",
+        "dropped_samples": 0,
+        "encoded_frames": 343,
+        "end_time_s": 34.19999999999975,
+        "error": null,
+        "experimental": true,
+        "fps": 10,
+        "received_samples": 343,
+        "schema": "roboenv/recording/1",
+        "state": "closed",
+        "status": "complete",
+        "views": [
+          {
+            "fov_y": 45.0,
+            "height": 360,
+            "name": "camera_head_left",
+            "pose": null,
+            "source": "camera_head_left",
+            "width": 640
+          },
+          {
+            "fov_y": 45.0,
+            "height": 360,
+            "name": "camera_head_right",
+            "pose": null,
+            "source": "camera_head_right",
+            "width": 640
+          }
+        ]
+      },
+      "view_names": [
+        "camera_head_left",
+        "camera_head_right"
+      ],
+      "sha256": "9ff6b51ac3df1d951d61e2c082370f22da0d92c477af7f6d7457011910bb9f4c",
+      "scope": "Native spectator recording postprocessed at 4x; no replay or rerender"
+    },
+    "analysis": {
+      "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002",
+      "duration": "\u4f7f\u7528 1,710 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002",
+      "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002"
+    },
+    "provenance": {
+      "sources": {
+        "RLE-Bench-inhouse": {
+          "build_inputs": [
+            "pyproject.toml",
+            "src",
+            "tasks",
+            "README.md",
+            "Makefile",
+            "tests",
+            "docs"
+          ],
+          "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc",
+          "dirty": true,
+          "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618",
+          "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse"
+        },
+        "RoboEnv": {
+          "build_inputs": [
+            "pyproject.toml",
+            "README.md",
+            "src",
+            "runtime/pyproject.toml",
+            "runtime/README.md",
+            "runtime/src",
+            "runtime/environments.json",
+            "runtime/locks",
+            "catalog",
+            "upstreams.lock.json",
+            "third_party/patches",
+            "docs/validation"
+          ],
+          "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2",
+          "dirty": true,
+          "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92",
+          "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv"
+        },
+        "kinex": {
+          "build_inputs": [
+            "package.json",
+            "package-lock.json",
+            ".nvmrc",
+            "tsconfig.json",
+            "VERSION",
+            "src",
+            "packages/core",
+            "packages/setup",
+            "script",
+            "assets",
+            "bin"
+          ],
+          "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38",
+          "dirty": false,
+          "revision": "caac19a8a36272972f762e0f74cfe381e8500048",
+          "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex"
+        }
+      },
+      "job": "task06-10-open-faucet-codex-seed0-attempt03",
+      "attempt": 3,
+      "harness": "stock Codex CLI",
+      "control_interface": "public RoboEnv SDK and CLI",
+      "kinex_agent_runtime_used": false,
+      "gpu_index": 6,
+      "gpu_model": "NVIDIA L40S",
+      "campaign_dispatch_concurrency_limit": 8,
+      "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.",
+      "codex_version": "0.160.0",
+      "model": "gpt-6-astra",
+      "effort": "high",
+      "service_tier": "default",
+      "fresh_session": true,
+      "source_jobs": [],
+      "resume_trajectory": false,
+      "imported_skills": [],
+      "automatic_harbor_retries": 0,
+      "request_policy": {
+        "max_request_retries": 50,
+        "configuration": "explicit retry50 SSE",
+        "usage_accounting": "reported-responses"
+      },
+      "classification": "formal",
+      "measured_images": {
+        "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217",
+        "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957"
+      },
+      "session_original_sha256": "b986ecabd7f40cc38364b5a3bbf7316dcc9e4f2aa2aa963bcedebacc501fbf8c",
+      "protocol_sha256": "3a6b2b718e41c395dbf3c891d8ed4164aa605473a8340f445d6ce59b9da0899a",
+      "deployment": "steps15000-20261010T012736Z",
+      "max_steps": 15000,
+      "budget_revision": "steps15000-20261010T012736Z",
+      "same_protocol_replacement": false,
+      "replaces_user_interrupted_attempt": "task06-10-open-faucet-codex-seed0-attempt01",
+      "replacement_reason": "User requested 300 seconds at 50 Hz with updated code."
+    },
+    "links": {
+      "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/agent/session.jsonl",
+      "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/agent/trajectory.json",
+      "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/agent/provider-usage.jsonl",
+      "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/transcript.json",
+      "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/verifier/episode.json",
+      "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/verifier/protocol.json",
+      "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/instructions.json",
+      "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/verifier/workspace.tar.gz",
+      "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/verifier/workspace.json",
+      "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl",
+      "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/evidence/episode/evidence/recording/manifest.json",
+      "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/usage.json",
+      "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/analysis.json",
+      "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/provenance.json",
+      "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/video.mp4",
+      "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/poster.jpg",
+      "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/media-validation.json",
+      "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/native/evidence/episode/evidence/final-observation.json"
+    },
+    "resources": [
+      {
+        "name": "tools/g1_control.py",
+        "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/resources/tools/g1_control.py",
+        "kind": "Created during this episode; final workspace snapshot."
+      },
+      {
+        "name": "memos/g1_faucet.md",
+        "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/resources/memos/g1_faucet.md",
+        "kind": "Created during this episode; final workspace snapshot."
+      }
+    ],
+    "session_counts": {
+      "visible_events": 138,
+      "observed_images": 30,
+      "tool_errors": 2
+    },
+    "selected_for_formal_metrics": true,
+    "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/10/seed-0/"
+  },
   {
     "id": "task06-11-seed0-formal",
     "task_key": "task06/11",

manifest.json

@@ -1,8 +1,8 @@
 {
   ".gitattributes": "51dec59a208c8656ce7d6614b7b547022f7a4720b1cdad6a20aeeda717ec090b",
   "DATA_FORMAT.md": "e2fd4f6b5c8be9d9e008db7d0c1f5d52b419d68328a1c86625c9f67c49b03403",
-  "README.md": "423ef820f315112755a0398a6749f33670e61950512197e52774cd0197e3377f",
-  "REPORT.md": "94caaf09e5f6d4612a628cdc53c6aebbd888e44a96db0ee0d09c687d1df9d4b5",
+  "README.md": "417b51bd1cbbb244a0097800b1eed8ad3865574d7e23ba6ad6f768dea461ce62",
+  "REPORT.md": "a41a0b35aaef619b0c143401f76668f46baeb3b4eee4fb384b7c0efbd1f615b4",
   "THIRD_PARTY_NOTICES.md": "8aa8e3db100d7a570931c5d87f44f78705b42cf17e8147016b35e9aeddb06bba",
   "app.js": "eb013cc9d24382410953a1a8c81e446a6592a81b32887749d17bfc863936556b",
   "attempt-history.json": "8555f458c1dd362f5a07447342994fddf265ff23bbc40e9bd6f9984a892ddb4e",
@@ -18,10 +18,10 @@
   "catalog/task06/10-open-faucet/task.yaml": "dcdd2228bfd8de298da36798856f5ff59b523c7104118103530eb7a641b52197",
   "catalog/task06/11-push-office-chair/task.yaml": "0e2de1c50619b0c58fd939f58362dfc0630f34d4e19587fe4927a2c4e5c1d785",
   "catalog/task06/12-open-trash-can/task.yaml": "e02833927892caa9c40ed193348aab1858e6ae6208c4c04892dec8eae4ad45f0",
-  "comparison.json": "335e11fa9d1587419f7ac8c0b155949c93c79dd07dd1c7013230076993527f3c",
-  "data.json": "48d75dc97294b2261bf09e88f1717c25dcde3b95c72e41e221a7150b909ee381",
-  "episodes.csv": "5a4306424027c4bae82fc6aac2473936009e9800250b5ff141c4b2dfecca30e3",
-  "episodes.json": "31b466b539cd0e8728fad9eec406ac41277823a804d3d2e70cb2fd4bd5034aac",
+  "comparison.json": "3b560729d2aa6ea55d9aeeb77140deb81532c7840b6c295cde3dcfc5b3ab25a0",
+  "data.json": "7af5e24cfb7b92970bed37b970344c1e6b32fa38cfae08214c969e132a60ae98",
+  "episodes.csv": "ce616fb9d22e6fb83cebfc3801a99ac64c0ceb6c78a073ce1c29777dd2cec72e",
+  "episodes.json": "9c6b6bb08aa7f6849f8824f814f7be1b358d1e389b6f46151073f7baa047723e",
   "failure-reviews.json": "c66a997c9ecd6f7f43f130a8a33cd64c7a5b2049033e9ccde653ee30e4796f23",
   "favicon.svg": "46063791218655f70c828ef71a1232a34fd38181da938338badec929ad226819",
   "index.html": "d29e3b0133b9af6378b8b3ab7a6bc04f8f985eff46a409078c4f8e6a521edfc2",
@@ -30,8 +30,8 @@
   "licenses/RoboTwin-LICENSE.txt": "c695d421718e54e6f3a60858f0c601125c3ecc60c199332c15947360561e2a9f",
   "licenses/robocasa-LICENSE.txt": "5da18670b3f00c59847b1ded9c28dee59940d963b1e03b528b0108d9c5a09885",
   "licenses/robosuite-LICENSE.txt": "177978cbece0a4c454c2aaec5b3f145b39270814874c43109da9e829c39d9cba",
-  "protocol.json": "14bdd3c9d8658f9ccf6ab6370868d88aeafa22fe7001695ac85b59710cbbec09",
+  "protocol.json": "484a85398cf50dd9b2396fa0ac4b360a1e608334f0fab2894cceae2366a8ecfe",
   "style.css": "dd31b30951c220ebb82bc96224b5e640913945679e42b2e0a42e2c373a329e6c",
-  "summary.json": "6e342bf3dd5914fe59b38e35fe34b49324355dcdcc7c1ec4c47beadef145b427",
-  "task-index.json": "5717c1a7d2cb924c490ae10a462bb4eb59222fdac7f211eaa35cac752a206067"
+  "summary.json": "338c9a23f8e8ed4d26545a2f0f3a307ea9c2aee45357e1edfc9958f31eec6a88",
+  "task-index.json": "eb61a9e67d6b7631e3de4903e2c81c385d8eb5630c65f6a0df76aaaee96b199b"
 }

protocol.json

@@ -95,12 +95,12 @@
       "id": "task06",
       "name": "SIMPLE G1",
       "total": 12,
-      "completed": 9,
-      "pending": 3,
-      "successes": 3,
-      "valid_results": 9,
-      "success_rate": 0.3333333333333333,
-      "usage_complete": 9,
+      "completed": 10,
+      "pending": 2,
+      "successes": 4,
+      "valid_results": 10,
+      "success_rate": 0.4,
+      "usage_complete": 10,
       "control_frequency_hz": 50,
       "max_control_steps": null,
       "max_control_steps_by_task": {
@@ -118,12 +118,12 @@
         "task06/12": 15000
       },
       "preflight_results": 0,
-      "formal_results": 9,
+      "formal_results": 10,
       "modified_results": 0,
-      "original_results": 9,
-      "input_tokens": 109041603,
-      "cached_input_tokens": 107832320,
-      "output_tokens": 286623
+      "original_results": 10,
+      "input_tokens": 111703380,
+      "cached_input_tokens": 110436224,
+      "output_tokens": 298235
     }
   ],
   "usage_accounting": "reported-responses",

publication-manifest.json

@@ -10,12 +10,12 @@
       "bytes": 1129
     },
     "README.md": {
-      "sha256": "423ef820f315112755a0398a6749f33670e61950512197e52774cd0197e3377f",
-      "bytes": 1606
+      "sha256": "417b51bd1cbbb244a0097800b1eed8ad3865574d7e23ba6ad6f768dea461ce62",
+      "bytes": 1609
     },
     "REPORT.md": {
-      "sha256": "94caaf09e5f6d4612a628cdc53c6aebbd888e44a96db0ee0d09c687d1df9d4b5",
-      "bytes": 1483
+      "sha256": "a41a0b35aaef619b0c143401f76668f46baeb3b4eee4fb384b7c0efbd1f615b4",
+      "bytes": 1486
     },
     "THIRD_PARTY_NOTICES.md": {
       "sha256": "8aa8e3db100d7a570931c5d87f44f78705b42cf17e8147016b35e9aeddb06bba",
@@ -78,20 +78,20 @@
       "bytes": 338
     },
     "comparison.json": {
-      "sha256": "335e11fa9d1587419f7ac8c0b155949c93c79dd07dd1c7013230076993527f3c",
-      "bytes": 144729
+      "sha256": "3b560729d2aa6ea55d9aeeb77140deb81532c7840b6c295cde3dcfc5b3ab25a0",
+      "bytes": 147980
     },
     "data.json": {
-      "sha256": "48d75dc97294b2261bf09e88f1717c25dcde3b95c72e41e221a7150b909ee381",
-      "bytes": 1183693
+      "sha256": "7af5e24cfb7b92970bed37b970344c1e6b32fa38cfae08214c969e132a60ae98",
+      "bytes": 1195544
     },
     "episodes.csv": {
-      "sha256": "5a4306424027c4bae82fc6aac2473936009e9800250b5ff141c4b2dfecca30e3",
-      "bytes": 8150
+      "sha256": "ce616fb9d22e6fb83cebfc3801a99ac64c0ceb6c78a073ce1c29777dd2cec72e",
+      "bytes": 8193
     },
     "episodes.json": {
-      "sha256": "31b466b539cd0e8728fad9eec406ac41277823a804d3d2e70cb2fd4bd5034aac",
-      "bytes": 886284
+      "sha256": "9c6b6bb08aa7f6849f8824f814f7be1b358d1e389b6f46151073f7baa047723e",
+      "bytes": 897610
     },
     "failure-reviews.json": {
       "sha256": "c66a997c9ecd6f7f43f130a8a33cd64c7a5b2049033e9ccde653ee30e4796f23",
@@ -126,23 +126,23 @@
       "bytes": 1474
     },
     "protocol.json": {
-      "sha256": "14bdd3c9d8658f9ccf6ab6370868d88aeafa22fe7001695ac85b59710cbbec09",
-      "bytes": 6409
+      "sha256": "484a85398cf50dd9b2396fa0ac4b360a1e608334f0fab2894cceae2366a8ecfe",
+      "bytes": 6399
     },
     "style.css": {
       "sha256": "dd31b30951c220ebb82bc96224b5e640913945679e42b2e0a42e2c373a329e6c",
       "bytes": 16560
     },
     "summary.json": {
-      "sha256": "6e342bf3dd5914fe59b38e35fe34b49324355dcdcc7c1ec4c47beadef145b427",
-      "bytes": 3603
+      "sha256": "338c9a23f8e8ed4d26545a2f0f3a307ea9c2aee45357e1edfc9958f31eec6a88",
+      "bytes": 3595
     },
     "task-index.json": {
-      "sha256": "5717c1a7d2cb924c490ae10a462bb4eb59222fdac7f211eaa35cac752a206067",
-      "bytes": 234352
+      "sha256": "eb61a9e67d6b7631e3de4903e2c81c385d8eb5630c65f6a0df76aaaee96b199b",
+      "bytes": 234353
     },
     "manifest.json": {
-      "sha256": "82f001cc7a7773677390d8fb63b069a78df3a5eee995cde9ad5f4c20a4a738aa",
+      "sha256": "cecf1d15624d35354eada7697377e911e7273f265efa5080ad64c5a8c92b4f23",
       "bytes": 3481
     }
   }

summary.json

@@ -1,7 +1,7 @@
 {
   "planned_tasks": 84,
-  "published_results": 81,
-  "pending_tasks": 3,
+  "published_results": 82,
+  "pending_tasks": 2,
   "families": [
     {
       "id": "task01",
@@ -88,12 +88,12 @@
       "id": "task06",
       "name": "SIMPLE G1",
       "total": 12,
-      "completed": 9,
-      "pending": 3,
-      "successes": 3,
-      "valid_results": 9,
-      "success_rate": 0.3333333333333333,
-      "usage_complete": 9,
+      "completed": 10,
+      "pending": 2,
+      "successes": 4,
+      "valid_results": 10,
+      "success_rate": 0.4,
+      "usage_complete": 10,
       "control_frequency_hz": 50,
       "max_control_steps": null,
       "max_control_steps_by_task": {
@@ -111,31 +111,31 @@
         "task06/12": 15000
       },
       "preflight_results": 0,
-      "formal_results": 9,
+      "formal_results": 10,
       "modified_results": 0,
-      "original_results": 9,
-      "input_tokens": 109041603,
-      "cached_input_tokens": 107832320,
-      "output_tokens": 286623
+      "original_results": 10,
+      "input_tokens": 111703380,
+      "cached_input_tokens": 110436224,
+      "output_tokens": 298235
     }
   ],
   "interrupted_attempts": 4,
   "usage_incomplete_tasks": [],
   "execution_incomplete_tasks": [],
   "deferred_tasks": [],
-  "new_evaluations": 9,
+  "new_evaluations": 10,
   "progress": {
-    "finished": 81,
+    "finished": 82,
     "running": 2,
-    "queued": 1,
+    "queued": 0,
     "needs_review": 0,
     "interrupted": 0,
-    "native_successes": 52,
+    "native_successes": 53,
     "native_failures": 29
   },
   "simple_campaign": {
     "planned": 12,
-    "published": 9,
+    "published": 10,
     "agent": "codex",
     "model": "gpt-6-astra",
     "effort": "high",

task-index.json

@@ -6448,10 +6448,10 @@
     "catalog_instruction": "turn the faucet",
     "native_instruction": "turn the faucet",
     "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.",
-    "status": "pending",
-    "episode_id": null,
-    "run_status": "running",
-    "status_note": "Authorized episode is queued, running, or awaiting evidence review.",
+    "status": "completed",
+    "episode_id": "task06-10-seed0-formal",
+    "run_status": "finished",
+    "status_note": "Native result, execution and usage complete.",
     "planned_protocol": {
       "episodes": 1,
       "seed": 0,
@@ -6538,7 +6538,7 @@
     "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.",
     "status": "pending",
     "episode_id": null,
-    "run_status": "queued",
+    "run_status": "running",
     "status_note": "Authorized episode is queued, running, or awaiting evidence review.",
     "planned_protocol": {
       "episodes": 1,