Publish codex SIMPLE seed-0 evaluation evidence

README.md

@@ -10,7 +10,7 @@ pinned: false
 
 # Codex Benchmark
 
-79 published results across 5 simulator families. This snapshot adds 7 of 12 authorized SIMPLE G1 results.
+80 published results across 5 simulator families. This snapshot adds 8 of 12 authorized SIMPLE G1 results.
 
 | Environment | Published / selected | Native successes / valid results |
 | --- | ---: | ---: |
@@ -18,7 +18,7 @@ pinned: false
 | LIBERO Long | 10 / 10 | 9 / 10 |
 | RoboTwin | 10 / 10 | 7 / 10 |
 | RoboDojo | 42 / 42 | 31 / 42 |
-| SIMPLE G1 | 7 / 12 | 2 / 7 |
+| SIMPLE G1 | 8 / 12 | 2 / 8 |
 
 SIMPLE uses GPT-6 Astra with high reasoning effort and default service tier, one fresh seed-0 episode per task, 10000 native control intervals at 50 Hz, stepped MuJoCo physics with Isaac rendering, domain randomization level 0, arm gravity compensation, stop-on-success, recording and an 8-hour wall-clock timeout. Request retries remain within the same session; whole episodes are never retried automatically.
 

REPORT.md

@@ -1,6 +1,6 @@
 # Codex Benchmark
 
-79 published results across 5 simulator families. This snapshot adds 7 of 12 authorized SIMPLE G1 results.
+80 published results across 5 simulator families. This snapshot adds 8 of 12 authorized SIMPLE G1 results.
 
 | Environment | Published / selected | Native successes / valid results |
 | --- | ---: | ---: |
@@ -8,7 +8,7 @@
 | LIBERO Long | 10 / 10 | 9 / 10 |
 | RoboTwin | 10 / 10 | 7 / 10 |
 | RoboDojo | 42 / 42 | 31 / 42 |
-| SIMPLE G1 | 7 / 12 | 2 / 7 |
+| SIMPLE G1 | 8 / 12 | 2 / 8 |
 
 SIMPLE uses GPT-6 Astra with high reasoning effort and default service tier, one fresh seed-0 episode per task, 10000 native control intervals at 50 Hz, stepped MuJoCo physics with Isaac rendering, domain randomization level 0, arm gravity compensation, stop-on-success, recording and an 8-hour wall-clock timeout. Request retries remain within the same session; whole episodes are never retried automatically.
 

data.json

@@ -6,7 +6,7 @@
   "benchmark_complete": false,
   "edition": "codex-simple-astra-high-seed0",
   "created_at": "2026-10-09T00:09:42.989838+00:00",
-  "updated_at": "2026-10-10T01:18:00.915385+00:00",
+  "updated_at": "2026-10-10T01:24:39.237016+00:00",
   "model": "gpt-6-astra",
   "effort": "high",
   "seed": 0,
@@ -17,8 +17,8 @@
   },
   "summary": {
     "planned_tasks": 84,
-    "published_results": 79,
-    "pending_tasks": 5,
+    "published_results": 80,
+    "pending_tasks": 4,
     "families": [
       {
         "id": "task01",
@@ -105,40 +105,40 @@
         "id": "task06",
         "name": "SIMPLE G1",
         "total": 12,
-        "completed": 7,
-        "pending": 5,
+        "completed": 8,
+        "pending": 4,
         "successes": 2,
-        "valid_results": 7,
-        "success_rate": 0.2857142857142857,
-        "usage_complete": 7,
+        "valid_results": 8,
+        "success_rate": 0.25,
+        "usage_complete": 8,
         "control_frequency_hz": 50,
         "max_control_steps": 10000,
         "preflight_results": 0,
-        "formal_results": 7,
+        "formal_results": 8,
         "modified_results": 0,
-        "original_results": 7,
-        "input_tokens": 91436538,
-        "cached_input_tokens": 90472576,
-        "output_tokens": 245134
+        "original_results": 8,
+        "input_tokens": 108175382,
+        "cached_input_tokens": 106998016,
+        "output_tokens": 283858
       }
     ],
     "interrupted_attempts": 4,
     "usage_incomplete_tasks": [],
     "execution_incomplete_tasks": [],
     "deferred_tasks": [],
-    "new_evaluations": 7,
+    "new_evaluations": 8,
     "progress": {
-      "finished": 79,
+      "finished": 80,
       "running": 3,
-      "queued": 2,
+      "queued": 1,
       "needs_review": 0,
       "interrupted": 0,
       "native_successes": 51,
-      "native_failures": 28
+      "native_failures": 29
     },
     "simple_campaign": {
       "planned": 12,
-      "published": 7,
+      "published": 8,
       "agent": "codex",
       "model": "gpt-6-astra",
       "effort": "high",
@@ -232,21 +232,21 @@
       "id": "task06",
       "name": "SIMPLE G1",
       "total": 12,
-      "completed": 7,
-      "pending": 5,
+      "completed": 8,
+      "pending": 4,
       "successes": 2,
-      "valid_results": 7,
-      "success_rate": 0.2857142857142857,
-      "usage_complete": 7,
+      "valid_results": 8,
+      "success_rate": 0.25,
+      "usage_complete": 8,
       "control_frequency_hz": 50,
       "max_control_steps": 10000,
       "preflight_results": 0,
-      "formal_results": 7,
+      "formal_results": 8,
       "modified_results": 0,
-      "original_results": 7,
-      "input_tokens": 91436538,
-      "cached_input_tokens": 90472576,
-      "output_tokens": 245134
+      "original_results": 8,
+      "input_tokens": 108175382,
+      "cached_input_tokens": 106998016,
+      "output_tokens": 283858
     }
   ],
   "tasks": [
@@ -6483,10 +6483,10 @@
       "catalog_instruction": "pick up the apple from table1,hug the container,walk to table2,and place  on table2.",
       "native_instruction": "pick up the apple from table1,hug the container,walk to table2,and place  on table2.",
       "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.",
-      "status": "pending",
-      "episode_id": null,
-      "run_status": "running",
-      "status_note": "Authorized episode is queued, running, or awaiting evidence review.",
+      "status": "completed",
+      "episode_id": "task06-07-seed0-formal",
+      "run_status": "finished",
+      "status_note": "Native result, execution and usage complete.",
       "planned_protocol": {
         "episodes": 1,
         "seed": 0,
@@ -6605,7 +6605,7 @@
       "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.",
       "status": "pending",
       "episode_id": null,
-      "run_status": "queued",
+      "run_status": "running",
       "status_note": "Authorized episode is queued, running, or awaiting evidence review.",
       "planned_protocol": {
         "episodes": 1,
@@ -26741,6 +26741,272 @@
       "selected_for_formal_metrics": true,
       "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/06/seed-0/"
     },
+    {
+      "id": "task06-07-seed0-formal",
+      "task_key": "task06/07",
+      "family": "task06",
+      "slot": "07",
+      "seed": 0,
+      "episode": 1,
+      "phase": "formal",
+      "status": "completed",
+      "success": false,
+      "native_reward": 0.0,
+      "valid": true,
+      "execution": {
+        "reason": null,
+        "status": "finished"
+      },
+      "verdict": {
+        "evidence_valid": true,
+        "steps": 8645,
+        "success": false,
+        "termination": "stopped"
+      },
+      "steps": 8645,
+      "simulation_time_s": null,
+      "wall_time_s": 2551.587007,
+      "model": "gpt-6-astra",
+      "effort": "high",
+      "harness": "codex",
+      "codex_version": "0.160.0",
+      "native_instruction": "pick up the apple from table1,hug the container,walk to table2,and place  on table2.",
+      "instruction": "pick up the apple from table1,hug the container,walk to table2,and place  on table2.",
+      "instruction_policy": "original_native",
+      "usage": {
+        "accounting": "reported-responses",
+        "audit_complete": true,
+        "cache_hit_rate": 0.9872509714529868,
+        "cache_reported_input_tokens": 16738844,
+        "cache_write_input_tokens": 0,
+        "cache_write_reported_input_tokens": 16738844,
+        "cached_input_tokens": 16525440,
+        "completed_turns": 1,
+        "cost_usd": null,
+        "failed_turns": 0,
+        "input_tokens": 16738844,
+        "known_cache_write_input_tokens": 0,
+        "known_cached_input_tokens": 16525440,
+        "known_input_tokens": 16738844,
+        "known_output_tokens": 38724,
+        "known_reasoning_output_tokens": 20279,
+        "output_tokens": 38724,
+        "reasoning_output_tokens": 20279,
+        "reasoning_reported_output_tokens": 38724,
+        "reported_responses": {
+          "cache_reported_input_tokens": 192,
+          "cache_write_input_tokens": 192,
+          "cache_write_reported_input_tokens": 192,
+          "cached_input_tokens": 192,
+          "input_tokens": 192,
+          "output_tokens": 192,
+          "reasoning_output_tokens": 192,
+          "reasoning_reported_output_tokens": 192
+        },
+        "response_count": 192,
+        "response_ids_complete": true,
+        "schema": "rlebench/token-usage/1",
+        "source": "Codex token_usage_record per response",
+        "uncached_input_tokens": 213404,
+        "unidentified_usage_records": 0
+      },
+      "call_activity": {
+        "model_tool_calls": 191,
+        "model_tool_calls_by_name": {
+          "exec": 191
+        },
+        "nested_python_tool_invocations": null,
+        "python_device_rpc_attempts": null,
+        "python_device_rpc_attempts_by_action": null,
+        "python_device_rpc_errors": null,
+        "python_instrumented_model_tool_calls": null,
+        "python_tool_invocations": null,
+        "python_tool_invocations_by_origin": null,
+        "schema": "rlebench/call-activity/1",
+        "source": "Codex native sessions"
+      },
+      "media": {
+        "passed": true,
+        "width": 1280,
+        "height": 360,
+        "duration_s": 43.25,
+        "speed": 4,
+        "source_fps": 10,
+        "output_fps": 20,
+        "recording": {
+          "accepted_samples": 1730,
+          "captured_samples": 1730,
+          "clock": "simulation",
+          "dropped_samples": 0,
+          "encoded_frames": 1730,
+          "end_time_s": 172.90000000001464,
+          "error": null,
+          "experimental": true,
+          "fps": 10,
+          "received_samples": 1730,
+          "schema": "roboenv/recording/1",
+          "state": "closed",
+          "status": "complete",
+          "views": [
+            {
+              "fov_y": 45.0,
+              "height": 360,
+              "name": "camera_head_left",
+              "pose": null,
+              "source": "camera_head_left",
+              "width": 640
+            },
+            {
+              "fov_y": 45.0,
+              "height": 360,
+              "name": "camera_head_right",
+              "pose": null,
+              "source": "camera_head_right",
+              "width": 640
+            }
+          ]
+        },
+        "view_names": [
+          "camera_head_left",
+          "camera_head_right"
+        ],
+        "sha256": "95adbcd6adbca86aa7e6a6cc90c6fe2930fa1d430ac07da60e5756b01ad3dd3b",
+        "scope": "Native spectator recording postprocessed at 4x; no replay or rerender"
+      },
+      "analysis": {
+        "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002",
+        "duration": "\u4f7f\u7528 8,645 / 10,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002",
+        "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002",
+        "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002"
+      },
+      "provenance": {
+        "sources": {
+          "RLE-Bench-inhouse": {
+            "build_inputs": [
+              "pyproject.toml",
+              "src",
+              "tasks",
+              "README.md",
+              "Makefile",
+              "tests",
+              "docs"
+            ],
+            "content_sha256": "5b1428b96104cb17b52215bce0ec33b880c5c591d01d4ecc2867cdc647940bda",
+            "dirty": true,
+            "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd",
+            "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse"
+          },
+          "RoboEnv": {
+            "build_inputs": [
+              "pyproject.toml",
+              "README.md",
+              "src",
+              "runtime/pyproject.toml",
+              "runtime/README.md",
+              "runtime/src",
+              "runtime/environments.json",
+              "runtime/locks",
+              "catalog",
+              "upstreams.lock.json",
+              "third_party/patches",
+              "docs/validation"
+            ],
+            "content_sha256": "5e90464ff5b67edba2a8e653e68cba5f7065c19f92f34a29689703d61e04d305",
+            "dirty": true,
+            "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92",
+            "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv"
+          },
+          "kinex": {
+            "build_inputs": [
+              "package.json",
+              "package-lock.json",
+              ".nvmrc",
+              "tsconfig.json",
+              "VERSION",
+              "src",
+              "packages/core",
+              "packages/setup",
+              "script",
+              "assets",
+              "bin"
+            ],
+            "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38",
+            "dirty": false,
+            "revision": "caac19a8a36272972f762e0f74cfe381e8500048",
+            "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex"
+          }
+        },
+        "job": "task06-07-pick-and-place-and-hug-container-codex-seed0-attempt01",
+        "attempt": 1,
+        "harness": "stock Codex CLI",
+        "control_interface": "public RoboEnv SDK and CLI",
+        "kinex_agent_runtime_used": false,
+        "gpu_index": 3,
+        "gpu_model": "NVIDIA L40S",
+        "campaign_dispatch_concurrency_limit": 5,
+        "concurrency_note": "One GPU per active SIMPLE episode; Kinex and Codex share a five-GPU dispatch pool.",
+        "codex_version": "0.160.0",
+        "model": "gpt-6-astra",
+        "effort": "high",
+        "service_tier": "default",
+        "fresh_session": true,
+        "source_jobs": [],
+        "resume_trajectory": false,
+        "imported_skills": [],
+        "automatic_harbor_retries": 0,
+        "request_policy": {
+          "max_request_retries": 50,
+          "configuration": "explicit retry50 SSE",
+          "usage_accounting": "reported-responses"
+        },
+        "classification": "formal",
+        "measured_images": {
+          "agent": "sha256:3fb135c4db8092900969023cfe52e4bddfc8960b6296318fda47c72e80e5f667",
+          "task": "sha256:730c5fc8ca66ce7bb07635943de71bd443d99c4f6ce962e01d35e90a3aec5c28"
+        },
+        "session_original_sha256": "f57a16755d9b599e6a03f0620b4092819f73f48643c586385c5d54ffa8ff4115",
+        "protocol_sha256": "6e24ceebccc59c6e91caf708994c458548bbb4c3539b206b9d85eebcb7ea6266"
+      },
+      "links": {
+        "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/agent/session.jsonl",
+        "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/agent/trajectory.json",
+        "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/agent/provider-usage.jsonl",
+        "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/transcript.json",
+        "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/verifier/episode.json",
+        "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/verifier/protocol.json",
+        "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/instructions.json",
+        "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/verifier/workspace.tar.gz",
+        "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/verifier/workspace.json",
+        "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl",
+        "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/evidence/episode/evidence/recording/manifest.json",
+        "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/usage.json",
+        "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/analysis.json",
+        "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/provenance.json",
+        "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/video.mp4",
+        "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/poster.jpg",
+        "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/media-validation.json",
+        "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/evidence/episode/evidence/final-observation.json"
+      },
+      "resources": [
+        {
+          "name": "tools/robot.py",
+          "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/resources/tools/robot.py",
+          "kind": "Created during this episode; final workspace snapshot."
+        },
+        {
+          "name": "memos/g1_simple_manipulation.md",
+          "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/resources/memos/g1_simple_manipulation.md",
+          "kind": "Created during this episode; final workspace snapshot."
+        }
+      ],
+      "session_counts": {
+        "visible_events": 414,
+        "observed_images": 108,
+        "tool_errors": 1
+      },
+      "selected_for_formal_metrics": true,
+      "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/"
+    },
     {
       "id": "task06-08-seed0-formal",
       "task_key": "task06/08",

episodes.csv

@@ -77,7 +77,7 @@ task06/03,simple/bend-pick,completed,0,True,False,9768,3956.495044,24248209,2404
 task06/04,simple/bend-pick-and-place,completed,0,True,False,8560,3261.927668,31457829,31121920,57128
 task06/05,simple/bend-handover,completed,0,True,False,9150,3541.240323,24778430,24583808,75261
 task06/06,simple/handover,completed,0,True,False,4040,1176.656816,5562825,5475584,17747
-task06/07,simple/pick-and-place-and-hug-container,pending,0,,,,,,,
+task06/07,simple/pick-and-place-and-hug-container,completed,0,True,False,8645,2551.587007,16738844,16525440,38724
 task06/08,simple/close-door,completed,0,True,True,1277,369.21498,1340798,1294208,6729
 task06/09,simple/open-oven,pending,0,,,,,,,
 task06/10,simple/open-faucet,pending,0,,,,,,,

episodes.json

@@ -20085,6 +20085,272 @@
     "selected_for_formal_metrics": true,
     "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/06/seed-0/"
   },
+  {
+    "id": "task06-07-seed0-formal",
+    "task_key": "task06/07",
+    "family": "task06",
+    "slot": "07",
+    "seed": 0,
+    "episode": 1,
+    "phase": "formal",
+    "status": "completed",
+    "success": false,
+    "native_reward": 0.0,
+    "valid": true,
+    "execution": {
+      "reason": null,
+      "status": "finished"
+    },
+    "verdict": {
+      "evidence_valid": true,
+      "steps": 8645,
+      "success": false,
+      "termination": "stopped"
+    },
+    "steps": 8645,
+    "simulation_time_s": null,
+    "wall_time_s": 2551.587007,
+    "model": "gpt-6-astra",
+    "effort": "high",
+    "harness": "codex",
+    "codex_version": "0.160.0",
+    "native_instruction": "pick up the apple from table1,hug the container,walk to table2,and place  on table2.",
+    "instruction": "pick up the apple from table1,hug the container,walk to table2,and place  on table2.",
+    "instruction_policy": "original_native",
+    "usage": {
+      "accounting": "reported-responses",
+      "audit_complete": true,
+      "cache_hit_rate": 0.9872509714529868,
+      "cache_reported_input_tokens": 16738844,
+      "cache_write_input_tokens": 0,
+      "cache_write_reported_input_tokens": 16738844,
+      "cached_input_tokens": 16525440,
+      "completed_turns": 1,
+      "cost_usd": null,
+      "failed_turns": 0,
+      "input_tokens": 16738844,
+      "known_cache_write_input_tokens": 0,
+      "known_cached_input_tokens": 16525440,
+      "known_input_tokens": 16738844,
+      "known_output_tokens": 38724,
+      "known_reasoning_output_tokens": 20279,
+      "output_tokens": 38724,
+      "reasoning_output_tokens": 20279,
+      "reasoning_reported_output_tokens": 38724,
+      "reported_responses": {
+        "cache_reported_input_tokens": 192,
+        "cache_write_input_tokens": 192,
+        "cache_write_reported_input_tokens": 192,
+        "cached_input_tokens": 192,
+        "input_tokens": 192,
+        "output_tokens": 192,
+        "reasoning_output_tokens": 192,
+        "reasoning_reported_output_tokens": 192
+      },
+      "response_count": 192,
+      "response_ids_complete": true,
+      "schema": "rlebench/token-usage/1",
+      "source": "Codex token_usage_record per response",
+      "uncached_input_tokens": 213404,
+      "unidentified_usage_records": 0
+    },
+    "call_activity": {
+      "model_tool_calls": 191,
+      "model_tool_calls_by_name": {
+        "exec": 191
+      },
+      "nested_python_tool_invocations": null,
+      "python_device_rpc_attempts": null,
+      "python_device_rpc_attempts_by_action": null,
+      "python_device_rpc_errors": null,
+      "python_instrumented_model_tool_calls": null,
+      "python_tool_invocations": null,
+      "python_tool_invocations_by_origin": null,
+      "schema": "rlebench/call-activity/1",
+      "source": "Codex native sessions"
+    },
+    "media": {
+      "passed": true,
+      "width": 1280,
+      "height": 360,
+      "duration_s": 43.25,
+      "speed": 4,
+      "source_fps": 10,
+      "output_fps": 20,
+      "recording": {
+        "accepted_samples": 1730,
+        "captured_samples": 1730,
+        "clock": "simulation",
+        "dropped_samples": 0,
+        "encoded_frames": 1730,
+        "end_time_s": 172.90000000001464,
+        "error": null,
+        "experimental": true,
+        "fps": 10,
+        "received_samples": 1730,
+        "schema": "roboenv/recording/1",
+        "state": "closed",
+        "status": "complete",
+        "views": [
+          {
+            "fov_y": 45.0,
+            "height": 360,
+            "name": "camera_head_left",
+            "pose": null,
+            "source": "camera_head_left",
+            "width": 640
+          },
+          {
+            "fov_y": 45.0,
+            "height": 360,
+            "name": "camera_head_right",
+            "pose": null,
+            "source": "camera_head_right",
+            "width": 640
+          }
+        ]
+      },
+      "view_names": [
+        "camera_head_left",
+        "camera_head_right"
+      ],
+      "sha256": "95adbcd6adbca86aa7e6a6cc90c6fe2930fa1d430ac07da60e5756b01ad3dd3b",
+      "scope": "Native spectator recording postprocessed at 4x; no replay or rerender"
+    },
+    "analysis": {
+      "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002",
+      "duration": "\u4f7f\u7528 8,645 / 10,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002",
+      "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002",
+      "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002"
+    },
+    "provenance": {
+      "sources": {
+        "RLE-Bench-inhouse": {
+          "build_inputs": [
+            "pyproject.toml",
+            "src",
+            "tasks",
+            "README.md",
+            "Makefile",
+            "tests",
+            "docs"
+          ],
+          "content_sha256": "5b1428b96104cb17b52215bce0ec33b880c5c591d01d4ecc2867cdc647940bda",
+          "dirty": true,
+          "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd",
+          "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse"
+        },
+        "RoboEnv": {
+          "build_inputs": [
+            "pyproject.toml",
+            "README.md",
+            "src",
+            "runtime/pyproject.toml",
+            "runtime/README.md",
+            "runtime/src",
+            "runtime/environments.json",
+            "runtime/locks",
+            "catalog",
+            "upstreams.lock.json",
+            "third_party/patches",
+            "docs/validation"
+          ],
+          "content_sha256": "5e90464ff5b67edba2a8e653e68cba5f7065c19f92f34a29689703d61e04d305",
+          "dirty": true,
+          "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92",
+          "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv"
+        },
+        "kinex": {
+          "build_inputs": [
+            "package.json",
+            "package-lock.json",
+            ".nvmrc",
+            "tsconfig.json",
+            "VERSION",
+            "src",
+            "packages/core",
+            "packages/setup",
+            "script",
+            "assets",
+            "bin"
+          ],
+          "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38",
+          "dirty": false,
+          "revision": "caac19a8a36272972f762e0f74cfe381e8500048",
+          "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex"
+        }
+      },
+      "job": "task06-07-pick-and-place-and-hug-container-codex-seed0-attempt01",
+      "attempt": 1,
+      "harness": "stock Codex CLI",
+      "control_interface": "public RoboEnv SDK and CLI",
+      "kinex_agent_runtime_used": false,
+      "gpu_index": 3,
+      "gpu_model": "NVIDIA L40S",
+      "campaign_dispatch_concurrency_limit": 5,
+      "concurrency_note": "One GPU per active SIMPLE episode; Kinex and Codex share a five-GPU dispatch pool.",
+      "codex_version": "0.160.0",
+      "model": "gpt-6-astra",
+      "effort": "high",
+      "service_tier": "default",
+      "fresh_session": true,
+      "source_jobs": [],
+      "resume_trajectory": false,
+      "imported_skills": [],
+      "automatic_harbor_retries": 0,
+      "request_policy": {
+        "max_request_retries": 50,
+        "configuration": "explicit retry50 SSE",
+        "usage_accounting": "reported-responses"
+      },
+      "classification": "formal",
+      "measured_images": {
+        "agent": "sha256:3fb135c4db8092900969023cfe52e4bddfc8960b6296318fda47c72e80e5f667",
+        "task": "sha256:730c5fc8ca66ce7bb07635943de71bd443d99c4f6ce962e01d35e90a3aec5c28"
+      },
+      "session_original_sha256": "f57a16755d9b599e6a03f0620b4092819f73f48643c586385c5d54ffa8ff4115",
+      "protocol_sha256": "6e24ceebccc59c6e91caf708994c458548bbb4c3539b206b9d85eebcb7ea6266"
+    },
+    "links": {
+      "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/agent/session.jsonl",
+      "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/agent/trajectory.json",
+      "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/agent/provider-usage.jsonl",
+      "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/transcript.json",
+      "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/verifier/episode.json",
+      "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/verifier/protocol.json",
+      "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/instructions.json",
+      "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/verifier/workspace.tar.gz",
+      "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/verifier/workspace.json",
+      "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl",
+      "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/evidence/episode/evidence/recording/manifest.json",
+      "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/usage.json",
+      "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/analysis.json",
+      "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/provenance.json",
+      "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/video.mp4",
+      "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/poster.jpg",
+      "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/media-validation.json",
+      "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/native/evidence/episode/evidence/final-observation.json"
+    },
+    "resources": [
+      {
+        "name": "tools/robot.py",
+        "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/resources/tools/robot.py",
+        "kind": "Created during this episode; final workspace snapshot."
+      },
+      {
+        "name": "memos/g1_simple_manipulation.md",
+        "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/resources/memos/g1_simple_manipulation.md",
+        "kind": "Created during this episode; final workspace snapshot."
+      }
+    ],
+    "session_counts": {
+      "visible_events": 414,
+      "observed_images": 108,
+      "tool_errors": 1
+    },
+    "selected_for_formal_metrics": true,
+    "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task06/07/seed-0/"
+  },
   {
     "id": "task06-08-seed0-formal",
     "task_key": "task06/08",

manifest.json

@@ -1,8 +1,8 @@
 {
   ".gitattributes": "51dec59a208c8656ce7d6614b7b547022f7a4720b1cdad6a20aeeda717ec090b",
   "DATA_FORMAT.md": "e2fd4f6b5c8be9d9e008db7d0c1f5d52b419d68328a1c86625c9f67c49b03403",
-  "README.md": "758ffe296585c8984d3e8444de92d9816752217e2dcdeaef8f608e258e88d552",
-  "REPORT.md": "dffa31e9f6c868a6916559fc813f56e62b35808e007f5b964d24e79772e25c06",
+  "README.md": "a69c662b67b32d762b71db8bfb458f33823a2ac65814cb13bf998f99c90cf53a",
+  "REPORT.md": "e8c55b5928c2ded32606f1552fd73634247d7921658cab0edbc320e0ffc84445",
   "THIRD_PARTY_NOTICES.md": "8aa8e3db100d7a570931c5d87f44f78705b42cf17e8147016b35e9aeddb06bba",
   "app.js": "eb013cc9d24382410953a1a8c81e446a6592a81b32887749d17bfc863936556b",
   "attempt-history.json": "8555f458c1dd362f5a07447342994fddf265ff23bbc40e9bd6f9984a892ddb4e",
@@ -19,9 +19,9 @@
   "catalog/task06/11-push-office-chair/task.yaml": "ab19bf9b7cec8992e2dcc1fb30ad9f6ce0f8343063b44777b2b68afc39d7f426",
   "catalog/task06/12-open-trash-can/task.yaml": "8d8b4b0ec848f91283e73416d257f5b0f816ac4b872fccd10b213e74d953085c",
   "comparison.json": "487d479f3d03e72efaf3c19c1e63f5894bcfea2954cb1cc46327c379b5315ad3",
-  "data.json": "778e49376f6133719890725e55fa81fde807327874d39f9d7741dad7a5808d8f",
-  "episodes.csv": "3d88b62d09ca6f64a6932942d21bbd46142f44c5403ab465abcbaf70227c3cd5",
-  "episodes.json": "66b5a163d5b639b1d3bd2e4232083e030d5b6e5e41caccf560a8617e973de2ee",
+  "data.json": "ac940eeb5e78302389d387aff8cccde4e81ecdf4ccea4d6fbb7b54f43e1ccd51",
+  "episodes.csv": "7f4d5f46d65689ff303609a98ffa8a34ad875e7cc9b6018628fc368e87070ec9",
+  "episodes.json": "41c1c467c7c36745454b1f1da54e2cbd25823af225581cfa168472878f4ea0f9",
   "failure-reviews.json": "c66a997c9ecd6f7f43f130a8a33cd64c7a5b2049033e9ccde653ee30e4796f23",
   "favicon.svg": "46063791218655f70c828ef71a1232a34fd38181da938338badec929ad226819",
   "index.html": "524b3f71cc36152a08125212ad1c9ae2ca30601907b378f69cce4d34390d0ee3",
@@ -30,8 +30,8 @@
   "licenses/RoboTwin-LICENSE.txt": "c695d421718e54e6f3a60858f0c601125c3ecc60c199332c15947360561e2a9f",
   "licenses/robocasa-LICENSE.txt": "5da18670b3f00c59847b1ded9c28dee59940d963b1e03b528b0108d9c5a09885",
   "licenses/robosuite-LICENSE.txt": "177978cbece0a4c454c2aaec5b3f145b39270814874c43109da9e829c39d9cba",
-  "protocol.json": "a3288345609e793384a1cf99eea6b07a451a302282082dc4c1e0714ea0b35859",
+  "protocol.json": "96be82ffa28eedefaa70bff4f7d4daa9e56f70fed407c1beac1ce8b725a00861",
   "style.css": "dd31b30951c220ebb82bc96224b5e640913945679e42b2e0a42e2c373a329e6c",
-  "summary.json": "71ec7e4438a0333ea048aad0d3bc8c4e5f558b03a885afdda327d7a74ce7b5c3",
-  "task-index.json": "26fea0071532a894ffee5fe88af6a7e80a8320bd0b1c6026c742ef5947a04f72"
+  "summary.json": "dbb20c916a74b687e14aece840ffe67138889d90d3bd4b7d389593d497c9aede",
+  "task-index.json": "1f9542d87396e5e80c842c021f081077ab3f8cbace59c1bf61593b587d9020bc"
 }

protocol.json

@@ -95,21 +95,21 @@
       "id": "task06",
       "name": "SIMPLE G1",
       "total": 12,
-      "completed": 7,
-      "pending": 5,
+      "completed": 8,
+      "pending": 4,
       "successes": 2,
-      "valid_results": 7,
-      "success_rate": 0.2857142857142857,
-      "usage_complete": 7,
+      "valid_results": 8,
+      "success_rate": 0.25,
+      "usage_complete": 8,
       "control_frequency_hz": 50,
       "max_control_steps": 10000,
       "preflight_results": 0,
-      "formal_results": 7,
+      "formal_results": 8,
       "modified_results": 0,
-      "original_results": 7,
-      "input_tokens": 91436538,
-      "cached_input_tokens": 90472576,
-      "output_tokens": 245134
+      "original_results": 8,
+      "input_tokens": 108175382,
+      "cached_input_tokens": 106998016,
+      "output_tokens": 283858
     }
   ],
   "usage_accounting": "reported-responses",

publication-manifest.json

@@ -10,11 +10,11 @@
       "bytes": 1129
     },
     "README.md": {
-      "sha256": "758ffe296585c8984d3e8444de92d9816752217e2dcdeaef8f608e258e88d552",
+      "sha256": "a69c662b67b32d762b71db8bfb458f33823a2ac65814cb13bf998f99c90cf53a",
       "bytes": 1544
     },
     "REPORT.md": {
-      "sha256": "dffa31e9f6c868a6916559fc813f56e62b35808e007f5b964d24e79772e25c06",
+      "sha256": "e8c55b5928c2ded32606f1552fd73634247d7921658cab0edbc320e0ffc84445",
       "bytes": 1421
     },
     "THIRD_PARTY_NOTICES.md": {
@@ -82,16 +82,16 @@
       "bytes": 134701
     },
     "data.json": {
-      "sha256": "778e49376f6133719890725e55fa81fde807327874d39f9d7741dad7a5808d8f",
-      "bytes": 1154073
+      "sha256": "ac940eeb5e78302389d387aff8cccde4e81ecdf4ccea4d6fbb7b54f43e1ccd51",
+      "bytes": 1166026
     },
     "episodes.csv": {
-      "sha256": "3d88b62d09ca6f64a6932942d21bbd46142f44c5403ab465abcbaf70227c3cd5",
-      "bytes": 8064
+      "sha256": "7f4d5f46d65689ff303609a98ffa8a34ad875e7cc9b6018628fc368e87070ec9",
+      "bytes": 8111
     },
     "episodes.json": {
-      "sha256": "66b5a163d5b639b1d3bd2e4232083e030d5b6e5e41caccf560a8617e973de2ee",
-      "bytes": 863439
+      "sha256": "41c1c467c7c36745454b1f1da54e2cbd25823af225581cfa168472878f4ea0f9",
+      "bytes": 874883
     },
     "failure-reviews.json": {
       "sha256": "c66a997c9ecd6f7f43f130a8a33cd64c7a5b2049033e9ccde653ee30e4796f23",
@@ -126,23 +126,23 @@
       "bytes": 1474
     },
     "protocol.json": {
-      "sha256": "a3288345609e793384a1cf99eea6b07a451a302282082dc4c1e0714ea0b35859",
-      "bytes": 5450
+      "sha256": "96be82ffa28eedefaa70bff4f7d4daa9e56f70fed407c1beac1ce8b725a00861",
+      "bytes": 5438
     },
     "style.css": {
       "sha256": "dd31b30951c220ebb82bc96224b5e640913945679e42b2e0a42e2c373a329e6c",
       "bytes": 16560
     },
     "summary.json": {
-      "sha256": "71ec7e4438a0333ea048aad0d3bc8c4e5f558b03a885afdda327d7a74ce7b5c3",
-      "bytes": 3221
+      "sha256": "dbb20c916a74b687e14aece840ffe67138889d90d3bd4b7d389593d497c9aede",
+      "bytes": 3209
     },
     "task-index.json": {
-      "sha256": "26fea0071532a894ffee5fe88af6a7e80a8320bd0b1c6026c742ef5947a04f72",
-      "bytes": 229779
+      "sha256": "1f9542d87396e5e80c842c021f081077ab3f8cbace59c1bf61593b587d9020bc",
+      "bytes": 229780
     },
     "manifest.json": {
-      "sha256": "887c8f7adde67c1b8241fc55b1a7989ec471bc81b41f3318d98864172bf09f9f",
+      "sha256": "f30322dfb31fe1049111c2bd2c2f026804ce81f9a7d025c7c90904e505fa5261",
       "bytes": 3481
     }
   }

summary.json

@@ -1,7 +1,7 @@
 {
   "planned_tasks": 84,
-  "published_results": 79,
-  "pending_tasks": 5,
+  "published_results": 80,
+  "pending_tasks": 4,
   "families": [
     {
       "id": "task01",
@@ -88,40 +88,40 @@
       "id": "task06",
       "name": "SIMPLE G1",
       "total": 12,
-      "completed": 7,
-      "pending": 5,
+      "completed": 8,
+      "pending": 4,
       "successes": 2,
-      "valid_results": 7,
-      "success_rate": 0.2857142857142857,
-      "usage_complete": 7,
+      "valid_results": 8,
+      "success_rate": 0.25,
+      "usage_complete": 8,
       "control_frequency_hz": 50,
       "max_control_steps": 10000,
       "preflight_results": 0,
-      "formal_results": 7,
+      "formal_results": 8,
       "modified_results": 0,
-      "original_results": 7,
-      "input_tokens": 91436538,
-      "cached_input_tokens": 90472576,
-      "output_tokens": 245134
+      "original_results": 8,
+      "input_tokens": 108175382,
+      "cached_input_tokens": 106998016,
+      "output_tokens": 283858
     }
   ],
   "interrupted_attempts": 4,
   "usage_incomplete_tasks": [],
   "execution_incomplete_tasks": [],
   "deferred_tasks": [],
-  "new_evaluations": 7,
+  "new_evaluations": 8,
   "progress": {
-    "finished": 79,
+    "finished": 80,
     "running": 3,
-    "queued": 2,
+    "queued": 1,
     "needs_review": 0,
     "interrupted": 0,
     "native_successes": 51,
-    "native_failures": 28
+    "native_failures": 29
   },
   "simple_campaign": {
     "planned": 12,
-    "published": 7,
+    "published": 8,
     "agent": "codex",
     "model": "gpt-6-astra",
     "effort": "high",

task-index.json

@@ -6232,10 +6232,10 @@
     "catalog_instruction": "pick up the apple from table1,hug the container,walk to table2,and place  on table2.",
     "native_instruction": "pick up the apple from table1,hug the container,walk to table2,and place  on table2.",
     "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.",
-    "status": "pending",
-    "episode_id": null,
-    "run_status": "running",
-    "status_note": "Authorized episode is queued, running, or awaiting evidence review.",
+    "status": "completed",
+    "episode_id": "task06-07-seed0-formal",
+    "run_status": "finished",
+    "status_note": "Native result, execution and usage complete.",
     "planned_protocol": {
       "episodes": 1,
       "seed": 0,
@@ -6354,7 +6354,7 @@
     "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.",
     "status": "pending",
     "episode_id": null,
-    "run_status": "queued",
+    "run_status": "running",
     "status_note": "Authorized episode is queued, running, or awaiting evidence review.",
     "planned_protocol": {
       "episodes": 1,