diff --git "a/data.json" "b/data.json" --- "a/data.json" +++ "b/data.json" @@ -6,14 +6,14 @@ "benchmark_complete": true, "edition": "codex-simple-astra-high-seed0", "created_at": "2026-10-09T00:09:42.989838+00:00", - "updated_at": "2026-10-10T05:34:41.504169+00:00", + "updated_at": "2026-10-10T20:20:40.424847+00:00", "model": "gpt-6-astra", "effort": "high", "seed": 0, "publication": { "dataset": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark", "space": "https://huggingface.co/spaces/RLE-Bench/codex-benchmark", - "dataset_resolve": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/" + "dataset_resolve": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/" }, "summary": { "planned_tasks": 84, @@ -49,9 +49,9 @@ "successes": 9, "valid_results": 10, "success_rate": 0.9, - "input_tokens": 12586499, - "cached_input_tokens": 12106752, - "output_tokens": 93969, + "input_tokens": 10541541, + "cached_input_tokens": 10098816, + "output_tokens": 87116, "usage_complete": 10, "control_frequency_hz": 20, "max_control_steps": 6000, @@ -247,9 +247,9 @@ "successes": 9, "valid_results": 10, "success_rate": 0.9, - "input_tokens": 12586499, - "cached_input_tokens": 12106752, - "output_tokens": 93969, + "input_tokens": 10541541, + "cached_input_tokens": 10098816, + "output_tokens": 87116, "usage_complete": 10, "control_frequency_hz": 20, "max_control_steps": 6000, @@ -449,7 +449,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task01/01-load-condiments-in-fridge/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/01-load-condiments-in-fridge/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -519,7 +519,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task01/02-filter-microwavable-item/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/02-filter-microwavable-item/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -589,7 +589,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task01/03-store-dumplings/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/03-store-dumplings/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -659,7 +659,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task01/04-divide-buffet-trays/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/04-divide-buffet-trays/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -724,7 +724,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task01/05-make-cheesecake-filling/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/05-make-cheesecake-filling/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -869,7 +869,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task01/06-multistep-steaming/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/06-multistep-steaming/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -959,7 +959,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task01/07-scale-portioning/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/07-scale-portioning/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -1134,14 +1134,599 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task01/08-scrub-cutting-board/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/08-scrub-cutting-board/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Clarify broad board coverage, maintained grasp/contact, and final retraction without numeric thresholds." }, "run_status": "finished", - "attempt_history": [], - "status_note": "Evaluation and native evidence complete." + "attempt_history": [ + { + "id": "task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01", + "task_key": "task01/08", + "family": "task01", + "slot": "08", + "seed": 0, + "episode": 1, + "phase": "diagnostic", + "status": "completed", + "success": false, + "native_reward": 0.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 1346, + "success": false, + "termination": "stopped" + }, + "steps": 1346, + "simulation_time_s": 67.3, + "wall_time_s": 523.717335, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "Pick up the sponge from the counter and clean the cutting board by briefly scrubbing or pressing down on the cutting board. Once finished, release the sponge.", + "instruction": "Pick up the sponge from the counter and scrub across a broad area of the cutting board, keeping the sponge grasped and in contact with the board throughout the scrubbing motion. Once finished, release the sponge and retract the gripper well away from it.", + "instruction_policy": "modified", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9727732412792149, + "cache_reported_input_tokens": 1201612, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 1201612, + "cached_input_tokens": 1168896, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 1201612, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 1168896, + "known_input_tokens": 1201612, + "known_output_tokens": 7027, + "known_reasoning_output_tokens": 1641, + "output_tokens": 7027, + "reasoning_output_tokens": 1641, + "reasoning_reported_output_tokens": 7027, + "reported_responses": { + "cache_reported_input_tokens": 39, + "cache_write_input_tokens": 39, + "cache_write_reported_input_tokens": 39, + "cached_input_tokens": 39, + "input_tokens": 39, + "output_tokens": 39, + "reasoning_output_tokens": 39, + "reasoning_reported_output_tokens": 39 + }, + "response_count": 39, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 32716, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 38, + "model_tool_calls_by_name": { + "exec": 38 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 674, + "published_frames": 754, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 673, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 18.85, + "sha256": "9e0425d2379caa2bc918ab3f9fc97562a4b2e17ac3264e0ccbcd59d4851a062b", + "source_sha256": { + "third_person.mp4": "12d92c962845a458e6af045da94396e825113a0eeccf30f09e93c963bda8e6ed", + "wrist.mp4": "16b77c14a44fa9ff7c7da86261d6440da01a1ad6f45ff78983dd0aa6395aa07b" + }, + "all_frames_compared": 674, + "minimum_frame_psnr_db": 41.69535624810065, + "maximum_frame_rgb_mae": 1.2265592813491821, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", + "recording": { + "accepted_samples": 674, + "captured_samples": 674, + "clock": "simulation", + "dropped_samples": 0, + "encoded_frames": 674, + "end_time_s": 67.29999999999832, + "error": null, + "experimental": true, + "fps": 10, + "overflow_policy": "bounded_backpressure", + "received_samples": 674, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 720, + "name": "third_person", + "pose": null, + "source": "third_person", + "width": 720 + }, + { + "fov_y": 45.0, + "height": 720, + "name": "wrist", + "pose": null, + "source": "wrist", + "width": 720 + } + ] + } + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 1,346 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002", + "diagnostics": "\u4fdd\u6301\u539f\u6709\u6307\u4ee4\u4e0e\u539f\u751f verifier\uff0c\u4ec5\u8bb0\u5f55\u5b9e\u9645\u5224\u5b9a\u8c03\u7528\u548c\u540c\u4e00\u72b6\u6001\u4e0b\u7684\u53ea\u8bfb\u6d4b\u91cf\u3002\u6b64\u8bca\u65ad\u56de\u5408\u4e0d\u66ff\u6362\u6b63\u5f0f\u7edf\u8ba1\u3002", + "diagnosis": "\u8bca\u65ad\u672a\u901a\u8fc7\uff1a\u6709\u6548\u63a5\u89e6\u4f4d\u7f6e 2 / \u81f3\u5c11 5\uff0c\u8986\u76d6\u8303\u56f4 0.0868 / \u81f3\u5c11 0.10 m\uff1b\u5df2\u91ca\u653e\u5e76\u64a4\u79bb\u3002\u66f4\u65b0\u540e\u7684\u53ea\u8bfb\u6d4b\u91cf\u663e\u793a\uff1a\u6d77\u7ef5\u63a5\u89e6\u677f\u9762 173 \u5e27\uff0c\u6293\u6301\u548c\u63a5\u89e6\u540c\u65f6\u6210\u7acb 3 \u5e27\uff1b\u539f\u751f\u8bb0\u5f55\u4ec5\u7d2f\u79ef 2 \u4e2a\u6709\u6548\u4f4d\u7f6e\u3002\u672a\u53d1\u73b0\u539f\u751f verifier \u4fee\u6539\u6216\u6307\u4ee4\u9057\u6f0f\uff1b\u65e7\u56de\u5408\u6ca1\u6709\u540c\u6837\u7684\u5386\u53f2\u8bb0\u5f55\uff0c\u65e0\u6cd5\u5bf9\u65e7\u56de\u5408\u4f5c\u51fa\u540c\u7b49\u5f3a\u5ea6\u7684\u7ed3\u8bba\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "tasks", + "tests", + "docs", + "Makefile" + ], + "content_sha256": "635adce00c840c49aac560ecdb0425a3a45ff8b4804b0620cca45e158bf1916f", + "dirty": true, + "revision": "f1935004c2fb23501b937cd29b826f69a06214b2", + "runtime_snapshot_content_sha256": "635adce00c840c49aac560ecdb0425a3a45ff8b4804b0620cca45e158bf1916f", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/RLE-Bench-inhouse", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse", + "live_content_sha256": "bbd03a94ca5776307435b6ed81c0f935a17f836b9ee80770069b58162d5e7e8f", + "snapshot_mode": "Current workspace plus owner-only diagnostic configuration; instruction unchanged." + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "b5e31f051d55e015746e645939e4cbcbbdc44ec111593ef2711c5360900d7768", + "dirty": true, + "revision": "612c1528c92065b8f2254a841595e65664dc957d", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/RoboEnv", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/kinex", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01", + "attempt": 1, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 7, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 6, + "concurrency_note": "Six authorized fresh repair and diagnostic episodes; idle GPUs only.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "owner-predicate-diagnostic", + "measured_images": { + "agent": "sha256:442b190234fb474b7ac7a1f0e03c268be10afc773a829c23bf421f8373f70dde", + "task": "sha256:0c2ee8e7600634872f86ea1a762bbfc1415371af3df5946bfc04263dc17bc388" + }, + "session_original_sha256": "172d709070540f2de39e5c95f586a7e239cdf49f9c08019a8a46b74f031fd481", + "protocol_sha256": "44bb17021918606c49586677dc6bf98272b20e0f9bb33f301645f4bd5b24918e" + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/final-observation.json", + "native_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", + "owner_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/terminal.jpg" + }, + "resources": [ + { + "name": "tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/tools/panda_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/robocasa_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/memos/robocasa_panda.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 89, + "observed_images": 17, + "tool_errors": 1 + }, + "selected_for_formal_metrics": false + } + ], + "status_note": "Evaluation and native evidence complete.", + "predicate_diagnostics": [ + { + "id": "task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01", + "task_key": "task01/08", + "family": "task01", + "slot": "08", + "seed": 0, + "episode": 1, + "phase": "diagnostic", + "status": "completed", + "success": false, + "native_reward": 0.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 1346, + "success": false, + "termination": "stopped" + }, + "steps": 1346, + "simulation_time_s": 67.3, + "wall_time_s": 523.717335, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "Pick up the sponge from the counter and clean the cutting board by briefly scrubbing or pressing down on the cutting board. Once finished, release the sponge.", + "instruction": "Pick up the sponge from the counter and scrub across a broad area of the cutting board, keeping the sponge grasped and in contact with the board throughout the scrubbing motion. Once finished, release the sponge and retract the gripper well away from it.", + "instruction_policy": "modified", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9727732412792149, + "cache_reported_input_tokens": 1201612, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 1201612, + "cached_input_tokens": 1168896, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 1201612, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 1168896, + "known_input_tokens": 1201612, + "known_output_tokens": 7027, + "known_reasoning_output_tokens": 1641, + "output_tokens": 7027, + "reasoning_output_tokens": 1641, + "reasoning_reported_output_tokens": 7027, + "reported_responses": { + "cache_reported_input_tokens": 39, + "cache_write_input_tokens": 39, + "cache_write_reported_input_tokens": 39, + "cached_input_tokens": 39, + "input_tokens": 39, + "output_tokens": 39, + "reasoning_output_tokens": 39, + "reasoning_reported_output_tokens": 39 + }, + "response_count": 39, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 32716, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 38, + "model_tool_calls_by_name": { + "exec": 38 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 674, + "published_frames": 754, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 673, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 18.85, + "sha256": "9e0425d2379caa2bc918ab3f9fc97562a4b2e17ac3264e0ccbcd59d4851a062b", + "source_sha256": { + "third_person.mp4": "12d92c962845a458e6af045da94396e825113a0eeccf30f09e93c963bda8e6ed", + "wrist.mp4": "16b77c14a44fa9ff7c7da86261d6440da01a1ad6f45ff78983dd0aa6395aa07b" + }, + "all_frames_compared": 674, + "minimum_frame_psnr_db": 41.69535624810065, + "maximum_frame_rgb_mae": 1.2265592813491821, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", + "recording": { + "accepted_samples": 674, + "captured_samples": 674, + "clock": "simulation", + "dropped_samples": 0, + "encoded_frames": 674, + "end_time_s": 67.29999999999832, + "error": null, + "experimental": true, + "fps": 10, + "overflow_policy": "bounded_backpressure", + "received_samples": 674, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 720, + "name": "third_person", + "pose": null, + "source": "third_person", + "width": 720 + }, + { + "fov_y": 45.0, + "height": 720, + "name": "wrist", + "pose": null, + "source": "wrist", + "width": 720 + } + ] + } + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 1,346 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002", + "diagnostics": "\u4fdd\u6301\u539f\u6709\u6307\u4ee4\u4e0e\u539f\u751f verifier\uff0c\u4ec5\u8bb0\u5f55\u5b9e\u9645\u5224\u5b9a\u8c03\u7528\u548c\u540c\u4e00\u72b6\u6001\u4e0b\u7684\u53ea\u8bfb\u6d4b\u91cf\u3002\u6b64\u8bca\u65ad\u56de\u5408\u4e0d\u66ff\u6362\u6b63\u5f0f\u7edf\u8ba1\u3002", + "diagnosis": "\u8bca\u65ad\u672a\u901a\u8fc7\uff1a\u6709\u6548\u63a5\u89e6\u4f4d\u7f6e 2 / \u81f3\u5c11 5\uff0c\u8986\u76d6\u8303\u56f4 0.0868 / \u81f3\u5c11 0.10 m\uff1b\u5df2\u91ca\u653e\u5e76\u64a4\u79bb\u3002\u66f4\u65b0\u540e\u7684\u53ea\u8bfb\u6d4b\u91cf\u663e\u793a\uff1a\u6d77\u7ef5\u63a5\u89e6\u677f\u9762 173 \u5e27\uff0c\u6293\u6301\u548c\u63a5\u89e6\u540c\u65f6\u6210\u7acb 3 \u5e27\uff1b\u539f\u751f\u8bb0\u5f55\u4ec5\u7d2f\u79ef 2 \u4e2a\u6709\u6548\u4f4d\u7f6e\u3002\u672a\u53d1\u73b0\u539f\u751f verifier \u4fee\u6539\u6216\u6307\u4ee4\u9057\u6f0f\uff1b\u65e7\u56de\u5408\u6ca1\u6709\u540c\u6837\u7684\u5386\u53f2\u8bb0\u5f55\uff0c\u65e0\u6cd5\u5bf9\u65e7\u56de\u5408\u4f5c\u51fa\u540c\u7b49\u5f3a\u5ea6\u7684\u7ed3\u8bba\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "tasks", + "tests", + "docs", + "Makefile" + ], + "content_sha256": "635adce00c840c49aac560ecdb0425a3a45ff8b4804b0620cca45e158bf1916f", + "dirty": true, + "revision": "f1935004c2fb23501b937cd29b826f69a06214b2", + "runtime_snapshot_content_sha256": "635adce00c840c49aac560ecdb0425a3a45ff8b4804b0620cca45e158bf1916f", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/RLE-Bench-inhouse", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse", + "live_content_sha256": "bbd03a94ca5776307435b6ed81c0f935a17f836b9ee80770069b58162d5e7e8f", + "snapshot_mode": "Current workspace plus owner-only diagnostic configuration; instruction unchanged." + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "b5e31f051d55e015746e645939e4cbcbbdc44ec111593ef2711c5360900d7768", + "dirty": true, + "revision": "612c1528c92065b8f2254a841595e65664dc957d", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/RoboEnv", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/kinex", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01", + "attempt": 1, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 7, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 6, + "concurrency_note": "Six authorized fresh repair and diagnostic episodes; idle GPUs only.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "owner-predicate-diagnostic", + "measured_images": { + "agent": "sha256:442b190234fb474b7ac7a1f0e03c268be10afc773a829c23bf421f8373f70dde", + "task": "sha256:0c2ee8e7600634872f86ea1a762bbfc1415371af3df5946bfc04263dc17bc388" + }, + "session_original_sha256": "172d709070540f2de39e5c95f586a7e239cdf49f9c08019a8a46b74f031fd481", + "protocol_sha256": "44bb17021918606c49586677dc6bf98272b20e0f9bb33f301645f4bd5b24918e" + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/final-observation.json", + "native_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", + "owner_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/terminal.jpg" + }, + "resources": [ + { + "name": "tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/tools/panda_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/robocasa_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/memos/robocasa_panda.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 89, + "observed_images": 17, + "tool_errors": 1 + }, + "selected_for_formal_metrics": false + } + ] }, { "key": "task01/09", @@ -1212,7 +1797,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task01/09-prepare-veggie-dip/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/09-prepare-veggie-dip/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -1282,7 +1867,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task01/10-prepare-vegetable-roasting/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/10-prepare-vegetable-roasting/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -1409,8 +1994,582 @@ "max_steps": 6000 }, "run_status": "finished", - "attempt_history": [], - "status_note": "Evaluation and native evidence complete." + "attempt_history": [ + { + "id": "task02-04-seed0-formal", + "task_key": "task02/04", + "family": "task02", + "slot": "04", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": false, + "native_reward": 0.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 3687, + "success": false, + "termination": "stopped" + }, + "steps": 3687, + "simulation_time_s": null, + "wall_time_s": 919.204412, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "put the black bowl in the bottom drawer of the cabinet and close it", + "instruction": "put the black bowl in the bottom drawer of the cabinet and close it", + "instruction_policy": "original_native", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9745746678635921, + "cache_reported_input_tokens": 3356377, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 3356377, + "cached_input_tokens": 3271040, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 3356377, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 3271040, + "known_input_tokens": 3356377, + "known_output_tokens": 23446, + "known_reasoning_output_tokens": 11600, + "output_tokens": 23446, + "reasoning_output_tokens": 11600, + "reasoning_reported_output_tokens": 23446, + "reported_responses": { + "cache_reported_input_tokens": 67, + "cache_write_input_tokens": 67, + "cache_write_reported_input_tokens": 67, + "cached_input_tokens": 67, + "input_tokens": 67, + "output_tokens": 67, + "reasoning_output_tokens": 67, + "reasoning_reported_output_tokens": 67 + }, + "response_count": 67, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 85337, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 66, + "model_tool_calls_by_name": { + "exec": 66 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 1842, + "published_frames": 1922, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 1841, + "terminal_hold_s": 2, + "source_dropped_samples": 10, + "duration_s": 48.05, + "sha256": "78128f89d1778f114232fcbd7f7bcbd99b651af608511496a8ec244e44985764", + "source_sha256": { + "third_person.mp4": "b64c98693c4e0732c1af18a9804ed4c6ad91658672172976c04e3be6d1b77330", + "wrist.mp4": "39da751d97f951af7e44f366ccd787ada328591c094ba1bf4bcfedea1f7dcef9" + }, + "all_frames_compared": 1842, + "minimum_frame_psnr_db": 39.815899173602, + "maximum_frame_rgb_mae": 1.6068315505981445, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", + "recording": { + "accepted_samples": 1832, + "captured_samples": 1842, + "clock": "simulation", + "dropped_samples": 10, + "encoded_frames": 1842, + "end_time_s": 184.1000000000076, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 1832, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 720, + "name": "third_person", + "pose": null, + "source": "third_person", + "width": 720 + }, + { + "fov_y": 45.0, + "height": 720, + "name": "wrist", + "pose": null, + "source": "wrist", + "width": 720 + } + ] + }, + "pts_verified": true + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 3,687 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "edd9d6bdc67136367a15c34949b94cf1bf11ab5d8454e74fd532076feb8032b1", + "dirty": true, + "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "742ec96151dd0d5549d4febb7e091167a35d7886af224fa26d4c464197ea6050", + "dirty": true, + "revision": "5f487ee5121a775aa6ea90919c8f5e3d31318906", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task02-04-libero-10-04-codex-seed0-attempt01", + "attempt": 1, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 4, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 20, + "concurrency_note": "One initial canary; then all remaining nineteen tasks launched concurrently under a twenty-task admission cap.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:5deb0795c2b18162bbcd75531b287469aabb0078db50a48bfe650ecf45868859", + "task": "sha256:4897d53efd57aea35b9c9ad36d3f29f819cedd5974e9c3f7b02775579a884922" + }, + "session_original_sha256": "b63b0b4dfb6ead64a30c353133825d5b20b8ebd3c17913be6864ed382ed9cf4a", + "protocol_sha256": "2cd301bfe0c1a1d6d4a420b1471374a8d33f9471ab060c1d6dcf66fbf74969bf" + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/terminal.jpg" + }, + "resources": [ + { + "name": "tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/tools/panda_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/memos/libero_panda.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 148, + "observed_images": 79, + "tool_errors": 1 + }, + "selected_for_formal_metrics": false, + "condition": "before-recording-repair" + } + ], + "status_note": "Evaluation and native evidence complete.", + "recording_replacements": [ + { + "id": "task02-04-seed0-formal", + "task_key": "task02/04", + "family": "task02", + "slot": "04", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": false, + "native_reward": 0.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 3687, + "success": false, + "termination": "stopped" + }, + "steps": 3687, + "simulation_time_s": null, + "wall_time_s": 919.204412, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "put the black bowl in the bottom drawer of the cabinet and close it", + "instruction": "put the black bowl in the bottom drawer of the cabinet and close it", + "instruction_policy": "original_native", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9745746678635921, + "cache_reported_input_tokens": 3356377, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 3356377, + "cached_input_tokens": 3271040, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 3356377, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 3271040, + "known_input_tokens": 3356377, + "known_output_tokens": 23446, + "known_reasoning_output_tokens": 11600, + "output_tokens": 23446, + "reasoning_output_tokens": 11600, + "reasoning_reported_output_tokens": 23446, + "reported_responses": { + "cache_reported_input_tokens": 67, + "cache_write_input_tokens": 67, + "cache_write_reported_input_tokens": 67, + "cached_input_tokens": 67, + "input_tokens": 67, + "output_tokens": 67, + "reasoning_output_tokens": 67, + "reasoning_reported_output_tokens": 67 + }, + "response_count": 67, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 85337, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 66, + "model_tool_calls_by_name": { + "exec": 66 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 1842, + "published_frames": 1922, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 1841, + "terminal_hold_s": 2, + "source_dropped_samples": 10, + "duration_s": 48.05, + "sha256": "78128f89d1778f114232fcbd7f7bcbd99b651af608511496a8ec244e44985764", + "source_sha256": { + "third_person.mp4": "b64c98693c4e0732c1af18a9804ed4c6ad91658672172976c04e3be6d1b77330", + "wrist.mp4": "39da751d97f951af7e44f366ccd787ada328591c094ba1bf4bcfedea1f7dcef9" + }, + "all_frames_compared": 1842, + "minimum_frame_psnr_db": 39.815899173602, + "maximum_frame_rgb_mae": 1.6068315505981445, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", + "recording": { + "accepted_samples": 1832, + "captured_samples": 1842, + "clock": "simulation", + "dropped_samples": 10, + "encoded_frames": 1842, + "end_time_s": 184.1000000000076, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 1832, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 720, + "name": "third_person", + "pose": null, + "source": "third_person", + "width": 720 + }, + { + "fov_y": 45.0, + "height": 720, + "name": "wrist", + "pose": null, + "source": "wrist", + "width": 720 + } + ] + }, + "pts_verified": true + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 3,687 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "edd9d6bdc67136367a15c34949b94cf1bf11ab5d8454e74fd532076feb8032b1", + "dirty": true, + "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "742ec96151dd0d5549d4febb7e091167a35d7886af224fa26d4c464197ea6050", + "dirty": true, + "revision": "5f487ee5121a775aa6ea90919c8f5e3d31318906", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task02-04-libero-10-04-codex-seed0-attempt01", + "attempt": 1, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 4, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 20, + "concurrency_note": "One initial canary; then all remaining nineteen tasks launched concurrently under a twenty-task admission cap.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:5deb0795c2b18162bbcd75531b287469aabb0078db50a48bfe650ecf45868859", + "task": "sha256:4897d53efd57aea35b9c9ad36d3f29f819cedd5974e9c3f7b02775579a884922" + }, + "session_original_sha256": "b63b0b4dfb6ead64a30c353133825d5b20b8ebd3c17913be6864ed382ed9cf4a", + "protocol_sha256": "2cd301bfe0c1a1d6d4a420b1471374a8d33f9471ab060c1d6dcf66fbf74969bf" + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/terminal.jpg" + }, + "resources": [ + { + "name": "tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/tools/panda_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/memos/libero_panda.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 148, + "observed_images": 79, + "tool_errors": 1 + }, + "selected_for_formal_metrics": false, + "condition": "before-recording-repair" + } + ], + "recording_rerun": { + "job": "task02-04-libero-10-04-codex-seed0-recording-replacement-attempt01", + "seed": 0, + "native_success": false, + "source_dropped_samples": 0, + "native_verifier_changed": false, + "prior_selection_retained": true, + "scope": "Fresh seed-0 recording replacement, with unchanged instruction; native outcomes are retained regardless of success." + } }, { "key": "task02/05", @@ -1498,7 +2657,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task02/05-libero-10-05/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/05-libero-10-05/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." @@ -1553,7 +2712,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task02/06-libero-10-06/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/06-libero-10-06/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." @@ -1633,14 +2792,586 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task02/07-libero-10-07/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/07-libero-10-07/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." }, "run_status": "finished", - "attempt_history": [], - "status_note": "Evaluation and native evidence complete." + "attempt_history": [ + { + "id": "task02-07-seed0-formal", + "task_key": "task02/07", + "family": "task02", + "slot": "07", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": true, + "native_reward": 1.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 1016, + "success": true, + "termination": "success" + }, + "steps": 1016, + "simulation_time_s": null, + "wall_time_s": 260.996992, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "put the white mug on the plate and put the chocolate pudding to the right of the plate", + "instruction": "put the white mug in the center of the plate and put the chocolate pudding immediately to the right of the plate", + "instruction_policy": "modified", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9487277032832223, + "cache_reported_input_tokens": 701841, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 701841, + "cached_input_tokens": 665856, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 701841, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 665856, + "known_input_tokens": 701841, + "known_output_tokens": 6754, + "known_reasoning_output_tokens": 1904, + "output_tokens": 6754, + "reasoning_output_tokens": 1904, + "reasoning_reported_output_tokens": 6754, + "reported_responses": { + "cache_reported_input_tokens": 24, + "cache_write_input_tokens": 24, + "cache_write_reported_input_tokens": 24, + "cached_input_tokens": 24, + "input_tokens": 24, + "output_tokens": 24, + "reasoning_output_tokens": 24, + "reasoning_reported_output_tokens": 24 + }, + "response_count": 24, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 35985, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 23, + "model_tool_calls_by_name": { + "exec": 23 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 506, + "published_frames": 586, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 505, + "terminal_hold_s": 2, + "source_dropped_samples": 6, + "duration_s": 14.65, + "sha256": "82195453e66453b792c04ec78daaf0dd4b8ded835d99620791c22fcf031d9f15", + "source_sha256": { + "third_person.mp4": "a78d6254c5808439fbd13a4370a4b6e990b0b5365594e6b6b67c0e51449d72af", + "wrist.mp4": "46aad4fd5251450ee8ae623620ac3d601d13e2a89198c7fa922586f680692d3c" + }, + "all_frames_compared": 506, + "minimum_frame_psnr_db": 41.213419705945526, + "maximum_frame_rgb_mae": 1.5328880548477173, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", + "recording": { + "accepted_samples": 501, + "captured_samples": 507, + "clock": "simulation", + "dropped_samples": 6, + "encoded_frames": 506, + "end_time_s": 50.549999999999265, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 501, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 720, + "name": "third_person", + "pose": null, + "source": "third_person", + "width": 720 + }, + { + "fov_y": 45.0, + "height": 720, + "name": "wrist", + "pose": null, + "source": "wrist", + "width": 720 + } + ] + }, + "pts_verified": true + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 1,016 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "edd9d6bdc67136367a15c34949b94cf1bf11ab5d8454e74fd532076feb8032b1", + "dirty": true, + "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "742ec96151dd0d5549d4febb7e091167a35d7886af224fa26d4c464197ea6050", + "dirty": true, + "revision": "5f487ee5121a775aa6ea90919c8f5e3d31318906", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task02-07-libero-10-07-codex-seed0-attempt01", + "attempt": 1, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 2, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 20, + "concurrency_note": "One initial canary; then all remaining nineteen tasks launched concurrently under a twenty-task admission cap.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:5deb0795c2b18162bbcd75531b287469aabb0078db50a48bfe650ecf45868859", + "task": "sha256:4897d53efd57aea35b9c9ad36d3f29f819cedd5974e9c3f7b02775579a884922" + }, + "session_original_sha256": "7587686fd933aa48160da3af4647d34dc914373521a51b368351fd134e6bf464", + "protocol_sha256": "bdb44af586667ee7eab0855e1a04f70e56609333b710668aa923999c6424aa55" + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/terminal.jpg" + }, + "resources": [ + { + "name": "tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/tools/panda_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/memos/libero_panda.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 58, + "observed_images": 18, + "tool_errors": 3 + }, + "selected_for_formal_metrics": false, + "condition": "before-recording-repair" + } + ], + "status_note": "Evaluation and native evidence complete.", + "recording_replacements": [ + { + "id": "task02-07-seed0-formal", + "task_key": "task02/07", + "family": "task02", + "slot": "07", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": true, + "native_reward": 1.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 1016, + "success": true, + "termination": "success" + }, + "steps": 1016, + "simulation_time_s": null, + "wall_time_s": 260.996992, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "put the white mug on the plate and put the chocolate pudding to the right of the plate", + "instruction": "put the white mug in the center of the plate and put the chocolate pudding immediately to the right of the plate", + "instruction_policy": "modified", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9487277032832223, + "cache_reported_input_tokens": 701841, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 701841, + "cached_input_tokens": 665856, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 701841, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 665856, + "known_input_tokens": 701841, + "known_output_tokens": 6754, + "known_reasoning_output_tokens": 1904, + "output_tokens": 6754, + "reasoning_output_tokens": 1904, + "reasoning_reported_output_tokens": 6754, + "reported_responses": { + "cache_reported_input_tokens": 24, + "cache_write_input_tokens": 24, + "cache_write_reported_input_tokens": 24, + "cached_input_tokens": 24, + "input_tokens": 24, + "output_tokens": 24, + "reasoning_output_tokens": 24, + "reasoning_reported_output_tokens": 24 + }, + "response_count": 24, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 35985, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 23, + "model_tool_calls_by_name": { + "exec": 23 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 506, + "published_frames": 586, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 505, + "terminal_hold_s": 2, + "source_dropped_samples": 6, + "duration_s": 14.65, + "sha256": "82195453e66453b792c04ec78daaf0dd4b8ded835d99620791c22fcf031d9f15", + "source_sha256": { + "third_person.mp4": "a78d6254c5808439fbd13a4370a4b6e990b0b5365594e6b6b67c0e51449d72af", + "wrist.mp4": "46aad4fd5251450ee8ae623620ac3d601d13e2a89198c7fa922586f680692d3c" + }, + "all_frames_compared": 506, + "minimum_frame_psnr_db": 41.213419705945526, + "maximum_frame_rgb_mae": 1.5328880548477173, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", + "recording": { + "accepted_samples": 501, + "captured_samples": 507, + "clock": "simulation", + "dropped_samples": 6, + "encoded_frames": 506, + "end_time_s": 50.549999999999265, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 501, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 720, + "name": "third_person", + "pose": null, + "source": "third_person", + "width": 720 + }, + { + "fov_y": 45.0, + "height": 720, + "name": "wrist", + "pose": null, + "source": "wrist", + "width": 720 + } + ] + }, + "pts_verified": true + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 1,016 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "edd9d6bdc67136367a15c34949b94cf1bf11ab5d8454e74fd532076feb8032b1", + "dirty": true, + "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "742ec96151dd0d5549d4febb7e091167a35d7886af224fa26d4c464197ea6050", + "dirty": true, + "revision": "5f487ee5121a775aa6ea90919c8f5e3d31318906", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task02-07-libero-10-07-codex-seed0-attempt01", + "attempt": 1, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 2, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 20, + "concurrency_note": "One initial canary; then all remaining nineteen tasks launched concurrently under a twenty-task admission cap.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:5deb0795c2b18162bbcd75531b287469aabb0078db50a48bfe650ecf45868859", + "task": "sha256:4897d53efd57aea35b9c9ad36d3f29f819cedd5974e9c3f7b02775579a884922" + }, + "session_original_sha256": "7587686fd933aa48160da3af4647d34dc914373521a51b368351fd134e6bf464", + "protocol_sha256": "bdb44af586667ee7eab0855e1a04f70e56609333b710668aa923999c6424aa55" + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/terminal.jpg" + }, + "resources": [ + { + "name": "tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/tools/panda_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/memos/libero_panda.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 58, + "observed_images": 18, + "tool_errors": 3 + }, + "selected_for_formal_metrics": false, + "condition": "before-recording-repair" + } + ], + "recording_rerun": { + "job": "task02-07-libero-10-07-codex-seed0-recording-replacement-attempt01", + "seed": 0, + "native_success": true, + "source_dropped_samples": 0, + "native_verifier_changed": false, + "prior_selection_retained": true, + "scope": "Fresh seed-0 recording replacement, with unchanged instruction; native outcomes are retained regardless of success." + } }, { "key": "task02/08", @@ -1703,14 +3434,586 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task02/08-libero-10-08/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/08-libero-10-08/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." }, "run_status": "finished", - "attempt_history": [], - "status_note": "Evaluation and native evidence complete." + "attempt_history": [ + { + "id": "task02-08-seed0-formal", + "task_key": "task02/08", + "family": "task02", + "slot": "08", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": true, + "native_reward": 1.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 1115, + "success": true, + "termination": "success" + }, + "steps": 1115, + "simulation_time_s": null, + "wall_time_s": 412.342365, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "put both the alphabet soup and the cream cheese box in the basket", + "instruction": "put both the alphabet soup and the cream cheese box fully inside the basket", + "instruction_policy": "modified", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9604073765886478, + "cache_reported_input_tokens": 1406070, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 1406070, + "cached_input_tokens": 1350400, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 1406070, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 1350400, + "known_input_tokens": 1406070, + "known_output_tokens": 9396, + "known_reasoning_output_tokens": 4471, + "output_tokens": 9396, + "reasoning_output_tokens": 4471, + "reasoning_reported_output_tokens": 9396, + "reported_responses": { + "cache_reported_input_tokens": 40, + "cache_write_input_tokens": 40, + "cache_write_reported_input_tokens": 40, + "cached_input_tokens": 40, + "input_tokens": 40, + "output_tokens": 40, + "reasoning_output_tokens": 40, + "reasoning_reported_output_tokens": 40 + }, + "response_count": 40, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 55670, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 39, + "model_tool_calls_by_name": { + "exec": 39 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 556, + "published_frames": 636, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 555, + "terminal_hold_s": 2, + "source_dropped_samples": 18, + "duration_s": 15.9, + "sha256": "2b67596f5fd3dc5617b35602be6de53f1393d093c6c79beb768d74066cbf0160", + "source_sha256": { + "third_person.mp4": "d9eea0594b6cebd92dd0a6037ce19366571548c6954fc3cbb25cb05aba170d59", + "wrist.mp4": "6420840a0e2c4eae8c9376b708a2644779db52f9f9ec14c5b7ff22e3d15f886f" + }, + "all_frames_compared": 556, + "minimum_frame_psnr_db": 41.863996296064386, + "maximum_frame_rgb_mae": 1.44694185256958, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", + "recording": { + "accepted_samples": 538, + "captured_samples": 556, + "clock": "simulation", + "dropped_samples": 18, + "encoded_frames": 556, + "end_time_s": 55.499999999998984, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 538, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 720, + "name": "third_person", + "pose": null, + "source": "third_person", + "width": 720 + }, + { + "fov_y": 45.0, + "height": 720, + "name": "wrist", + "pose": null, + "source": "wrist", + "width": 720 + } + ] + }, + "pts_verified": true + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 1,115 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "edd9d6bdc67136367a15c34949b94cf1bf11ab5d8454e74fd532076feb8032b1", + "dirty": true, + "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "742ec96151dd0d5549d4febb7e091167a35d7886af224fa26d4c464197ea6050", + "dirty": true, + "revision": "5f487ee5121a775aa6ea90919c8f5e3d31318906", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task02-08-libero-10-08-codex-seed0-attempt01", + "attempt": 1, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 3, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 20, + "concurrency_note": "One initial canary; then all remaining nineteen tasks launched concurrently under a twenty-task admission cap.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:5deb0795c2b18162bbcd75531b287469aabb0078db50a48bfe650ecf45868859", + "task": "sha256:4897d53efd57aea35b9c9ad36d3f29f819cedd5974e9c3f7b02775579a884922" + }, + "session_original_sha256": "2be2b70496fa5cd49637b5f93111331068e67ccc7e3b261df25b663d09f162aa", + "protocol_sha256": "b0e2388679180c5a165f77c1cc90a2c5bfce0b2b1582b118c86392d435c21543" + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/terminal.jpg" + }, + "resources": [ + { + "name": "tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/tools/panda_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/memos/libero_panda.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 91, + "observed_images": 27, + "tool_errors": 2 + }, + "selected_for_formal_metrics": false, + "condition": "before-recording-repair" + } + ], + "status_note": "Evaluation and native evidence complete.", + "recording_replacements": [ + { + "id": "task02-08-seed0-formal", + "task_key": "task02/08", + "family": "task02", + "slot": "08", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": true, + "native_reward": 1.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 1115, + "success": true, + "termination": "success" + }, + "steps": 1115, + "simulation_time_s": null, + "wall_time_s": 412.342365, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "put both the alphabet soup and the cream cheese box in the basket", + "instruction": "put both the alphabet soup and the cream cheese box fully inside the basket", + "instruction_policy": "modified", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9604073765886478, + "cache_reported_input_tokens": 1406070, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 1406070, + "cached_input_tokens": 1350400, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 1406070, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 1350400, + "known_input_tokens": 1406070, + "known_output_tokens": 9396, + "known_reasoning_output_tokens": 4471, + "output_tokens": 9396, + "reasoning_output_tokens": 4471, + "reasoning_reported_output_tokens": 9396, + "reported_responses": { + "cache_reported_input_tokens": 40, + "cache_write_input_tokens": 40, + "cache_write_reported_input_tokens": 40, + "cached_input_tokens": 40, + "input_tokens": 40, + "output_tokens": 40, + "reasoning_output_tokens": 40, + "reasoning_reported_output_tokens": 40 + }, + "response_count": 40, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 55670, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 39, + "model_tool_calls_by_name": { + "exec": 39 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 556, + "published_frames": 636, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 555, + "terminal_hold_s": 2, + "source_dropped_samples": 18, + "duration_s": 15.9, + "sha256": "2b67596f5fd3dc5617b35602be6de53f1393d093c6c79beb768d74066cbf0160", + "source_sha256": { + "third_person.mp4": "d9eea0594b6cebd92dd0a6037ce19366571548c6954fc3cbb25cb05aba170d59", + "wrist.mp4": "6420840a0e2c4eae8c9376b708a2644779db52f9f9ec14c5b7ff22e3d15f886f" + }, + "all_frames_compared": 556, + "minimum_frame_psnr_db": 41.863996296064386, + "maximum_frame_rgb_mae": 1.44694185256958, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", + "recording": { + "accepted_samples": 538, + "captured_samples": 556, + "clock": "simulation", + "dropped_samples": 18, + "encoded_frames": 556, + "end_time_s": 55.499999999998984, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 538, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 720, + "name": "third_person", + "pose": null, + "source": "third_person", + "width": 720 + }, + { + "fov_y": 45.0, + "height": 720, + "name": "wrist", + "pose": null, + "source": "wrist", + "width": 720 + } + ] + }, + "pts_verified": true + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 1,115 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "edd9d6bdc67136367a15c34949b94cf1bf11ab5d8454e74fd532076feb8032b1", + "dirty": true, + "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "742ec96151dd0d5549d4febb7e091167a35d7886af224fa26d4c464197ea6050", + "dirty": true, + "revision": "5f487ee5121a775aa6ea90919c8f5e3d31318906", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task02-08-libero-10-08-codex-seed0-attempt01", + "attempt": 1, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 3, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 20, + "concurrency_note": "One initial canary; then all remaining nineteen tasks launched concurrently under a twenty-task admission cap.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:5deb0795c2b18162bbcd75531b287469aabb0078db50a48bfe650ecf45868859", + "task": "sha256:4897d53efd57aea35b9c9ad36d3f29f819cedd5974e9c3f7b02775579a884922" + }, + "session_original_sha256": "2be2b70496fa5cd49637b5f93111331068e67ccc7e3b261df25b663d09f162aa", + "protocol_sha256": "b0e2388679180c5a165f77c1cc90a2c5bfce0b2b1582b118c86392d435c21543" + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/terminal.jpg" + }, + "resources": [ + { + "name": "tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/tools/panda_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/memos/libero_panda.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 91, + "observed_images": 27, + "tool_errors": 2 + }, + "selected_for_formal_metrics": false, + "condition": "before-recording-repair" + } + ], + "recording_rerun": { + "job": "task02-08-libero-10-08-codex-seed0-recording-replacement-attempt01", + "seed": 0, + "native_success": true, + "source_dropped_samples": 0, + "native_verifier_changed": false, + "prior_selection_retained": true, + "scope": "Fresh seed-0 recording replacement, with unchanged instruction; native outcomes are retained regardless of success." + } }, { "key": "task02/09", @@ -1758,14 +4061,586 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task02/09-libero-10-09/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/09-libero-10-09/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." }, "run_status": "finished", - "attempt_history": [], - "status_note": "Evaluation and native evidence complete." + "attempt_history": [ + { + "id": "task02-09-seed0-formal", + "task_key": "task02/09", + "family": "task02", + "slot": "09", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": true, + "native_reward": 1.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 560, + "success": true, + "termination": "success" + }, + "steps": 560, + "simulation_time_s": null, + "wall_time_s": 216.244925, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "put both moka pots on the stove", + "instruction": "put both moka pots on the stove and turn the stove on", + "instruction_policy": "modified", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9551717429852196, + "cache_reported_input_tokens": 714527, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 714527, + "cached_input_tokens": 682496, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 714527, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 682496, + "known_input_tokens": 714527, + "known_output_tokens": 5355, + "known_reasoning_output_tokens": 1306, + "output_tokens": 5355, + "reasoning_output_tokens": 1306, + "reasoning_reported_output_tokens": 5355, + "reported_responses": { + "cache_reported_input_tokens": 24, + "cache_write_input_tokens": 24, + "cache_write_reported_input_tokens": 24, + "cached_input_tokens": 24, + "input_tokens": 24, + "output_tokens": 24, + "reasoning_output_tokens": 24, + "reasoning_reported_output_tokens": 24 + }, + "response_count": 24, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 32031, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 23, + "model_tool_calls_by_name": { + "exec": 23 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 278, + "published_frames": 358, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 277, + "terminal_hold_s": 2, + "source_dropped_samples": 7, + "duration_s": 8.95, + "sha256": "afd7678319792100afb17e59d344caea7f5464dcea0374db2bd3517f64041259", + "source_sha256": { + "third_person.mp4": "4f33956a561150f0a352a150d4f9da0b8a30bb6754182de57a658603b4a2dbc5", + "wrist.mp4": "07fc26ccac3f0faf8f736d1e193fced8b9644d6832f88e501105e183b9941e6f" + }, + "all_frames_compared": 278, + "minimum_frame_psnr_db": 41.48512676102155, + "maximum_frame_rgb_mae": 1.4647440910339355, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", + "recording": { + "accepted_samples": 272, + "captured_samples": 279, + "clock": "simulation", + "dropped_samples": 7, + "encoded_frames": 278, + "end_time_s": 27.75000000000026, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 272, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 720, + "name": "third_person", + "pose": null, + "source": "third_person", + "width": 720 + }, + { + "fov_y": 45.0, + "height": 720, + "name": "wrist", + "pose": null, + "source": "wrist", + "width": 720 + } + ] + }, + "pts_verified": true + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 560 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "edd9d6bdc67136367a15c34949b94cf1bf11ab5d8454e74fd532076feb8032b1", + "dirty": true, + "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "742ec96151dd0d5549d4febb7e091167a35d7886af224fa26d4c464197ea6050", + "dirty": true, + "revision": "5f487ee5121a775aa6ea90919c8f5e3d31318906", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task02-09-libero-10-09-codex-seed0-attempt01", + "attempt": 1, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 6, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 20, + "concurrency_note": "One initial canary; then all remaining nineteen tasks launched concurrently under a twenty-task admission cap.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:5deb0795c2b18162bbcd75531b287469aabb0078db50a48bfe650ecf45868859", + "task": "sha256:4897d53efd57aea35b9c9ad36d3f29f819cedd5974e9c3f7b02775579a884922" + }, + "session_original_sha256": "b63a5267201015666c8fcdac5e4b3884a10661b76638beb854a1b405e91f2719", + "protocol_sha256": "acc9470d622733def9d480c2fb756b1724249077e1ad1f73cf80967246cabf7a" + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/terminal.jpg" + }, + "resources": [ + { + "name": "tools/panda.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/tools/panda.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/libero-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/memos/libero-control.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 57, + "observed_images": 21, + "tool_errors": 2 + }, + "selected_for_formal_metrics": false, + "condition": "before-recording-repair" + } + ], + "status_note": "Evaluation and native evidence complete.", + "recording_replacements": [ + { + "id": "task02-09-seed0-formal", + "task_key": "task02/09", + "family": "task02", + "slot": "09", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": true, + "native_reward": 1.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 560, + "success": true, + "termination": "success" + }, + "steps": 560, + "simulation_time_s": null, + "wall_time_s": 216.244925, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "put both moka pots on the stove", + "instruction": "put both moka pots on the stove and turn the stove on", + "instruction_policy": "modified", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9551717429852196, + "cache_reported_input_tokens": 714527, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 714527, + "cached_input_tokens": 682496, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 714527, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 682496, + "known_input_tokens": 714527, + "known_output_tokens": 5355, + "known_reasoning_output_tokens": 1306, + "output_tokens": 5355, + "reasoning_output_tokens": 1306, + "reasoning_reported_output_tokens": 5355, + "reported_responses": { + "cache_reported_input_tokens": 24, + "cache_write_input_tokens": 24, + "cache_write_reported_input_tokens": 24, + "cached_input_tokens": 24, + "input_tokens": 24, + "output_tokens": 24, + "reasoning_output_tokens": 24, + "reasoning_reported_output_tokens": 24 + }, + "response_count": 24, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 32031, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 23, + "model_tool_calls_by_name": { + "exec": 23 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-camera-media/3", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 278, + "published_frames": 358, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 277, + "terminal_hold_s": 2, + "source_dropped_samples": 7, + "duration_s": 8.95, + "sha256": "afd7678319792100afb17e59d344caea7f5464dcea0374db2bd3517f64041259", + "source_sha256": { + "third_person.mp4": "4f33956a561150f0a352a150d4f9da0b8a30bb6754182de57a658603b4a2dbc5", + "wrist.mp4": "07fc26ccac3f0faf8f736d1e193fced8b9644d6832f88e501105e183b9941e6f" + }, + "all_frames_compared": 278, + "minimum_frame_psnr_db": 41.48512676102155, + "maximum_frame_rgb_mae": 1.4647440910339355, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", + "recording": { + "accepted_samples": 272, + "captured_samples": 279, + "clock": "simulation", + "dropped_samples": 7, + "encoded_frames": 278, + "end_time_s": 27.75000000000026, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 272, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 720, + "name": "third_person", + "pose": null, + "source": "third_person", + "width": 720 + }, + { + "fov_y": 45.0, + "height": 720, + "name": "wrist", + "pose": null, + "source": "wrist", + "width": 720 + } + ] + }, + "pts_verified": true + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 560 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "edd9d6bdc67136367a15c34949b94cf1bf11ab5d8454e74fd532076feb8032b1", + "dirty": true, + "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "742ec96151dd0d5549d4febb7e091167a35d7886af224fa26d4c464197ea6050", + "dirty": true, + "revision": "5f487ee5121a775aa6ea90919c8f5e3d31318906", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task02-09-libero-10-09-codex-seed0-attempt01", + "attempt": 1, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 6, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 20, + "concurrency_note": "One initial canary; then all remaining nineteen tasks launched concurrently under a twenty-task admission cap.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:5deb0795c2b18162bbcd75531b287469aabb0078db50a48bfe650ecf45868859", + "task": "sha256:4897d53efd57aea35b9c9ad36d3f29f819cedd5974e9c3f7b02775579a884922" + }, + "session_original_sha256": "b63a5267201015666c8fcdac5e4b3884a10661b76638beb854a1b405e91f2719", + "protocol_sha256": "acc9470d622733def9d480c2fb756b1724249077e1ad1f73cf80967246cabf7a" + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/terminal.jpg" + }, + "resources": [ + { + "name": "tools/panda.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/tools/panda.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/libero-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/memos/libero-control.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 57, + "observed_images": 21, + "tool_errors": 2 + }, + "selected_for_formal_metrics": false, + "condition": "before-recording-repair" + } + ], + "recording_rerun": { + "job": "task02-09-libero-10-09-codex-seed0-recording-replacement-attempt01", + "seed": 0, + "native_success": true, + "source_dropped_samples": 0, + "native_verifier_changed": false, + "prior_selection_retained": true, + "scope": "Fresh seed-0 recording replacement, with unchanged instruction; native outcomes are retained regardless of success." + } }, { "key": "task02/10", @@ -1850,7 +4725,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task03/01-handover-block/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/01-handover-block/task.yaml" }, "attempt_history": [], "run_status": "finished", @@ -1933,7 +4808,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task03/02-place-dual-shoes/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/02-place-dual-shoes/task.yaml" }, "attempt_history": [], "run_status": "finished", @@ -2063,7 +4938,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task03/04-scan-object/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/04-scan-object/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Clarify the close, centered face-to-face scan pose and keeping both items grasped; native scan geometry thresholds are unchanged." @@ -2214,7 +5089,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task03/05-stack-bowls-three/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/05-stack-bowls-three/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Retain the existing concise instruction describing the compact bowl stack, release and open grippers; rerun using the current Kinex source." @@ -2402,7 +5277,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task03/07-hanging-mug/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/07-hanging-mug/task.yaml" }, "attempt_history": [], "run_status": "finished", @@ -2497,7 +5372,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task03/09-blocks-ranking-size/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/09-blocks-ranking-size/task.yaml" }, "attempt_history": [], "run_status": "finished", @@ -2565,7 +5440,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task03/10-lift-pot/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/10-lift-pot/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Retain the existing instruction describing two-handle grasping, upright orientation and sufficient lift; rerun using the current Kinex source." @@ -2668,7 +5543,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/01-robodojo-make-toast/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/01-robodojo-make-toast/task.yaml" }, "display_slot": "01", "display_key": "task04/01", @@ -2865,7 +5740,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/03-robodojo-store-laptop-and-headphones/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/03-robodojo-store-laptop-and-headphones/task.yaml" }, "display_slot": "03", "display_key": "task04/03", @@ -3007,7 +5882,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/04-robodojo-cover-blocks/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/04-robodojo-cover-blocks/task.yaml" }, "display_slot": "04", "display_key": "task04/04", @@ -3150,7 +6025,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/06-robodojo-store-tools-in-toolbox/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/06-robodojo-store-tools-in-toolbox/task.yaml" }, "display_slot": "06", "display_key": "task04/06", @@ -3217,7 +6092,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/07-robodojo-insert-tubes/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/07-robodojo-insert-tubes/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Describe fully seated tubes without exposing the verifier insertion-depth threshold." @@ -3331,7 +6206,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/08-robodojo-deposit-coin/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/08-robodojo-deposit-coin/task.yaml" }, "display_slot": "08", "display_key": "task04/08", @@ -3423,7 +6298,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/09-robodojo-fasten-screws/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/09-robodojo-fasten-screws/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Use a natural paragraph for matching and seating nuts; omit geometric scoring thresholds." @@ -3498,7 +6373,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/10-robodojo-play-stacking-toy/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/10-robodojo-play-stacking-toy/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Leave piece counts and peg matching to the agent while retaining the intended completed arrangement." @@ -3679,7 +6554,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/12-robodojo-arrange-largest-number/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/12-robodojo-arrange-largest-number/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Leave the numerical ordering strategy to the agent while retaining readable placement on the pads." @@ -3839,7 +6714,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/14-robodojo-build-tower/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/14-robodojo-build-tower/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Shorten the successful instruction while preserving the tower structure, original bottom faces, horizontal orientation, centering, plank alignment, perpendicular roof ridge, release and arm return requirements." @@ -3899,7 +6774,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/15-robodojo-classify-objects/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/15-robodojo-classify-objects/task.yaml" }, "display_slot": "14", "display_key": "task04/14", @@ -3976,7 +6851,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/16-robodojo-fill-egg-holder/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/16-robodojo-fill-egg-holder/task.yaml" }, "display_slot": "15", "display_key": "task04/15", @@ -4118,7 +6993,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/17-robodojo-fill-pen-holder/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/17-robodojo-fill-pen-holder/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Retain the native two-hand roles and final pen orientation without the insertion-depth threshold." @@ -4188,7 +7063,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/18-robodojo-fold-clothes/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/18-robodojo-fold-clothes/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Describe the folded garment without enumerating tracked point correspondences or grasping steps." @@ -4294,7 +7169,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/21-robodojo-hang-mugs/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/21-robodojo-hang-mugs/task.yaml" }, "display_slot": "19", "display_key": "task04/19", @@ -4426,7 +7301,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/23-robodojo-imitate-sorting-sequence/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/23-robodojo-imitate-sorting-sequence/task.yaml" }, "display_slot": "20", "display_key": "task04/20", @@ -4513,7 +7388,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/24-robodojo-insert-key/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/24-robodojo-insert-key/task.yaml" }, "display_slot": "21", "display_key": "task04/21", @@ -4630,7 +7505,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/25-robodojo-make-kong/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/25-robodojo-make-kong/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "State the turn and replacement-tile rules in one paragraph without a manipulation plan." @@ -4826,7 +7701,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/28-robodojo-organize-table/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/28-robodojo-organize-table/task.yaml" }, "display_slot": "24", "display_key": "task04/24", @@ -4903,7 +7778,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/29-robodojo-pack-objects-into-box/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/29-robodojo-pack-objects-into-box/task.yaml" }, "display_slot": "25", "display_key": "task04/25", @@ -5005,7 +7880,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/31-robodojo-pick-from-conveyor-by-image/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/31-robodojo-pick-from-conveyor-by-image/task.yaml" }, "display_slot": "26", "display_key": "task04/26", @@ -5122,7 +7997,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/32-robodojo-play-tic-tac-toe/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/32-robodojo-play-tic-tac-toe/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "State alternating turns and waiting rules without supplying the inferred ring counts." @@ -5197,7 +8072,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/33-robodojo-plug-in-charger/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/33-robodojo-plug-in-charger/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Describe a fully plugged-in charger without the verifier insertion-depth threshold." @@ -5257,7 +8132,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/34-robodojo-pour-balls-into-vase/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/34-robodojo-pour-balls-into-vase/task.yaml" }, "display_slot": "29", "display_key": "task04/29", @@ -5360,7 +8235,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/36-robodojo-pour-liquid-into-cup/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/36-robodojo-pour-liquid-into-cup/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Wait for flow to stop and liquid to settle while tilted before returning the bottle upright." @@ -5525,7 +8400,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/38-robodojo-press-by-number/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/38-robodojo-press-by-number/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Retain the per-button counting and confirmation rules without button-joint thresholds or card answers." @@ -5630,7 +8505,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/39-robodojo-push-t/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/39-robodojo-push-t/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Describe matching the target shape along the table without internal position and angle tolerances." @@ -5725,7 +8600,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/41-robodojo-put-bottles-into-dustbin/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/41-robodojo-put-bottles-into-dustbin/task.yaml" }, "display_slot": "34", "display_key": "task04/34", @@ -5828,7 +8703,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/43-robodojo-sort-nesting-dolls-by-size/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/43-robodojo-sort-nesting-dolls-by-size/task.yaml" }, "display_slot": "36", "display_key": "task04/36", @@ -5925,7 +8800,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/45-robodojo-stack-blocks/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/45-robodojo-stack-blocks/task.yaml" }, "display_slot": "37", "display_key": "task04/37", @@ -5998,7 +8873,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/48-robodojo-stack-bowls/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/48-robodojo-stack-bowls/task.yaml" }, "display_slot": "39", "display_key": "task04/39", @@ -6080,7 +8955,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/51-robodojo-swap-t/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/51-robodojo-swap-t/task.yaml" }, "display_slot": "40", "display_key": "task04/40", @@ -6242,7 +9117,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/52-robodojo-swap-blocks/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/52-robodojo-swap-blocks/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Clarify the three moves and complete press, withdrawal and button rebound after each placement; native release thresholds remain unchanged." @@ -6346,7 +9221,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/catalog/task04/53-robodojo-sweep-blocks/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/53-robodojo-sweep-blocks/task.yaml" }, "display_slot": "42", "display_key": "task04/42", @@ -6484,25 +9359,25 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/history/simple-10000/02/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/02/episode.json" }, "selected_for_formal_metrics": false } @@ -6570,25 +9445,25 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/history/simple-10000/03/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/03/episode.json" }, "selected_for_formal_metrics": false } @@ -6656,25 +9531,25 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/history/simple-10000/04/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/04/episode.json" }, "selected_for_formal_metrics": false } @@ -6742,25 +9617,25 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/history/simple-10000/05/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/05/episode.json" }, "selected_for_formal_metrics": false } @@ -6828,25 +9703,25 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/history/simple-10000/06/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/06/episode.json" }, "selected_for_formal_metrics": false } @@ -6914,25 +9789,25 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/history/simple-10000/07/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/07/episode.json" }, "selected_for_formal_metrics": false } @@ -7245,13 +10120,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 68.7, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 2749, + "published_frames": 2829, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 2748, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 70.725, + "sha256": "8434dd4ce662dfe2fcb5079ea977c7342eb82293dc072ec07462c503278eca9a", + "source_sha256": { + "third_person.mp4": "138d4f7303594c610111e9bdb46196eab0fcfda826baece5d19e894ddec15830", + "wrist.mp4": "306f147d313639d1337cffd8802420b5509f903974e31d4757a70a6f262dfbed" + }, + "all_frames_compared": 2749, + "minimum_frame_psnr_db": 40.83108475387529, + "maximum_frame_rgb_mae": 1.445607304573059, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 2749, "captured_samples": 2749, @@ -7285,12 +10179,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "f5bed61912d5964665c8153f7025f6e7fc77b6cf69bf940b5721224ae5610326", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", @@ -7387,34 +10276,35 @@ "protocol_sha256": "a080e2d2fe0139ffe12d237f92d7ed6874269c53d0239657accc8b22d9ea3c9f" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/resources/memos/robocasa-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/resources/memos/robocasa-control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -7423,8 +10313,7 @@ "observed_images": 84, "tool_errors": 5 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/01/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task01-02-seed0-formal", @@ -7511,18 +10400,37 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 73.8, "speed": 4, "source_fps": 10, - "output_fps": 20, - "recording": { - "accepted_samples": 2953, - "captured_samples": 2953, - "clock": "simulation", - "dropped_samples": 0, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 2952, + "published_frames": 3032, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 2951, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 75.8, + "sha256": "3352ee6d3aabae0bc0d6c4223dee347061275900c92991eaccd5a64b315bdb44", + "source_sha256": { + "third_person.mp4": "11b07ca658ef9a326d2cb29da8ed73e0cf6f147fb9359b8cad8c1cfbc8d21564", + "wrist.mp4": "24166f4da1fb0dbf3f46cc6c6745846af06bdf94c9738711c8dc54a27b25f736" + }, + "all_frames_compared": 2952, + "minimum_frame_psnr_db": 40.21783018038982, + "maximum_frame_rgb_mae": 1.5107555389404297, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", + "recording": { + "accepted_samples": 2953, + "captured_samples": 2953, + "clock": "simulation", + "dropped_samples": 0, "encoded_frames": 2952, "end_time_s": 295.15000000003283, "error": null, @@ -7551,12 +10459,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "e60624e8afd162c74aaf3bbedbad70727acaa7fc0e99c744bd8fae140dadde29", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", @@ -7653,34 +10556,35 @@ "protocol_sha256": "1574aac678202802d3a7f391e140c19708280b6bbc85fa11f3d59e948633a852" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/resources/tools/control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/resources/tools/control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/resources/memos/robocasa.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/resources/memos/robocasa.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -7689,8 +10593,7 @@ "observed_images": 119, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/02/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task01-03-seed0-formal", @@ -7777,13 +10680,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 63.15, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 2527, + "published_frames": 2607, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 2526, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 65.175, + "sha256": "bb370aeec770a0779101489ca54c642e59043b1154a77944f9c16128c0ccb1a6", + "source_sha256": { + "third_person.mp4": "f2edd772f07b63d73e819f7f4ebf4a73c6ae596ea414f7558918a05066896ceb", + "wrist.mp4": "31465ff1bf698067633f347139d17aaaab596ad3646c43ce2e42c4231d8223fe" + }, + "all_frames_compared": 2527, + "minimum_frame_psnr_db": 38.0021855976479, + "maximum_frame_rgb_mae": 2.147566556930542, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 2528, "captured_samples": 2528, @@ -7817,12 +10739,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "846d4b0f59787f8085bb4d75001947b38c98a921df6a24a7b4725b1c5defeb7e", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", @@ -7919,34 +10836,35 @@ "protocol_sha256": "66c49605362349c72c2effedd9355205c3fce52ef9b59b59c366c0fdd30aa3e1" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/resources/memos/robocasa-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/resources/memos/robocasa-control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -7955,8 +10873,7 @@ "observed_images": 113, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/03/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task01-04-seed0-formal", @@ -8043,13 +10960,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 67.95, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 2718, + "published_frames": 2798, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 2717, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 69.95, + "sha256": "684612dd30e2c586f6deb01b6b74955c597cfea56a5bb00caceb212c46263cca", + "source_sha256": { + "third_person.mp4": "90df5bd6f19ed94c25ec34092417a7845ee8534e9d3ea461077eb6bf16c6f6f4", + "wrist.mp4": "e3f1feb0d0f43db7c931838b1d84b38cbde954031d19211c7fc3cc95abb63a58" + }, + "all_frames_compared": 2718, + "minimum_frame_psnr_db": 37.46343455083394, + "maximum_frame_rgb_mae": 2.2885396480560303, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 2719, "captured_samples": 2719, @@ -8083,12 +11019,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "e0f50542a3275a5478c3d58d8b6802ca053ef8584e50d91bdf6833ddf14c4517", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", @@ -8185,34 +11116,35 @@ "protocol_sha256": "1ee5b218734e6a1617f3d05d489c52600880cddfc683e60accdf644d04e51bbb" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/resources/memos/robocasa-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/resources/memos/robocasa-control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -8221,8 +11153,7 @@ "observed_images": 122, "tool_errors": 7 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/04/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task01-05-seed0-formal", @@ -8309,13 +11240,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 65.3, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 2613, + "published_frames": 2693, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 2612, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 67.325, + "sha256": "2ee140b8365320f6893680402f15d63d475571c41c3daf19620c0de225174a70", + "source_sha256": { + "third_person.mp4": "c690f8554dc0cf2c3fe95a7e3855b737a1f47d94e490e8fd72dc31c5a7ae1780", + "wrist.mp4": "12fbeaed9f517867e91b8f4fb9ad9e66bcab5bfe59e5bb900c1783ec40ebe3e9" + }, + "all_frames_compared": 2613, + "minimum_frame_psnr_db": 38.241204359424536, + "maximum_frame_rgb_mae": 1.9368914365768433, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 2613, "captured_samples": 2613, @@ -8349,12 +11299,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "64bd66f69c3baf1ad63aa89ccf9c1fa6c0097f553d6fb0b855c213454b12d0c1", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", @@ -8450,34 +11395,35 @@ "protocol_sha256": "09896ba5b0aa8989cea18da115f01eb7144582b8c97b2953b5d3ea2dd0eb42fa" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa_manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/resources/memos/robocasa_manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/resources/memos/robocasa_manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -8486,8 +11432,7 @@ "observed_images": 97, "tool_errors": 5 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/05/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task01-06-seed0-formal", @@ -8574,13 +11519,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 67.7, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 2709, + "published_frames": 2789, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 2708, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 69.725, + "sha256": "35c0d3c3ce913fb355a6a8fb50bd4cd26c9df41340806c1410e1389dd236422e", + "source_sha256": { + "third_person.mp4": "b53a2b29b9594bde19861818b56f25327d2f6140774ae5128ec5ebc92ec81bd3", + "wrist.mp4": "40fde6737510cff0da74a5e1af9736ad4276642e6075b3b3fc4f927a59dab16c" + }, + "all_frames_compared": 2709, + "minimum_frame_psnr_db": 37.057278088285955, + "maximum_frame_rgb_mae": 2.494063138961792, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 2709, "captured_samples": 2709, @@ -8614,12 +11578,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "5b5c91a8fe6002978329674a82d9571b8b84e164f886d2e4ef6c55dbefe4aa27", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", @@ -8716,39 +11675,40 @@ "protocol_sha256": "30fb270b4f0830e49eae5d8c77ee418edc397414725233a4c11f2985fee95fe7" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "skills/robocasa-manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/resources/skills/robocasa-manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/resources/skills/robocasa-manipulation.md", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/resources/memos/robocasa-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/resources/memos/robocasa-control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -8757,8 +11717,7 @@ "observed_images": 93, "tool_errors": 1 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/06/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task01-07-seed0-formal", @@ -8845,13 +11804,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 17.85, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 714, + "published_frames": 794, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 713, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 19.85, + "sha256": "e74d7940fbc5369896194e26b6f59e2169a8e6f717977dd31a60d6219af9fe36", + "source_sha256": { + "third_person.mp4": "d91ecca1b9173d6b999cc624bdc8e4c8510886fcfe5808be56e7de00ddc96d8c", + "wrist.mp4": "4a78e4e2810929c0e0068944ef1613a9fbec584c2e3cdd4854e25725c4c81e22" + }, + "all_frames_compared": 714, + "minimum_frame_psnr_db": 38.685168755373944, + "maximum_frame_rgb_mae": 1.9230414628982544, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 714, "captured_samples": 714, @@ -8885,12 +11863,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "a8988c3e18e487a59ae8c59443039e5e090072a7231c107d7d992606053c496d", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", @@ -8987,34 +11960,35 @@ "protocol_sha256": "d8853a96f748501bccf41fbca04175e69b3b4b557976e14e245ba24ea58d7323" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa_control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/resources/memos/robocasa_control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/resources/memos/robocasa_control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -9023,8 +11997,7 @@ "observed_images": 35, "tool_errors": 1 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/07/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task01-08-seed0-formal", @@ -9111,13 +12084,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 15.15, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 607, + "published_frames": 687, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 606, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 17.175, + "sha256": "8cb379fe8fd05c3f3809ec2e0baa94de25466930fbbc307e37a97fc0a49dc39b", + "source_sha256": { + "third_person.mp4": "78c9dbf1e853522cf0a5b5c954c225427b6e1162f4c605b06b22d37034aa642b", + "wrist.mp4": "d285adf3bc8b962be7f5f0d15cfff23abb620c27af07befb34925de26b230cd3" + }, + "all_frames_compared": 607, + "minimum_frame_psnr_db": 42.55739246406893, + "maximum_frame_rgb_mae": 1.2078295946121216, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 607, "captured_samples": 607, @@ -9151,12 +12143,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "3932519e46bad4fc6c57f44afb37499f07429e5594712e7d6f096a381a7327b6", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", @@ -9253,34 +12240,35 @@ "protocol_sha256": "828302ab34955790504cb974ea07d080685f76d638b3f581c76d06dba7106974" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/resources/memos/robocasa.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/resources/memos/robocasa.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -9289,8 +12277,7 @@ "observed_images": 17, "tool_errors": 6 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/08/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task01-09-seed0-formal", @@ -9377,13 +12364,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 27.7, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 1109, + "published_frames": 1189, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 1108, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 29.725, + "sha256": "faa9cc05c5cdaea638fa2b4a20503ef1ee2eee2c5701b7749cf775dec83f6f7e", + "source_sha256": { + "third_person.mp4": "9a6102841cd30b0ec444bf97d71314b0e274a2a8640f9c55da57393091efbf3a", + "wrist.mp4": "efdb53d8dc8b6cf857cd9033762149a088e65d43704bfa8f97fb96413f0fb603" + }, + "all_frames_compared": 1109, + "minimum_frame_psnr_db": 40.26705996642036, + "maximum_frame_rgb_mae": 1.5987242460250854, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 1109, "captured_samples": 1109, @@ -9417,12 +12423,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "f5b075fc26a06533abe3252b781cdd3fc659cac45d40349afb5bd7b9d8a9dd49", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", @@ -9519,34 +12520,35 @@ "protocol_sha256": "fdeb69aecef69e1512136a89c7f9cc51fde28b55423ac16dc6948b17fdb6fcec" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/resources/tools/control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/resources/tools/control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/resources/memos/robocasa.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/resources/memos/robocasa.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -9555,8 +12557,7 @@ "observed_images": 61, "tool_errors": 0 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/09/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task01-10-seed0-formal", @@ -9643,13 +12644,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 49.6, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 1985, + "published_frames": 2065, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 1984, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 51.625, + "sha256": "25179c986a7201b43cf28c9533cd0b72107a3a0e086233e163399730435fb3bb", + "source_sha256": { + "third_person.mp4": "6527fceea5a0d8d47b9e4d5a82afc453c1089df609e6f0ff3c3c9f2ab9ae1d47", + "wrist.mp4": "88aa7ebe0accc51f8d5e7778fdb592361e2ecab3e21b84dea6f9c0fc9ce3e56d" + }, + "all_frames_compared": 1985, + "minimum_frame_psnr_db": 40.172689032627545, + "maximum_frame_rgb_mae": 1.5079092979431152, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 1986, "captured_samples": 1986, @@ -9683,12 +12703,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "97a9b174702a706c69d3a864da852c271e0f2de38c547df647380152268d016d", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", @@ -9784,34 +12799,35 @@ "protocol_sha256": "12c94809e7b2e98486cd0036d528a828a0c211418b0d365710fd263517dec825" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa_control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/resources/memos/robocasa_control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/resources/memos/robocasa_control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -9820,8 +12836,7 @@ "observed_images": 87, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task01/10/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task02-01-seed0-formal", @@ -9908,13 +12923,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 12.05, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 483, + "published_frames": 563, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 482, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 14.075, + "sha256": "e2ad053cb3a6f5b07c3e793898fb32591098bb9ade819f07be97c6460588bddb", + "source_sha256": { + "third_person.mp4": "55d66feaeedd2a7d372371836b975869fa1a0d707d35ad9783d9f276258143d6", + "wrist.mp4": "3751337173e58dc4c3eaabe1f5e8e54828abb0fe371a97111831949fefafcb1b" + }, + "all_frames_compared": 483, + "minimum_frame_psnr_db": 40.92736650465333, + "maximum_frame_rgb_mae": 1.5692113637924194, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 483, "captured_samples": 483, @@ -9948,12 +12982,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "37b2314d769aed5120526c59804f2ec83f09afb1c4aa26397d2668a0ece7434c", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", @@ -10049,34 +13078,35 @@ "protocol_sha256": "12310099caf8c22cd35d1b1702b90f7780034b1347863b2f0b0dd36c0986aadb" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -10085,8 +13115,7 @@ "observed_images": 34, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/01/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task02-02-seed0-formal", @@ -10173,13 +13202,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 8.95, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 359, + "published_frames": 439, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 358, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 10.975, + "sha256": "4e613cb6dc6dad95d47b0cf0b1c85fd6d7f9db563eb7325a36517a154cab3066", + "source_sha256": { + "third_person.mp4": "e52e829f8c04238f0f3225520f4f4d5e3a0bc90fe40f619652495578fb2eeef6", + "wrist.mp4": "29b92a509c788bfc0b12214f2ec4205b6fbc32cd02f00a2977bf72565c65121f" + }, + "all_frames_compared": 359, + "minimum_frame_psnr_db": 40.865708684473034, + "maximum_frame_rgb_mae": 1.5611512660980225, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 360, "captured_samples": 360, @@ -10213,12 +13261,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "0c7db49b1d23b819f8ec6e5ddb8ce44fd5c893c97b32e315e7cdfa3fa832b623", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", @@ -10314,34 +13357,35 @@ "protocol_sha256": "12cc7fa7e225614b41a5c3fda75bdf62f9e9b39b16c730cef3d5920ad9440392" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/resources/tools/panda.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/resources/tools/panda.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -10350,8 +13394,7 @@ "observed_images": 29, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/02/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task02-03-seed0-formal", @@ -10438,13 +13481,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 25.1, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 1005, + "published_frames": 1085, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 1004, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 27.125, + "sha256": "683ad67e8dbb251f9f862a7e087f84ccd6c4d265e0c2fc8eb01afaddd2d029dd", + "source_sha256": { + "third_person.mp4": "5b8f6723d00ee426fb4cb6473a05b4e0972027055775c3f2d50c9827247295ad", + "wrist.mp4": "bf4acb9f6d2d9a4cd9532d9c5efe67d28f90b3494af7a274d595ed9aeee7dd23" + }, + "all_frames_compared": 1005, + "minimum_frame_psnr_db": 40.85824027475528, + "maximum_frame_rgb_mae": 1.5785024166107178, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 1005, "captured_samples": 1005, @@ -10478,17 +13540,13 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "7406566cb47cae744e1b3419f06e02902b3d0960f44bb43f834f854be2856a8b", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", "duration": "\u4f7f\u7528 2,013 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "native_goal_scope": "\u539f\u751f\u6210\u529f\u6761\u4ef6\u4ec5\u68c0\u67e5\u76ee\u6807\u4f4d\u7f6e\uff0f\u63a5\u89e6\u5173\u7cfb\uff1b\u771f\u5b9e\u7ec8\u5e27\u4ecd\u663e\u793a\u5939\u6301\u3002\u539f\u751f\u6761\u4ef6\u4e0d\u8981\u6c42\u677e\u624b\u6216\u7a33\u5b9a\u843d\u4f4d\uff0c\u6b64\u5904\u4fdd\u7559\u539f\u751f\u901a\u8fc7\u5224\u5b9a\u3002" }, "provenance": { "sources": { @@ -10579,39 +13637,40 @@ "protocol_sha256": "4ee86729ea6301a80cd237762045640ac298b6dfd27faf0547387137274f2fcd" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/resources/tools/panda.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/resources/tools/panda.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/triangulate.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/resources/tools/triangulate.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/resources/tools/triangulate.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -10620,8 +13679,7 @@ "observed_images": 62, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/03/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task02-04-seed0-formal", @@ -10641,13 +13699,13 @@ }, "verdict": { "evidence_valid": true, - "steps": 3687, + "steps": 2081, "success": false, "termination": "stopped" }, - "steps": 3687, + "steps": 2081, "simulation_time_s": null, - "wall_time_s": 919.204412, + "wall_time_s": 425.202323, "model": "gpt-6-astra", "effort": "high", "harness": "codex", @@ -10658,44 +13716,44 @@ "usage": { "accounting": "reported-responses", "audit_complete": true, - "cache_hit_rate": 0.9745746678635921, - "cache_reported_input_tokens": 3356377, + "cache_hit_rate": 0.9656174137975932, + "cache_reported_input_tokens": 1382444, "cache_write_input_tokens": 0, - "cache_write_reported_input_tokens": 3356377, - "cached_input_tokens": 3271040, + "cache_write_reported_input_tokens": 1382444, + "cached_input_tokens": 1334912, "completed_turns": 1, "cost_usd": null, "failed_turns": 0, - "input_tokens": 3356377, + "input_tokens": 1382444, "known_cache_write_input_tokens": 0, - "known_cached_input_tokens": 3271040, - "known_input_tokens": 3356377, - "known_output_tokens": 23446, - "known_reasoning_output_tokens": 11600, - "output_tokens": 23446, - "reasoning_output_tokens": 11600, - "reasoning_reported_output_tokens": 23446, + "known_cached_input_tokens": 1334912, + "known_input_tokens": 1382444, + "known_output_tokens": 12747, + "known_reasoning_output_tokens": 5769, + "output_tokens": 12747, + "reasoning_output_tokens": 5769, + "reasoning_reported_output_tokens": 12747, "reported_responses": { - "cache_reported_input_tokens": 67, - "cache_write_input_tokens": 67, - "cache_write_reported_input_tokens": 67, - "cached_input_tokens": 67, - "input_tokens": 67, - "output_tokens": 67, - "reasoning_output_tokens": 67, - "reasoning_reported_output_tokens": 67 - }, - "response_count": 67, + "cache_reported_input_tokens": 40, + "cache_write_input_tokens": 40, + "cache_write_reported_input_tokens": 40, + "cached_input_tokens": 40, + "input_tokens": 40, + "output_tokens": 40, + "reasoning_output_tokens": 40, + "reasoning_reported_output_tokens": 40 + }, + "response_count": 40, "response_ids_complete": true, "schema": "rlebench/token-usage/1", "source": "Codex token_usage_record per response", - "uncached_input_tokens": 85337, + "uncached_input_tokens": 47532, "unidentified_usage_records": 0 }, "call_activity": { - "model_tool_calls": 66, + "model_tool_calls": 39, "model_tool_calls_by_name": { - "exec": 66 + "exec": 39 }, "nested_python_tool_invocations": null, "python_device_rpc_attempts": null, @@ -10708,24 +13766,45 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 46.05, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 1039, + "published_frames": 1119, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 1038, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 27.975, + "sha256": "5ab1deff54475f6dd503514998fd78940c252ea4f1716f8962b8ab94eb9eb6f5", + "source_sha256": { + "third_person.mp4": "0a88e5c4b9189415ccd4bc819af7bbfabc04557738e1bb1d683b8f7be353ea29", + "wrist.mp4": "85dae1c48d2e13c13ea0e663a4a50aedd7fced526aaf44bae6dc75e9bcb07a56" + }, + "all_frames_compared": 1039, + "minimum_frame_psnr_db": 39.6070605417452, + "maximum_frame_rgb_mae": 1.6335564851760864, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { - "accepted_samples": 1832, - "captured_samples": 1842, + "accepted_samples": 1039, + "captured_samples": 1039, "clock": "simulation", - "dropped_samples": 10, - "encoded_frames": 1842, - "end_time_s": 184.1000000000076, + "dropped_samples": 0, + "encoded_frames": 1039, + "end_time_s": 103.79999999999625, "error": null, "experimental": true, "fps": 10, - "received_samples": 1832, + "overflow_policy": "bounded_backpressure", + "received_samples": 1039, "schema": "roboenv/recording/1", "state": "closed", "status": "complete", @@ -10747,36 +13826,35 @@ "width": 720 } ] - }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "5cdfb9b994575cd13740d3acedab7e848ddd65dcb78eb827d77d610714fd0097", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + } }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", - "duration": "\u4f7f\u7528 3,687 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", + "duration": "\u4f7f\u7528 2,081 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", - "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002" + "cause": "\u771f\u5b9e\u7ec8\u5e27\u7684\u5e95\u5c42\u62bd\u5c49\u4ecd\u6709\u7f1d\u9699\uff0cagent \u4e5f\u62a5\u544a\u672a\u5b8c\u5168\u5173\u95ed\uff1b\u4e0e\u539f\u751f\u5931\u8d25\u4e00\u81f4\u3002\u6307\u4ee4\u660e\u786e\u8981\u6c42\u5173\u4e0a\u62bd\u5c49\uff0c\u6ca1\u6709\u786e\u8ba4\u7684\u6307\u4ee4\u9057\u6f0f\u6216 verifier \u8bef\u5224\u3002", + "recording": "\u4f7f\u7528\u66f4\u65b0\u540e\u7684 stepped \u5f55\u5236\u961f\u5217\u91cd\u65b0\u6267\u884c\uff1b\u6e90\u91c7\u6837\u4e22\u5931\u4e3a 0\u3002\u539f\u59cb\u56de\u5408\u53ca\u5176\u539f\u751f\u5224\u5b9a\u5b8c\u6574\u4fdd\u7559\u3002" }, "provenance": { "sources": { "RLE-Bench-inhouse": { "build_inputs": [ "pyproject.toml", + "README.md", "src", "tasks", - "README.md", - "Makefile", "tests", - "docs" + "docs", + "Makefile" ], - "content_sha256": "edd9d6bdc67136367a15c34949b94cf1bf11ab5d8454e74fd532076feb8032b1", + "content_sha256": "635adce00c840c49aac560ecdb0425a3a45ff8b4804b0620cca45e158bf1916f", "dirty": true, - "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd", - "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + "revision": "f1935004c2fb23501b937cd29b826f69a06214b2", + "runtime_snapshot_content_sha256": "635adce00c840c49aac560ecdb0425a3a45ff8b4804b0620cca45e158bf1916f", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/RLE-Bench-inhouse", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse", + "live_content_sha256": "bbd03a94ca5776307435b6ed81c0f935a17f836b9ee80770069b58162d5e7e8f", + "snapshot_mode": "Current workspace plus owner-only diagnostic configuration; instruction unchanged." }, "RoboEnv": { "build_inputs": [ @@ -10793,9 +13871,10 @@ "third_party/patches", "docs/validation" ], - "content_sha256": "742ec96151dd0d5549d4febb7e091167a35d7886af224fa26d4c464197ea6050", + "content_sha256": "b5e31f051d55e015746e645939e4cbcbbdc44ec111593ef2711c5360900d7768", "dirty": true, - "revision": "5f487ee5121a775aa6ea90919c8f5e3d31318906", + "revision": "612c1528c92065b8f2254a841595e65664dc957d", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/RoboEnv", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" }, "kinex": { @@ -10815,18 +13894,19 @@ "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", "dirty": false, "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/kinex", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" } }, - "job": "task02-04-libero-10-04-codex-seed0-attempt01", + "job": "task02-04-libero-10-04-codex-seed0-recording-replacement-attempt01", "attempt": 1, "harness": "stock Codex CLI", "control_interface": "public RoboEnv SDK and CLI", "kinex_agent_runtime_used": false, - "gpu_index": 4, + "gpu_index": 1, "gpu_model": "NVIDIA L40S", - "campaign_dispatch_concurrency_limit": 20, - "concurrency_note": "One initial canary; then all remaining nineteen tasks launched concurrently under a twenty-task admission cap.", + "campaign_dispatch_concurrency_limit": 6, + "concurrency_note": "Six authorized fresh repair and diagnostic episodes; idle GPUs only.", "codex_version": "0.160.0", "model": "gpt-6-astra", "effort": "high", @@ -10841,53 +13921,54 @@ "configuration": "explicit retry50 SSE", "usage_accounting": "reported-responses" }, - "classification": "formal", + "classification": "recording-replacement", "measured_images": { - "agent": "sha256:5deb0795c2b18162bbcd75531b287469aabb0078db50a48bfe650ecf45868859", - "task": "sha256:4897d53efd57aea35b9c9ad36d3f29f819cedd5974e9c3f7b02775579a884922" + "agent": "sha256:442b190234fb474b7ac7a1f0e03c268be10afc773a829c23bf421f8373f70dde", + "task": "sha256:43e8b22eabf2f490bb210e6f13663bc6e5f49cc65be67174edf268d03cc41f6d" }, - "session_original_sha256": "b63b0b4dfb6ead64a30c353133825d5b20b8ebd3c17913be6864ed382ed9cf4a", - "protocol_sha256": "2cd301bfe0c1a1d6d4a420b1471374a8d33f9471ab060c1d6dcf66fbf74969bf" + "session_original_sha256": "23f688fddf5fe3d5b02b5df9b9992b412bc25e10bb3a33e2edbcf8ce54c56c2e", + "protocol_sha256": "2cd301bfe0c1a1d6d4a420b1471374a8d33f9471ab060c1d6dcf66fbf74969bf", + "replaced_job": "task02-04-libero-10-04-codex-seed0-attempt01" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], "session_counts": { - "visible_events": 148, - "observed_images": 79, - "tool_errors": 1 + "visible_events": 91, + "observed_images": 45, + "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/04/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task02-05-seed0-formal", @@ -10974,13 +14055,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 7.85, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 315, + "published_frames": 395, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 314, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 9.875, + "sha256": "2dee7acaa30050fba7a9c0a4a37bb58e7bef519788af617aa568ec29dfc9c4dc", + "source_sha256": { + "third_person.mp4": "98f062ecdd508144734397696f8427c17b2a8c8b770d93068d4f1350ab3bd1b7", + "wrist.mp4": "8858407a29d56a0e938fb075ac12c00634f14c71dcf3ab39227e6913b711df80" + }, + "all_frames_compared": 315, + "minimum_frame_psnr_db": 41.50050843030813, + "maximum_frame_rgb_mae": 1.4998823404312134, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 316, "captured_samples": 316, @@ -11014,17 +14114,13 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "abfc4504430e57b850b3723fe170f3a6ca3457a8abd6ffa73a17a1bf601023b7", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", "duration": "\u4f7f\u7528 634 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "native_goal_scope": "\u539f\u751f\u6210\u529f\u6761\u4ef6\u4ec5\u68c0\u67e5\u76ee\u6807\u4f4d\u7f6e\uff0f\u63a5\u89e6\u5173\u7cfb\uff1b\u771f\u5b9e\u7ec8\u5e27\u4ecd\u663e\u793a\u5939\u6301\u3002\u539f\u751f\u6761\u4ef6\u4e0d\u8981\u6c42\u677e\u624b\u6216\u7a33\u5b9a\u843d\u4f4d\uff0c\u6b64\u5904\u4fdd\u7559\u539f\u751f\u901a\u8fc7\u5224\u5b9a\u3002" }, "provenance": { "sources": { @@ -11115,34 +14211,35 @@ "protocol_sha256": "aa8612b5784bd68176b7ef74c2a6b2df6f1923236ef02e59a7af155a463dd0ef" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/resources/tools/panda.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/resources/tools/panda.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -11151,8 +14248,7 @@ "observed_images": 19, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/05/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task02-06-seed0-formal", @@ -11239,13 +14335,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 4.0, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 161, + "published_frames": 241, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 160, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 6.025, + "sha256": "bb58ca0e4e16abe53c16a1084bba73da1a01ba1066d183c3c420a942677de2d6", + "source_sha256": { + "third_person.mp4": "9daab516aa8fde2a7d5d830c99ce081bb7a940a025bde7f473df9327626552ab", + "wrist.mp4": "878e6626dec38093ec97e5a24417d80c1a985bb4e1e8274798f4eea3b0c87dcd" + }, + "all_frames_compared": 161, + "minimum_frame_psnr_db": 42.44780443534815, + "maximum_frame_rgb_mae": 1.3797035217285156, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 161, "captured_samples": 161, @@ -11279,12 +14394,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "21e728b9fce1a2954030e87b82dc418320e12675a0e28f89a136a5582f5063fa", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", @@ -11380,34 +14490,35 @@ "protocol_sha256": "00df44aea40ecfa4fdb9b2d79ea89ee3bac07b53ecc4b93a1ffc313e0c1f4e98" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -11416,8 +14527,7 @@ "observed_images": 13, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/06/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task02-07-seed0-formal", @@ -11437,13 +14547,13 @@ }, "verdict": { "evidence_valid": true, - "steps": 1016, + "steps": 1109, "success": true, "termination": "success" }, - "steps": 1016, + "steps": 1109, "simulation_time_s": null, - "wall_time_s": 260.996992, + "wall_time_s": 265.471056, "model": "gpt-6-astra", "effort": "high", "harness": "codex", @@ -11454,44 +14564,44 @@ "usage": { "accounting": "reported-responses", "audit_complete": true, - "cache_hit_rate": 0.9487277032832223, - "cache_reported_input_tokens": 701841, + "cache_hit_rate": 0.9516503131606452, + "cache_reported_input_tokens": 759674, "cache_write_input_tokens": 0, - "cache_write_reported_input_tokens": 701841, - "cached_input_tokens": 665856, + "cache_write_reported_input_tokens": 759674, + "cached_input_tokens": 722944, "completed_turns": 1, "cost_usd": null, "failed_turns": 0, - "input_tokens": 701841, + "input_tokens": 759674, "known_cache_write_input_tokens": 0, - "known_cached_input_tokens": 665856, - "known_input_tokens": 701841, - "known_output_tokens": 6754, - "known_reasoning_output_tokens": 1904, - "output_tokens": 6754, - "reasoning_output_tokens": 1904, - "reasoning_reported_output_tokens": 6754, + "known_cached_input_tokens": 722944, + "known_input_tokens": 759674, + "known_output_tokens": 6893, + "known_reasoning_output_tokens": 2037, + "output_tokens": 6893, + "reasoning_output_tokens": 2037, + "reasoning_reported_output_tokens": 6893, "reported_responses": { - "cache_reported_input_tokens": 24, - "cache_write_input_tokens": 24, - "cache_write_reported_input_tokens": 24, - "cached_input_tokens": 24, - "input_tokens": 24, - "output_tokens": 24, - "reasoning_output_tokens": 24, - "reasoning_reported_output_tokens": 24 - }, - "response_count": 24, + "cache_reported_input_tokens": 26, + "cache_write_input_tokens": 26, + "cache_write_reported_input_tokens": 26, + "cached_input_tokens": 26, + "input_tokens": 26, + "output_tokens": 26, + "reasoning_output_tokens": 26, + "reasoning_reported_output_tokens": 26 + }, + "response_count": 26, "response_ids_complete": true, "schema": "rlebench/token-usage/1", "source": "Codex token_usage_record per response", - "uncached_input_tokens": 35985, + "uncached_input_tokens": 36730, "unidentified_usage_records": 0 }, "call_activity": { - "model_tool_calls": 23, + "model_tool_calls": 25, "model_tool_calls_by_name": { - "exec": 23 + "exec": 25 }, "nested_python_tool_invocations": null, "python_device_rpc_attempts": null, @@ -11504,24 +14614,45 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 12.65, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 553, + "published_frames": 633, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 552, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 15.825, + "sha256": "be2280b03ea7ad4fd0bb943a92d920d52cae98380218fb693ae7f2a0b94c1905", + "source_sha256": { + "third_person.mp4": "c37421010ad4bdbef0d995f4f3c85932700c047420c91efabc41112b01974b1d", + "wrist.mp4": "259cf5f9caff794242fdd437aa71a79cf22030db509be97ccb37ee8c01237182" + }, + "all_frames_compared": 553, + "minimum_frame_psnr_db": 41.38950814581598, + "maximum_frame_rgb_mae": 1.5420993566513062, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { - "accepted_samples": 501, - "captured_samples": 507, + "accepted_samples": 553, + "captured_samples": 553, "clock": "simulation", - "dropped_samples": 6, - "encoded_frames": 506, - "end_time_s": 50.549999999999265, + "dropped_samples": 0, + "encoded_frames": 553, + "end_time_s": 55.199999999999, "error": null, "experimental": true, "fps": 10, - "received_samples": 501, + "overflow_policy": "bounded_backpressure", + "received_samples": 553, "schema": "roboenv/recording/1", "state": "closed", "status": "complete", @@ -11543,35 +14674,34 @@ "width": 720 } ] - }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "e56bc148cb2905f49aa7cc78124462b86130dedaa18a8195efe0e08b0d0501a7", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + } }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", - "duration": "\u4f7f\u7528 1,016 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + "duration": "\u4f7f\u7528 1,109 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "recording": "\u4f7f\u7528\u66f4\u65b0\u540e\u7684 stepped \u5f55\u5236\u961f\u5217\u91cd\u65b0\u6267\u884c\uff1b\u6e90\u91c7\u6837\u4e22\u5931\u4e3a 0\u3002\u539f\u59cb\u56de\u5408\u53ca\u5176\u539f\u751f\u5224\u5b9a\u5b8c\u6574\u4fdd\u7559\u3002" }, "provenance": { "sources": { "RLE-Bench-inhouse": { "build_inputs": [ "pyproject.toml", + "README.md", "src", "tasks", - "README.md", - "Makefile", "tests", - "docs" + "docs", + "Makefile" ], - "content_sha256": "edd9d6bdc67136367a15c34949b94cf1bf11ab5d8454e74fd532076feb8032b1", + "content_sha256": "635adce00c840c49aac560ecdb0425a3a45ff8b4804b0620cca45e158bf1916f", "dirty": true, - "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd", - "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + "revision": "f1935004c2fb23501b937cd29b826f69a06214b2", + "runtime_snapshot_content_sha256": "635adce00c840c49aac560ecdb0425a3a45ff8b4804b0620cca45e158bf1916f", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/RLE-Bench-inhouse", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse", + "live_content_sha256": "bbd03a94ca5776307435b6ed81c0f935a17f836b9ee80770069b58162d5e7e8f", + "snapshot_mode": "Current workspace plus owner-only diagnostic configuration; instruction unchanged." }, "RoboEnv": { "build_inputs": [ @@ -11588,9 +14718,10 @@ "third_party/patches", "docs/validation" ], - "content_sha256": "742ec96151dd0d5549d4febb7e091167a35d7886af224fa26d4c464197ea6050", + "content_sha256": "b5e31f051d55e015746e645939e4cbcbbdc44ec111593ef2711c5360900d7768", "dirty": true, - "revision": "5f487ee5121a775aa6ea90919c8f5e3d31318906", + "revision": "612c1528c92065b8f2254a841595e65664dc957d", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/RoboEnv", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" }, "kinex": { @@ -11610,18 +14741,19 @@ "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", "dirty": false, "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/kinex", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" } }, - "job": "task02-07-libero-10-07-codex-seed0-attempt01", + "job": "task02-07-libero-10-07-codex-seed0-recording-replacement-attempt01", "attempt": 1, "harness": "stock Codex CLI", "control_interface": "public RoboEnv SDK and CLI", "kinex_agent_runtime_used": false, - "gpu_index": 2, + "gpu_index": 6, "gpu_model": "NVIDIA L40S", - "campaign_dispatch_concurrency_limit": 20, - "concurrency_note": "One initial canary; then all remaining nineteen tasks launched concurrently under a twenty-task admission cap.", + "campaign_dispatch_concurrency_limit": 6, + "concurrency_note": "Six authorized fresh repair and diagnostic episodes; idle GPUs only.", "codex_version": "0.160.0", "model": "gpt-6-astra", "effort": "high", @@ -11636,53 +14768,54 @@ "configuration": "explicit retry50 SSE", "usage_accounting": "reported-responses" }, - "classification": "formal", + "classification": "recording-replacement", "measured_images": { - "agent": "sha256:5deb0795c2b18162bbcd75531b287469aabb0078db50a48bfe650ecf45868859", - "task": "sha256:4897d53efd57aea35b9c9ad36d3f29f819cedd5974e9c3f7b02775579a884922" + "agent": "sha256:442b190234fb474b7ac7a1f0e03c268be10afc773a829c23bf421f8373f70dde", + "task": "sha256:43e8b22eabf2f490bb210e6f13663bc6e5f49cc65be67174edf268d03cc41f6d" }, - "session_original_sha256": "7587686fd933aa48160da3af4647d34dc914373521a51b368351fd134e6bf464", - "protocol_sha256": "bdb44af586667ee7eab0855e1a04f70e56609333b710668aa923999c6424aa55" + "session_original_sha256": "41cf7d89429bdc21e0eb8883ae2286a4ea0ed345d482b4cd2a045ffe4588d691", + "protocol_sha256": "bdb44af586667ee7eab0855e1a04f70e56609333b710668aa923999c6424aa55", + "replaced_job": "task02-07-libero-10-07-codex-seed0-attempt01" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], "session_counts": { - "visible_events": 58, - "observed_images": 18, - "tool_errors": 3 + "visible_events": 62, + "observed_images": 25, + "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/07/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task02-08-seed0-formal", @@ -11702,13 +14835,13 @@ }, "verdict": { "evidence_valid": true, - "steps": 1115, + "steps": 749, "success": true, "termination": "success" }, - "steps": 1115, + "steps": 749, "simulation_time_s": null, - "wall_time_s": 412.342365, + "wall_time_s": 194.219873, "model": "gpt-6-astra", "effort": "high", "harness": "codex", @@ -11719,44 +14852,44 @@ "usage": { "accounting": "reported-responses", "audit_complete": true, - "cache_hit_rate": 0.9604073765886478, - "cache_reported_input_tokens": 1406070, + "cache_hit_rate": 0.9524417760029756, + "cache_reported_input_tokens": 569954, "cache_write_input_tokens": 0, - "cache_write_reported_input_tokens": 1406070, - "cached_input_tokens": 1350400, + "cache_write_reported_input_tokens": 569954, + "cached_input_tokens": 542848, "completed_turns": 1, "cost_usd": null, "failed_turns": 0, - "input_tokens": 1406070, + "input_tokens": 569954, "known_cache_write_input_tokens": 0, - "known_cached_input_tokens": 1350400, - "known_input_tokens": 1406070, - "known_output_tokens": 9396, - "known_reasoning_output_tokens": 4471, - "output_tokens": 9396, - "reasoning_output_tokens": 4471, - "reasoning_reported_output_tokens": 9396, + "known_cached_input_tokens": 542848, + "known_input_tokens": 569954, + "known_output_tokens": 6419, + "known_reasoning_output_tokens": 2083, + "output_tokens": 6419, + "reasoning_output_tokens": 2083, + "reasoning_reported_output_tokens": 6419, "reported_responses": { - "cache_reported_input_tokens": 40, - "cache_write_input_tokens": 40, - "cache_write_reported_input_tokens": 40, - "cached_input_tokens": 40, - "input_tokens": 40, - "output_tokens": 40, - "reasoning_output_tokens": 40, - "reasoning_reported_output_tokens": 40 + "cache_reported_input_tokens": 22, + "cache_write_input_tokens": 22, + "cache_write_reported_input_tokens": 22, + "cached_input_tokens": 22, + "input_tokens": 22, + "output_tokens": 22, + "reasoning_output_tokens": 22, + "reasoning_reported_output_tokens": 22 }, - "response_count": 40, + "response_count": 22, "response_ids_complete": true, "schema": "rlebench/token-usage/1", "source": "Codex token_usage_record per response", - "uncached_input_tokens": 55670, + "uncached_input_tokens": 27106, "unidentified_usage_records": 0 }, "call_activity": { - "model_tool_calls": 39, + "model_tool_calls": 21, "model_tool_calls_by_name": { - "exec": 39 + "exec": 21 }, "nested_python_tool_invocations": null, "python_device_rpc_attempts": null, @@ -11769,24 +14902,45 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 13.9, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 373, + "published_frames": 453, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 372, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 11.325, + "sha256": "dceb019aac59787c44ed776949f7a17988ef68ab77920e96cc24d4e983a10b03", + "source_sha256": { + "third_person.mp4": "3dd6952f500810b9960e874b923a2ae4d8ebb16fc3dabbda713522c29016dc6c", + "wrist.mp4": "531f18c47fc4a6dc362d9cf91c492962d48854cc22171184c84a3664fcefdabe" + }, + "all_frames_compared": 373, + "minimum_frame_psnr_db": 41.92577356463522, + "maximum_frame_rgb_mae": 1.4596598148345947, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { - "accepted_samples": 538, - "captured_samples": 556, + "accepted_samples": 373, + "captured_samples": 373, "clock": "simulation", - "dropped_samples": 18, - "encoded_frames": 556, - "end_time_s": 55.499999999998984, + "dropped_samples": 0, + "encoded_frames": 373, + "end_time_s": 37.200000000000024, "error": null, "experimental": true, "fps": 10, - "received_samples": 538, + "overflow_policy": "bounded_backpressure", + "received_samples": 373, "schema": "roboenv/recording/1", "state": "closed", "status": "complete", @@ -11808,35 +14962,34 @@ "width": 720 } ] - }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "ebc33c52431806b4f772f68b9d94dbaa7dc7eb16be8583fc51f4761281c22e16", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + } }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", - "duration": "\u4f7f\u7528 1,115 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + "duration": "\u4f7f\u7528 749 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "recording": "\u4f7f\u7528\u66f4\u65b0\u540e\u7684 stepped \u5f55\u5236\u961f\u5217\u91cd\u65b0\u6267\u884c\uff1b\u6e90\u91c7\u6837\u4e22\u5931\u4e3a 0\u3002\u539f\u59cb\u56de\u5408\u53ca\u5176\u539f\u751f\u5224\u5b9a\u5b8c\u6574\u4fdd\u7559\u3002" }, "provenance": { "sources": { "RLE-Bench-inhouse": { "build_inputs": [ "pyproject.toml", + "README.md", "src", "tasks", - "README.md", - "Makefile", "tests", - "docs" + "docs", + "Makefile" ], - "content_sha256": "edd9d6bdc67136367a15c34949b94cf1bf11ab5d8454e74fd532076feb8032b1", + "content_sha256": "635adce00c840c49aac560ecdb0425a3a45ff8b4804b0620cca45e158bf1916f", "dirty": true, - "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd", - "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + "revision": "f1935004c2fb23501b937cd29b826f69a06214b2", + "runtime_snapshot_content_sha256": "635adce00c840c49aac560ecdb0425a3a45ff8b4804b0620cca45e158bf1916f", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/RLE-Bench-inhouse", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse", + "live_content_sha256": "bbd03a94ca5776307435b6ed81c0f935a17f836b9ee80770069b58162d5e7e8f", + "snapshot_mode": "Current workspace plus owner-only diagnostic configuration; instruction unchanged." }, "RoboEnv": { "build_inputs": [ @@ -11853,9 +15006,10 @@ "third_party/patches", "docs/validation" ], - "content_sha256": "742ec96151dd0d5549d4febb7e091167a35d7886af224fa26d4c464197ea6050", + "content_sha256": "b5e31f051d55e015746e645939e4cbcbbdc44ec111593ef2711c5360900d7768", "dirty": true, - "revision": "5f487ee5121a775aa6ea90919c8f5e3d31318906", + "revision": "612c1528c92065b8f2254a841595e65664dc957d", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/RoboEnv", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" }, "kinex": { @@ -11875,18 +15029,19 @@ "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", "dirty": false, "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/kinex", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" } }, - "job": "task02-08-libero-10-08-codex-seed0-attempt01", + "job": "task02-08-libero-10-08-codex-seed0-recording-replacement-attempt01", "attempt": 1, "harness": "stock Codex CLI", "control_interface": "public RoboEnv SDK and CLI", "kinex_agent_runtime_used": false, - "gpu_index": 3, + "gpu_index": 7, "gpu_model": "NVIDIA L40S", - "campaign_dispatch_concurrency_limit": 20, - "concurrency_note": "One initial canary; then all remaining nineteen tasks launched concurrently under a twenty-task admission cap.", + "campaign_dispatch_concurrency_limit": 6, + "concurrency_note": "Six authorized fresh repair and diagnostic episodes; idle GPUs only.", "codex_version": "0.160.0", "model": "gpt-6-astra", "effort": "high", @@ -11901,53 +15056,54 @@ "configuration": "explicit retry50 SSE", "usage_accounting": "reported-responses" }, - "classification": "formal", + "classification": "recording-replacement", "measured_images": { - "agent": "sha256:5deb0795c2b18162bbcd75531b287469aabb0078db50a48bfe650ecf45868859", - "task": "sha256:4897d53efd57aea35b9c9ad36d3f29f819cedd5974e9c3f7b02775579a884922" + "agent": "sha256:442b190234fb474b7ac7a1f0e03c268be10afc773a829c23bf421f8373f70dde", + "task": "sha256:43e8b22eabf2f490bb210e6f13663bc6e5f49cc65be67174edf268d03cc41f6d" }, - "session_original_sha256": "2be2b70496fa5cd49637b5f93111331068e67ccc7e3b261df25b663d09f162aa", - "protocol_sha256": "b0e2388679180c5a165f77c1cc90a2c5bfce0b2b1582b118c86392d435c21543" + "session_original_sha256": "1a0bb0f8a4bd09402a82d45533ec038a5f2606ba6c0bc4ce69fc39bfc04de807", + "protocol_sha256": "b0e2388679180c5a165f77c1cc90a2c5bfce0b2b1582b118c86392d435c21543", + "replaced_job": "task02-08-libero-10-08-codex-seed0-attempt01" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/terminal.jpg" }, "resources": [ { - "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/resources/tools/panda_control.py", + "name": "tools/libero_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/resources/tools/libero_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], "session_counts": { - "visible_events": 91, + "visible_events": 55, "observed_images": 27, - "tool_errors": 2 + "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/08/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task02-09-seed0-formal", @@ -11967,13 +15123,13 @@ }, "verdict": { "evidence_valid": true, - "steps": 560, + "steps": 1678, "success": true, "termination": "success" }, - "steps": 560, + "steps": 1678, "simulation_time_s": null, - "wall_time_s": 216.244925, + "wall_time_s": 468.030664, "model": "gpt-6-astra", "effort": "high", "harness": "codex", @@ -11984,44 +15140,44 @@ "usage": { "accounting": "reported-responses", "audit_complete": true, - "cache_hit_rate": 0.9551717429852196, - "cache_reported_input_tokens": 714527, + "cache_hit_rate": 0.957354311657529, + "cache_reported_input_tokens": 1421785, "cache_write_input_tokens": 0, - "cache_write_reported_input_tokens": 714527, - "cached_input_tokens": 682496, + "cache_write_reported_input_tokens": 1421785, + "cached_input_tokens": 1361152, "completed_turns": 1, "cost_usd": null, "failed_turns": 0, - "input_tokens": 714527, + "input_tokens": 1421785, "known_cache_write_input_tokens": 0, - "known_cached_input_tokens": 682496, - "known_input_tokens": 714527, - "known_output_tokens": 5355, - "known_reasoning_output_tokens": 1306, - "output_tokens": 5355, - "reasoning_output_tokens": 1306, - "reasoning_reported_output_tokens": 5355, + "known_cached_input_tokens": 1361152, + "known_input_tokens": 1421785, + "known_output_tokens": 12039, + "known_reasoning_output_tokens": 5862, + "output_tokens": 12039, + "reasoning_output_tokens": 5862, + "reasoning_reported_output_tokens": 12039, "reported_responses": { - "cache_reported_input_tokens": 24, - "cache_write_input_tokens": 24, - "cache_write_reported_input_tokens": 24, - "cached_input_tokens": 24, - "input_tokens": 24, - "output_tokens": 24, - "reasoning_output_tokens": 24, - "reasoning_reported_output_tokens": 24 - }, - "response_count": 24, + "cache_reported_input_tokens": 40, + "cache_write_input_tokens": 40, + "cache_write_reported_input_tokens": 40, + "cached_input_tokens": 40, + "input_tokens": 40, + "output_tokens": 40, + "reasoning_output_tokens": 40, + "reasoning_reported_output_tokens": 40 + }, + "response_count": 40, "response_ids_complete": true, "schema": "rlebench/token-usage/1", "source": "Codex token_usage_record per response", - "uncached_input_tokens": 32031, + "uncached_input_tokens": 60633, "unidentified_usage_records": 0 }, "call_activity": { - "model_tool_calls": 23, + "model_tool_calls": 39, "model_tool_calls_by_name": { - "exec": 23 + "exec": 39 }, "nested_python_tool_invocations": null, "python_device_rpc_attempts": null, @@ -12034,24 +15190,45 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 6.95, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 837, + "published_frames": 917, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 836, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 22.925, + "sha256": "e8f02e3f80449ca54a67ac6e365f60ce697ca68475e9d8d5ad20162c7313a56f", + "source_sha256": { + "third_person.mp4": "9819be7d7d915d6f1497d9494abc6781afb76beb4f0eee13d95756e8f49cfb19", + "wrist.mp4": "d4f9827efe589d67a1897762834c2df2b56ecf7f7292a8b15355106f1117d6e4" + }, + "all_frames_compared": 837, + "minimum_frame_psnr_db": 40.15954932563524, + "maximum_frame_rgb_mae": 1.6357674598693848, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { - "accepted_samples": 272, - "captured_samples": 279, + "accepted_samples": 838, + "captured_samples": 838, "clock": "simulation", - "dropped_samples": 7, - "encoded_frames": 278, - "end_time_s": 27.75000000000026, + "dropped_samples": 0, + "encoded_frames": 837, + "end_time_s": 83.64999999999739, "error": null, "experimental": true, "fps": 10, - "received_samples": 272, + "overflow_policy": "bounded_backpressure", + "received_samples": 838, "schema": "roboenv/recording/1", "state": "closed", "status": "complete", @@ -12073,35 +15250,35 @@ "width": 720 } ] - }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "a76a2e7a791e193195741516810fd124b1d545fd74ab1aa6bc729c40f2f1d1fc", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + } }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", - "duration": "\u4f7f\u7528 560 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + "duration": "\u4f7f\u7528 1,678 / 6,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "recording": "\u4f7f\u7528\u66f4\u65b0\u540e\u7684 stepped \u5f55\u5236\u961f\u5217\u91cd\u65b0\u6267\u884c\uff1b\u6e90\u91c7\u6837\u4e22\u5931\u4e3a 0\u3002\u539f\u59cb\u56de\u5408\u53ca\u5176\u539f\u751f\u5224\u5b9a\u5b8c\u6574\u4fdd\u7559\u3002", + "native_goal_scope": "\u539f\u751f\u6210\u529f\u6761\u4ef6\u4ec5\u68c0\u67e5\u76ee\u6807\u4f4d\u7f6e\uff0f\u63a5\u89e6\u5173\u7cfb\uff1b\u771f\u5b9e\u7ec8\u5e27\u4ecd\u663e\u793a\u5939\u6301\u3002\u539f\u751f\u6761\u4ef6\u4e0d\u8981\u6c42\u677e\u624b\u6216\u7a33\u5b9a\u843d\u4f4d\uff0c\u6b64\u5904\u4fdd\u7559\u539f\u751f\u901a\u8fc7\u5224\u5b9a\u3002" }, "provenance": { "sources": { "RLE-Bench-inhouse": { "build_inputs": [ "pyproject.toml", + "README.md", "src", "tasks", - "README.md", - "Makefile", "tests", - "docs" + "docs", + "Makefile" ], - "content_sha256": "edd9d6bdc67136367a15c34949b94cf1bf11ab5d8454e74fd532076feb8032b1", + "content_sha256": "635adce00c840c49aac560ecdb0425a3a45ff8b4804b0620cca45e158bf1916f", "dirty": true, - "revision": "6e75c8cdf4d737087d727c6627645d0aa7e2d1dd", - "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + "revision": "f1935004c2fb23501b937cd29b826f69a06214b2", + "runtime_snapshot_content_sha256": "635adce00c840c49aac560ecdb0425a3a45ff8b4804b0620cca45e158bf1916f", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/RLE-Bench-inhouse", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse", + "live_content_sha256": "bbd03a94ca5776307435b6ed81c0f935a17f836b9ee80770069b58162d5e7e8f", + "snapshot_mode": "Current workspace plus owner-only diagnostic configuration; instruction unchanged." }, "RoboEnv": { "build_inputs": [ @@ -12118,9 +15295,10 @@ "third_party/patches", "docs/validation" ], - "content_sha256": "742ec96151dd0d5549d4febb7e091167a35d7886af224fa26d4c464197ea6050", + "content_sha256": "b5e31f051d55e015746e645939e4cbcbbdc44ec111593ef2711c5360900d7768", "dirty": true, - "revision": "5f487ee5121a775aa6ea90919c8f5e3d31318906", + "revision": "612c1528c92065b8f2254a841595e65664dc957d", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/RoboEnv", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" }, "kinex": { @@ -12140,18 +15318,19 @@ "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", "dirty": false, "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "snapshot": "/shared/evaluations/roboenv-main-media-repair-20261010T191800Z/sources/kinex", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" } }, - "job": "task02-09-libero-10-09-codex-seed0-attempt01", + "job": "task02-09-libero-10-09-codex-seed0-recording-replacement-attempt01", "attempt": 1, "harness": "stock Codex CLI", "control_interface": "public RoboEnv SDK and CLI", "kinex_agent_runtime_used": false, - "gpu_index": 6, + "gpu_index": 1, "gpu_model": "NVIDIA L40S", - "campaign_dispatch_concurrency_limit": 20, - "concurrency_note": "One initial canary; then all remaining nineteen tasks launched concurrently under a twenty-task admission cap.", + "campaign_dispatch_concurrency_limit": 6, + "concurrency_note": "Six authorized fresh repair and diagnostic episodes; idle GPUs only.", "codex_version": "0.160.0", "model": "gpt-6-astra", "effort": "high", @@ -12166,53 +15345,54 @@ "configuration": "explicit retry50 SSE", "usage_accounting": "reported-responses" }, - "classification": "formal", + "classification": "recording-replacement", "measured_images": { - "agent": "sha256:5deb0795c2b18162bbcd75531b287469aabb0078db50a48bfe650ecf45868859", - "task": "sha256:4897d53efd57aea35b9c9ad36d3f29f819cedd5974e9c3f7b02775579a884922" + "agent": "sha256:442b190234fb474b7ac7a1f0e03c268be10afc773a829c23bf421f8373f70dde", + "task": "sha256:43e8b22eabf2f490bb210e6f13663bc6e5f49cc65be67174edf268d03cc41f6d" }, - "session_original_sha256": "b63a5267201015666c8fcdac5e4b3884a10661b76638beb854a1b405e91f2719", - "protocol_sha256": "acc9470d622733def9d480c2fb756b1724249077e1ad1f73cf80967246cabf7a" + "session_original_sha256": "079769f3340d9083683e927b402ea18ee307e6c8b70d9833851cff45de20b978", + "protocol_sha256": "acc9470d622733def9d480c2fb756b1724249077e1ad1f73cf80967246cabf7a", + "replaced_job": "task02-09-libero-10-09-codex-seed0-attempt01" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/terminal.jpg" }, "resources": [ { - "name": "tools/panda.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/resources/tools/panda.py", + "name": "tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { - "name": "memos/libero-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/resources/memos/libero-control.md", + "name": "memos/libero_control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/resources/memos/libero_control.md", "kind": "Created during this episode; final workspace snapshot." } ], "session_counts": { - "visible_events": 57, - "observed_images": 21, - "tool_errors": 2 + "visible_events": 91, + "observed_images": 37, + "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/09/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task02-10-seed0-formal", @@ -12299,13 +15479,32 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 1440, - "height": 720, - "duration_s": 15.05, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 1440, + "height": 720, + "native_frames": 603, + "published_frames": 683, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 602, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 17.075, + "sha256": "f19ef0619ee1a49d549f7862e2d95c39fb44eb7c8e2517972372986ce9906314", + "source_sha256": { + "third_person.mp4": "777c72c592a083ab831f8f8291b1cff9af2d4092006ca7b674e2e579c31d4f99", + "wrist.mp4": "6d4a21aca2ea442709c71c63108875e9ac00f46a660ae4271e3828d9d985f875" + }, + "all_frames_compared": 603, + "minimum_frame_psnr_db": 41.572665775005014, + "maximum_frame_rgb_mae": 1.422065019607544, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 604, "captured_samples": 604, @@ -12339,12 +15538,7 @@ } ] }, - "view_names": [ - "third_person", - "wrist" - ], - "sha256": "73dac6240e4da279d5f73ff9c67815dc19eb01421a5c3914c17e160fdeb7a86e", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", @@ -12440,34 +15634,35 @@ "protocol_sha256": "67091e2f8027e0c35cabd7bed5126858b5ff9603ccb55d40bccc34bb4b005d61" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -12476,8 +15671,7 @@ "observed_images": 42, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task02/10/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task03-01-seed0-formal", @@ -12564,13 +15758,33 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 2880, - "height": 720, - "duration_s": 8.05, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 322, + "published_frames": 402, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 321, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 10.05, + "sha256": "88c31bda9ccd18b48a689cb05de7ed1ec2f0060565f686a1584790cabeb7266e", + "source_sha256": { + "third_person.mp4": "91a39898808effe72a9dc268897a11e2b8d4881f5fc786d3ca465a94b49ace41", + "left_wrist.mp4": "eb123dd3ab9557d8e64d47407121e52a9f1deca0c895f583a65874eee46a39d1", + "right_wrist.mp4": "1580ee3fe4b1f43c1c38066f3c21eefb170564bfa56b7f199d8bed459024071a" + }, + "all_frames_compared": 322, + "minimum_frame_psnr_db": 43.653306171640274, + "maximum_frame_rgb_mae": 1.2375693321228027, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 322, "captured_samples": 322, @@ -12612,13 +15826,7 @@ } ] }, - "view_names": [ - "third_person", - "left_wrist", - "right_wrist" - ], - "sha256": "cf4e4fb86af1c0f99b066153506d5f07bd96d5e2936b643887be8d92e5a3fe31", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", @@ -12714,34 +15922,35 @@ "protocol_sha256": "ad98ebf78ce3849d130c3fe4cf620520d1f7e5cfa6ef852f901223e0ffea50fd" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/native/evidence/episode/evidence/native-episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/resources/tools/aloha.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/resources/tools/aloha.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/aloha_handover.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/resources/memos/aloha_handover.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/resources/memos/aloha_handover.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -12750,8 +15959,7 @@ "observed_images": 14, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/01/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task03-02-seed0-formal", @@ -12838,13 +16046,33 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 2880, - "height": 720, - "duration_s": 7.75, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 311, + "published_frames": 391, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 310, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 9.775, + "sha256": "dd736ea02fcfbc005655e27beef38ae7e6f7d1cb0cacbe65039b26e8ba0ddc77", + "source_sha256": { + "third_person.mp4": "ac9f30444c0dfdde14affb3887762664bf61d40c1f82158b33e7c9921c2ce255", + "left_wrist.mp4": "557a30229ae662000a8a5ace6243d3001841aa9fd31a5973baf759cc3049ca9f", + "right_wrist.mp4": "3e290c41e7f43ba29691d5ffe042c3c1e3eae52feefe4fda0d780eb44d422206" + }, + "all_frames_compared": 311, + "minimum_frame_psnr_db": 41.35740377565105, + "maximum_frame_rgb_mae": 1.5965594053268433, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 312, "captured_samples": 312, @@ -12886,13 +16114,7 @@ } ] }, - "view_names": [ - "third_person", - "left_wrist", - "right_wrist" - ], - "sha256": "3688cb50d8a51e4d82cf708190bceffa2534192f41f336a6ba6046c4f015bdb1", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", @@ -12988,34 +16210,35 @@ "protocol_sha256": "6ba3948ba616c4ca3d353bff11cd576ebc0c5187c28471204be413fa95999676" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/native/evidence/episode/evidence/native-episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/resources/tools/aloha_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/resources/tools/aloha_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/aloha_shoe_placement.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/resources/memos/aloha_shoe_placement.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/resources/memos/aloha_shoe_placement.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -13024,8 +16247,7 @@ "observed_images": 19, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/02/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task03-03-seed0-formal", @@ -13112,13 +16334,33 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 2880, - "height": 720, - "duration_s": 27.9, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 1117, + "published_frames": 1197, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 1116, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 29.925, + "sha256": "8f931f2e72b7ad113113be6cf16ef9786a13e93edafc580ee4ee7592e520a192", + "source_sha256": { + "third_person.mp4": "1eb56cf62f067c1470e1d2d635fa88f61d16739169642b6cc975d4c538f1485f", + "left_wrist.mp4": "735241b382f238ea4910e2041c5a5ce7d4aa7c099f59dba7845270b373abca4d", + "right_wrist.mp4": "65a3277c183df0a04660fe5c379f54a08969a93a25639fa796121fb743222709" + }, + "all_frames_compared": 1117, + "minimum_frame_psnr_db": 41.02636410816193, + "maximum_frame_rgb_mae": 1.6107410192489624, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 1117, "captured_samples": 1117, @@ -13160,13 +16402,7 @@ } ] }, - "view_names": [ - "third_person", - "left_wrist", - "right_wrist" - ], - "sha256": "e363412425448230e053fa034cad62505f61a65e7fcbac94bacd2393c8c7df4c", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", @@ -13263,34 +16499,35 @@ "protocol_sha256": "6f2e06434fc34ac77c80c6ab5f307ef627ef25d814317530a0d122781a52346b" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/native/evidence/episode/evidence/native-episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/resources/tools/robot_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/resources/tools/robot_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robotwin_aloha.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/resources/memos/robotwin_aloha.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/resources/memos/robotwin_aloha.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -13299,8 +16536,7 @@ "observed_images": 37, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/03/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task03-04-seed0-formal", @@ -13387,13 +16623,33 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 2880, - "height": 720, - "duration_s": 12.75, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 511, + "published_frames": 591, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 510, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 14.775, + "sha256": "778a8c753efe2ad095338901a02b7e775ad1bababe1174d02e8d31be12947880", + "source_sha256": { + "third_person.mp4": "3df3f5832c8dc7628929bc203e4648e052d626285c4476318cc275d0dda8267c", + "left_wrist.mp4": "97b442c429b2f8c647225ec8dc9fdf1eb60d673e3aa0c155d66440ec1b4442b7", + "right_wrist.mp4": "1128f89ae973b39378855d9775f85ce476e63d6ae36a2da8dbcd8d5dc9194267" + }, + "all_frames_compared": 511, + "minimum_frame_psnr_db": 42.14528928367494, + "maximum_frame_rgb_mae": 1.4536242485046387, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 512, "captured_samples": 512, @@ -13435,13 +16691,7 @@ } ] }, - "view_names": [ - "third_person", - "left_wrist", - "right_wrist" - ], - "sha256": "628c7593e28d6959821b90a4ef67b50865108053895e5240b73a4a7d25b1d046", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", @@ -13538,34 +16788,35 @@ "protocol_sha256": "80bf39a41667e4718ca9014cb14c7b3825863df9a915cee99415aad8ccf711fd" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/native/evidence/episode/evidence/native-episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/resources/tools/aloha.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/resources/tools/aloha.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/aloha_scanning.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/resources/memos/aloha_scanning.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/resources/memos/aloha_scanning.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -13574,8 +16825,7 @@ "observed_images": 27, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/04/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task03-05-seed0-formal", @@ -13662,13 +16912,33 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 2880, - "height": 720, - "duration_s": 32.65, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 1306, + "published_frames": 1386, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 1305, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 34.65, + "sha256": "6aa91f96033c2ca5dc6808f00886cd42b7678aa5f4b44fc6d540726de54a9396", + "source_sha256": { + "third_person.mp4": "b4730790d901ccf94d27666ff66a0fb682697ceec6e9d6debba077980819addf", + "left_wrist.mp4": "24747433a8e41150d6de1ead4bc0cd4481b9ea7c2ad95337e154eac17985466d", + "right_wrist.mp4": "8a0780725fad342d9826f457667883309036f9548678cb9fd656eaecc4dd496a" + }, + "all_frames_compared": 1306, + "minimum_frame_psnr_db": 43.37254990881951, + "maximum_frame_rgb_mae": 1.3213776350021362, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 1306, "captured_samples": 1306, @@ -13710,13 +16980,7 @@ } ] }, - "view_names": [ - "third_person", - "left_wrist", - "right_wrist" - ], - "sha256": "34a75c4b3104015f6faeb22a4180608a3d88632c7e8dd5d707776d4e3d4f35fd", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", @@ -13813,34 +17077,35 @@ "protocol_sha256": "e2de18f5e058a9b288accd36917ebbdd8cb5b957bee0cff2637d8a732df39495" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/native/evidence/episode/evidence/native-episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/resources/tools/aloha.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/resources/tools/aloha.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robotwin_aloha_bowls.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/resources/memos/robotwin_aloha_bowls.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/resources/memos/robotwin_aloha_bowls.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -13849,8 +17114,7 @@ "observed_images": 48, "tool_errors": 1 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/05/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task03-06-seed0-formal", @@ -13937,13 +17201,33 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 2880, - "height": 720, - "duration_s": 9.05, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 363, + "published_frames": 443, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 362, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 11.075, + "sha256": "60bb450da8e4e4bb9c054e623b4c1c1a70d4b2c2a6c443a26a30d1cf44ac1e6c", + "source_sha256": { + "third_person.mp4": "24f1e9175b6f4a1f58156ab4a8b706be2ebd00f02ba0c3594a692bc81d911fbc", + "left_wrist.mp4": "270cf4333975c161faafe96de7fc1cc8a32907a1f12a54feb209b922b3b9ba04", + "right_wrist.mp4": "8a6f9fe23b2e5282b225736a93d5a2611d4ab505db2b82bd506e1805cd0a22d2" + }, + "all_frames_compared": 363, + "minimum_frame_psnr_db": 43.69486927411482, + "maximum_frame_rgb_mae": 1.2050166130065918, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 363, "captured_samples": 363, @@ -13985,13 +17269,7 @@ } ] }, - "view_names": [ - "third_person", - "left_wrist", - "right_wrist" - ], - "sha256": "b187a9302202675de1b85393beec8662720d11866a1cad19180b90105f2f5ce5", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", @@ -14087,34 +17365,35 @@ "protocol_sha256": "fb73feaa0822af598381331e7f3add1ee2f53b407bc3c4214783b9cf05e75533" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/native/evidence/episode/evidence/native-episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robotwin_motion.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/resources/tools/robotwin_motion.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/resources/tools/robotwin_motion.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robotwin-aloha.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/resources/memos/robotwin-aloha.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/resources/memos/robotwin-aloha.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -14123,8 +17402,7 @@ "observed_images": 10, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/06/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task03-07-seed0-formal", @@ -14211,13 +17489,33 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 2880, - "height": 720, - "duration_s": 8.7, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 349, + "published_frames": 429, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 348, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 10.725, + "sha256": "fc2eae87e69acc3bba2b68931219731f2a812e3c2420b20e3b7a6bdf96525595", + "source_sha256": { + "third_person.mp4": "eb5fec2c4060f3d1ec830549e8a91bf73d291095ecba5fb5b721adaaa704c2ca", + "left_wrist.mp4": "44eb5c467cdc76ec3af06c062beb07f0e9b82ed7e118d702082ce8c761ad271f", + "right_wrist.mp4": "18622e8e574ec482fc3130d7f4d14248def32cd2625687391aeb2b2c0709f2ea" + }, + "all_frames_compared": 349, + "minimum_frame_psnr_db": 43.8473311124103, + "maximum_frame_rgb_mae": 1.1971997022628784, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 349, "captured_samples": 349, @@ -14259,13 +17557,7 @@ } ] }, - "view_names": [ - "third_person", - "left_wrist", - "right_wrist" - ], - "sha256": "52923b888c65ddc772be0f45ac6371520beb026e99dbe4a46a854714bb2e3d3e", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", @@ -14361,34 +17653,35 @@ "protocol_sha256": "9f648bb671b982843084f745100d3537e63eec4ee492747082c29eb3c34b9ec7" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/native/evidence/episode/evidence/native-episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/resources/tools/robot_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/resources/tools/robot_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robotwin-hanging-mug.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/resources/memos/robotwin-hanging-mug.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/resources/memos/robotwin-hanging-mug.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -14397,8 +17690,7 @@ "observed_images": 19, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/07/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task03-08-seed0-formal", @@ -14485,13 +17777,33 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 2880, - "height": 720, - "duration_s": 4.35, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 174, + "published_frames": 254, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 173, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 6.35, + "sha256": "6d075fe6a10d81076cfef169c5e2b260dbcb630a9a82320c8129bb48a5ac461e", + "source_sha256": { + "third_person.mp4": "279aab3e81805c5f093b6fdb3bdc953bdefbc0f0b2736c62ce80b87a30ed1ba0", + "left_wrist.mp4": "e8ac3e566cbfc85101f3a346af06810fab38bf518b5903890f7ead212efe5f82", + "right_wrist.mp4": "c879040fcec1e99c9a28f7317aaf58d8c72e3dcd5c03f7e6fe7e0636abbf187e" + }, + "all_frames_compared": 174, + "minimum_frame_psnr_db": 41.85481030674896, + "maximum_frame_rgb_mae": 1.5419611930847168, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 174, "captured_samples": 174, @@ -14533,13 +17845,7 @@ } ] }, - "view_names": [ - "third_person", - "left_wrist", - "right_wrist" - ], - "sha256": "c00cba68f9c9dd043f8f0f461b3350b151fb00b6d7e02651f8da6e0d227e792d", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", @@ -14635,34 +17941,35 @@ "protocol_sha256": "7010fe9319e6d7eb43c3c32683da9925dfb429340fb89866aaa38621945ae8d2" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/native/evidence/episode/evidence/native-episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha_motion.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/resources/tools/aloha_motion.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/resources/tools/aloha_motion.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robotwin_aloha.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/resources/memos/robotwin_aloha.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/resources/memos/robotwin_aloha.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -14671,8 +17978,7 @@ "observed_images": 11, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/08/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task03-09-seed0-formal", @@ -14759,13 +18065,33 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 2880, - "height": 720, - "duration_s": 10.95, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 439, + "published_frames": 519, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 438, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 12.975, + "sha256": "f7d9cda13ba0948ba0ac55d34c9a415a652cf41b13fc8a4ffcf9bd7427e20429", + "source_sha256": { + "third_person.mp4": "fd8c3dc8125ce87e28c52e4a42f43a7274c7bf0171f3ab02fa4d68dee9bbd7fc", + "left_wrist.mp4": "f17cc82c7b77fbee451dd517578f60baa8f37e444231ad9a6f4dbf0a56b79b07", + "right_wrist.mp4": "fa0e79338b9a5927df3bde38cc33ff7a6789207fae173175700856038aa67bb7" + }, + "all_frames_compared": 439, + "minimum_frame_psnr_db": 43.48948953380447, + "maximum_frame_rgb_mae": 1.4038331508636475, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 440, "captured_samples": 440, @@ -14807,13 +18133,7 @@ } ] }, - "view_names": [ - "third_person", - "left_wrist", - "right_wrist" - ], - "sha256": "713984b2a54c3f2bad023edcb3cab6160cb1b7998de51c9348a1bb1cd9c0a8e6", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", @@ -14909,34 +18229,35 @@ "protocol_sha256": "f2f18aacd79fb816308cd5e0ea1962b42bd5d82e4ee624b2b424b7cab9b0357b" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/native/evidence/episode/evidence/native-episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/resources/tools/aloha.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/resources/tools/aloha.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/aloha_manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/resources/memos/aloha_manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/resources/memos/aloha_manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -14945,8 +18266,7 @@ "observed_images": 9, "tool_errors": 5 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/09/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task03-10-seed0-formal", @@ -15033,13 +18353,33 @@ "source": "Codex native sessions" }, "media": { + "schema": "rlebench/native-camera-media/3", "passed": true, - "width": 2880, - "height": 720, - "duration_s": 6.5, "speed": 4, "source_fps": 10, - "output_fps": 20, + "output_fps": 40, + "width": 2880, + "height": 720, + "native_frames": 260, + "published_frames": 340, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 259, + "terminal_hold_s": 2, + "source_dropped_samples": 0, + "duration_s": 8.5, + "sha256": "0a3a1378e718b0fe70098144f0e91a86aa5f13b020bfef365760651536c8d171", + "source_sha256": { + "third_person.mp4": "e7d52460db2ce31fb4a78d5f0feeb0e74a9f11533d0470357865e2a535125c33", + "left_wrist.mp4": "8bdb28ba49a0dd185ae4df58a2cf5397a0c36f7ba2b0eae9b1e8cc31df818cec", + "right_wrist.mp4": "6fc2b4865a2686bf583c07f61f0b8779b81c76b03e5b84bc2b4f2eb551ca0ffc" + }, + "all_frames_compared": 260, + "minimum_frame_psnr_db": 42.81118120345254, + "maximum_frame_rgb_mae": 1.3442394733428955, + "decode_ok": true, + "frame_mapping": "Each native index once, followed by copies of native index N-1.", + "scope": "Original native recording; no simulation replay, rerender or invented frames.", "recording": { "accepted_samples": 260, "captured_samples": 260, @@ -15081,13 +18421,7 @@ } ] }, - "view_names": [ - "third_person", - "left_wrist", - "right_wrist" - ], - "sha256": "2723e73b315270446683bec817add276b7cc483b231075081b4cef5535dad005", - "scope": "Native spectator recording postprocessed at 4x; no replay or rerender" + "pts_verified": true }, "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\uff0c\u7ec8\u6001\u590d\u6838\u7684\u56db\u9879\u6761\u4ef6\u5747\u901a\u8fc7\u3002", @@ -15185,41 +18519,42 @@ "protocol_sha256": "23c4b22476372d9b7eedabe33380721f5a26f1ba4dd5b7057155c0abd56b3590" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/media-validation.json", - "native_snapshot": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/native/evidence/episode/evidence/native-episode.json", - "terminal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/native/evidence/episode/evidence/terminal.json", - "terminal_checks": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/terminal-checks.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/media-validation.json", + "native_snapshot": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/evidence/episode/evidence/terminal.json", + "terminal_checks": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/terminal-checks.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha_motion.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/resources/tools/aloha_motion.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/resources/tools/aloha_motion.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "skills/aloha-pot/SKILL.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/resources/skills/aloha-pot/SKILL.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/resources/skills/aloha-pot/SKILL.md", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/aloha-pot.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/resources/memos/aloha-pot.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/resources/memos/aloha-pot.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -15228,8 +18563,7 @@ "observed_images": 11, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task03/10/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-01-seed0-formal", @@ -15440,33 +18774,33 @@ "protocol_sha256": "558b51d6786cd5ba0a989f7b2b872a3e37424dda668128a679113df86f67e43d" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/resources/tools/arx.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/resources/tools/arx.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/resources/memos/robodojo-arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/resources/memos/robodojo-arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -15475,8 +18809,7 @@ "observed_images": 76, "tool_errors": 6 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/01/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-02-seed0-formal", @@ -15685,33 +19018,33 @@ "protocol_sha256": "3af4e8c103ad2c73d236f4fa8afce82c78df7a6d0f34c6e7c4664d7a9b7ba30f" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/media-validation.json" }, "resources": [ { "name": "tools/manipulate.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/resources/tools/manipulate.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/resources/tools/manipulate.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx-x5.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/resources/memos/robodojo-arx-x5.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/resources/memos/robodojo-arx-x5.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -15720,8 +19053,7 @@ "observed_images": 18, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/02/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-03-seed0-formal", @@ -15932,33 +19264,33 @@ "protocol_sha256": "1ede5afa307a8048edacac6c0bed72eba2d894ed61887aefbaf2cd0cdbf903d5" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/resources/memos/robodojo_manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/resources/memos/robodojo_manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -15967,8 +19299,7 @@ "observed_images": 81, "tool_errors": 8 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/03/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-04-seed0-formal", @@ -16178,33 +19509,33 @@ "protocol_sha256": "f82ccffd492114e7545cc0020de438e9656e0206bbec42d7b35a87b1ac2969a4" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/resources/memos/robodojo_arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/resources/memos/robodojo_arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -16213,8 +19544,7 @@ "observed_images": 17, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/04/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-05-seed0-formal", @@ -16423,38 +19753,38 @@ "protocol_sha256": "6e8f773de082b048547d9d092a6e48aab2bbfa59b7bb0fab422bd78a958354b5" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/xylophone.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/resources/tools/xylophone.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/resources/tools/xylophone.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -16463,8 +19793,7 @@ "observed_images": 14, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/05/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-06-seed0-formal", @@ -16675,38 +20004,38 @@ "protocol_sha256": "fa5ac36c9b8059eb095876d7da44d3a1fcdb68df35a6ef72c3800d96d5292357" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "skills/robodojo-arx-manipulation/SKILL.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/resources/skills/robodojo-arx-manipulation/SKILL.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/resources/skills/robodojo-arx-manipulation/SKILL.md", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/resources/memos/robodojo-arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/resources/memos/robodojo-arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -16715,8 +20044,7 @@ "observed_images": 61, "tool_errors": 7 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/06/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-07-seed0-formal", @@ -16926,38 +20254,38 @@ "protocol_sha256": "331cfc33f7aa656ce04ffa0d8e7927d625ea36dd825616041760649c239cee4b" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/tubes.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/resources/tools/tubes.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/resources/tools/tubes.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/insert-tubes.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/resources/memos/insert-tubes.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/resources/memos/insert-tubes.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -16966,8 +20294,7 @@ "observed_images": 84, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/07/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-08-seed0-formal", @@ -17176,33 +20503,33 @@ "protocol_sha256": "a5257eeb791b38dc956b69d4b7720388b44fff6d4ebee9de4202fec62c582203" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/resources/tools/arx.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/resources/tools/arx.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/resources/memos/robodojo-arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/resources/memos/robodojo-arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -17211,8 +20538,7 @@ "observed_images": 35, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/08/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-09-seed0-formal", @@ -17422,43 +20748,43 @@ "protocol_sha256": "3ef339de3038dcc6f878291f55e69c8bdfa168ab01175b93c61087d631f4cd43" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/resources/tools/arx.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/resources/tools/arx.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/thread.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/resources/tools/thread.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/resources/tools/thread.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/vision.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/resources/tools/vision.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/resources/tools/vision.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/fasten-screws.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/resources/memos/fasten-screws.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/resources/memos/fasten-screws.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -17467,8 +20793,7 @@ "observed_images": 46, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/09/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-10-seed0-formal", @@ -17678,43 +21003,43 @@ "protocol_sha256": "151595e8813018fa139d3fa1302f064a6f3147285b0afa155b9e316dac2a8cef" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/scene.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/resources/tools/scene.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/resources/tools/scene.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/star_pose.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/resources/tools/star_pose.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/resources/tools/star_pose.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/stacking-toy.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/resources/memos/stacking-toy.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/resources/memos/stacking-toy.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -17723,8 +21048,7 @@ "observed_images": 42, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/10/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-11-seed0-formal", @@ -17934,33 +21258,33 @@ "protocol_sha256": "620ebf01c8128411b840fd2d8f4f2fe1218e00277e6acd3cde5539758a420a74" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/resources/memos/robodojo_arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/resources/memos/robodojo_arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -17969,8 +21293,7 @@ "observed_images": 11, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/11/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-12-seed0-formal", @@ -18180,33 +21503,33 @@ "protocol_sha256": "e82a721587b91661ee5bcdb485c3e6573d246b35f68ab10c3db9adcfe9fbbe1f" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -18215,8 +21538,7 @@ "observed_images": 34, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/12/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-14-seed0-formal", @@ -18426,33 +21748,33 @@ "protocol_sha256": "f23cb3045132545976e459babecca972c759abe27461c2ba6453d38a7e4d1360" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -18461,8 +21783,7 @@ "observed_images": 26, "tool_errors": 8 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/14/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-15-seed0-formal", @@ -18672,33 +21993,33 @@ "protocol_sha256": "d4764df5340422c36eecca651903cdb4fbd291218376a4e4117bd3dc8acfe06b" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -18707,8 +22028,7 @@ "observed_images": 29, "tool_errors": 7 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/15/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-16-seed0-formal", @@ -18918,38 +22238,38 @@ "protocol_sha256": "e0a19fd656f3a7648fb37353a65d615d5764a23477c26b9ef17a28e052f6af6c" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/vision.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/resources/tools/vision.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/resources/tools/vision.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/egg-holder.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/resources/memos/egg-holder.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/resources/memos/egg-holder.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -18958,8 +22278,7 @@ "observed_images": 62, "tool_errors": 5 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/16/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-17-seed0-formal", @@ -19170,33 +22489,33 @@ "protocol_sha256": "8db05ad726248326912895583e4fe0eb492747d9f7702eb2b524abfb8da02a0a" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/resources/memos/robodojo-manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/resources/memos/robodojo-manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -19205,8 +22524,7 @@ "observed_images": 123, "tool_errors": 20 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/17/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-18-seed0-formal", @@ -19416,33 +22734,33 @@ "protocol_sha256": "4d88cc5a999dce74aa071c90a44dc1dd29d91e85bb470ce457003c0efec6e5f3" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/media-validation.json" }, "resources": [ { "name": "tools/cloth_robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/resources/tools/cloth_robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/resources/tools/cloth_robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/fold-clothes.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/resources/memos/fold-clothes.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/resources/memos/fold-clothes.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -19451,8 +22769,7 @@ "observed_images": 18, "tool_errors": 5 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/18/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-20-seed0-formal", @@ -19662,33 +22979,33 @@ "protocol_sha256": "eb6397849e2080ca6ce830dcd4ed4e76f31023c07b80325b71f2a0444ae4aeb0" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -19697,8 +23014,7 @@ "observed_images": 10, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/20/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-21-seed0-formal", @@ -19908,33 +23224,33 @@ "protocol_sha256": "c503521ec7cadab57ffa2b9e37d6e21b1429512d1233fec140dd86f0f171f1ce" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/hang-mugs.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/resources/memos/hang-mugs.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/resources/memos/hang-mugs.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -19943,8 +23259,7 @@ "observed_images": 76, "tool_errors": 7 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/21/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-23-seed0-formal", @@ -20154,33 +23469,33 @@ "protocol_sha256": "32d862b0761e0d956a45f371a10e4e0140fd59918700869986626301b85a8aec" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/resources/memos/robodojo_arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/resources/memos/robodojo_arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -20189,8 +23504,7 @@ "observed_images": 20, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/23/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-24-seed0-formal", @@ -20400,38 +23714,38 @@ "protocol_sha256": "e21d3a817474cba6513493d9ccfe2fec68ee69d186cf329dcd7700bcd9cf21bb" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/vision.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/resources/tools/vision.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/resources/tools/vision.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -20440,8 +23754,7 @@ "observed_images": 166, "tool_errors": 19 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/24/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-25-seed0-formal", @@ -20652,33 +23965,33 @@ "protocol_sha256": "1fc4df2b6175d238be961281622ba215bef1bcdb21ea23cbe6f10a9795467980" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -20687,8 +24000,7 @@ "observed_images": 33, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/25/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-27-seed0-formal", @@ -20898,33 +24210,33 @@ "protocol_sha256": "4c9a990682a0a89bb584861b32a8cfe09170766f00e78e5e9a92588d42764808" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/media-validation.json" }, "resources": [ { "name": "tools/conveyor.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/resources/tools/conveyor.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/resources/tools/conveyor.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/conveyor.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/resources/memos/conveyor.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/resources/memos/conveyor.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -20933,8 +24245,7 @@ "observed_images": 18, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/27/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-28-seed0-formal", @@ -21145,33 +24456,33 @@ "protocol_sha256": "5547f2ca84e4a90c7e8d6f7972573843c4cf5c907cf296c4d9cbd5a83caec610" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -21180,8 +24491,7 @@ "observed_images": 60, "tool_errors": 7 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/28/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-29-seed0-formal", @@ -21391,33 +24701,33 @@ "protocol_sha256": "d344c9afa60711fa33950eb1a8cfc805ff7c78c5ad9b476e4f3f524638f5980c" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arm_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/resources/tools/arm_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/resources/tools/arm_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -21426,8 +24736,7 @@ "observed_images": 56, "tool_errors": 9 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/29/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-31-seed0-formal", @@ -21638,33 +24947,33 @@ "protocol_sha256": "9ce80cb246bf47f91aa2889530d7ef56aa5498f223bbd77c69f483653254c63d" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -21673,8 +24982,7 @@ "observed_images": 78, "tool_errors": 7 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/31/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-32-seed0-formal", @@ -21884,38 +25192,38 @@ "protocol_sha256": "00eed76b0a0f2e74c99ae3f70a20940787ab4c6232c63f6d71e5834abb43b1df" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/tic_tac_toe.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/resources/tools/tic_tac_toe.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/resources/tools/tic_tac_toe.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-tic-tac-toe.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/resources/memos/robodojo-tic-tac-toe.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/resources/memos/robodojo-tic-tac-toe.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -21924,8 +25232,7 @@ "observed_images": 19, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/32/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-33-seed0-formal", @@ -22135,33 +25442,33 @@ "protocol_sha256": "3ad1206a40141e1e795d8afd438e9e3d126c9c09ac4c8c256bb2fcb5970c2411" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/resources/tools/arx.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/resources/tools/arx.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/resources/memos/robodojo-arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/resources/memos/robodojo-arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -22170,8 +25477,7 @@ "observed_images": 25, "tool_errors": 5 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/33/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-34-seed0-formal", @@ -22381,33 +25687,33 @@ "protocol_sha256": "9b55de7e87971566acdb0c291a2500699e8a0a888faf6e71d4ce3fff56b5480b" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/pour_balls.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/resources/memos/pour_balls.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/resources/memos/pour_balls.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -22416,8 +25722,7 @@ "observed_images": 75, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/34/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-35-seed0-formal", @@ -22628,33 +25933,33 @@ "protocol_sha256": "b10751bbc24694b2b82ddcb85171f0806a4d2c357f0001a79f4780ffb7e227aa" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/resources/tools/arx.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/resources/tools/arx.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -22663,8 +25968,7 @@ "observed_images": 17, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/35/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-36-seed0-formal", @@ -22874,38 +26178,38 @@ "protocol_sha256": "b149476eb7b742f99fdc2849cf33dae7045952a02b90dd86fc3756a18c01a07b" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/media-validation.json" }, "resources": [ { "name": "tools/pour_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/resources/tools/pour_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/resources/tools/pour_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/robot_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/resources/tools/robot_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/resources/tools/robot_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/pouring.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/resources/memos/pouring.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/resources/memos/pouring.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -22914,8 +26218,7 @@ "observed_images": 30, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/36/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-38-seed0-formal", @@ -23125,38 +26428,38 @@ "protocol_sha256": "9fce1006e083eccaf342031dee9ee536219916e20037c3b853f01379591895a3" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/media-validation.json" }, "resources": [ { "name": "tools/press_sequence.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/resources/tools/press_sequence.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/resources/tools/press_sequence.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/robot_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/resources/tools/robot_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/resources/tools/robot_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -23165,8 +26468,7 @@ "observed_images": 19, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/38/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-39-seed0-formal", @@ -23377,33 +26679,33 @@ "protocol_sha256": "67d36ac75c05a3dace78b479283200ce78a0d5fa330bd3deb44a3033a0feeec3" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_push_t.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/resources/memos/robodojo_push_t.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/resources/memos/robodojo_push_t.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -23412,8 +26714,7 @@ "observed_images": 16, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/39/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-41-seed0-formal", @@ -23623,38 +26924,38 @@ "protocol_sha256": "b9dfde3386ef956c89bac335893d836bd52984cc886a9fcd5f7152575935fca3" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "skills/robodojo-arx/SKILL.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/resources/skills/robodojo-arx/SKILL.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/resources/skills/robodojo-arx/SKILL.md", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/resources/memos/robodojo-arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/resources/memos/robodojo-arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -23663,8 +26964,7 @@ "observed_images": 29, "tool_errors": 8 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/41/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-42-seed0-formal", @@ -23874,33 +27174,33 @@ "protocol_sha256": "438173dad57f0b8c50f92d2d9903c94dc1b872d213d505617081782fca15e41d" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -23909,8 +27209,7 @@ "observed_images": 16, "tool_errors": 5 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/42/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-43-seed0-formal", @@ -24120,33 +27419,33 @@ "protocol_sha256": "6330a7ba676fbbe37e3eeaa32e9e720360f342588aebda1a3d017df413d83b5d" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx-x5.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/resources/memos/robodojo-arx-x5.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/resources/memos/robodojo-arx-x5.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -24155,8 +27454,7 @@ "observed_images": 25, "tool_errors": 8 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/43/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-45-seed0-formal", @@ -24366,33 +27664,33 @@ "protocol_sha256": "739b225b6df68661a4a1e1d7b03b8746eb82f4857dd7917dbc4a32d2fa99e6b0" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arm_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/resources/tools/arm_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/resources/tools/arm_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/stack_blocks.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/resources/memos/stack_blocks.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/resources/memos/stack_blocks.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -24401,8 +27699,7 @@ "observed_images": 10, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/45/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-46-seed0-formal", @@ -24612,33 +27909,33 @@ "protocol_sha256": "0353fcd3c3f4918cb27247a7ab486a7a8257fe72da1367bd5e66477b4333d346" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/resources/memos/robodojo_arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/resources/memos/robodojo_arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -24647,8 +27944,7 @@ "observed_images": 17, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/46/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-48-seed0-formal", @@ -24858,38 +28154,38 @@ "protocol_sha256": "e276da876239aed4f22db9100d1674254fb5c5893a8ab8bde9004a87801e33da" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/media-validation.json" }, "resources": [ { "name": "tools/bowl_vision.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/resources/tools/bowl_vision.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/resources/tools/bowl_vision.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/stack-bowls.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/resources/memos/stack-bowls.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/resources/memos/stack-bowls.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -24898,8 +28194,7 @@ "observed_images": 19, "tool_errors": 4 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/48/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-51-seed0-formal", @@ -25109,38 +28404,38 @@ "protocol_sha256": "1b0efaab6f68b573e59b4805686ca303c55abe32e88e5dd336e1df6e2cc12c60" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/arx_vision.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/resources/tools/arx_vision.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/resources/tools/arx_vision.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/resources/memos/robodojo_arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/resources/memos/robodojo_arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -25149,8 +28444,7 @@ "observed_images": 18, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/51/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-52-seed0-formal", @@ -25361,33 +28655,33 @@ "protocol_sha256": "b13933dd9228ca4b0ce123b8d301924aef1635fa85b203cac66a3525eab23346" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -25396,8 +28690,7 @@ "observed_images": 19, "tool_errors": 1 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/52/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task04-53-seed0-formal", @@ -25608,38 +28901,38 @@ "protocol_sha256": "56d44fedec316c2ce14ed4b5c39044883d7c867957d2d59950232a1041e8cfbd" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "skills/robodojo/SKILL.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/resources/skills/robodojo/SKILL.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/resources/skills/robodojo/SKILL.md", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -25648,8 +28941,7 @@ "observed_images": 53, "tool_errors": 9 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/main/episodes/task04/53/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task06-01-seed0-formal", @@ -25890,34 +29182,34 @@ "protocol_sha256": "00eb3775803cfae2d59017b38de6849c91902a1b82cb9d64b59a0fab03cc5f20" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_simple.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/resources/memos/g1_simple.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/resources/memos/g1_simple.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -25926,8 +29218,7 @@ "observed_images": 20, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/01/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task06-02-seed0-formal-attempt02", @@ -26177,34 +29468,34 @@ "previous_max_steps": 10000 }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_table_transfer.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/resources/memos/g1_table_transfer.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/resources/memos/g1_table_transfer.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -26213,8 +29504,7 @@ "observed_images": 36, "tool_errors": 1 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/02/seed-0-attempt02/" + "selected_for_formal_metrics": true }, { "id": "task06-03-seed0-formal-attempt02", @@ -26464,34 +29754,34 @@ "previous_max_steps": 10000 }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/resources/memos/g1_manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/resources/memos/g1_manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -26500,8 +29790,7 @@ "observed_images": 140, "tool_errors": 1 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/03/seed-0-attempt02/" + "selected_for_formal_metrics": true }, { "id": "task06-04-seed0-formal-attempt02", @@ -26750,34 +30039,34 @@ "previous_max_steps": 10000 }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_bend_pick.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/resources/memos/g1_bend_pick.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/resources/memos/g1_bend_pick.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -26786,8 +30075,7 @@ "observed_images": 5, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/04/seed-0-attempt02/" + "selected_for_formal_metrics": true }, { "id": "task06-05-seed0-formal-attempt02", @@ -27036,34 +30324,34 @@ "previous_max_steps": 10000 }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_dex3.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/resources/memos/g1_dex3.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/resources/memos/g1_dex3.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -27072,8 +30360,7 @@ "observed_images": 97, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/05/seed-0-attempt02/" + "selected_for_formal_metrics": true }, { "id": "task06-06-seed0-formal-attempt02", @@ -27323,34 +30610,34 @@ "previous_max_steps": 10000 }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/resources/memos/g1_manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/resources/memos/g1_manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -27359,8 +30646,7 @@ "observed_images": 73, "tool_errors": 0 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/06/seed-0-attempt02/" + "selected_for_formal_metrics": true }, { "id": "task06-07-seed0-formal-attempt02", @@ -27610,34 +30896,34 @@ "previous_max_steps": 10000 }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_simple.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/resources/memos/g1_simple.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/resources/memos/g1_simple.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -27646,8 +30932,7 @@ "observed_images": 88, "tool_errors": 12 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/07/seed-0-attempt02/" + "selected_for_formal_metrics": true }, { "id": "task06-08-seed0-formal", @@ -27888,39 +31173,39 @@ "protocol_sha256": "7459222297772303726f117c2d722012c84ef5c2b01e4057b1a8961d3ee23688" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/arm_ik.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/resources/tools/arm_ik.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/resources/tools/arm_ik.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/robot_step.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/resources/tools/robot_step.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/resources/tools/robot_step.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/close-door.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/resources/memos/close-door.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/resources/memos/close-door.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -27929,8 +31214,7 @@ "observed_images": 19, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/08/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task06-09-seed0-formal", @@ -28178,34 +31462,34 @@ "replacement_reason": "User requested 300 seconds at 50 Hz with updated code." }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_oven.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/resources/memos/g1_oven.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/resources/memos/g1_oven.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -28214,8 +31498,7 @@ "observed_images": 31, "tool_errors": 1 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/09/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task06-10-seed0-formal", @@ -28462,34 +31745,34 @@ "replacement_reason": "User requested 300 seconds at 50 Hz with updated code." }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_faucet.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/resources/memos/g1_faucet.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/resources/memos/g1_faucet.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -28498,8 +31781,7 @@ "observed_images": 30, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/10/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task06-11-seed0-formal", @@ -28746,34 +32028,34 @@ "replacement_reason": "User requested 300 seconds at 50 Hz with updated code." }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/robot_step.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/resources/tools/robot_step.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/resources/tools/robot_step.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/roboenv.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/resources/memos/roboenv.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/resources/memos/roboenv.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -28782,8 +32064,7 @@ "observed_images": 7, "tool_errors": 2 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/11/seed-0/" + "selected_for_formal_metrics": true }, { "id": "task06-12-seed0-formal", @@ -29030,34 +32311,34 @@ "replacement_reason": "User requested 300 seconds at 50 Hz with updated code." }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1-trash-can.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/resources/memos/g1-trash-can.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/resources/memos/g1-trash-can.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -29066,8 +32347,7 @@ "observed_images": 26, "tool_errors": 3 }, - "selected_for_formal_metrics": true, - "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/6c81560ce5510ba3e84c0ca1aa66ffe49de0d18f/episodes/task06/12/seed-0/" + "selected_for_formal_metrics": true } ], "failure_review": { @@ -29135,5 +32415,16 @@ } ] }, - "authorized_rerun_selection_complete": true + "authorized_rerun_selection_complete": true, + "presentation_updated_at": "2026-10-10T20:20:40.424864+00:00", + "media_repair": { + "time": "2026-10-10T20:20:40.424866+00:00", + "videos": 30, + "source_sample_coverage": 1.0, + "terminal_hold_s": 2, + "source_drops_retained_in_old_recordings": 41, + "fresh_recording_replacements": 4, + "independent_native_diagnostics": 1, + "native_verifier_changed": false + } }