diff --git "a/data.json" "b/data.json" --- "a/data.json" +++ "b/data.json" @@ -6,14 +6,14 @@ "benchmark_complete": true, "edition": "codex-simple-astra-high-seed0", "created_at": "2026-10-09T00:09:42.989838+00:00", - "updated_at": "2026-10-10T20:20:40.424847+00:00", + "updated_at": "2026-10-10T20:38:18.201818+00:00", "model": "gpt-6-astra", "effort": "high", "seed": 0, "publication": { "dataset": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark", "space": "https://huggingface.co/spaces/RLE-Bench/codex-benchmark", - "dataset_resolve": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/" + "dataset_resolve": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/" }, "summary": { "planned_tasks": 84, @@ -107,9 +107,9 @@ "total": 12, "completed": 12, "pending": 0, - "successes": 7, + "successes": 8, "valid_results": 12, - "success_rate": 0.5833333333333334, + "success_rate": 0.6666666666666666, "usage_complete": 12, "control_frequency_hz": 50, "max_control_steps": null, @@ -129,18 +129,18 @@ }, "preflight_results": 0, "formal_results": 12, - "modified_results": 0, - "original_results": 12, - "input_tokens": 106763761, - "cached_input_tokens": 105331968, - "output_tokens": 303566 + "modified_results": 4, + "original_results": 8, + "input_tokens": 89900808, + "cached_input_tokens": 88661760, + "output_tokens": 244198 } ], "interrupted_attempts": 4, "usage_incomplete_tasks": [], "execution_incomplete_tasks": [], "deferred_tasks": [], - "new_evaluations": 12, + "new_evaluations": 4, "progress": { "finished": 84, "running": 0, @@ -215,6 +215,27 @@ "success": false } ] + }, + "current_modified_results": 60, + "new_modified_results": 4, + "simple_instruction_reruns": { + "condition": "modified-instructions", + "tasks": [ + "task06/02", + "task06/09", + "task06/04", + "task06/05" + ], + "new_evaluations": 4, + "seed": 0, + "max_steps": 15000, + "native_verifier_changed": false, + "native_successes": 3, + "visible_goal_completed": 2, + "confirmed_false_positives": 1, + "confirmed_false_negatives": 0, + "scope": "Single-seed instruction revision, with earlier runs preserved. Native success and independent visible completion are reported separately.", + "comparison_limit": "Earlier selected runs have mixed budgets and use an earlier account; this is not a multi-seed causal success-rate estimate." } }, "families": [ @@ -305,9 +326,9 @@ "total": 12, "completed": 12, "pending": 0, - "successes": 7, + "successes": 8, "valid_results": 12, - "success_rate": 0.5833333333333334, + "success_rate": 0.6666666666666666, "usage_complete": 12, "control_frequency_hz": 50, "max_control_steps": null, @@ -327,11 +348,11 @@ }, "preflight_results": 0, "formal_results": 12, - "modified_results": 0, - "original_results": 12, - "input_tokens": 106763761, - "cached_input_tokens": 105331968, - "output_tokens": 303566 + "modified_results": 4, + "original_results": 8, + "input_tokens": 89900808, + "cached_input_tokens": 88661760, + "output_tokens": 244198 } ], "tasks": [ @@ -449,7 +470,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/01-load-condiments-in-fridge/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/01-load-condiments-in-fridge/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -519,7 +540,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/02-filter-microwavable-item/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/02-filter-microwavable-item/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -589,7 +610,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/03-store-dumplings/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/03-store-dumplings/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -659,7 +680,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/04-divide-buffet-trays/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/04-divide-buffet-trays/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -724,7 +745,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/05-make-cheesecake-filling/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/05-make-cheesecake-filling/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -869,7 +890,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/06-multistep-steaming/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/06-multistep-steaming/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -959,7 +980,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/07-scale-portioning/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/07-scale-portioning/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -1134,7 +1155,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/08-scrub-cutting-board/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/08-scrub-cutting-board/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Clarify broad board coverage, maintained grasp/contact, and final retraction without numeric thresholds." @@ -1391,37 +1412,37 @@ "protocol_sha256": "44bb17021918606c49586677dc6bf98272b20e0f9bb33f301645f4bd5b24918e" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/final-observation.json", - "native_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", - "owner_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/final-observation.json", + "native_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", + "owner_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/memos/robocasa_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/memos/robocasa_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -1685,37 +1706,37 @@ "protocol_sha256": "44bb17021918606c49586677dc6bf98272b20e0f9bb33f301645f4bd5b24918e" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/final-observation.json", - "native_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", - "owner_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/final-observation.json", + "native_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", + "owner_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/memos/robocasa_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/memos/robocasa_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -1797,7 +1818,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/09-prepare-veggie-dip/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/09-prepare-veggie-dip/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -1867,7 +1888,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/10-prepare-vegetable-roasting/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/10-prepare-vegetable-roasting/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -2236,35 +2257,35 @@ "protocol_sha256": "2cd301bfe0c1a1d6d4a420b1471374a8d33f9471ab060c1d6dcf66fbf74969bf" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -2520,35 +2541,35 @@ "protocol_sha256": "2cd301bfe0c1a1d6d4a420b1471374a8d33f9471ab060c1d6dcf66fbf74969bf" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -2657,7 +2678,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/05-libero-10-05/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task02/05-libero-10-05/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." @@ -2712,7 +2733,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/06-libero-10-06/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task02/06-libero-10-06/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." @@ -2792,7 +2813,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/07-libero-10-07/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task02/07-libero-10-07/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." @@ -3039,35 +3060,35 @@ "protocol_sha256": "bdb44af586667ee7eab0855e1a04f70e56609333b710668aa923999c6424aa55" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -3322,35 +3343,35 @@ "protocol_sha256": "bdb44af586667ee7eab0855e1a04f70e56609333b710668aa923999c6424aa55" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -3434,7 +3455,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/08-libero-10-08/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task02/08-libero-10-08/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." @@ -3681,35 +3702,35 @@ "protocol_sha256": "b0e2388679180c5a165f77c1cc90a2c5bfce0b2b1582b118c86392d435c21543" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -3964,35 +3985,35 @@ "protocol_sha256": "b0e2388679180c5a165f77c1cc90a2c5bfce0b2b1582b118c86392d435c21543" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -4061,7 +4082,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/09-libero-10-09/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task02/09-libero-10-09/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." @@ -4308,35 +4329,35 @@ "protocol_sha256": "acc9470d622733def9d480c2fb756b1724249077e1ad1f73cf80967246cabf7a" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/tools/panda.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/tools/panda.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/memos/libero-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/memos/libero-control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -4591,35 +4612,35 @@ "protocol_sha256": "acc9470d622733def9d480c2fb756b1724249077e1ad1f73cf80967246cabf7a" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/tools/panda.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/tools/panda.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/memos/libero-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/memos/libero-control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -4725,7 +4746,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/01-handover-block/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/01-handover-block/task.yaml" }, "attempt_history": [], "run_status": "finished", @@ -4808,7 +4829,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/02-place-dual-shoes/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/02-place-dual-shoes/task.yaml" }, "attempt_history": [], "run_status": "finished", @@ -4938,7 +4959,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/04-scan-object/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/04-scan-object/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Clarify the close, centered face-to-face scan pose and keeping both items grasped; native scan geometry thresholds are unchanged." @@ -5089,7 +5110,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/05-stack-bowls-three/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/05-stack-bowls-three/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Retain the existing concise instruction describing the compact bowl stack, release and open grippers; rerun using the current Kinex source." @@ -5277,7 +5298,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/07-hanging-mug/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/07-hanging-mug/task.yaml" }, "attempt_history": [], "run_status": "finished", @@ -5372,7 +5393,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/09-blocks-ranking-size/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/09-blocks-ranking-size/task.yaml" }, "attempt_history": [], "run_status": "finished", @@ -5440,7 +5461,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/10-lift-pot/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/10-lift-pot/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Retain the existing instruction describing two-handle grasping, upright orientation and sufficient lift; rerun using the current Kinex source." @@ -5543,7 +5564,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/01-robodojo-make-toast/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/01-robodojo-make-toast/task.yaml" }, "display_slot": "01", "display_key": "task04/01", @@ -5740,7 +5761,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/03-robodojo-store-laptop-and-headphones/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/03-robodojo-store-laptop-and-headphones/task.yaml" }, "display_slot": "03", "display_key": "task04/03", @@ -5882,7 +5903,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/04-robodojo-cover-blocks/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/04-robodojo-cover-blocks/task.yaml" }, "display_slot": "04", "display_key": "task04/04", @@ -6025,7 +6046,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/06-robodojo-store-tools-in-toolbox/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/06-robodojo-store-tools-in-toolbox/task.yaml" }, "display_slot": "06", "display_key": "task04/06", @@ -6092,7 +6113,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/07-robodojo-insert-tubes/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/07-robodojo-insert-tubes/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Describe fully seated tubes without exposing the verifier insertion-depth threshold." @@ -6206,7 +6227,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/08-robodojo-deposit-coin/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/08-robodojo-deposit-coin/task.yaml" }, "display_slot": "08", "display_key": "task04/08", @@ -6298,7 +6319,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/09-robodojo-fasten-screws/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/09-robodojo-fasten-screws/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Use a natural paragraph for matching and seating nuts; omit geometric scoring thresholds." @@ -6373,7 +6394,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/10-robodojo-play-stacking-toy/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/10-robodojo-play-stacking-toy/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Leave piece counts and peg matching to the agent while retaining the intended completed arrangement." @@ -6554,7 +6575,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/12-robodojo-arrange-largest-number/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/12-robodojo-arrange-largest-number/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Leave the numerical ordering strategy to the agent while retaining readable placement on the pads." @@ -6714,7 +6735,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/14-robodojo-build-tower/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/14-robodojo-build-tower/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Shorten the successful instruction while preserving the tower structure, original bottom faces, horizontal orientation, centering, plank alignment, perpendicular roof ridge, release and arm return requirements." @@ -6774,7 +6795,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/15-robodojo-classify-objects/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/15-robodojo-classify-objects/task.yaml" }, "display_slot": "14", "display_key": "task04/14", @@ -6851,7 +6872,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/16-robodojo-fill-egg-holder/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/16-robodojo-fill-egg-holder/task.yaml" }, "display_slot": "15", "display_key": "task04/15", @@ -6993,7 +7014,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/17-robodojo-fill-pen-holder/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/17-robodojo-fill-pen-holder/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Retain the native two-hand roles and final pen orientation without the insertion-depth threshold." @@ -7063,7 +7084,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/18-robodojo-fold-clothes/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/18-robodojo-fold-clothes/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Describe the folded garment without enumerating tracked point correspondences or grasping steps." @@ -7169,7 +7190,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/21-robodojo-hang-mugs/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/21-robodojo-hang-mugs/task.yaml" }, "display_slot": "19", "display_key": "task04/19", @@ -7301,7 +7322,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/23-robodojo-imitate-sorting-sequence/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/23-robodojo-imitate-sorting-sequence/task.yaml" }, "display_slot": "20", "display_key": "task04/20", @@ -7388,7 +7409,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/24-robodojo-insert-key/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/24-robodojo-insert-key/task.yaml" }, "display_slot": "21", "display_key": "task04/21", @@ -7505,7 +7526,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/25-robodojo-make-kong/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/25-robodojo-make-kong/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "State the turn and replacement-tile rules in one paragraph without a manipulation plan." @@ -7701,7 +7722,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/28-robodojo-organize-table/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/28-robodojo-organize-table/task.yaml" }, "display_slot": "24", "display_key": "task04/24", @@ -7778,7 +7799,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/29-robodojo-pack-objects-into-box/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/29-robodojo-pack-objects-into-box/task.yaml" }, "display_slot": "25", "display_key": "task04/25", @@ -7880,7 +7901,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/31-robodojo-pick-from-conveyor-by-image/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/31-robodojo-pick-from-conveyor-by-image/task.yaml" }, "display_slot": "26", "display_key": "task04/26", @@ -7997,7 +8018,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/32-robodojo-play-tic-tac-toe/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/32-robodojo-play-tic-tac-toe/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "State alternating turns and waiting rules without supplying the inferred ring counts." @@ -8072,7 +8093,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/33-robodojo-plug-in-charger/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/33-robodojo-plug-in-charger/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Describe a fully plugged-in charger without the verifier insertion-depth threshold." @@ -8132,7 +8153,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/34-robodojo-pour-balls-into-vase/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/34-robodojo-pour-balls-into-vase/task.yaml" }, "display_slot": "29", "display_key": "task04/29", @@ -8235,7 +8256,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/36-robodojo-pour-liquid-into-cup/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/36-robodojo-pour-liquid-into-cup/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Wait for flow to stop and liquid to settle while tilted before returning the bottle upright." @@ -8400,7 +8421,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/38-robodojo-press-by-number/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/38-robodojo-press-by-number/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Retain the per-button counting and confirmation rules without button-joint thresholds or card answers." @@ -8505,7 +8526,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/39-robodojo-push-t/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/39-robodojo-push-t/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Describe matching the target shape along the table without internal position and angle tolerances." @@ -8600,7 +8621,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/41-robodojo-put-bottles-into-dustbin/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/41-robodojo-put-bottles-into-dustbin/task.yaml" }, "display_slot": "34", "display_key": "task04/34", @@ -8703,7 +8724,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/43-robodojo-sort-nesting-dolls-by-size/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/43-robodojo-sort-nesting-dolls-by-size/task.yaml" }, "display_slot": "36", "display_key": "task04/36", @@ -8800,7 +8821,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/45-robodojo-stack-blocks/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/45-robodojo-stack-blocks/task.yaml" }, "display_slot": "37", "display_key": "task04/37", @@ -8873,7 +8894,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/48-robodojo-stack-bowls/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/48-robodojo-stack-bowls/task.yaml" }, "display_slot": "39", "display_key": "task04/39", @@ -8955,7 +8976,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/51-robodojo-swap-t/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/51-robodojo-swap-t/task.yaml" }, "display_slot": "40", "display_key": "task04/40", @@ -9117,7 +9138,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/52-robodojo-swap-blocks/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/52-robodojo-swap-blocks/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Clarify the three moves and complete press, withdrawal and button rebound after each placement; native release thresholds remain unchanged." @@ -9221,7 +9242,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/53-robodojo-sweep-blocks/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/53-robodojo-sweep-blocks/task.yaml" }, "display_slot": "42", "display_key": "task04/42", @@ -9304,11 +9325,11 @@ "native_id": "simple/transfer-between-tables", "catalog_id": "simple/transfer-between-tables", "title": "Transfer between tables", - "catalog_instruction": "pick up the cracker box from table1,locomotion to table2,and place on table2.", + "catalog_instruction": "Pick up the cracker box from table1, carry it to table2, and release it inside the basket on table2.", "native_instruction": "pick up the cracker box from table1,locomotion to table2,and place on table2.", - "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.", + "instruction_source": "Approved benchmark instruction revision; native simulator wording and verifier remain frozen.", "status": "completed", - "episode_id": "task06-02-seed0-formal-attempt02", + "episode_id": "task06-02-transfer-between-tables-codex-seed0-instruction-attempt04", "run_status": "finished", "status_note": "Native result, execution and usage complete.", "planned_protocol": { @@ -9332,7 +9353,7 @@ }, "timeout_s": 28800, "mode": "stepped", - "instruction_policy": "original_native" + "instruction_policy": "modified" }, "native_identity": { "arm_gravity_compensation": true, @@ -9341,11 +9362,11 @@ "sim_mode": "mujoco_isaac" }, "simple_rerun": { - "attempt": 2, + "attempt": 4, "status": "published", "target_max_steps": 15000, - "previous_max_steps": 10000, - "note": "Fresh 15000-step rerun selected; previous 10000-step native failure retained below." + "previous_max_steps": 15000, + "note": "Fresh modified-instruction result selected. Earlier native-instruction runs and their unchanged native verdicts are retained below." }, "attempt_history": [ { @@ -9359,29 +9380,389 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/02/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/simple-10000/02/episode.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, - "selected_for_formal_metrics": false + "selected_for_formal_metrics": false, + "visual_review": { + "status": "instruction_false_negative", + "confidence": "high", + "review": "Box visibly released on table2 near the basket; native failure requires basket contact rather than ordinary placement on table2. Historical instruction/verifier disagreement." + } + }, + { + "id": "task06-02-seed0-formal-attempt02", + "task_key": "task06/02", + "family": "task06", + "slot": "02", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": false, + "native_reward": 0.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 3875, + "success": false, + "termination": "stopped" + }, + "steps": 3875, + "simulation_time_s": null, + "wall_time_s": 1106.044938, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "pick up the cracker box from table1,locomotion to table2,and place on table2.", + "instruction": "pick up the cracker box from table1,locomotion to table2,and place on table2.", + "instruction_policy": "original_native", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9820486387779543, + "cache_reported_input_tokens": 4989928, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 4989928, + "cached_input_tokens": 4900352, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 4989928, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 4900352, + "known_input_tokens": 4989928, + "known_output_tokens": 18385, + "known_reasoning_output_tokens": 6657, + "output_tokens": 18385, + "reasoning_output_tokens": 6657, + "reasoning_reported_output_tokens": 18385, + "reported_responses": { + "cache_reported_input_tokens": 90, + "cache_write_input_tokens": 90, + "cache_write_reported_input_tokens": 90, + "cached_input_tokens": 90, + "input_tokens": 90, + "output_tokens": 90, + "reasoning_output_tokens": 90, + "reasoning_reported_output_tokens": 90 + }, + "response_count": 90, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 89576, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 89, + "model_tool_calls_by_name": { + "exec": 89 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-stereo-media/2", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1280, + "height": 360, + "native_frames": 776, + "published_frames": 776, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 775, + "duration_s": 19.4, + "sha256": "d5de93036b922175abed15e1d59c2060c131133cb379cd05a322a5f6431fdbe1", + "source_sha256": { + "camera_head_left.mp4": "53abfd74f074d3ee748dbd647ba095793ebd713ba16e2504e2b17a3b5d44ae27", + "camera_head_right.mp4": "a17439e097d2e3ef0aafa45191610b16454bbfe2c3214b6e3628c7088d0f7457" + }, + "all_frames_compared": 776, + "minimum_frame_psnr_db": 37.273305160584975, + "maximum_frame_rgb_mae": 2.372187614440918, + "terminal_frame_psnr_db": 41.26179590726265, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "One decoded stereo pair for each native index; raw RGB input at source_fps * speed", + "scope": "Original native stereo recording, all samples at 4x speed; no simulation replay or rerender", + "recording": { + "accepted_samples": 776, + "captured_samples": 776, + "clock": "simulation", + "dropped_samples": 0, + "encoded_frames": 776, + "end_time_s": 77.50000000000172, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 776, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_left", + "pose": null, + "source": "camera_head_left", + "width": 640 + }, + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_right", + "pose": null, + "source": "camera_head_right", + "width": 640 + } + ] + } + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 3,875 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc", + "dirty": true, + "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2", + "dirty": true, + "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task06-02-transfer-between-tables-codex-seed0-attempt02", + "attempt": 2, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 4, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 8, + "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217", + "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957" + }, + "session_original_sha256": "efee4c63cca91fe21230a740f9b45eb87ac9c145977761b56ae6e54f04b051ef", + "protocol_sha256": "87f34e1db6395a96c29d02fda9e5da869a64e77792776cf2f5d133806f37ede8", + "deployment": "failed10000-rerun15000-20261010T020844Z", + "max_steps": 15000, + "budget_revision": "failed10000-rerun15000-20261010T020844Z", + "same_protocol_replacement": false, + "replaces_user_interrupted_attempt": null, + "replacement_reason": "Explicit user-authorized fresh rerun of a completed 10000-step native failure with 15000 steps.", + "previous_native_failure": "task06-02-transfer-between-tables-codex-seed0-attempt01", + "previous_max_steps": 10000 + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" + }, + "resources": [ + { + "name": "tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/resources/tools/robot.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/g1_table_transfer.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/resources/memos/g1_table_transfer.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 198, + "observed_images": 36, + "tool_errors": 1 + }, + "selected_for_formal_metrics": false, + "max_steps": 15000, + "condition": "original-native-instruction", + "visual_review": { + "status": "instruction_false_negative", + "confidence": "high", + "review": "Box is visibly released onto table2 outside the basket. The instruction asks for placement on table2, but active reward requires basket contact and no hand contact; the table2 contact helper is not used." + } } - ] + ], + "instruction_revision": { + "native_instruction": "pick up the cracker box from table1,locomotion to table2,and place on table2.", + "instruction": "Pick up the cracker box from table1, carry it to table2, and release it inside the basket on table2.", + "native_instruction_kind": "source", + "diff": [ + { + "op": "replace", + "original": "pick", + "modified": "Pick" + }, + { + "op": "equal", + "original": " up the cracker box from ", + "modified": " up the cracker box from " + }, + { + "op": "replace", + "original": "table1,locomotion", + "modified": "table1, carry it" + }, + { + "op": "equal", + "original": " to ", + "modified": " to " + }, + { + "op": "replace", + "original": "table2,and", + "modified": "table2," + }, + { + "op": "equal", + "original": " ", + "modified": " " + }, + { + "op": "replace", + "original": "place ", + "modified": "and release it inside the basket " + }, + { + "op": "equal", + "original": "on table2.", + "modified": "on table2." + } + ], + "changed": true, + "preserve_whitespace": true, + "label": "Modified instruction", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task06/02-transfer-between-tables/task.yaml" + }, + "instruction_rerun": { + "attempt": 4, + "condition": "modified-instructions", + "max_steps": 15000, + "seed": 0, + "status": "published", + "native_verifier_changed": false, + "prior_selection_retained": true + } }, { "key": "task06/03", @@ -9445,27 +9826,33 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/03/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/simple-10000/03/episode.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, - "selected_for_formal_metrics": false + "selected_for_formal_metrics": false, + "visual_review": { + "status": "no_confirmed_mismatch", + "confidence": "limited", + "review": "No verified sustained pickup above the native lift threshold. Native failure is not contradicted by the sampled footage." + } } ] }, @@ -9476,11 +9863,11 @@ "native_id": "simple/bend-pick-and-place", "catalog_id": "simple/bend-pick-and-place", "title": "Bend pick and place", - "catalog_instruction": "bend to grasp the cracker box and drop it in the basket.", + "catalog_instruction": "Bend down, pick up the cracker box, and release it above the center of the basket so it drops inside.", "native_instruction": "bend to grasp the cracker box and drop it in the basket.", - "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.", + "instruction_source": "Approved benchmark instruction revision; native simulator wording and verifier remain frozen.", "status": "completed", - "episode_id": "task06-04-seed0-formal-attempt02", + "episode_id": "task06-04-bend-pick-and-place-codex-seed0-instruction-attempt04", "run_status": "finished", "status_note": "Native result, execution and usage complete.", "planned_protocol": { @@ -9504,7 +9891,7 @@ }, "timeout_s": 28800, "mode": "stepped", - "instruction_policy": "original_native" + "instruction_policy": "modified" }, "native_identity": { "arm_gravity_compensation": true, @@ -9513,11 +9900,11 @@ "sim_mode": "mujoco_isaac" }, "simple_rerun": { - "attempt": 2, + "attempt": 4, "status": "published", "target_max_steps": 15000, - "previous_max_steps": 10000, - "note": "Fresh 15000-step rerun selected; previous 10000-step native failure retained below." + "previous_max_steps": 15000, + "note": "Fresh modified-instruction result selected. Earlier native-instruction runs and their unchanged native verdicts are retained below." }, "attempt_history": [ { @@ -9531,42 +9918,426 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/04/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/simple-10000/04/episode.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, - "selected_for_formal_metrics": false - } - ] - }, - { - "key": "task06/05", - "family": "task06", - "slot": "05", - "native_id": "simple/bend-handover", - "catalog_id": "simple/bend-handover", + "selected_for_formal_metrics": false, + "visual_review": { + "status": "no_confirmed_mismatch", + "confidence": "high", + "review": "Box finishes on the floor, outside the basket. Native failure agrees with the final state." + } + }, + { + "id": "task06-04-seed0-formal-attempt02", + "task_key": "task06/04", + "family": "task06", + "slot": "04", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": true, + "native_reward": 1.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 116, + "success": true, + "termination": "success" + }, + "steps": 116, + "simulation_time_s": null, + "wall_time_s": 111.95334, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "bend to grasp the cracker box and drop it in the basket.", + "instruction": "bend to grasp the cracker box and drop it in the basket.", + "instruction_policy": "original_native", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9163462297790655, + "cache_reported_input_tokens": 329238, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 329238, + "cached_input_tokens": 301696, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 329238, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 301696, + "known_input_tokens": 329238, + "known_output_tokens": 2595, + "known_reasoning_output_tokens": 187, + "output_tokens": 2595, + "reasoning_output_tokens": 187, + "reasoning_reported_output_tokens": 2595, + "reported_responses": { + "cache_reported_input_tokens": 11, + "cache_write_input_tokens": 11, + "cache_write_reported_input_tokens": 11, + "cached_input_tokens": 11, + "input_tokens": 11, + "output_tokens": 11, + "reasoning_output_tokens": 11, + "reasoning_reported_output_tokens": 11 + }, + "response_count": 11, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 27542, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 10, + "model_tool_calls_by_name": { + "exec": 10 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-stereo-media/2", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1280, + "height": 360, + "native_frames": 24, + "published_frames": 24, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 23, + "duration_s": 0.6, + "sha256": "011fb24c546f4314b7c695a0f553544772086a397afa8dd50a9f0a0c12869e2e", + "source_sha256": { + "camera_head_left.mp4": "1cc8b62ab1d645f322702797da39c5b9c638b1a7b64fac0525e75c81001f3b33", + "camera_head_right.mp4": "ddf92efd54b082dcd847602221236c0de5a90d3e0c596148941f38205314c20f" + }, + "all_frames_compared": 24, + "minimum_frame_psnr_db": 36.0947630714779, + "maximum_frame_rgb_mae": 2.956716537475586, + "terminal_frame_psnr_db": 36.0947630714779, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "One decoded stereo pair for each native index; raw RGB input at source_fps * speed", + "scope": "Original native stereo recording, all samples at 4x speed; no simulation replay or rerender", + "recording": { + "accepted_samples": 25, + "captured_samples": 25, + "clock": "simulation", + "dropped_samples": 0, + "encoded_frames": 24, + "end_time_s": 2.3200000000000016, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 25, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_left", + "pose": null, + "source": "camera_head_left", + "width": 640 + }, + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_right", + "pose": null, + "source": "camera_head_right", + "width": 640 + } + ] + } + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 116 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc", + "dirty": true, + "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2", + "dirty": true, + "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task06-04-bend-pick-and-place-codex-seed0-attempt02", + "attempt": 2, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 2, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 8, + "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217", + "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957" + }, + "session_original_sha256": "ec64abe7c35301ed7902c9a352d65d221a43eda56dfdfd37ff537c0f98ce58d0", + "protocol_sha256": "f1f597064f79f90d61d0cd7b77a96ead0cf0c9c83b37d5abfc9541366e21b97c", + "deployment": "failed10000-rerun15000-20261010T020844Z", + "max_steps": 15000, + "budget_revision": "failed10000-rerun15000-20261010T020844Z", + "same_protocol_replacement": false, + "replaces_user_interrupted_attempt": null, + "replacement_reason": "Explicit user-authorized fresh rerun of a completed 10000-step native failure with 15000 steps.", + "previous_native_failure": "task06-04-bend-pick-and-place-codex-seed0-attempt01", + "previous_max_steps": 10000 + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" + }, + "resources": [ + { + "name": "tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/resources/tools/g1_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/g1_bend_pick.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/resources/memos/g1_bend_pick.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 29, + "observed_images": 5, + "tool_errors": 3 + }, + "selected_for_formal_metrics": false, + "max_steps": 15000, + "condition": "original-native-instruction", + "visual_review": { + "status": "instruction_false_positive", + "confidence": "high", + "review": "Only 116 action frames (2.32 simulation seconds), 12 published frames. Native terminal images show the box next to the basket exterior and forearm, without a grasp-and-drop sequence. Contact-only predicate plus two reward updates per step allows one qualifying step to reach success." + } + } + ], + "instruction_revision": { + "native_instruction": "bend to grasp the cracker box and drop it in the basket.", + "instruction": "Bend down, pick up the cracker box, and release it above the center of the basket so it drops inside.", + "native_instruction_kind": "source", + "diff": [ + { + "op": "replace", + "original": "bend", + "modified": "Bend" + }, + { + "op": "equal", + "original": " ", + "modified": " " + }, + { + "op": "replace", + "original": "to", + "modified": "down," + }, + { + "op": "equal", + "original": " ", + "modified": " " + }, + { + "op": "replace", + "original": "grasp", + "modified": "pick up" + }, + { + "op": "equal", + "original": " the cracker ", + "modified": " the cracker " + }, + { + "op": "replace", + "original": "box", + "modified": "box," + }, + { + "op": "equal", + "original": " and ", + "modified": " and " + }, + { + "op": "replace", + "original": "drop", + "modified": "release" + }, + { + "op": "equal", + "original": " it ", + "modified": " it " + }, + { + "op": "replace", + "original": "in", + "modified": "above" + }, + { + "op": "equal", + "original": " the ", + "modified": " the " + }, + { + "op": "replace", + "original": "basket.", + "modified": "center of the basket so it drops inside." + } + ], + "changed": true, + "preserve_whitespace": true, + "label": "Modified instruction", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task06/04-bend-pick-and-place/task.yaml" + }, + "instruction_rerun": { + "attempt": 4, + "condition": "modified-instructions", + "max_steps": 15000, + "seed": 0, + "status": "published", + "native_verifier_changed": false, + "prior_selection_retained": true + } + }, + { + "key": "task06/05", + "family": "task06", + "slot": "05", + "native_id": "simple/bend-handover", + "catalog_id": "simple/bend-handover", "title": "Bend handover", - "catalog_instruction": "bend the robot and pick up the cracker box, then place it on the container.", + "catalog_instruction": "Bend down, pick up the cracker box, then release it above the basket opening so it drops inside.", "native_instruction": "bend the robot and pick up the cracker box, then place it on the container.", - "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.", + "instruction_source": "Approved benchmark instruction revision; native simulator wording and verifier remain frozen.", "status": "completed", - "episode_id": "task06-05-seed0-formal-attempt02", + "episode_id": "task06-05-bend-handover-codex-seed0-instruction-attempt04", "run_status": "finished", "status_note": "Native result, execution and usage complete.", "planned_protocol": { @@ -9590,7 +10361,7 @@ }, "timeout_s": 28800, "mode": "stepped", - "instruction_policy": "original_native" + "instruction_policy": "modified" }, "native_identity": { "arm_gravity_compensation": true, @@ -9599,11 +10370,11 @@ "sim_mode": "mujoco_isaac" }, "simple_rerun": { - "attempt": 2, + "attempt": 4, "status": "published", "target_max_steps": 15000, - "previous_max_steps": 10000, - "note": "Fresh 15000-step rerun selected; previous 10000-step native failure retained below." + "previous_max_steps": 15000, + "note": "Fresh modified-instruction result selected. Earlier native-instruction runs and their unchanged native verdicts are retained below." }, "attempt_history": [ { @@ -9615,31 +10386,395 @@ "reason": null, "status": "finished" }, - "usage_complete": true, - "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/05/episode.json" + "usage_complete": true, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/simple-10000/05/episode.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" + }, + "selected_for_formal_metrics": false, + "visual_review": { + "status": "no_confirmed_mismatch", + "confidence": "high", + "review": "Box finishes released on the table, outside the basket; full required placement is not shown. No confirmed false negative." + } + }, + { + "id": "task06-05-seed0-formal-attempt02", + "task_key": "task06/05", + "family": "task06", + "slot": "05", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": true, + "native_reward": 1.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 12320, + "success": true, + "termination": "success" + }, + "steps": 12320, + "simulation_time_s": null, + "wall_time_s": 4359.883363, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "bend the robot and pick up the cracker box, then place it on the container.", + "instruction": "bend the robot and pick up the cracker box, then place it on the container.", + "instruction_policy": "original_native", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9926789916948668, + "cache_reported_input_tokens": 32371907, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 32371907, + "cached_input_tokens": 32134912, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 32371907, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 32134912, + "known_input_tokens": 32371907, + "known_output_tokens": 103454, + "known_reasoning_output_tokens": 69820, + "output_tokens": 103454, + "reasoning_output_tokens": 69820, + "reasoning_reported_output_tokens": 103454, + "reported_responses": { + "cache_reported_input_tokens": 297, + "cache_write_input_tokens": 297, + "cache_write_reported_input_tokens": 297, + "cached_input_tokens": 297, + "input_tokens": 297, + "output_tokens": 297, + "reasoning_output_tokens": 297, + "reasoning_reported_output_tokens": 297 + }, + "response_count": 297, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 236995, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 296, + "model_tool_calls_by_name": { + "exec": 296 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-stereo-media/2", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1280, + "height": 360, + "native_frames": 2465, + "published_frames": 2465, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 2464, + "duration_s": 61.625, + "sha256": "6e375814036f0e820e1bc519f368f7b8bcb8acb0fdc425eaa7b26e37b8322bf5", + "source_sha256": { + "camera_head_left.mp4": "99324d095b871bb7856ba04c0897deee67a7d094b5f7c5340be942c90643cc19", + "camera_head_right.mp4": "bd6a79e4d41523fc54dbf0eb78a4f228429bb16ae6db859e364cddddb65aa26c" + }, + "all_frames_compared": 2465, + "minimum_frame_psnr_db": 36.42045017056597, + "maximum_frame_rgb_mae": 2.767667770385742, + "terminal_frame_psnr_db": 36.993661080255514, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "One decoded stereo pair for each native index; raw RGB input at source_fps * speed", + "scope": "Original native stereo recording, all samples at 4x speed; no simulation replay or rerender", + "recording": { + "accepted_samples": 2465, + "captured_samples": 2465, + "clock": "simulation", + "dropped_samples": 0, + "encoded_frames": 2465, + "end_time_s": 246.40000000005224, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 2465, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_left", + "pose": null, + "source": "camera_head_left", + "width": 640 + }, + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_right", + "pose": null, + "source": "camera_head_right", + "width": 640 + } + ] + } + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 12,320 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc", + "dirty": true, + "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2", + "dirty": true, + "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task06-05-bend-handover-codex-seed0-attempt02", + "attempt": 2, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 6, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 8, + "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217", + "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957" + }, + "session_original_sha256": "dd94125fcdce620f874702dd716c5257502cf1aef0288654232ff4baf07b2a5b", + "protocol_sha256": "01bd486cc627e7126d7a549e63ec36a66d3b828d4b374814f730b17f8680edc9", + "deployment": "failed10000-rerun15000-20261010T020844Z", + "max_steps": 15000, + "budget_revision": "failed10000-rerun15000-20261010T020844Z", + "same_protocol_replacement": false, + "replaces_user_interrupted_attempt": null, + "replacement_reason": "Explicit user-authorized fresh rerun of a completed 10000-step native failure with 15000 steps.", + "previous_native_failure": "task06-05-bend-handover-codex-seed0-attempt01", + "previous_max_steps": 10000 + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" + }, + "resources": [ + { + "name": "tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/resources/tools/g1_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/g1_dex3.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/resources/memos/g1_dex3.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 639, + "observed_images": 97, + "tool_errors": 3 + }, + "selected_for_formal_metrics": false, + "max_steps": 15000, + "condition": "original-native-instruction", + "visual_review": { + "status": "instruction_false_positive", + "confidence": "high", + "review": "Native terminal stereo frames still show the right hand holding the box against/beside the container, without release and completed placement. Success tests only box-container contact; hand release, containment, stable support and handover are absent from the active predicate." + } + } + ], + "instruction_revision": { + "native_instruction": "bend the robot and pick up the cracker box, then place it on the container.", + "instruction": "Bend down, pick up the cracker box, then release it above the basket opening so it drops inside.", + "native_instruction_kind": "source", + "diff": [ + { + "op": "replace", + "original": "bend", + "modified": "Bend" + }, + { + "op": "equal", + "original": " ", + "modified": " " + }, + { + "op": "replace", + "original": "the robot and", + "modified": "down," + }, + { + "op": "equal", + "original": " pick up the cracker box, then ", + "modified": " pick up the cracker box, then " }, - "selected_for_formal_metrics": false - } - ] + { + "op": "replace", + "original": "place", + "modified": "release" + }, + { + "op": "equal", + "original": " it ", + "modified": " it " + }, + { + "op": "replace", + "original": "on", + "modified": "above" + }, + { + "op": "equal", + "original": " the ", + "modified": " the " + }, + { + "op": "replace", + "original": "container.", + "modified": "basket opening so it drops inside." + } + ], + "changed": true, + "preserve_whitespace": true, + "label": "Modified instruction", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task06/05-bend-handover/task.yaml" + }, + "instruction_rerun": { + "attempt": 4, + "condition": "modified-instructions", + "max_steps": 15000, + "seed": 0, + "status": "published", + "native_verifier_changed": false, + "prior_selection_retained": true + } }, { "key": "task06/06", @@ -9703,27 +10838,33 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/06/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/simple-10000/06/episode.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, - "selected_for_formal_metrics": false + "selected_for_formal_metrics": false, + "visual_review": { + "status": "visibility_problem", + "confidence": "high", + "review": "Late viewpoint becomes almost black; original native terminal frames are also black. No completed handover and placement is demonstrated." + } } ] }, @@ -9789,27 +10930,33 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/07/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/simple-10000/07/episode.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, - "selected_for_formal_metrics": false + "selected_for_formal_metrics": false, + "visual_review": { + "status": "verifier_mismatch_but_run_failed", + "confidence": "high", + "review": "Apple and basket finish on the floor apart. The task predicate has hidden distractor contact requirements, but this run is visibly unsuccessful." + } } ] }, @@ -9864,11 +11011,11 @@ "native_id": "simple/open-oven", "catalog_id": "simple/open-oven", "title": "Open oven", - "catalog_instruction": "move forward to the oven and open it", + "catalog_instruction": "Move to the oven and open either door fully, to roughly a right angle.", "native_instruction": "move forward to the oven and open it", - "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.", + "instruction_source": "Approved benchmark instruction revision; native simulator wording and verifier remain frozen.", "status": "completed", - "episode_id": "task06-09-seed0-formal", + "episode_id": "task06-09-open-oven-codex-seed0-instruction-attempt04", "run_status": "finished", "status_note": "Native result, execution and usage complete.", "planned_protocol": { @@ -9892,13 +11039,349 @@ }, "timeout_s": 28800, "mode": "stepped", - "instruction_policy": "original_native" + "instruction_policy": "modified" + }, + "native_identity": { + "arm_gravity_compensation": true, + "dr_level": 0, + "env_id": "simple/G1WholebodyOpenOvenTeleop-v0", + "sim_mode": "mujoco_isaac" + }, + "instruction_revision": { + "native_instruction": "move forward to the oven and open it", + "instruction": "Move to the oven and open either door fully, to roughly a right angle.", + "native_instruction_kind": "source", + "diff": [ + { + "op": "replace", + "original": "move forward", + "modified": "Move" + }, + { + "op": "equal", + "original": " to the oven and open ", + "modified": " to the oven and open " + }, + { + "op": "replace", + "original": "it", + "modified": "either door fully, to roughly a right angle." + } + ], + "changed": true, + "preserve_whitespace": true, + "label": "Modified instruction", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task06/09-open-oven/task.yaml" + }, + "attempt_history": [ + { + "id": "task06-09-seed0-formal", + "task_key": "task06/09", + "family": "task06", + "slot": "09", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": false, + "native_reward": 0.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 3170, + "success": false, + "termination": "stopped" + }, + "steps": 3170, + "simulation_time_s": null, + "wall_time_s": 918.375583, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "move forward to the oven and open it", + "instruction": "move forward to the oven and open it", + "instruction_policy": "original_native", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9660542143458919, + "cache_reported_input_tokens": 4133002, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 4133002, + "cached_input_tokens": 3992704, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 4133002, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 3992704, + "known_input_tokens": 4133002, + "known_output_tokens": 16710, + "known_reasoning_output_tokens": 5193, + "output_tokens": 16710, + "reasoning_output_tokens": 5193, + "reasoning_reported_output_tokens": 16710, + "reported_responses": { + "cache_reported_input_tokens": 84, + "cache_write_input_tokens": 84, + "cache_write_reported_input_tokens": 84, + "cached_input_tokens": 84, + "input_tokens": 84, + "output_tokens": 84, + "reasoning_output_tokens": 84, + "reasoning_reported_output_tokens": 84 + }, + "response_count": 84, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 140298, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 83, + "model_tool_calls_by_name": { + "exec": 83 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-stereo-media/2", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1280, + "height": 360, + "native_frames": 635, + "published_frames": 635, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 634, + "duration_s": 15.875, + "sha256": "0cab12ef33849200981cc426d00338fa21e96c5883935cd1ef860e036685f5d5", + "source_sha256": { + "camera_head_left.mp4": "06cf5c373fb2d02854812e0cebe90e8d15e72c76fa928245282d3a7950d822fa", + "camera_head_right.mp4": "8dce7464960955f707ab9e9ae109ec8286cf86652a576e582ce6e5f724bf30ec" + }, + "all_frames_compared": 635, + "minimum_frame_psnr_db": 38.42669493473994, + "maximum_frame_rgb_mae": 2.0199508666992188, + "terminal_frame_psnr_db": 39.10580975354201, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "One decoded stereo pair for each native index; raw RGB input at source_fps * speed", + "scope": "Original native stereo recording, all samples at 4x speed; no simulation replay or rerender", + "recording": { + "accepted_samples": 635, + "captured_samples": 635, + "clock": "simulation", + "dropped_samples": 0, + "encoded_frames": 635, + "end_time_s": 63.40000000000431, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 635, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_left", + "pose": null, + "source": "camera_head_left", + "width": 640 + }, + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_right", + "pose": null, + "source": "camera_head_right", + "width": 640 + } + ] + } + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 3,170 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc", + "dirty": true, + "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2", + "dirty": true, + "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task06-09-open-oven-codex-seed0-attempt03", + "attempt": 3, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 2, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 8, + "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217", + "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957" + }, + "session_original_sha256": "6fb961f0b29e07b7221771021776cddb69173e038f2ec5beba7e983b4c767efb", + "protocol_sha256": "cabc50e33ebf5f7a2e34a075aff7a1f66d2b2ffc5f84e941d688a929d7f70681", + "deployment": "steps15000-20261010T012736Z", + "max_steps": 15000, + "budget_revision": "steps15000-20261010T012736Z", + "same_protocol_replacement": false, + "replaces_user_interrupted_attempt": "task06-09-open-oven-codex-seed0-attempt01", + "replacement_reason": "User requested 300 seconds at 50 Hz with updated code." + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" + }, + "resources": [ + { + "name": "tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/resources/tools/g1_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/g1_oven.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/resources/memos/g1_oven.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 183, + "observed_images": 31, + "tool_errors": 1 + }, + "selected_for_formal_metrics": false, + "max_steps": 15000, + "condition": "original-native-instruction", + "visual_review": { + "status": "threshold_explained", + "confidence": "high", + "review": "Door looks open, but terminal oven joints are 0.63390 rad (36.32 degrees) and 0.04089 rad (2.34 degrees); native requires either >1.4 rad (80.21 degrees). Not a proven native false negative; instruction omits the required opening angle." + } + } + ], + "instruction_rerun": { + "attempt": 4, + "condition": "modified-instructions", + "max_steps": 15000, + "seed": 0, + "status": "published", + "native_verifier_changed": false, + "prior_selection_retained": true }, - "native_identity": { - "arm_gravity_compensation": true, - "dr_level": 0, - "env_id": "simple/G1WholebodyOpenOvenTeleop-v0", - "sim_mode": "mujoco_isaac" + "simple_rerun": { + "attempt": 4, + "status": "published", + "target_max_steps": 15000, + "previous_max_steps": 15000, + "note": "Fresh modified-instruction result selected. Earlier native-instruction runs and their unchanged native verdicts are retained below." } }, { @@ -10276,35 +11759,35 @@ "protocol_sha256": "a080e2d2fe0139ffe12d237f92d7ed6874269c53d0239657accc8b22d9ea3c9f" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/01/seed-0/resources/memos/robocasa-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/01/seed-0/resources/memos/robocasa-control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -10556,35 +12039,35 @@ "protocol_sha256": "1574aac678202802d3a7f391e140c19708280b6bbc85fa11f3d59e948633a852" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/resources/tools/control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/resources/tools/control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/02/seed-0/resources/memos/robocasa.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/02/seed-0/resources/memos/robocasa.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -10836,35 +12319,35 @@ "protocol_sha256": "66c49605362349c72c2effedd9355205c3fce52ef9b59b59c366c0fdd30aa3e1" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/03/seed-0/resources/memos/robocasa-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/03/seed-0/resources/memos/robocasa-control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -11116,35 +12599,35 @@ "protocol_sha256": "1ee5b218734e6a1617f3d05d489c52600880cddfc683e60accdf644d04e51bbb" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/04/seed-0/resources/memos/robocasa-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/04/seed-0/resources/memos/robocasa-control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -11395,35 +12878,35 @@ "protocol_sha256": "09896ba5b0aa8989cea18da115f01eb7144582b8c97b2953b5d3ea2dd0eb42fa" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa_manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/05/seed-0/resources/memos/robocasa_manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/05/seed-0/resources/memos/robocasa_manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -11675,40 +13158,40 @@ "protocol_sha256": "30fb270b4f0830e49eae5d8c77ee418edc397414725233a4c11f2985fee95fe7" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "skills/robocasa-manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/resources/skills/robocasa-manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/resources/skills/robocasa-manipulation.md", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/06/seed-0/resources/memos/robocasa-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/06/seed-0/resources/memos/robocasa-control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -11960,35 +13443,35 @@ "protocol_sha256": "d8853a96f748501bccf41fbca04175e69b3b4b557976e14e245ba24ea58d7323" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa_control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/07/seed-0/resources/memos/robocasa_control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/07/seed-0/resources/memos/robocasa_control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -12240,35 +13723,35 @@ "protocol_sha256": "828302ab34955790504cb974ea07d080685f76d638b3f581c76d06dba7106974" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/08/seed-0/resources/memos/robocasa.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/08/seed-0/resources/memos/robocasa.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -12520,35 +14003,35 @@ "protocol_sha256": "fdeb69aecef69e1512136a89c7f9cc51fde28b55423ac16dc6948b17fdb6fcec" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/resources/tools/control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/resources/tools/control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/09/seed-0/resources/memos/robocasa.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/09/seed-0/resources/memos/robocasa.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -12799,35 +14282,35 @@ "protocol_sha256": "12c94809e7b2e98486cd0036d528a828a0c211418b0d365710fd263517dec825" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa_control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task01/10/seed-0/resources/memos/robocasa_control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task01/10/seed-0/resources/memos/robocasa_control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -13078,35 +14561,35 @@ "protocol_sha256": "12310099caf8c22cd35d1b1702b90f7780034b1347863b2f0b0dd36c0986aadb" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/01/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/01/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -13357,35 +14840,35 @@ "protocol_sha256": "12cc7fa7e225614b41a5c3fda75bdf62f9e9b39b16c730cef3d5920ad9440392" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/resources/tools/panda.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/resources/tools/panda.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/02/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/02/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -13637,40 +15120,40 @@ "protocol_sha256": "4ee86729ea6301a80cd237762045640ac298b6dfd27faf0547387137274f2fcd" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/resources/tools/panda.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/resources/tools/panda.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/triangulate.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/resources/tools/triangulate.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/resources/tools/triangulate.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/03/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/03/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -13931,35 +15414,35 @@ "replaced_job": "task02-04-libero-10-04-codex-seed0-attempt01" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/04/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/04/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -14211,35 +15694,35 @@ "protocol_sha256": "aa8612b5784bd68176b7ef74c2a6b2df6f1923236ef02e59a7af155a463dd0ef" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/resources/tools/panda.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/resources/tools/panda.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/05/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/05/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -14490,35 +15973,35 @@ "protocol_sha256": "00df44aea40ecfa4fdb9b2d79ea89ee3bac07b53ecc4b93a1ffc313e0c1f4e98" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/06/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/06/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -14778,35 +16261,35 @@ "replaced_job": "task02-07-libero-10-07-codex-seed0-attempt01" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/07/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/07/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -15066,35 +16549,35 @@ "replaced_job": "task02-08-libero-10-08-codex-seed0-attempt01" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/libero_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/resources/tools/libero_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/resources/tools/libero_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/08/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/08/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -15355,35 +16838,35 @@ "replaced_job": "task02-09-libero-10-09-codex-seed0-attempt01" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/09/seed-0/resources/memos/libero_control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/09/seed-0/resources/memos/libero_control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -15634,35 +17117,35 @@ "protocol_sha256": "67091e2f8027e0c35cabd7bed5126858b5ff9603ccb55d40bccc34bb4b005d61" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task02/10/seed-0/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task02/10/seed-0/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -15922,35 +17405,35 @@ "protocol_sha256": "ad98ebf78ce3849d130c3fe4cf620520d1f7e5cfa6ef852f901223e0ffea50fd" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/native/evidence/episode/evidence/native-episode.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/resources/tools/aloha.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/resources/tools/aloha.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/aloha_handover.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/01/seed-0/resources/memos/aloha_handover.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/01/seed-0/resources/memos/aloha_handover.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -16210,35 +17693,35 @@ "protocol_sha256": "6ba3948ba616c4ca3d353bff11cd576ebc0c5187c28471204be413fa95999676" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/native/evidence/episode/evidence/native-episode.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/resources/tools/aloha_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/resources/tools/aloha_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/aloha_shoe_placement.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/02/seed-0/resources/memos/aloha_shoe_placement.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/02/seed-0/resources/memos/aloha_shoe_placement.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -16499,35 +17982,35 @@ "protocol_sha256": "6f2e06434fc34ac77c80c6ab5f307ef627ef25d814317530a0d122781a52346b" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/native/evidence/episode/evidence/native-episode.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/resources/tools/robot_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/resources/tools/robot_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robotwin_aloha.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/03/seed-0/resources/memos/robotwin_aloha.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/03/seed-0/resources/memos/robotwin_aloha.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -16788,35 +18271,35 @@ "protocol_sha256": "80bf39a41667e4718ca9014cb14c7b3825863df9a915cee99415aad8ccf711fd" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/native/evidence/episode/evidence/native-episode.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/resources/tools/aloha.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/resources/tools/aloha.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/aloha_scanning.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/04/seed-0/resources/memos/aloha_scanning.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/04/seed-0/resources/memos/aloha_scanning.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -17077,35 +18560,35 @@ "protocol_sha256": "e2de18f5e058a9b288accd36917ebbdd8cb5b957bee0cff2637d8a732df39495" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/native/evidence/episode/evidence/native-episode.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/resources/tools/aloha.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/resources/tools/aloha.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robotwin_aloha_bowls.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/05/seed-0/resources/memos/robotwin_aloha_bowls.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/05/seed-0/resources/memos/robotwin_aloha_bowls.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -17365,35 +18848,35 @@ "protocol_sha256": "fb73feaa0822af598381331e7f3add1ee2f53b407bc3c4214783b9cf05e75533" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/native/evidence/episode/evidence/native-episode.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robotwin_motion.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/resources/tools/robotwin_motion.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/resources/tools/robotwin_motion.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robotwin-aloha.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/06/seed-0/resources/memos/robotwin-aloha.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/06/seed-0/resources/memos/robotwin-aloha.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -17653,35 +19136,35 @@ "protocol_sha256": "9f648bb671b982843084f745100d3537e63eec4ee492747082c29eb3c34b9ec7" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/native/evidence/episode/evidence/native-episode.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/robot_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/resources/tools/robot_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/resources/tools/robot_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robotwin-hanging-mug.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/07/seed-0/resources/memos/robotwin-hanging-mug.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/07/seed-0/resources/memos/robotwin-hanging-mug.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -17941,35 +19424,35 @@ "protocol_sha256": "7010fe9319e6d7eb43c3c32683da9925dfb429340fb89866aaa38621945ae8d2" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/native/evidence/episode/evidence/native-episode.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha_motion.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/resources/tools/aloha_motion.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/resources/tools/aloha_motion.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robotwin_aloha.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/08/seed-0/resources/memos/robotwin_aloha.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/08/seed-0/resources/memos/robotwin_aloha.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -18229,35 +19712,35 @@ "protocol_sha256": "f2f18aacd79fb816308cd5e0ea1962b42bd5d82e4ee624b2b424b7cab9b0357b" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/media-validation.json", - "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/native/evidence/episode/evidence/native-episode.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/media-validation.json", + "native_episode": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/resources/tools/aloha.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/resources/tools/aloha.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/aloha_manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/09/seed-0/resources/memos/aloha_manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/09/seed-0/resources/memos/aloha_manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -18519,42 +20002,42 @@ "protocol_sha256": "23c4b22476372d9b7eedabe33380721f5a26f1ba4dd5b7057155c0abd56b3590" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/media-validation.json", - "native_snapshot": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/evidence/episode/evidence/native-episode.json", - "terminal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/native/evidence/episode/evidence/terminal.json", - "terminal_checks": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/terminal-checks.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/media-validation.json", + "native_snapshot": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/native/evidence/episode/evidence/native-episode.json", + "terminal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/native/evidence/episode/evidence/terminal.json", + "terminal_checks": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/terminal-checks.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/terminal.jpg" }, "resources": [ { "name": "tools/aloha_motion.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/resources/tools/aloha_motion.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/resources/tools/aloha_motion.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "skills/aloha-pot/SKILL.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/resources/skills/aloha-pot/SKILL.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/resources/skills/aloha-pot/SKILL.md", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/aloha-pot.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task03/10/seed-0/resources/memos/aloha-pot.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task03/10/seed-0/resources/memos/aloha-pot.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -18774,33 +20257,33 @@ "protocol_sha256": "558b51d6786cd5ba0a989f7b2b872a3e37424dda668128a679113df86f67e43d" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/resources/tools/arx.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/resources/tools/arx.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/01/seed-0/resources/memos/robodojo-arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/01/seed-0/resources/memos/robodojo-arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -19018,33 +20501,33 @@ "protocol_sha256": "3af4e8c103ad2c73d236f4fa8afce82c78df7a6d0f34c6e7c4664d7a9b7ba30f" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/media-validation.json" }, "resources": [ { "name": "tools/manipulate.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/resources/tools/manipulate.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/resources/tools/manipulate.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx-x5.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/02/seed-0/resources/memos/robodojo-arx-x5.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/02/seed-0/resources/memos/robodojo-arx-x5.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -19264,33 +20747,33 @@ "protocol_sha256": "1ede5afa307a8048edacac6c0bed72eba2d894ed61887aefbaf2cd0cdbf903d5" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/03/seed-0/resources/memos/robodojo_manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/03/seed-0/resources/memos/robodojo_manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -19509,33 +20992,33 @@ "protocol_sha256": "f82ccffd492114e7545cc0020de438e9656e0206bbec42d7b35a87b1ac2969a4" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/04/seed-0/resources/memos/robodojo_arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/04/seed-0/resources/memos/robodojo_arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -19753,38 +21236,38 @@ "protocol_sha256": "6e8f773de082b048547d9d092a6e48aab2bbfa59b7bb0fab422bd78a958354b5" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/xylophone.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/resources/tools/xylophone.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/resources/tools/xylophone.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/05/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/05/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -20004,38 +21487,38 @@ "protocol_sha256": "fa5ac36c9b8059eb095876d7da44d3a1fcdb68df35a6ef72c3800d96d5292357" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "skills/robodojo-arx-manipulation/SKILL.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/resources/skills/robodojo-arx-manipulation/SKILL.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/resources/skills/robodojo-arx-manipulation/SKILL.md", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/06/seed-0/resources/memos/robodojo-arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/06/seed-0/resources/memos/robodojo-arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -20254,38 +21737,38 @@ "protocol_sha256": "331cfc33f7aa656ce04ffa0d8e7927d625ea36dd825616041760649c239cee4b" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/tubes.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/resources/tools/tubes.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/resources/tools/tubes.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/insert-tubes.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/07/seed-0/resources/memos/insert-tubes.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/07/seed-0/resources/memos/insert-tubes.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -20503,33 +21986,33 @@ "protocol_sha256": "a5257eeb791b38dc956b69d4b7720388b44fff6d4ebee9de4202fec62c582203" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/resources/tools/arx.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/resources/tools/arx.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/08/seed-0/resources/memos/robodojo-arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/08/seed-0/resources/memos/robodojo-arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -20748,43 +22231,43 @@ "protocol_sha256": "3ef339de3038dcc6f878291f55e69c8bdfa168ab01175b93c61087d631f4cd43" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/resources/tools/arx.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/resources/tools/arx.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/thread.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/resources/tools/thread.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/resources/tools/thread.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/vision.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/resources/tools/vision.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/resources/tools/vision.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/fasten-screws.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/09/seed-0/resources/memos/fasten-screws.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/09/seed-0/resources/memos/fasten-screws.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -21003,43 +22486,43 @@ "protocol_sha256": "151595e8813018fa139d3fa1302f064a6f3147285b0afa155b9e316dac2a8cef" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/scene.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/resources/tools/scene.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/resources/tools/scene.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/star_pose.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/resources/tools/star_pose.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/resources/tools/star_pose.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/stacking-toy.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/10/seed-0/resources/memos/stacking-toy.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/10/seed-0/resources/memos/stacking-toy.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -21258,33 +22741,33 @@ "protocol_sha256": "620ebf01c8128411b840fd2d8f4f2fe1218e00277e6acd3cde5539758a420a74" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/11/seed-0/resources/memos/robodojo_arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/11/seed-0/resources/memos/robodojo_arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -21503,33 +22986,33 @@ "protocol_sha256": "e82a721587b91661ee5bcdb485c3e6573d246b35f68ab10c3db9adcfe9fbbe1f" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/12/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/12/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -21748,33 +23231,33 @@ "protocol_sha256": "f23cb3045132545976e459babecca972c759abe27461c2ba6453d38a7e4d1360" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/14/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/14/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -21993,33 +23476,33 @@ "protocol_sha256": "d4764df5340422c36eecca651903cdb4fbd291218376a4e4117bd3dc8acfe06b" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/15/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/15/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -22238,38 +23721,38 @@ "protocol_sha256": "e0a19fd656f3a7648fb37353a65d615d5764a23477c26b9ef17a28e052f6af6c" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/vision.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/resources/tools/vision.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/resources/tools/vision.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/egg-holder.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/16/seed-0/resources/memos/egg-holder.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/16/seed-0/resources/memos/egg-holder.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -22489,33 +23972,33 @@ "protocol_sha256": "8db05ad726248326912895583e4fe0eb492747d9f7702eb2b524abfb8da02a0a" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/17/seed-0/resources/memos/robodojo-manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/17/seed-0/resources/memos/robodojo-manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -22734,33 +24217,33 @@ "protocol_sha256": "4d88cc5a999dce74aa071c90a44dc1dd29d91e85bb470ce457003c0efec6e5f3" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/media-validation.json" }, "resources": [ { "name": "tools/cloth_robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/resources/tools/cloth_robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/resources/tools/cloth_robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/fold-clothes.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/18/seed-0/resources/memos/fold-clothes.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/18/seed-0/resources/memos/fold-clothes.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -22979,33 +24462,33 @@ "protocol_sha256": "eb6397849e2080ca6ce830dcd4ed4e76f31023c07b80325b71f2a0444ae4aeb0" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/20/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/20/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -23224,33 +24707,33 @@ "protocol_sha256": "c503521ec7cadab57ffa2b9e37d6e21b1429512d1233fec140dd86f0f171f1ce" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/hang-mugs.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/21/seed-0/resources/memos/hang-mugs.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/21/seed-0/resources/memos/hang-mugs.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -23469,33 +24952,33 @@ "protocol_sha256": "32d862b0761e0d956a45f371a10e4e0140fd59918700869986626301b85a8aec" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/23/seed-0/resources/memos/robodojo_arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/23/seed-0/resources/memos/robodojo_arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -23714,38 +25197,38 @@ "protocol_sha256": "e21d3a817474cba6513493d9ccfe2fec68ee69d186cf329dcd7700bcd9cf21bb" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/vision.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/resources/tools/vision.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/resources/tools/vision.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/24/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/24/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -23965,33 +25448,33 @@ "protocol_sha256": "1fc4df2b6175d238be961281622ba215bef1bcdb21ea23cbe6f10a9795467980" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/25/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/25/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -24210,33 +25693,33 @@ "protocol_sha256": "4c9a990682a0a89bb584861b32a8cfe09170766f00e78e5e9a92588d42764808" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/media-validation.json" }, "resources": [ { "name": "tools/conveyor.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/resources/tools/conveyor.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/resources/tools/conveyor.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/conveyor.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/27/seed-0/resources/memos/conveyor.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/27/seed-0/resources/memos/conveyor.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -24456,33 +25939,33 @@ "protocol_sha256": "5547f2ca84e4a90c7e8d6f7972573843c4cf5c907cf296c4d9cbd5a83caec610" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/28/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/28/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -24701,33 +26184,33 @@ "protocol_sha256": "d344c9afa60711fa33950eb1a8cfc805ff7c78c5ad9b476e4f3f524638f5980c" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arm_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/resources/tools/arm_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/resources/tools/arm_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/29/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/29/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -24947,33 +26430,33 @@ "protocol_sha256": "9ce80cb246bf47f91aa2889530d7ef56aa5498f223bbd77c69f483653254c63d" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/31/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/31/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -25192,38 +26675,38 @@ "protocol_sha256": "00eed76b0a0f2e74c99ae3f70a20940787ab4c6232c63f6d71e5834abb43b1df" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/tic_tac_toe.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/resources/tools/tic_tac_toe.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/resources/tools/tic_tac_toe.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-tic-tac-toe.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/32/seed-0/resources/memos/robodojo-tic-tac-toe.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/32/seed-0/resources/memos/robodojo-tic-tac-toe.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -25442,33 +26925,33 @@ "protocol_sha256": "3ad1206a40141e1e795d8afd438e9e3d126c9c09ac4c8c256bb2fcb5970c2411" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/resources/tools/arx.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/resources/tools/arx.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/33/seed-0/resources/memos/robodojo-arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/33/seed-0/resources/memos/robodojo-arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -25687,33 +27170,33 @@ "protocol_sha256": "9b55de7e87971566acdb0c291a2500699e8a0a888faf6e71d4ce3fff56b5480b" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/pour_balls.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/34/seed-0/resources/memos/pour_balls.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/34/seed-0/resources/memos/pour_balls.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -25933,33 +27416,33 @@ "protocol_sha256": "b10751bbc24694b2b82ddcb85171f0806a4d2c357f0001a79f4780ffb7e227aa" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/resources/tools/arx.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/resources/tools/arx.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/35/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/35/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -26178,38 +27661,38 @@ "protocol_sha256": "b149476eb7b742f99fdc2849cf33dae7045952a02b90dd86fc3756a18c01a07b" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/media-validation.json" }, "resources": [ { "name": "tools/pour_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/resources/tools/pour_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/resources/tools/pour_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/robot_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/resources/tools/robot_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/resources/tools/robot_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/pouring.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/36/seed-0/resources/memos/pouring.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/36/seed-0/resources/memos/pouring.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -26428,38 +27911,38 @@ "protocol_sha256": "9fce1006e083eccaf342031dee9ee536219916e20037c3b853f01379591895a3" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/media-validation.json" }, "resources": [ { "name": "tools/press_sequence.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/resources/tools/press_sequence.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/resources/tools/press_sequence.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/robot_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/resources/tools/robot_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/resources/tools/robot_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/38/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/38/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -26679,33 +28162,33 @@ "protocol_sha256": "67d36ac75c05a3dace78b479283200ce78a0d5fa330bd3deb44a3033a0feeec3" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_push_t.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/39/seed-0/resources/memos/robodojo_push_t.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/39/seed-0/resources/memos/robodojo_push_t.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -26924,38 +28407,38 @@ "protocol_sha256": "b9dfde3386ef956c89bac335893d836bd52984cc886a9fcd5f7152575935fca3" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "skills/robodojo-arx/SKILL.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/resources/skills/robodojo-arx/SKILL.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/resources/skills/robodojo-arx/SKILL.md", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/41/seed-0/resources/memos/robodojo-arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/41/seed-0/resources/memos/robodojo-arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -27174,33 +28657,33 @@ "protocol_sha256": "438173dad57f0b8c50f92d2d9903c94dc1b872d213d505617081782fca15e41d" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/42/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/42/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -27419,33 +28902,33 @@ "protocol_sha256": "6330a7ba676fbbe37e3eeaa32e9e720360f342588aebda1a3d017df413d83b5d" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo-arx-x5.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/43/seed-0/resources/memos/robodojo-arx-x5.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/43/seed-0/resources/memos/robodojo-arx-x5.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -27664,33 +29147,33 @@ "protocol_sha256": "739b225b6df68661a4a1e1d7b03b8746eb82f4857dd7917dbc4a32d2fa99e6b0" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arm_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/resources/tools/arm_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/resources/tools/arm_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/stack_blocks.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/45/seed-0/resources/memos/stack_blocks.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/45/seed-0/resources/memos/stack_blocks.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -27909,33 +29392,33 @@ "protocol_sha256": "0353fcd3c3f4918cb27247a7ab486a7a8257fe72da1367bd5e66477b4333d346" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/46/seed-0/resources/memos/robodojo_arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/46/seed-0/resources/memos/robodojo_arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -28154,38 +29637,38 @@ "protocol_sha256": "e276da876239aed4f22db9100d1674254fb5c5893a8ab8bde9004a87801e33da" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/media-validation.json" }, "resources": [ { "name": "tools/bowl_vision.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/resources/tools/bowl_vision.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/resources/tools/bowl_vision.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/stack-bowls.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/48/seed-0/resources/memos/stack-bowls.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/48/seed-0/resources/memos/stack-bowls.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -28404,38 +29887,38 @@ "protocol_sha256": "1b0efaab6f68b573e59b4805686ca303c55abe32e88e5dd336e1df6e2cc12c60" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/arx_vision.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/resources/tools/arx_vision.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/resources/tools/arx_vision.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo_arx.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/51/seed-0/resources/memos/robodojo_arx.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/51/seed-0/resources/memos/robodojo_arx.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -28655,33 +30138,33 @@ "protocol_sha256": "b13933dd9228ca4b0ce123b8d301924aef1635fa85b203cac66a3525eab23346" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/media-validation.json" }, "resources": [ { "name": "tools/arx_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/resources/tools/arx_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/resources/tools/arx_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/52/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/52/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -28901,38 +30384,38 @@ "protocol_sha256": "56d44fedec316c2ce14ed4b5c39044883d7c867957d2d59950232a1041e8cfbd" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/media-validation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/media-validation.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "skills/robodojo/SKILL.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/resources/skills/robodojo/SKILL.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/resources/skills/robodojo/SKILL.md", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robodojo.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task04/53/seed-0/resources/memos/robodojo.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task04/53/seed-0/resources/memos/robodojo.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -29091,7 +30574,8 @@ "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", "duration": "\u4f7f\u7528 1,647 / 10,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "independent_verifier_review": "Visible grasp and lift agree with native success. Predicate measures apple height, without proving grasp quality or a stable hold." }, "provenance": { "sources": { @@ -29182,34 +30666,35 @@ "protocol_sha256": "00eb3775803cfae2d59017b38de6849c91902a1b82cb9d64b59a0fab03cc5f20" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_simple.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/01/seed-0/resources/memos/g1_simple.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/01/seed-0/resources/memos/g1_simple.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -29218,10 +30703,16 @@ "observed_images": 20, "tool_errors": 2 }, - "selected_for_formal_metrics": true + "selected_for_formal_metrics": true, + "visual_review": { + "status": "no_confirmed_mismatch", + "confidence": "medium", + "native_verdict_retained": true, + "review": "Visible grasp and lift agree with native success. Predicate measures apple height, without proving grasp quality or a stable hold." + } }, { - "id": "task06-02-seed0-formal-attempt02", + "id": "task06-02-transfer-between-tables-codex-seed0-instruction-attempt04", "task_key": "task06/02", "family": "task06", "slot": "02", @@ -29229,8 +30720,8 @@ "episode": 1, "phase": "formal", "status": "completed", - "success": false, - "native_reward": 0.0, + "success": true, + "native_reward": 1.0, "valid": true, "execution": { "reason": null, @@ -29238,61 +30729,61 @@ }, "verdict": { "evidence_valid": true, - "steps": 3875, - "success": false, - "termination": "stopped" + "steps": 1181, + "success": true, + "termination": "success" }, - "steps": 3875, + "steps": 1181, "simulation_time_s": null, - "wall_time_s": 1106.044938, + "wall_time_s": 399.316915, "model": "gpt-6-astra", "effort": "high", "harness": "codex", "codex_version": "0.160.0", "native_instruction": "pick up the cracker box from table1,locomotion to table2,and place on table2.", - "instruction": "pick up the cracker box from table1,locomotion to table2,and place on table2.", - "instruction_policy": "original_native", + "instruction": "Pick up the cracker box from table1, carry it to table2, and release it inside the basket on table2.", + "instruction_policy": "modified", "usage": { "accounting": "reported-responses", "audit_complete": true, - "cache_hit_rate": 0.9820486387779543, - "cache_reported_input_tokens": 4989928, + "cache_hit_rate": 0.9715884524975027, + "cache_reported_input_tokens": 1609768, "cache_write_input_tokens": 0, - "cache_write_reported_input_tokens": 4989928, - "cached_input_tokens": 4900352, + "cache_write_reported_input_tokens": 1609768, + "cached_input_tokens": 1564032, "completed_turns": 1, "cost_usd": null, "failed_turns": 0, - "input_tokens": 4989928, + "input_tokens": 1609768, "known_cache_write_input_tokens": 0, - "known_cached_input_tokens": 4900352, - "known_input_tokens": 4989928, - "known_output_tokens": 18385, - "known_reasoning_output_tokens": 6657, - "output_tokens": 18385, - "reasoning_output_tokens": 6657, - "reasoning_reported_output_tokens": 18385, + "known_cached_input_tokens": 1564032, + "known_input_tokens": 1609768, + "known_output_tokens": 6000, + "known_reasoning_output_tokens": 1350, + "output_tokens": 6000, + "reasoning_output_tokens": 1350, + "reasoning_reported_output_tokens": 6000, "reported_responses": { - "cache_reported_input_tokens": 90, - "cache_write_input_tokens": 90, - "cache_write_reported_input_tokens": 90, - "cached_input_tokens": 90, - "input_tokens": 90, - "output_tokens": 90, - "reasoning_output_tokens": 90, - "reasoning_reported_output_tokens": 90 + "cache_reported_input_tokens": 40, + "cache_write_input_tokens": 40, + "cache_write_reported_input_tokens": 40, + "cached_input_tokens": 40, + "input_tokens": 40, + "output_tokens": 40, + "reasoning_output_tokens": 40, + "reasoning_reported_output_tokens": 40 }, - "response_count": 90, + "response_count": 40, "response_ids_complete": true, "schema": "rlebench/token-usage/1", "source": "Codex token_usage_record per response", - "uncached_input_tokens": 89576, + "uncached_input_tokens": 45736, "unidentified_usage_records": 0 }, "call_activity": { - "model_tool_calls": 89, + "model_tool_calls": 39, "model_tool_calls_by_name": { - "exec": 89 + "exec": 39 }, "nested_python_tool_invocations": null, "python_device_rpc_attempts": null, @@ -29312,36 +30803,36 @@ "output_fps": 40, "width": 1280, "height": 360, - "native_frames": 776, - "published_frames": 776, + "native_frames": 237, + "published_frames": 237, "source_sample_coverage": 1.0, "terminal_frame_retained": true, - "terminal_source_index": 775, - "duration_s": 19.4, - "sha256": "d5de93036b922175abed15e1d59c2060c131133cb379cd05a322a5f6431fdbe1", + "terminal_source_index": 236, + "duration_s": 5.925, + "sha256": "be70b506fc5ec3a8fc14017702a9293a46fabe90b767a828a515f6051715bd7a", "source_sha256": { - "camera_head_left.mp4": "53abfd74f074d3ee748dbd647ba095793ebd713ba16e2504e2b17a3b5d44ae27", - "camera_head_right.mp4": "a17439e097d2e3ef0aafa45191610b16454bbfe2c3214b6e3628c7088d0f7457" + "camera_head_left.mp4": "b71debdd0fba3fce507aa0117a1e94b4d4f47914b05de2672e25d07bb00880ba", + "camera_head_right.mp4": "afeff34674dbb7460648c7f8861e26cdd63da59da44bd4b9f2d0866bba734deb" }, - "all_frames_compared": 776, - "minimum_frame_psnr_db": 37.273305160584975, - "maximum_frame_rgb_mae": 2.372187614440918, - "terminal_frame_psnr_db": 41.26179590726265, + "all_frames_compared": 237, + "minimum_frame_psnr_db": 37.47335323653394, + "maximum_frame_rgb_mae": 2.4073662757873535, + "terminal_frame_psnr_db": 40.89991027673416, "decode_ok": true, "pts_verified": true, "frame_mapping": "One decoded stereo pair for each native index; raw RGB input at source_fps * speed", "scope": "Original native stereo recording, all samples at 4x speed; no simulation replay or rerender", "recording": { - "accepted_samples": 776, - "captured_samples": 776, + "accepted_samples": 238, + "captured_samples": 238, "clock": "simulation", "dropped_samples": 0, - "encoded_frames": 776, - "end_time_s": 77.50000000000172, + "encoded_frames": 237, + "end_time_s": 23.619999999999585, "error": null, "experimental": true, "fps": 10, - "received_samples": 776, + "received_samples": 238, "schema": "roboenv/recording/1", "state": "closed", "status": "complete", @@ -29366,10 +30857,12 @@ } }, "analysis": { - "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", - "duration": "\u4f7f\u7528 3,875 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", - "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002" + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\uff0c\u7ec8\u6001\u8bc1\u636e\u6709\u6548\uff0c\u7528\u91cf\u5b8c\u6574\u3002", + "instruction": "\u672c\u56de\u5408\u660e\u786e\u8981\u6c42\u5c06\u7bb1\u5b50\u653e\u5165 table2 \u4e0a\u7684\u7bee\u5b50\u5e76\u677e\u624b\u3002", + "visible_completion": "\u5b8c\u6574\u53cc\u76ee\u89c6\u9891\u663e\u793a\u4ece table1 \u6293\u53d6\u7bb1\u5b50\u3001\u79fb\u52a8\u5230 table2\uff0c\u7136\u540e\u6253\u5f00\u53f3\u624b\u5c06\u7bb1\u5b50\u7559\u5728\u7bee\u5b50\u4e2d\u592e\u9644\u8fd1\u3002\u96f6\u6a21\u578b\u3001\u96f6\u7269\u7406\u6b65\u7684\u5c01\u5b58\u7ec8\u6001\u63a5\u89e6\u68c0\u67e5\u786e\u8ba4\uff1a\u7bb1\u5b50\u4ec5\u4e0e\u7bee\u5b50\u63a5\u89e6\uff0c\u65e0\u624b\u6307\u6216\u624b\u638c\u63a5\u89e6\uff0c\u5df2\u8131\u79bb\u624b\u90e8\u3002", + "duration": "\u4f7f\u7528 1,181 / 15,000 \u4e2a\u63a7\u5236\u6b65\uff0cnative termination=success\u3002", + "comparison": "\u6b64\u524d Codex \u56de\u5408\u628a\u7bb1\u5b50\u653e\u5728 table2 \u7684\u7bee\u5b50\u4e4b\u5916\uff0c\u6b64\u6b21\u660e\u786e\u8fdb\u5165\u7bee\u5b50\u3002\u8be5\u5355\u6b21 fresh seed-0 \u5bf9\u6bd4\u4e0d\u80fd\u72ec\u7acb\u8bc1\u660e instruction \u7684\u56e0\u679c\u589e\u76ca\u3002", + "limitation": "\u539f\u751f\u6210\u529f\u4f1a\u7acb\u5373\u7ec8\u6b62\u3002\u5f00\u624b\u8f68\u8ff9\u5c1a\u672a\u5b8c\u5168\u5230\u8fbe\u5168\u96f6\u76ee\u6807\uff0c\u4f46\u7ec8\u6001\u5df2\u7ecf\u6ca1\u6709\u624b\u90e8\u63a5\u89e6\uff1b\u89c6\u9891\u6ca1\u6709\u63d0\u4f9b\u957f\u65f6\u95f4\u677e\u624b\u540e\u7684\u7a33\u5b9a\u6027\u89c2\u5bdf\u3002" }, "provenance": { "sources": { @@ -29383,9 +30876,10 @@ "tests", "docs" ], - "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc", + "content_sha256": "bbd03a94ca5776307435b6ed81c0f935a17f836b9ee80770069b58162d5e7e8f", "dirty": true, - "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618", + "revision": "f1935004c2fb23501b937cd29b826f69a06214b2", + "snapshot": "/shared/evaluations/kinex-v010-task04-20261006T151944Z/additional-campaigns/simple-g1-media-instruction-fix-20261010T181909Z/sources/RLE-Bench-inhouse", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" }, "RoboEnv": { @@ -29405,7 +30899,9 @@ ], "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2", "dirty": true, + "reuse": "Exact 15000-step deployment owner source; no edits", "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92", + "snapshot": "/shared/evaluations/kinex-v010-task04-20261006T151944Z/additional-campaigns/simple-g1-media-instruction-fix-20261010T181909Z/sources/RoboEnv", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" }, "kinex": { @@ -29424,19 +30920,20 @@ ], "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", "dirty": false, + "reuse": "Exact existing agent image and frozen Kinex revision", "revision": "caac19a8a36272972f762e0f74cfe381e8500048", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" } }, - "job": "task06-02-transfer-between-tables-codex-seed0-attempt02", - "attempt": 2, + "job": "task06-02-transfer-between-tables-codex-seed0-instruction-attempt04", + "attempt": 4, "harness": "stock Codex CLI", "control_interface": "public RoboEnv SDK and CLI", "kinex_agent_runtime_used": false, "gpu_index": 4, "gpu_model": "NVIDIA L40S", "campaign_dispatch_concurrency_limit": 8, - "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.", + "concurrency_note": "At most one added SIMPLE episode per GPU, admitted with spare memory and low utilization; existing environments remain active.", "codex_version": "0.160.0", "model": "gpt-6-astra", "effort": "high", @@ -29456,55 +30953,208 @@ "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217", "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957" }, - "session_original_sha256": "efee4c63cca91fe21230a740f9b45eb87ac9c145977761b56ae6e54f04b051ef", - "protocol_sha256": "87f34e1db6395a96c29d02fda9e5da869a64e77792776cf2f5d133806f37ede8", - "deployment": "failed10000-rerun15000-20261010T020844Z", + "session_original_sha256": "e6faf0ffb7832be5675b921da751e9150812a2f55a71dc9f2e230a0e06e5fc34", + "protocol_sha256": "65790369d3ac63adf8cf3493d7a2da1412870db04b55465d1cc8f7dd66f489ef", + "condition": "modified-instructions", "max_steps": 15000, - "budget_revision": "failed10000-rerun15000-20261010T020844Z", "same_protocol_replacement": false, - "replaces_user_interrupted_attempt": null, - "replacement_reason": "Explicit user-authorized fresh rerun of a completed 10000-step native failure with 15000 steps.", - "previous_native_failure": "task06-02-transfer-between-tables-codex-seed0-attempt01", - "previous_max_steps": 10000 + "previous_selection": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02", + "native_verifier_changed": false, + "setup_failure_predecessor": "task06-02-transfer-between-tables-codex-seed0-instruction-attempt03", + "replacement_reason": "Eight simultaneous cold starts exceeded the initialization deadline under shared CPU load.", + "predecessor_model_calls": 0, + "predecessor_measured_episodes": 0 }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { - "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/resources/tools/robot.py", + "name": "tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { - "name": "memos/g1_table_transfer.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0-attempt02/resources/memos/g1_table_transfer.md", + "name": "memos/g1_manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/resources/memos/g1_manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], "session_counts": { - "visible_events": 198, - "observed_images": 36, - "tool_errors": 1 + "visible_events": 91, + "observed_images": 14, + "tool_errors": 2 }, - "selected_for_formal_metrics": true + "selected_for_formal_metrics": true, + "visual_review": { + "status": "consistent_pass", + "visible_goal_completed": true, + "false_positive": false, + "false_negative": false, + "confidence": "high", + "native_verifier_changed": false, + "release_command_observed": true, + "release_command_target": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "reviewed_samples": [ + 0, + 47, + 94, + 142, + 156, + 189, + 196, + 236 + ], + "terminal_state_review": "Placement is visible. Independent reconstruction finds three basket contacts and no finger or palm contacts at the sealed pose. The opening trajectory did not need to reach its zero target to release the box. No extended settling period was recorded.", + "release_command_completed": false, + "actual_terminal_hand_positions": { + "left_hand": { + "left_hand_thumb_0_joint": 0.014547852255075253, + "left_hand_thumb_1_joint": -0.018311079482944432, + "left_hand_thumb_2_joint": -0.000553219101044073, + "left_hand_index_0_joint": -0.0018576487346638059, + "left_hand_index_1_joint": 4.9241592551884685e-06, + "left_hand_middle_0_joint": 0.0005199073215739375, + "left_hand_middle_1_joint": 0.000419273511670888 + }, + "right_hand": { + "right_hand_thumb_0_joint": 0.008073401926713226, + "right_hand_thumb_1_joint": 0.2841115032473764, + "right_hand_thumb_2_joint": -0.5911596277555279, + "right_hand_index_0_joint": 0.5665201739483607, + "right_hand_index_1_joint": 0.6128393323342631, + "right_hand_middle_0_joint": 0.5634056302364534, + "right_hand_middle_1_joint": 0.6103175058222502 + } + }, + "independent_terminal_contact_check": { + "passed": true, + "model_calls": 0, + "physics_steps": 0, + "reward_checks_invoked": 0, + "input_sha256": "d317396709b8dd6ac6ebcc830c3a5ae4883aa6c9991025869f4afda0aac45c94", + "input_task": "simple/transfer-between-tables", + "native_implementation": { + "backend": "simple", + "dependencies_sha256": "b655ef7dd8620ec89bc34238efb9a1ea70178f569afe1f429a6de9494840ee5f", + "implementation_sha256": "b184ced6cf472c3554fda97879a31cffcce98416acc28b4d6a4672529017d19b", + "selection": "shared-and-selected-backend/1", + "sources_sha256": "aae5710e1a7dd1e87d63a827b5e93881bc71c19aba1f6f0555fdf41c552c572a" + }, + "model_state_size": 709, + "nq": 92, + "sealed_time": 26.50499999999901, + "target_contacts": [ + { + "pair": [ + { + "body": "brasket", + "geom": "brasket_convex_1", + "mesh": "brasket_mesh_convex1" + }, + { + "body": "cracker_box", + "geom": "cracker_box_convex_0", + "mesh": "cracker_box_mesh_convex0" + } + ], + "distance_m": -0.00044468320433578275, + "position": [ + -2.503625753334981, + 0.13804563772605238, + 0.6715523495548237 + ], + "recognized_as_hand_by_native": false, + "palm_collision_geometry": false, + "container_contact": true + }, + { + "pair": [ + { + "body": "brasket", + "geom": "brasket_convex_6", + "mesh": "brasket_mesh_convex6" + }, + { + "body": "cracker_box", + "geom": "cracker_box_convex_0", + "mesh": "cracker_box_mesh_convex0" + } + ], + "distance_m": -0.0010050880855029367, + "position": [ + -2.4980351299194843, + 0.14453658542393416, + 0.671122208883927 + ], + "recognized_as_hand_by_native": false, + "palm_collision_geometry": false, + "container_contact": true + }, + { + "pair": [ + { + "body": "brasket", + "geom": "brasket_convex_17", + "mesh": "brasket_mesh_convex17" + }, + { + "body": "cracker_box", + "geom": "cracker_box_convex_0", + "mesh": "cracker_box_mesh_convex0" + } + ], + "distance_m": -0.0005650584038646472, + "position": [ + -2.532908968654665, + 0.2860293551313309, + 0.7253940312330712 + ], + "recognized_as_hand_by_native": false, + "palm_collision_geometry": false, + "container_contact": true + } + ], + "recognized_hand_contacts": 0, + "palm_contacts": 0, + "container_contacts": 3, + "source_files_edited": false, + "native_scene_assembly_used": true, + "diagnostic_bypasses": [ + "Unused controller setup", + "Unused renderers" + ], + "limitation": "Contacts are recomputed at the exact sealed pose with mj_forward, rather than reusing the prior substep contact cache.", + "measured_task_image": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957", + "input_hash_scope": "Original sealed native observation before any public redaction" + } + }, + "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-instruction-attempt04/" }, { "id": "task06-03-seed0-formal-attempt02", @@ -29655,7 +31305,8 @@ "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", "duration": "\u4f7f\u7528 13,225 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", - "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002" + "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002", + "independent_verifier_review": "No confirmed successful pickup. Very dark/occluded intervals at published 30.95-34.25 s and 35.60-37.55 s impair inspection; no codec failure or freeze is detected." }, "provenance": { "sources": { @@ -29754,34 +31405,35 @@ "previous_max_steps": 10000 }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0-attempt02/resources/memos/g1_manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0-attempt02/resources/memos/g1_manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -29790,10 +31442,16 @@ "observed_images": 140, "tool_errors": 1 }, - "selected_for_formal_metrics": true + "selected_for_formal_metrics": true, + "visual_review": { + "status": "visibility_problem", + "confidence": "medium", + "native_verdict_retained": true, + "review": "No confirmed successful pickup. Very dark/occluded intervals at published 30.95-34.25 s and 35.60-37.55 s impair inspection; no codec failure or freeze is detected." + } }, { - "id": "task06-04-seed0-formal-attempt02", + "id": "task06-04-bend-pick-and-place-codex-seed0-instruction-attempt04", "task_key": "task06/04", "family": "task06", "slot": "04", @@ -29810,61 +31468,61 @@ }, "verdict": { "evidence_valid": true, - "steps": 116, + "steps": 15, "success": true, "termination": "success" }, - "steps": 116, + "steps": 15, "simulation_time_s": null, - "wall_time_s": 111.95334, + "wall_time_s": 67.473848, "model": "gpt-6-astra", "effort": "high", "harness": "codex", "codex_version": "0.160.0", "native_instruction": "bend to grasp the cracker box and drop it in the basket.", - "instruction": "bend to grasp the cracker box and drop it in the basket.", - "instruction_policy": "original_native", + "instruction": "Bend down, pick up the cracker box, and release it above the center of the basket so it drops inside.", + "instruction_policy": "modified", "usage": { "accounting": "reported-responses", "audit_complete": true, - "cache_hit_rate": 0.9163462297790655, - "cache_reported_input_tokens": 329238, + "cache_hit_rate": 0.9039477677563947, + "cache_reported_input_tokens": 257006, "cache_write_input_tokens": 0, - "cache_write_reported_input_tokens": 329238, - "cached_input_tokens": 301696, + "cache_write_reported_input_tokens": 257006, + "cached_input_tokens": 232320, "completed_turns": 1, "cost_usd": null, "failed_turns": 0, - "input_tokens": 329238, + "input_tokens": 257006, "known_cache_write_input_tokens": 0, - "known_cached_input_tokens": 301696, - "known_input_tokens": 329238, - "known_output_tokens": 2595, - "known_reasoning_output_tokens": 187, - "output_tokens": 2595, - "reasoning_output_tokens": 187, - "reasoning_reported_output_tokens": 2595, + "known_cached_input_tokens": 232320, + "known_input_tokens": 257006, + "known_output_tokens": 1562, + "known_reasoning_output_tokens": 162, + "output_tokens": 1562, + "reasoning_output_tokens": 162, + "reasoning_reported_output_tokens": 1562, "reported_responses": { - "cache_reported_input_tokens": 11, - "cache_write_input_tokens": 11, - "cache_write_reported_input_tokens": 11, - "cached_input_tokens": 11, - "input_tokens": 11, - "output_tokens": 11, - "reasoning_output_tokens": 11, - "reasoning_reported_output_tokens": 11 - }, - "response_count": 11, + "cache_reported_input_tokens": 9, + "cache_write_input_tokens": 9, + "cache_write_reported_input_tokens": 9, + "cached_input_tokens": 9, + "input_tokens": 9, + "output_tokens": 9, + "reasoning_output_tokens": 9, + "reasoning_reported_output_tokens": 9 + }, + "response_count": 9, "response_ids_complete": true, "schema": "rlebench/token-usage/1", "source": "Codex token_usage_record per response", - "uncached_input_tokens": 27542, + "uncached_input_tokens": 24686, "unidentified_usage_records": 0 }, "call_activity": { - "model_tool_calls": 10, + "model_tool_calls": 8, "model_tool_calls_by_name": { - "exec": 10 + "exec": 8 }, "nested_python_tool_invocations": null, "python_device_rpc_attempts": null, @@ -29884,36 +31542,36 @@ "output_fps": 40, "width": 1280, "height": 360, - "native_frames": 24, - "published_frames": 24, + "native_frames": 4, + "published_frames": 4, "source_sample_coverage": 1.0, "terminal_frame_retained": true, - "terminal_source_index": 23, - "duration_s": 0.6, - "sha256": "011fb24c546f4314b7c695a0f553544772086a397afa8dd50a9f0a0c12869e2e", + "terminal_source_index": 3, + "duration_s": 0.1, + "sha256": "36dfc24e74866c3bf1cc5df293fa8f30226c4aa20cebfb52e4ffa20062d472fb", "source_sha256": { - "camera_head_left.mp4": "1cc8b62ab1d645f322702797da39c5b9c638b1a7b64fac0525e75c81001f3b33", - "camera_head_right.mp4": "ddf92efd54b082dcd847602221236c0de5a90d3e0c596148941f38205314c20f" + "camera_head_left.mp4": "12dda1b9000f90f5610385e0c6e39f00ec3af84383f51793e886e8eaee88cc48", + "camera_head_right.mp4": "a20620246ab20a3350fb81ee3af798ff00d55a6f4cf8cd0f4a1608b5ae2bd26a" }, - "all_frames_compared": 24, - "minimum_frame_psnr_db": 36.0947630714779, - "maximum_frame_rgb_mae": 2.956716537475586, - "terminal_frame_psnr_db": 36.0947630714779, + "all_frames_compared": 4, + "minimum_frame_psnr_db": 36.85036165937504, + "maximum_frame_rgb_mae": 2.667937755584717, + "terminal_frame_psnr_db": 36.85036165937504, "decode_ok": true, "pts_verified": true, "frame_mapping": "One decoded stereo pair for each native index; raw RGB input at source_fps * speed", "scope": "Original native stereo recording, all samples at 4x speed; no simulation replay or rerender", "recording": { - "accepted_samples": 25, - "captured_samples": 25, + "accepted_samples": 4, + "captured_samples": 4, "clock": "simulation", "dropped_samples": 0, - "encoded_frames": 24, - "end_time_s": 2.3200000000000016, + "encoded_frames": 4, + "end_time_s": 0.3, "error": null, "experimental": true, "fps": 10, - "received_samples": 25, + "received_samples": 4, "schema": "roboenv/recording/1", "state": "closed", "status": "complete", @@ -29938,9 +31596,12 @@ } }, "analysis": { - "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", - "duration": "\u4f7f\u7528 116 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\uff0c\u7ec8\u6001\u8bc1\u636e\u6709\u6548\uff0c\u7528\u91cf\u5b8c\u6574\uff1b\u72ec\u7acb\u89c6\u9891\u5ba1\u67e5\u786e\u8ba4\u8bef\u901a\u8fc7\u3002", + "instruction": "\u672c\u56de\u5408\u5df2\u660e\u786e\u8981\u6c42\u5f2f\u8170\u6293\u53d6\u7bb1\u5b50\uff0c\u5e76\u5728\u7bee\u5b50\u4e2d\u592e\u4e0a\u65b9\u677e\u624b\u4f7f\u5176\u843d\u5165\u3002", + "visible_completion": "\u5168\u90e8 4 \u4e2a\u539f\u751f\u53cc\u76ee\u91c7\u6837\u53ea\u663e\u793a\u5f2f\u8170\uff0c\u7bb1\u5b50\u4f4d\u4e8e\u673a\u5668\u4eba\u817f\u65c1\u548c\u7bee\u5b50\u5916\u4fa7\uff0c\u6ca1\u6709\u6293\u53d6\u6216\u653e\u5165\u3002\u53cc\u624b\u59cb\u7ec8\u5f20\u5f00\uff0c\u5c01\u5b58\u65e5\u5fd7\u4e2d\u6ca1\u6709\u624b\u90e8\u5199\u5165\u52a8\u4f5c\u3002 \u96f6\u6a21\u578b\u3001\u96f6\u7269\u7406\u6b65\u7684\u7ec8\u6001\u63a5\u89e6\u91cd\u5efa\u786e\u8ba4\uff0c\u7bb1\u5b50\u4ecd\u540c\u65f6\u63a5\u89e6\u539f\u53f0\u9762\u548c\u7bee\u5b50\u3002", + "duration": "\u4ec5\u4f7f\u7528 15 / 15,000 \u4e2a\u63a7\u5236\u6b65\uff080.30 \u79d2\u539f\u751f\u63a7\u5236\u65f6\u95f4\uff09\u5373 native success\uff0c\u9996\u6b21\u5f2f\u8170\u52a8\u4f5c\u5c1a\u672a\u5b8c\u6210\u3002", + "cause": "\u51bb\u7ed3 verifier \u4e0d\u8981\u6c42\u6293\u53d6\u5386\u53f2\u548c\u7a33\u5b9a\u7684\u7bee\u5185\u653e\u7f6e\uff1b\u521d\u59cb\u7bb1\u5b50\u5728\u7bee\u5b50\u65c1\uff0c\u8f7b\u5fae\u59ff\u6001\u53d8\u5316\u5373\u53ef\u89e6\u53d1\u6210\u529f\u7d2f\u79ef\u53ca\u63d0\u524d\u7ec8\u6b62\u3002\u660e\u786e instruction \u65e0\u6cd5\u963b\u6b62\u8fd9\u79cd\u539f\u751f\u63d0\u524d\u63a5\u53d7\u3002", + "comparison": "\u4fee\u8ba2 instruction \u540e\u4ecd\u53d1\u751f\u8bef\u901a\u8fc7\uff0c\u4e0d\u80fd\u5c06\u6b64\u539f\u751f PASS \u5f53\u4f5c\u4efb\u52a1\u5b8c\u6210\u7684\u53ef\u9760\u8bc1\u636e\u3002" }, "provenance": { "sources": { @@ -29954,9 +31615,10 @@ "tests", "docs" ], - "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc", + "content_sha256": "bbd03a94ca5776307435b6ed81c0f935a17f836b9ee80770069b58162d5e7e8f", "dirty": true, - "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618", + "revision": "f1935004c2fb23501b937cd29b826f69a06214b2", + "snapshot": "/shared/evaluations/kinex-v010-task04-20261006T151944Z/additional-campaigns/simple-g1-media-instruction-fix-20261010T181909Z/sources/RLE-Bench-inhouse", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" }, "RoboEnv": { @@ -29976,7 +31638,9 @@ ], "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2", "dirty": true, + "reuse": "Exact 15000-step deployment owner source; no edits", "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92", + "snapshot": "/shared/evaluations/kinex-v010-task04-20261006T151944Z/additional-campaigns/simple-g1-media-instruction-fix-20261010T181909Z/sources/RoboEnv", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" }, "kinex": { @@ -29995,19 +31659,20 @@ ], "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", "dirty": false, + "reuse": "Exact existing agent image and frozen Kinex revision", "revision": "caac19a8a36272972f762e0f74cfe381e8500048", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" } }, - "job": "task06-04-bend-pick-and-place-codex-seed0-attempt02", - "attempt": 2, + "job": "task06-04-bend-pick-and-place-codex-seed0-instruction-attempt04", + "attempt": 4, "harness": "stock Codex CLI", "control_interface": "public RoboEnv SDK and CLI", "kinex_agent_runtime_used": false, - "gpu_index": 2, + "gpu_index": 1, "gpu_model": "NVIDIA L40S", "campaign_dispatch_concurrency_limit": 8, - "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.", + "concurrency_note": "At most one added SIMPLE episode per GPU, admitted with spare memory and low utilization; existing environments remain active.", "codex_version": "0.160.0", "model": "gpt-6-astra", "effort": "high", @@ -30027,58 +31692,214 @@ "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217", "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957" }, - "session_original_sha256": "ec64abe7c35301ed7902c9a352d65d221a43eda56dfdfd37ff537c0f98ce58d0", - "protocol_sha256": "f1f597064f79f90d61d0cd7b77a96ead0cf0c9c83b37d5abfc9541366e21b97c", - "deployment": "failed10000-rerun15000-20261010T020844Z", + "session_original_sha256": "8b284735b8b58c5b2e8e9b2f2e416e277b1136072b70d9cd38b169f1e9997386", + "protocol_sha256": "f81ffa2593ccaca210aa75fc87935c2363f3281b87c24263ca9963b479ad1582", + "condition": "modified-instructions", "max_steps": 15000, - "budget_revision": "failed10000-rerun15000-20261010T020844Z", "same_protocol_replacement": false, - "replaces_user_interrupted_attempt": null, - "replacement_reason": "Explicit user-authorized fresh rerun of a completed 10000-step native failure with 15000 steps.", - "previous_native_failure": "task06-04-bend-pick-and-place-codex-seed0-attempt01", - "previous_max_steps": 10000 + "previous_selection": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02", + "native_verifier_changed": false, + "setup_failure_predecessor": "task06-04-bend-pick-and-place-codex-seed0-instruction-attempt03", + "replacement_reason": "Eight simultaneous cold starts exceeded the initialization deadline under shared CPU load.", + "predecessor_model_calls": 0, + "predecessor_measured_episodes": 0 }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { - "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/resources/tools/g1_control.py", + "name": "tools/roboenv_snapshot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/resources/tools/roboenv_snapshot.py", "kind": "Created during this episode; final workspace snapshot." }, { - "name": "memos/g1_bend_pick.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0-attempt02/resources/memos/g1_bend_pick.md", + "name": "memos/roboenv.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/resources/memos/roboenv.md", "kind": "Created during this episode; final workspace snapshot." } ], "session_counts": { - "visible_events": 29, - "observed_images": 5, - "tool_errors": 3 + "visible_events": 24, + "observed_images": 2, + "tool_errors": 2 }, - "selected_for_formal_metrics": true + "selected_for_formal_metrics": true, + "visual_review": { + "status": "confirmed_false_positive", + "visible_goal_completed": false, + "false_positive": true, + "false_negative": false, + "confidence": "high", + "native_verifier_changed": false, + "pickup_history_observed": false, + "hand_write_count": 0, + "reviewed_samples": [ + 0, + 1, + 2, + 3 + ], + "terminal_object_poses": { + "container": { + "label": "brasket", + "state_index": 51, + "terminal_qpos": [ + -0.2192241395896768, + -0.08051240134912414, + 0.4012197448499592, + 0.49693016431140025, + 0.4998224769879486, + 0.5018351419957602, + 0.5013974407126259 + ], + "mapping": "Unique exact match to previously audited qpos, with unchanged frozen model and state layout." + }, + "target": { + "label": "cracker_box", + "state_index": 58, + "terminal_qpos": [ + -0.3872390149185321, + -0.06050962204007641, + 0.5109277494224526, + 0.9612961230719299, + -0.020147332828399256, + 0.22467939942400153, + 0.1581866499463472 + ], + "mapping": "Unique exact match to previously audited qpos, with unchanged frozen model and state layout." + }, + "distractor_0": { + "label": "toy_monkey", + "state_index": 65, + "terminal_qpos": [ + 0.3871862651882573, + 0.17948979767739442, + 0.41085712291481896, + 0.25555106648329645, + -0.2706974837100474, + 0.03198346580793792, + 0.9275740308176009 + ], + "mapping": "Unique exact match to previously audited qpos, with unchanged frozen model and state layout." + }, + "distractor_1": { + "label": "bowl", + "state_index": 72, + "terminal_qpos": [ + 0.07686490834651107, + 0.16832786432720373, + 0.4256186606233802, + 0.9948862879375171, + -0.0007298166460625917, + -0.0009248053279437012, + -0.10099448587263041 + ], + "mapping": "Unique exact match to previously audited qpos, with unchanged frozen model and state layout." + } + }, + "measured_control_steps": 15, + "native_verdict_retained": true, + "independent_terminal_contact_check": { + "passed": true, + "model_calls": 0, + "physics_steps": 0, + "reward_checks_invoked": 0, + "input_sha256": "6794b3810e898d4724cda23f4eb348169b3c4225675e4551d023d2316ec83282", + "input_task": "simple/bend-pick-and-place", + "native_implementation": { + "backend": "simple", + "dependencies_sha256": "b655ef7dd8620ec89bc34238efb9a1ea70178f569afe1f429a6de9494840ee5f", + "implementation_sha256": "b184ced6cf472c3554fda97879a31cffcce98416acc28b4d6a4672529017d19b", + "selection": "shared-and-selected-backend/1", + "sources_sha256": "aae5710e1a7dd1e87d63a827b5e93881bc71c19aba1f6f0555fdf41c552c572a" + }, + "model_state_size": 641, + "nq": 78, + "sealed_time": 3.564999999999946, + "target_contacts": [ + { + "pair": [ + { + "body": "table", + "geom": "table_geom", + "mesh": null + }, + { + "body": "cracker_box", + "geom": "cracker_box_convex_0", + "mesh": "cracker_box_mesh_convex0" + } + ], + "distance_m": -0.003547762458395054, + "position": [ + -0.3844545935131092, + -0.13806303473028383, + 0.3982261187708025 + ], + "recognized_as_hand_by_native": false, + "palm_collision_geometry": false, + "container_contact": false + }, + { + "pair": [ + { + "body": "brasket", + "geom": "brasket_convex_5", + "mesh": "brasket_mesh_convex5" + }, + { + "body": "cracker_box", + "geom": "cracker_box_convex_1", + "mesh": "cracker_box_mesh_convex1" + } + ], + "distance_m": -0.0016191378146212826, + "position": [ + -0.3162593205479579, + -0.1218073606990798, + 0.5435856863845453 + ], + "recognized_as_hand_by_native": false, + "palm_collision_geometry": false, + "container_contact": true + } + ], + "recognized_hand_contacts": 0, + "palm_contacts": 0, + "container_contacts": 1, + "source_files_edited": false, + "native_scene_assembly_used": true, + "diagnostic_bypasses": [ + "Unused controller setup", + "Unused renderers" + ], + "limitation": "Contacts are recomputed at the exact sealed pose with mj_forward, rather than reusing the prior substep contact cache.", + "measured_task_image": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957", + "input_hash_scope": "Original sealed native observation before any public redaction" + } + }, + "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-instruction-attempt04/" }, { - "id": "task06-05-seed0-formal-attempt02", + "id": "task06-05-bend-handover-codex-seed0-instruction-attempt04", "task_key": "task06/05", "family": "task06", "slot": "05", @@ -30086,8 +31907,8 @@ "episode": 1, "phase": "formal", "status": "completed", - "success": true, - "native_reward": 1.0, + "success": false, + "native_reward": 0.0, "valid": true, "execution": { "reason": null, @@ -30095,61 +31916,61 @@ }, "verdict": { "evidence_valid": true, - "steps": 12320, - "success": true, - "termination": "success" + "steps": 9265, + "success": false, + "termination": "stopped" }, - "steps": 12320, + "steps": 9265, "simulation_time_s": null, - "wall_time_s": 4359.883363, + "wall_time_s": 3565.594147, "model": "gpt-6-astra", "effort": "high", "harness": "codex", "codex_version": "0.160.0", "native_instruction": "bend the robot and pick up the cracker box, then place it on the container.", - "instruction": "bend the robot and pick up the cracker box, then place it on the container.", - "instruction_policy": "original_native", + "instruction": "Bend down, pick up the cracker box, then release it above the basket opening so it drops inside.", + "instruction_policy": "modified", "usage": { "accounting": "reported-responses", "audit_complete": true, - "cache_hit_rate": 0.9926789916948668, - "cache_reported_input_tokens": 32371907, + "cache_hit_rate": 0.9912073526633496, + "cache_reported_input_tokens": 22421916, "cache_write_input_tokens": 0, - "cache_write_reported_input_tokens": 32371907, - "cached_input_tokens": 32134912, + "cache_write_reported_input_tokens": 22421916, + "cached_input_tokens": 22224768, "completed_turns": 1, "cost_usd": null, "failed_turns": 0, - "input_tokens": 32371907, + "input_tokens": 22421916, "known_cache_write_input_tokens": 0, - "known_cached_input_tokens": 32134912, - "known_input_tokens": 32371907, - "known_output_tokens": 103454, - "known_reasoning_output_tokens": 69820, - "output_tokens": 103454, - "reasoning_output_tokens": 69820, - "reasoning_reported_output_tokens": 103454, + "known_cached_input_tokens": 22224768, + "known_input_tokens": 22421916, + "known_output_tokens": 69606, + "known_reasoning_output_tokens": 40045, + "output_tokens": 69606, + "reasoning_output_tokens": 40045, + "reasoning_reported_output_tokens": 69606, "reported_responses": { - "cache_reported_input_tokens": 297, - "cache_write_input_tokens": 297, - "cache_write_reported_input_tokens": 297, - "cached_input_tokens": 297, - "input_tokens": 297, - "output_tokens": 297, - "reasoning_output_tokens": 297, - "reasoning_reported_output_tokens": 297 - }, - "response_count": 297, + "cache_reported_input_tokens": 218, + "cache_write_input_tokens": 218, + "cache_write_reported_input_tokens": 218, + "cached_input_tokens": 218, + "input_tokens": 218, + "output_tokens": 218, + "reasoning_output_tokens": 218, + "reasoning_reported_output_tokens": 218 + }, + "response_count": 218, "response_ids_complete": true, "schema": "rlebench/token-usage/1", "source": "Codex token_usage_record per response", - "uncached_input_tokens": 236995, + "uncached_input_tokens": 197148, "unidentified_usage_records": 0 }, "call_activity": { - "model_tool_calls": 296, + "model_tool_calls": 217, "model_tool_calls_by_name": { - "exec": 296 + "exec": 217 }, "nested_python_tool_invocations": null, "python_device_rpc_attempts": null, @@ -30169,36 +31990,36 @@ "output_fps": 40, "width": 1280, "height": 360, - "native_frames": 2465, - "published_frames": 2465, + "native_frames": 1854, + "published_frames": 1854, "source_sample_coverage": 1.0, "terminal_frame_retained": true, - "terminal_source_index": 2464, - "duration_s": 61.625, - "sha256": "6e375814036f0e820e1bc519f368f7b8bcb8acb0fdc425eaa7b26e37b8322bf5", + "terminal_source_index": 1853, + "duration_s": 46.35, + "sha256": "ec9cca03f71d79cb055c4fe0699b63b56ac8dd53a9b0fe15954872af1daad2f6", "source_sha256": { - "camera_head_left.mp4": "99324d095b871bb7856ba04c0897deee67a7d094b5f7c5340be942c90643cc19", - "camera_head_right.mp4": "bd6a79e4d41523fc54dbf0eb78a4f228429bb16ae6db859e364cddddb65aa26c" + "camera_head_left.mp4": "32dbf0b2f767ec3fa7ce257df08d1f2ba085f3d246401b01a9af593122a58a9d", + "camera_head_right.mp4": "d90ccaed1f6a76998595a123e669301d6b830d824a9bdf3b35561c7b20e7d0dc" }, - "all_frames_compared": 2465, - "minimum_frame_psnr_db": 36.42045017056597, - "maximum_frame_rgb_mae": 2.767667770385742, - "terminal_frame_psnr_db": 36.993661080255514, + "all_frames_compared": 1854, + "minimum_frame_psnr_db": 36.120242550654126, + "maximum_frame_rgb_mae": 2.9573140144348145, + "terminal_frame_psnr_db": 38.7119463701233, "decode_ok": true, "pts_verified": true, "frame_mapping": "One decoded stereo pair for each native index; raw RGB input at source_fps * speed", "scope": "Original native stereo recording, all samples at 4x speed; no simulation replay or rerender", "recording": { - "accepted_samples": 2465, - "captured_samples": 2465, + "accepted_samples": 1854, + "captured_samples": 1854, "clock": "simulation", "dropped_samples": 0, - "encoded_frames": 2465, - "end_time_s": 246.40000000005224, + "encoded_frames": 1854, + "end_time_s": 185.300000000021, "error": null, "experimental": true, "fps": 10, - "received_samples": 2465, + "received_samples": 1854, "schema": "roboenv/recording/1", "state": "closed", "status": "complete", @@ -30223,9 +32044,12 @@ } }, "analysis": { - "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", - "duration": "\u4f7f\u7528 12,320 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + "outcome": "\u539f\u751f\u5224\u5b9a\u5931\u8d25\uff0c\u7ec8\u6001\u8bc1\u636e\u6709\u6548\uff0c\u7528\u91cf\u5b8c\u6574\uff1b\u72ec\u7acb\u753b\u9762\u590d\u6838\u4e0e\u5931\u8d25\u4e00\u81f4\u3002", + "instruction": "\u672c\u56de\u5408\u660e\u786e\u8981\u6c42\u6293\u53d6\u7bb1\u5b50\uff0c\u5e76\u5728\u7bee\u5b50\u5f00\u53e3\u4e0a\u65b9\u677e\u624b\u4f7f\u5176\u843d\u5165\u3002", + "visible_completion": "\u89c6\u9891\u663e\u793a\u64cd\u4f5c\u8fc7\u7a0b\u4e2d\u7bb1\u5b50\u6389\u79bb\u53f0\u9762\uff0c\u6700\u7ec8\u7559\u5728\u5730\u677f\u4e0a\u3001\u7bee\u5b50\u5916\u3002\u5c01\u5b58\u7ec8\u6001\u7bb1\u5b50\u4e2d\u5fc3 z=0.03755 m\uff0c\u7bee\u5b50\u4e2d\u5fc3 z=0.45316 m\uff0c\u76f8\u5dee\u7ea6 0.416 m\uff0c\u6ca1\u6709\u5b8c\u6210\u7bee\u5185\u653e\u7f6e\u3002", + "duration": "\u4f7f\u7528 9,265 / 15,000 \u4e2a\u63a7\u5236\u6b65\uff08185.3 \u79d2\u63a7\u5236\u65f6\u95f4\uff09\uff0cagent \u4e3b\u52a8\u505c\u6b62\uff1bnative_reward=0.0\uff0cnative_success=false\u3002", + "cause": "\u8be5\u56de\u5408\u672a\u628a\u6389\u5728\u5730\u677f\u4e0a\u7684\u7bb1\u5b50\u91cd\u65b0\u62fe\u8d77\u5e76\u653e\u5165\u7bee\u5b50\uff0c\u662f\u5b9e\u9645\u63a7\u5236\u5931\u8d25\uff0c\u6ca1\u6709\u8bc1\u636e\u652f\u6301\u8bef\u62d2\u7edd\u3002", + "comparison": "\u6b64\u524d Codex 05 \u7684\u539f\u751f PASS \u5df2\u88ab\u539f\u59cb\u89c6\u9891\u590d\u6838\u5224\u4e3a\u8bef\u901a\u8fc7\uff1b\u6b64\u6b21\u4fee\u8ba2 instruction \u540e\u7684 fresh seed-0 \u56de\u5408\u4ecd\u672a\u5b8c\u6210\uff0c\u4f46\u539f\u751f\u5931\u8d25\u4e0e\u5b9e\u9645\u7ec8\u6001\u4e00\u81f4\u3002\u8be5\u5355\u6b21\u5dee\u5f02\u4e0d\u80fd\u8bc1\u660e instruction \u7684\u56e0\u679c\u6548\u679c\u3002" }, "provenance": { "sources": { @@ -30239,9 +32063,10 @@ "tests", "docs" ], - "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc", + "content_sha256": "bbd03a94ca5776307435b6ed81c0f935a17f836b9ee80770069b58162d5e7e8f", "dirty": true, - "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618", + "revision": "f1935004c2fb23501b937cd29b826f69a06214b2", + "snapshot": "/shared/evaluations/kinex-v010-task04-20261006T151944Z/additional-campaigns/simple-g1-media-instruction-fix-20261010T181909Z/sources/RLE-Bench-inhouse", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" }, "RoboEnv": { @@ -30261,7 +32086,9 @@ ], "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2", "dirty": true, + "reuse": "Exact 15000-step deployment owner source; no edits", "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92", + "snapshot": "/shared/evaluations/kinex-v010-task04-20261006T151944Z/additional-campaigns/simple-g1-media-instruction-fix-20261010T181909Z/sources/RoboEnv", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" }, "kinex": { @@ -30280,19 +32107,20 @@ ], "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", "dirty": false, + "reuse": "Exact existing agent image and frozen Kinex revision", "revision": "caac19a8a36272972f762e0f74cfe381e8500048", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" } }, - "job": "task06-05-bend-handover-codex-seed0-attempt02", - "attempt": 2, + "job": "task06-05-bend-handover-codex-seed0-instruction-attempt04", + "attempt": 4, "harness": "stock Codex CLI", "control_interface": "public RoboEnv SDK and CLI", "kinex_agent_runtime_used": false, - "gpu_index": 6, + "gpu_index": 5, "gpu_model": "NVIDIA L40S", "campaign_dispatch_concurrency_limit": 8, - "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.", + "concurrency_note": "At most one added SIMPLE episode per GPU, admitted with spare memory and low utilization; existing environments remain active.", "codex_version": "0.160.0", "model": "gpt-6-astra", "effort": "high", @@ -30312,55 +32140,135 @@ "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217", "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957" }, - "session_original_sha256": "dd94125fcdce620f874702dd716c5257502cf1aef0288654232ff4baf07b2a5b", - "protocol_sha256": "01bd486cc627e7126d7a549e63ec36a66d3b828d4b374814f730b17f8680edc9", - "deployment": "failed10000-rerun15000-20261010T020844Z", + "session_original_sha256": "4a87897945962236c6215dd7caa6dfcd59bdf8e8c0064420dca0ccb0beb1624a", + "protocol_sha256": "8597c49a0e89454533b82c22a68bb423d3893436d65f9ae5e9153cb256a37446", + "condition": "modified-instructions", "max_steps": 15000, - "budget_revision": "failed10000-rerun15000-20261010T020844Z", "same_protocol_replacement": false, - "replaces_user_interrupted_attempt": null, - "replacement_reason": "Explicit user-authorized fresh rerun of a completed 10000-step native failure with 15000 steps.", - "previous_native_failure": "task06-05-bend-handover-codex-seed0-attempt01", - "previous_max_steps": 10000 + "previous_selection": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02", + "native_verifier_changed": false, + "setup_failure_predecessor": "task06-05-bend-handover-codex-seed0-instruction-attempt03", + "replacement_reason": "Eight simultaneous cold starts exceeded the initialization deadline under shared CPU load.", + "predecessor_model_calls": 0, + "predecessor_measured_episodes": 0 }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { - "name": "memos/g1_dex3.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0-attempt02/resources/memos/g1_dex3.md", + "name": "memos/g1_manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/resources/memos/g1_manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], "session_counts": { - "visible_events": 639, - "observed_images": 97, - "tool_errors": 3 + "visible_events": 460, + "observed_images": 78, + "tool_errors": 4 }, - "selected_for_formal_metrics": true + "selected_for_formal_metrics": true, + "visual_review": { + "status": "consistent_failure", + "visible_goal_completed": false, + "false_positive": false, + "false_negative": false, + "confidence": "high", + "native_verifier_changed": false, + "native_verdict_retained": true, + "reviewed_samples": [ + 0, + 371, + 741, + 1112, + 1482, + 1773, + 1813, + 1853 + ], + "terminal_object_poses": { + "container": { + "label": "brasket", + "state_index": 51, + "terminal_qpos": [ + -0.26493907009176226, + 0.18714995964212794, + 0.4531581685400422, + 0.7692535612414195, + 0.14830581794173264, + 0.11090923111177861, + 0.6115173630701103 + ], + "mapping": "Unique exact match to previously audited qpos, with unchanged frozen model and state layout." + }, + "target": { + "label": "cracker_box", + "state_index": 58, + "terminal_qpos": [ + -0.24760553509000038, + 0.11285413622116917, + 0.037552045862085374, + 0.6211811537453864, + 0.32956603595130746, + -0.6305501710730999, + 0.3285219687305184 + ], + "mapping": "Unique exact match to previously audited qpos, with unchanged frozen model and state layout." + }, + "distractor_0": { + "label": "toy_monkey", + "state_index": 65, + "terminal_qpos": [ + -0.08987553547033712, + 0.19956656428749517, + 0.41085712291478843, + 0.2555510664836823, + -0.27069748370501917, + 0.031983465807329615, + 0.9275740308189836 + ], + "mapping": "Unique exact match to previously audited qpos, with unchanged frozen model and state layout." + }, + "distractor_1": { + "label": "bowl", + "state_index": 72, + "terminal_qpos": [ + 0.28905567795928083, + 0.22201764217831088, + 0.42561866062338144, + 0.9948862879375167, + -0.0007298166460610347, + -0.0009248053283265724, + -0.10099448587262949 + ], + "mapping": "Unique exact match to previously audited qpos, with unchanged frozen model and state layout." + } + }, + "target_on_floor_outside_container": true + }, + "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-instruction-attempt04/" }, { "id": "task06-06-seed0-formal-attempt02", @@ -30511,7 +32419,8 @@ "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", "duration": "\u4f7f\u7528 8,170 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", - "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002" + "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002", + "independent_verifier_review": "A completed right-to-left handover plus released container placement is not shown. Active predicate omits handover and release, but this failed run is not evidence of a false negative." }, "provenance": { "sources": { @@ -30610,34 +32519,35 @@ "previous_max_steps": 10000 }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_manipulation.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0-attempt02/resources/memos/g1_manipulation.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0-attempt02/resources/memos/g1_manipulation.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -30646,7 +32556,13 @@ "observed_images": 73, "tool_errors": 0 }, - "selected_for_formal_metrics": true + "selected_for_formal_metrics": true, + "visual_review": { + "status": "no_confirmed_mismatch", + "confidence": "medium", + "native_verdict_retained": true, + "review": "A completed right-to-left handover plus released container placement is not shown. Active predicate omits handover and release, but this failed run is not evidence of a false negative." + } }, { "id": "task06-07-seed0-formal-attempt02", @@ -30797,7 +32713,8 @@ "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", "duration": "\u4f7f\u7528 13,688 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", - "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002" + "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002", + "independent_verifier_review": "Apple finishes on the floor and away from the basket. Actual run failure is consistent; the verifier additionally has an unmentioned tomato-can condition and table-name inconsistency." }, "provenance": { "sources": { @@ -30896,34 +32813,35 @@ "previous_max_steps": 10000 }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_simple.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0-attempt02/resources/memos/g1_simple.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0-attempt02/resources/memos/g1_simple.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -30932,7 +32850,13 @@ "observed_images": 88, "tool_errors": 12 }, - "selected_for_formal_metrics": true + "selected_for_formal_metrics": true, + "visual_review": { + "status": "verifier_mismatch_but_run_failed", + "confidence": "high", + "native_verdict_retained": true, + "review": "Apple finishes on the floor and away from the basket. Actual run failure is consistent; the verifier additionally has an unmentioned tomato-can condition and table-name inconsistency." + } }, { "id": "task06-08-seed0-formal", @@ -31082,7 +33006,8 @@ "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", "duration": "\u4f7f\u7528 1,277 / 10,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "independent_verifier_review": "Door leaf obstructs the head camera near termination; terminal joint angle is approximately -0.174 rad, below -0.16. Source and terminal state support success despite weak visual observability." }, "provenance": { "sources": { @@ -31173,39 +33098,40 @@ "protocol_sha256": "7459222297772303726f117c2d722012c84ef5c2b01e4057b1a8961d3ee23688" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, "resources": [ { "name": "tools/arm_ik.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/resources/tools/arm_ik.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/resources/tools/arm_ik.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "tools/robot_step.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/resources/tools/robot_step.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/resources/tools/robot_step.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/close-door.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/08/seed-0/resources/memos/close-door.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/08/seed-0/resources/memos/close-door.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -31214,10 +33140,16 @@ "observed_images": 19, "tool_errors": 2 }, - "selected_for_formal_metrics": true + "selected_for_formal_metrics": true, + "visual_review": { + "status": "threshold_explained", + "confidence": "high", + "native_verdict_retained": true, + "review": "Door leaf obstructs the head camera near termination; terminal joint angle is approximately -0.174 rad, below -0.16. Source and terminal state support success despite weak visual observability." + } }, { - "id": "task06-09-seed0-formal", + "id": "task06-09-open-oven-codex-seed0-instruction-attempt04", "task_key": "task06/09", "family": "task06", "slot": "09", @@ -31225,8 +33157,8 @@ "episode": 1, "phase": "formal", "status": "completed", - "success": false, - "native_reward": 0.0, + "success": true, + "native_reward": 1.0, "valid": true, "execution": { "reason": null, @@ -31234,61 +33166,61 @@ }, "verdict": { "evidence_valid": true, - "steps": 3170, - "success": false, - "termination": "stopped" + "steps": 521, + "success": true, + "termination": "success" }, - "steps": 3170, + "steps": 521, "simulation_time_s": null, - "wall_time_s": 918.375583, + "wall_time_s": 216.963646, "model": "gpt-6-astra", "effort": "high", "harness": "codex", "codex_version": "0.160.0", "native_instruction": "move forward to the oven and open it", - "instruction": "move forward to the oven and open it", - "instruction_policy": "original_native", + "instruction": "Move to the oven and open either door fully, to roughly a right angle.", + "instruction_policy": "modified", "usage": { "accounting": "reported-responses", "audit_complete": true, - "cache_hit_rate": 0.9660542143458919, - "cache_reported_input_tokens": 4133002, + "cache_hit_rate": 0.94929450115402, + "cache_reported_input_tokens": 672432, "cache_write_input_tokens": 0, - "cache_write_reported_input_tokens": 4133002, - "cached_input_tokens": 3992704, + "cache_write_reported_input_tokens": 672432, + "cached_input_tokens": 638336, "completed_turns": 1, "cost_usd": null, "failed_turns": 0, - "input_tokens": 4133002, + "input_tokens": 672432, "known_cache_write_input_tokens": 0, - "known_cached_input_tokens": 3992704, - "known_input_tokens": 4133002, - "known_output_tokens": 16710, - "known_reasoning_output_tokens": 5193, - "output_tokens": 16710, - "reasoning_output_tokens": 5193, - "reasoning_reported_output_tokens": 16710, + "known_cached_input_tokens": 638336, + "known_input_tokens": 672432, + "known_output_tokens": 4608, + "known_reasoning_output_tokens": 854, + "output_tokens": 4608, + "reasoning_output_tokens": 854, + "reasoning_reported_output_tokens": 4608, "reported_responses": { - "cache_reported_input_tokens": 84, - "cache_write_input_tokens": 84, - "cache_write_reported_input_tokens": 84, - "cached_input_tokens": 84, - "input_tokens": 84, - "output_tokens": 84, - "reasoning_output_tokens": 84, - "reasoning_reported_output_tokens": 84 - }, - "response_count": 84, + "cache_reported_input_tokens": 21, + "cache_write_input_tokens": 21, + "cache_write_reported_input_tokens": 21, + "cached_input_tokens": 21, + "input_tokens": 21, + "output_tokens": 21, + "reasoning_output_tokens": 21, + "reasoning_reported_output_tokens": 21 + }, + "response_count": 21, "response_ids_complete": true, "schema": "rlebench/token-usage/1", "source": "Codex token_usage_record per response", - "uncached_input_tokens": 140298, + "uncached_input_tokens": 34096, "unidentified_usage_records": 0 }, "call_activity": { - "model_tool_calls": 83, + "model_tool_calls": 20, "model_tool_calls_by_name": { - "exec": 83 + "exec": 20 }, "nested_python_tool_invocations": null, "python_device_rpc_attempts": null, @@ -31308,36 +33240,36 @@ "output_fps": 40, "width": 1280, "height": 360, - "native_frames": 635, - "published_frames": 635, + "native_frames": 105, + "published_frames": 105, "source_sample_coverage": 1.0, "terminal_frame_retained": true, - "terminal_source_index": 634, - "duration_s": 15.875, - "sha256": "0cab12ef33849200981cc426d00338fa21e96c5883935cd1ef860e036685f5d5", + "terminal_source_index": 104, + "duration_s": 2.625, + "sha256": "f11eef665a5e76a79181f64ccb3fe80acd732f55badeb12c03f3ad731acc2dbb", "source_sha256": { - "camera_head_left.mp4": "06cf5c373fb2d02854812e0cebe90e8d15e72c76fa928245282d3a7950d822fa", - "camera_head_right.mp4": "8dce7464960955f707ab9e9ae109ec8286cf86652a576e582ce6e5f724bf30ec" + "camera_head_left.mp4": "cb84f4aefbaa1a712e46ac475c631e7c38c272be9690bb6d1e5e176fa7b05061", + "camera_head_right.mp4": "e05c30a466f568aeaa52c08c0760d1571fca8065e4a6befdd562fef52dc80d2e" }, - "all_frames_compared": 635, - "minimum_frame_psnr_db": 38.42669493473994, - "maximum_frame_rgb_mae": 2.0199508666992188, - "terminal_frame_psnr_db": 39.10580975354201, + "all_frames_compared": 105, + "minimum_frame_psnr_db": 38.759188327260404, + "maximum_frame_rgb_mae": 1.9476063251495361, + "terminal_frame_psnr_db": 39.245646459091276, "decode_ok": true, "pts_verified": true, "frame_mapping": "One decoded stereo pair for each native index; raw RGB input at source_fps * speed", "scope": "Original native stereo recording, all samples at 4x speed; no simulation replay or rerender", "recording": { - "accepted_samples": 635, - "captured_samples": 635, + "accepted_samples": 106, + "captured_samples": 106, "clock": "simulation", "dropped_samples": 0, - "encoded_frames": 635, - "end_time_s": 63.40000000000431, + "encoded_frames": 105, + "end_time_s": 10.419999999999867, "error": null, "experimental": true, "fps": 10, - "received_samples": 635, + "received_samples": 106, "schema": "roboenv/recording/1", "state": "closed", "status": "complete", @@ -31362,10 +33294,11 @@ } }, "analysis": { - "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", - "duration": "\u4f7f\u7528 3,170 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", - "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002" + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\uff0c\u7ec8\u6001\u8bc1\u636e\u6709\u6548\uff0c\u7528\u91cf\u5b8c\u6574\u3002", + "instruction": "\u672c\u56de\u5408\u4f7f\u7528\u4fee\u8ba2 instruction\uff1a\u5c06\u4efb\u610f\u4e00\u4e2a\u70e4\u7bb1\u95e8\u5145\u5206\u6253\u5f00\uff0c\u63a5\u8fd1\u76f4\u89d2\u3002", + "visible_completion": "\u5b8c\u6574\u53cc\u76ee\u89c6\u9891\u663e\u793a\u4e0b\u65b9\u70e4\u7bb1\u95e8\u6253\u5f00\uff0c\u771f\u5b9e\u672b\u5e27\u4e0e\u8be5\u7ec8\u6001\u4e00\u81f4\u3002\u5c01\u5b58\u7ec8\u6001\u5173\u8282\u89d2\u4e3a 1.4119 rad\uff0880.90\u00b0\uff09\uff0c\u8d85\u8fc7\u51bb\u7ed3 verifier \u7684 1.4 rad \u95e8\u69db\u3002", + "duration": "\u4f7f\u7528 521 / 15,000 \u4e2a\u63a7\u5236\u6b65\uff0cnative termination=success\u3002", + "comparison": "\u6b64\u524d Codex \u56de\u5408\u7ec8\u6001\u7ea6\u4e3a 36.32\u00b0\uff0c\u4e0d\u8db3\u901a\u8fc7\u95e8\u69db\uff1b\u6b64\u6b21 fresh seed-0 \u56de\u5408\u6210\u529f\u3002\u5355\u6b21\u7ed3\u679c\u4e0d\u80fd\u8bc1\u660e instruction \u7684\u56e0\u679c\u589e\u76ca\u3002" }, "provenance": { "sources": { @@ -31379,9 +33312,10 @@ "tests", "docs" ], - "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc", + "content_sha256": "bbd03a94ca5776307435b6ed81c0f935a17f836b9ee80770069b58162d5e7e8f", "dirty": true, - "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618", + "revision": "f1935004c2fb23501b937cd29b826f69a06214b2", + "snapshot": "/shared/evaluations/kinex-v010-task04-20261006T151944Z/additional-campaigns/simple-g1-media-instruction-fix-20261010T181909Z/sources/RLE-Bench-inhouse", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" }, "RoboEnv": { @@ -31401,7 +33335,9 @@ ], "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2", "dirty": true, + "reuse": "Exact 15000-step deployment owner source; no edits", "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92", + "snapshot": "/shared/evaluations/kinex-v010-task04-20261006T151944Z/additional-campaigns/simple-g1-media-instruction-fix-20261010T181909Z/sources/RoboEnv", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" }, "kinex": { @@ -31420,19 +33356,20 @@ ], "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", "dirty": false, + "reuse": "Exact existing agent image and frozen Kinex revision", "revision": "caac19a8a36272972f762e0f74cfe381e8500048", "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" } }, - "job": "task06-09-open-oven-codex-seed0-attempt03", - "attempt": 3, + "job": "task06-09-open-oven-codex-seed0-instruction-attempt04", + "attempt": 4, "harness": "stock Codex CLI", "control_interface": "public RoboEnv SDK and CLI", "kinex_agent_runtime_used": false, - "gpu_index": 2, + "gpu_index": 1, "gpu_model": "NVIDIA L40S", "campaign_dispatch_concurrency_limit": 8, - "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.", + "concurrency_note": "At most one added SIMPLE episode per GPU, admitted with spare memory and low utilization; existing environments remain active.", "codex_version": "0.160.0", "model": "gpt-6-astra", "effort": "high", @@ -31452,53 +33389,77 @@ "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217", "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957" }, - "session_original_sha256": "6fb961f0b29e07b7221771021776cddb69173e038f2ec5beba7e983b4c767efb", - "protocol_sha256": "cabc50e33ebf5f7a2e34a075aff7a1f66d2b2ffc5f84e941d688a929d7f70681", - "deployment": "steps15000-20261010T012736Z", + "session_original_sha256": "808b5ff828f98fbacb7527fac405e034551887dbd28e078576d3504b96f1e2a2", + "protocol_sha256": "7c71ca7c3d124a2ec23a538d7400af4e36c321a8a48d6fef5e3a9675f9a65096", + "condition": "modified-instructions", "max_steps": 15000, - "budget_revision": "steps15000-20261010T012736Z", "same_protocol_replacement": false, - "replaces_user_interrupted_attempt": "task06-09-open-oven-codex-seed0-attempt01", - "replacement_reason": "User requested 300 seconds at 50 Hz with updated code." + "previous_selection": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0", + "native_verifier_changed": false }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/native/evidence/episode/evidence/final-observation.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_oven.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/09/seed-0/resources/memos/g1_oven.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/resources/memos/g1_oven.md", "kind": "Created during this episode; final workspace snapshot." } ], "session_counts": { - "visible_events": 183, - "observed_images": 31, - "tool_errors": 1 + "visible_events": 50, + "observed_images": 15, + "tool_errors": 2 }, - "selected_for_formal_metrics": true + "selected_for_formal_metrics": true, + "visual_review": { + "status": "consistent_pass", + "visible_goal_completed": true, + "false_positive": false, + "false_negative": false, + "confidence": "high", + "terminal_joint_angles": { + "upper_door_rad": -6.761838114045569e-18, + "lower_door_rad": 1.4119226048563112 + }, + "native_threshold_rad": 1.4, + "native_verifier_changed": false, + "reviewed_samples": [ + 0, + 21, + 24, + 42, + 62, + 64, + 83, + 104 + ], + "state_mapping": "Frozen oven XML body preorder and native integration-state layout, unchanged from the original audited seed-0 task." + }, + "asset_base": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0-instruction-attempt04/" }, { "id": "task06-10-seed0-formal", @@ -31648,7 +33609,8 @@ "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", "duration": "\u4f7f\u7528 1,710 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "independent_verifier_review": "Faucet handle is visibly turned; terminal joint magnitude exceeds 0.7 rad. The instruction is turn the faucet; water-flow semantics are not requested or established." }, "provenance": { "sources": { @@ -31745,34 +33707,35 @@ "replacement_reason": "User requested 300 seconds at 50 Hz with updated code." }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, "resources": [ { "name": "tools/g1_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/resources/tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/resources/tools/g1_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1_faucet.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/10/seed-0/resources/memos/g1_faucet.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/10/seed-0/resources/memos/g1_faucet.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -31781,7 +33744,13 @@ "observed_images": 30, "tool_errors": 2 }, - "selected_for_formal_metrics": true + "selected_for_formal_metrics": true, + "visual_review": { + "status": "no_confirmed_mismatch", + "confidence": "high", + "native_verdict_retained": true, + "review": "Faucet handle is visibly turned; terminal joint magnitude exceeds 0.7 rad. The instruction is turn the faucet; water-flow semantics are not requested or established." + } }, { "id": "task06-11-seed0-formal", @@ -31931,7 +33900,8 @@ "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", "duration": "\u4f7f\u7528 581 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "independent_verifier_review": "Chair visibly moves toward the table, but arrival is not established. Native success only requires absolute x displacement >0.8 m, in either direction; it does not test distance/contact/alignment with the table. Suspected early acceptance, not a confirmed failed instruction." }, "provenance": { "sources": { @@ -32028,34 +33998,35 @@ "replacement_reason": "User requested 300 seconds at 50 Hz with updated code." }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, "resources": [ { "name": "tools/robot_step.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/resources/tools/robot_step.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/resources/tools/robot_step.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/roboenv.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/11/seed-0/resources/memos/roboenv.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/11/seed-0/resources/memos/roboenv.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -32064,7 +34035,13 @@ "observed_images": 7, "tool_errors": 2 }, - "selected_for_formal_metrics": true + "selected_for_formal_metrics": true, + "visual_review": { + "status": "weak_success_criteria", + "confidence": "high_for_code_only", + "native_verdict_retained": true, + "review": "Chair visibly moves toward the table, but arrival is not established. Native success only requires absolute x displacement >0.8 m, in either direction; it does not test distance/contact/alignment with the table. Suspected early acceptance, not a confirmed failed instruction." + } }, { "id": "task06-12-seed0-formal", @@ -32214,7 +34191,8 @@ "analysis": { "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", "duration": "\u4f7f\u7528 1,237 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", - "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "independent_verifier_review": "Trash lid is visibly opened; terminal lid joint exceeds 0.5 rad. Native success agrees with the observed opening." }, "provenance": { "sources": { @@ -32311,34 +34289,35 @@ "replacement_reason": "User requested 300 seconds at 50 Hz with updated code." }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/native/evidence/episode/evidence/final-observation.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, "resources": [ { "name": "tools/robot.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/resources/tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/resources/tools/robot.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/g1-trash-can.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/12/seed-0/resources/memos/g1-trash-can.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/12/seed-0/resources/memos/g1-trash-can.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -32347,7 +34326,13 @@ "observed_images": 26, "tool_errors": 3 }, - "selected_for_formal_metrics": true + "selected_for_formal_metrics": true, + "visual_review": { + "status": "no_confirmed_mismatch", + "confidence": "high", + "native_verdict_retained": true, + "review": "Trash lid is visibly opened; terminal lid joint exceeds 0.5 rad. Native success agrees with the observed opening." + } } ], "failure_review": { @@ -32355,7 +34340,27 @@ "reviewed": 0 }, "instruction_review": { - "policy": "Lift pot uses native RoboTwin TCP frames after the RoboEnv correction, with the existing reviewed instruction, unchanged native verifier, seed 0, 7500 action frames at 25 Hz, and one fresh stock Codex 0.160.0 / GPT-6 Astra high episode. Other results retain their historical runtime and protocol." + "policy": "Lift pot uses native RoboTwin TCP frames after the RoboEnv correction, with the existing reviewed instruction, unchanged native verifier, seed 0, 7500 action frames at 25 Hz, and one fresh stock Codex 0.160.0 / GPT-6 Astra high episode. Other results retain their historical runtime and protocol.", + "simple_instruction_revision": { + "condition": "modified-instructions", + "tasks": [ + "task06/02", + "task06/09", + "task06/04", + "task06/05" + ], + "new_evaluations": 4, + "seed": 0, + "max_steps": 15000, + "native_verifier_changed": false, + "native_successes": 3, + "visible_goal_completed": 2, + "confirmed_false_positives": 1, + "confirmed_false_negatives": 0, + "scope": "Single-seed instruction revision, with earlier runs preserved. Native success and independent visible completion are reported separately.", + "comparison_limit": "Earlier selected runs have mixed budgets and use an earlier account; this is not a multi-seed causal success-rate estimate." + }, + "modified_tasks": 60 }, "authorized_selection_complete": true, "simple_reruns": { @@ -32416,7 +34421,7 @@ ] }, "authorized_rerun_selection_complete": true, - "presentation_updated_at": "2026-10-10T20:20:40.424864+00:00", + "presentation_updated_at": "2026-10-10T20:38:18.201846+00:00", "media_repair": { "time": "2026-10-10T20:20:40.424866+00:00", "videos": 30, @@ -32426,5 +34431,24 @@ "fresh_recording_replacements": 4, "independent_native_diagnostics": 1, "native_verifier_changed": false + }, + "simple_instruction_reruns": { + "condition": "modified-instructions", + "tasks": [ + "task06/02", + "task06/09", + "task06/04", + "task06/05" + ], + "new_evaluations": 4, + "seed": 0, + "max_steps": 15000, + "native_verifier_changed": false, + "native_successes": 3, + "visible_goal_completed": 2, + "confirmed_false_positives": 1, + "confirmed_false_negatives": 0, + "scope": "Single-seed instruction revision, with earlier runs preserved. Native success and independent visible completion are reported separately.", + "comparison_limit": "Earlier selected runs have mixed budgets and use an earlier account; this is not a multi-seed causal success-rate estimate." } }