diff --git "a/task-index.json" "b/task-index.json" --- "a/task-index.json" +++ "b/task-index.json" @@ -113,7 +113,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/01-load-condiments-in-fridge/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/01-load-condiments-in-fridge/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -183,7 +183,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/02-filter-microwavable-item/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/02-filter-microwavable-item/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -253,7 +253,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/03-store-dumplings/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/03-store-dumplings/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -323,7 +323,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/04-divide-buffet-trays/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/04-divide-buffet-trays/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -388,7 +388,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/05-make-cheesecake-filling/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/05-make-cheesecake-filling/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -533,7 +533,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/06-multistep-steaming/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/06-multistep-steaming/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -623,7 +623,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/07-scale-portioning/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/07-scale-portioning/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -798,7 +798,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/08-scrub-cutting-board/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/08-scrub-cutting-board/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Clarify broad board coverage, maintained grasp/contact, and final retraction without numeric thresholds." @@ -1055,37 +1055,37 @@ "protocol_sha256": "44bb17021918606c49586677dc6bf98272b20e0f9bb33f301645f4bd5b24918e" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/final-observation.json", - "native_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", - "owner_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/final-observation.json", + "native_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", + "owner_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/memos/robocasa_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/memos/robocasa_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -1349,37 +1349,37 @@ "protocol_sha256": "44bb17021918606c49586677dc6bf98272b20e0f9bb33f301645f4bd5b24918e" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/final-observation.json", - "native_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", - "owner_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/evidence/episode/evidence/final-observation.json", + "native_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", + "owner_diagnostics": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/native/verifier/native-diagnostics.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/robocasa_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/memos/robocasa_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/diagnostics/task01/08/task01-08-scrub-cutting-board-codex-seed0-diagnostic-attempt01/resources/memos/robocasa_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -1461,7 +1461,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/09-prepare-veggie-dip/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/09-prepare-veggie-dip/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -1531,7 +1531,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task01/10-prepare-vegetable-roasting/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task01/10-prepare-vegetable-roasting/task.yaml" }, "run_status": "finished", "attempt_history": [], @@ -1900,35 +1900,35 @@ "protocol_sha256": "2cd301bfe0c1a1d6d4a420b1471374a8d33f9471ab060c1d6dcf66fbf74969bf" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -2184,35 +2184,35 @@ "protocol_sha256": "2cd301bfe0c1a1d6d4a420b1471374a8d33f9471ab060c1d6dcf66fbf74969bf" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-04-libero-10-04-codex-seed0-attempt01/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -2321,7 +2321,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/05-libero-10-05/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task02/05-libero-10-05/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." @@ -2376,7 +2376,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/06-libero-10-06/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task02/06-libero-10-06/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." @@ -2456,7 +2456,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/07-libero-10-07/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task02/07-libero-10-07/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." @@ -2703,35 +2703,35 @@ "protocol_sha256": "bdb44af586667ee7eab0855e1a04f70e56609333b710668aa923999c6424aa55" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -2986,35 +2986,35 @@ "protocol_sha256": "bdb44af586667ee7eab0855e1a04f70e56609333b710668aa923999c6424aa55" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-07-libero-10-07-codex-seed0-attempt01/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -3098,7 +3098,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/08-libero-10-08/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task02/08-libero-10-08/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." @@ -3345,35 +3345,35 @@ "protocol_sha256": "b0e2388679180c5a165f77c1cc90a2c5bfce0b2b1582b118c86392d435c21543" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -3628,35 +3628,35 @@ "protocol_sha256": "b0e2388679180c5a165f77c1cc90a2c5bfce0b2b1582b118c86392d435c21543" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda_control.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/tools/panda_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/tools/panda_control.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero_panda.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/memos/libero_panda.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-08-libero-10-08-codex-seed0-attempt01/resources/memos/libero_panda.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -3725,7 +3725,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task02/09-libero-10-09/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task02/09-libero-10-09/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Authorized full current-source LIBERO comparison refresh." @@ -3972,35 +3972,35 @@ "protocol_sha256": "acc9470d622733def9d480c2fb756b1724249077e1ad1f73cf80967246cabf7a" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/tools/panda.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/tools/panda.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/memos/libero-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/memos/libero-control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -4255,35 +4255,35 @@ "protocol_sha256": "acc9470d622733def9d480c2fb756b1724249077e1ad1f73cf80967246cabf7a" }, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", - "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/terminal.jpg" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/native/evidence/episode/evidence/final-observation.json", + "terminal_frame": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/terminal.jpg" }, "resources": [ { "name": "tools/panda.py", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/tools/panda.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/tools/panda.py", "kind": "Created during this episode; final workspace snapshot." }, { "name": "memos/libero-control.md", - "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/memos/libero-control.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/recording-repair/task02-09-libero-10-09-codex-seed0-attempt01/resources/memos/libero-control.md", "kind": "Created during this episode; final workspace snapshot." } ], @@ -4389,7 +4389,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/01-handover-block/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/01-handover-block/task.yaml" }, "attempt_history": [], "run_status": "finished", @@ -4472,7 +4472,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/02-place-dual-shoes/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/02-place-dual-shoes/task.yaml" }, "attempt_history": [], "run_status": "finished", @@ -4602,7 +4602,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/04-scan-object/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/04-scan-object/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Clarify the close, centered face-to-face scan pose and keeping both items grasped; native scan geometry thresholds are unchanged." @@ -4753,7 +4753,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/05-stack-bowls-three/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/05-stack-bowls-three/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Retain the existing concise instruction describing the compact bowl stack, release and open grippers; rerun using the current Kinex source." @@ -4941,7 +4941,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/07-hanging-mug/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/07-hanging-mug/task.yaml" }, "attempt_history": [], "run_status": "finished", @@ -5036,7 +5036,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/09-blocks-ranking-size/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/09-blocks-ranking-size/task.yaml" }, "attempt_history": [], "run_status": "finished", @@ -5104,7 +5104,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task03/10-lift-pot/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task03/10-lift-pot/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Retain the existing instruction describing two-handle grasping, upright orientation and sufficient lift; rerun using the current Kinex source." @@ -5207,7 +5207,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/01-robodojo-make-toast/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/01-robodojo-make-toast/task.yaml" }, "display_slot": "01", "display_key": "task04/01", @@ -5404,7 +5404,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/03-robodojo-store-laptop-and-headphones/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/03-robodojo-store-laptop-and-headphones/task.yaml" }, "display_slot": "03", "display_key": "task04/03", @@ -5546,7 +5546,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/04-robodojo-cover-blocks/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/04-robodojo-cover-blocks/task.yaml" }, "display_slot": "04", "display_key": "task04/04", @@ -5689,7 +5689,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/06-robodojo-store-tools-in-toolbox/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/06-robodojo-store-tools-in-toolbox/task.yaml" }, "display_slot": "06", "display_key": "task04/06", @@ -5756,7 +5756,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/07-robodojo-insert-tubes/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/07-robodojo-insert-tubes/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Describe fully seated tubes without exposing the verifier insertion-depth threshold." @@ -5870,7 +5870,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/08-robodojo-deposit-coin/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/08-robodojo-deposit-coin/task.yaml" }, "display_slot": "08", "display_key": "task04/08", @@ -5962,7 +5962,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/09-robodojo-fasten-screws/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/09-robodojo-fasten-screws/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Use a natural paragraph for matching and seating nuts; omit geometric scoring thresholds." @@ -6037,7 +6037,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/10-robodojo-play-stacking-toy/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/10-robodojo-play-stacking-toy/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Leave piece counts and peg matching to the agent while retaining the intended completed arrangement." @@ -6218,7 +6218,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/12-robodojo-arrange-largest-number/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/12-robodojo-arrange-largest-number/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Leave the numerical ordering strategy to the agent while retaining readable placement on the pads." @@ -6378,7 +6378,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/14-robodojo-build-tower/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/14-robodojo-build-tower/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Shorten the successful instruction while preserving the tower structure, original bottom faces, horizontal orientation, centering, plank alignment, perpendicular roof ridge, release and arm return requirements." @@ -6438,7 +6438,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/15-robodojo-classify-objects/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/15-robodojo-classify-objects/task.yaml" }, "display_slot": "14", "display_key": "task04/14", @@ -6515,7 +6515,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/16-robodojo-fill-egg-holder/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/16-robodojo-fill-egg-holder/task.yaml" }, "display_slot": "15", "display_key": "task04/15", @@ -6657,7 +6657,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/17-robodojo-fill-pen-holder/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/17-robodojo-fill-pen-holder/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Retain the native two-hand roles and final pen orientation without the insertion-depth threshold." @@ -6727,7 +6727,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/18-robodojo-fold-clothes/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/18-robodojo-fold-clothes/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Describe the folded garment without enumerating tracked point correspondences or grasping steps." @@ -6833,7 +6833,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/21-robodojo-hang-mugs/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/21-robodojo-hang-mugs/task.yaml" }, "display_slot": "19", "display_key": "task04/19", @@ -6965,7 +6965,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/23-robodojo-imitate-sorting-sequence/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/23-robodojo-imitate-sorting-sequence/task.yaml" }, "display_slot": "20", "display_key": "task04/20", @@ -7052,7 +7052,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/24-robodojo-insert-key/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/24-robodojo-insert-key/task.yaml" }, "display_slot": "21", "display_key": "task04/21", @@ -7169,7 +7169,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/25-robodojo-make-kong/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/25-robodojo-make-kong/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "State the turn and replacement-tile rules in one paragraph without a manipulation plan." @@ -7365,7 +7365,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/28-robodojo-organize-table/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/28-robodojo-organize-table/task.yaml" }, "display_slot": "24", "display_key": "task04/24", @@ -7442,7 +7442,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/29-robodojo-pack-objects-into-box/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/29-robodojo-pack-objects-into-box/task.yaml" }, "display_slot": "25", "display_key": "task04/25", @@ -7544,7 +7544,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/31-robodojo-pick-from-conveyor-by-image/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/31-robodojo-pick-from-conveyor-by-image/task.yaml" }, "display_slot": "26", "display_key": "task04/26", @@ -7661,7 +7661,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/32-robodojo-play-tic-tac-toe/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/32-robodojo-play-tic-tac-toe/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "State alternating turns and waiting rules without supplying the inferred ring counts." @@ -7736,7 +7736,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/33-robodojo-plug-in-charger/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/33-robodojo-plug-in-charger/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Describe a fully plugged-in charger without the verifier insertion-depth threshold." @@ -7796,7 +7796,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/34-robodojo-pour-balls-into-vase/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/34-robodojo-pour-balls-into-vase/task.yaml" }, "display_slot": "29", "display_key": "task04/29", @@ -7899,7 +7899,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/36-robodojo-pour-liquid-into-cup/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/36-robodojo-pour-liquid-into-cup/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Wait for flow to stop and liquid to settle while tilted before returning the bottle upright." @@ -8064,7 +8064,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/38-robodojo-press-by-number/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/38-robodojo-press-by-number/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Retain the per-button counting and confirmation rules without button-joint thresholds or card answers." @@ -8169,7 +8169,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/39-robodojo-push-t/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/39-robodojo-push-t/task.yaml", "evaluation_status": "Not yet evaluated with this revised instruction. The published result used the earlier instruction.", "native_predicates_unchanged": true, "review_reason": "Describe matching the target shape along the table without internal position and angle tolerances." @@ -8264,7 +8264,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/41-robodojo-put-bottles-into-dustbin/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/41-robodojo-put-bottles-into-dustbin/task.yaml" }, "display_slot": "34", "display_key": "task04/34", @@ -8367,7 +8367,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/43-robodojo-sort-nesting-dolls-by-size/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/43-robodojo-sort-nesting-dolls-by-size/task.yaml" }, "display_slot": "36", "display_key": "task04/36", @@ -8464,7 +8464,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/45-robodojo-stack-blocks/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/45-robodojo-stack-blocks/task.yaml" }, "display_slot": "37", "display_key": "task04/37", @@ -8537,7 +8537,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/48-robodojo-stack-bowls/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/48-robodojo-stack-bowls/task.yaml" }, "display_slot": "39", "display_key": "task04/39", @@ -8619,7 +8619,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/51-robodojo-swap-t/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/51-robodojo-swap-t/task.yaml" }, "display_slot": "40", "display_key": "task04/40", @@ -8781,7 +8781,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/52-robodojo-swap-blocks/task.yaml", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/52-robodojo-swap-blocks/task.yaml", "native_predicates_unchanged": true, "evaluation_status": "Evaluated with this modified instruction.", "review_reason": "Clarify the three moves and complete press, withdrawal and button rebound after each placement; native release thresholds remain unchanged." @@ -8885,7 +8885,7 @@ "changed": true, "preserve_whitespace": true, "label": "Modified instruction", - "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/catalog/task04/53-robodojo-sweep-blocks/task.yaml" + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task04/53-robodojo-sweep-blocks/task.yaml" }, "display_slot": "42", "display_key": "task04/42", @@ -8968,11 +8968,11 @@ "native_id": "simple/transfer-between-tables", "catalog_id": "simple/transfer-between-tables", "title": "Transfer between tables", - "catalog_instruction": "pick up the cracker box from table1,locomotion to table2,and place on table2.", + "catalog_instruction": "Pick up the cracker box from table1, carry it to table2, and release it inside the basket on table2.", "native_instruction": "pick up the cracker box from table1,locomotion to table2,and place on table2.", - "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.", + "instruction_source": "Approved benchmark instruction revision; native simulator wording and verifier remain frozen.", "status": "completed", - "episode_id": "task06-02-seed0-formal-attempt02", + "episode_id": "task06-02-transfer-between-tables-codex-seed0-instruction-attempt04", "run_status": "finished", "status_note": "Native result, execution and usage complete.", "planned_protocol": { @@ -8996,7 +8996,7 @@ }, "timeout_s": 28800, "mode": "stepped", - "instruction_policy": "original_native" + "instruction_policy": "modified" }, "native_identity": { "arm_gravity_compensation": true, @@ -9005,11 +9005,11 @@ "sim_mode": "mujoco_isaac" }, "simple_rerun": { - "attempt": 2, + "attempt": 4, "status": "published", "target_max_steps": 15000, - "previous_max_steps": 10000, - "note": "Fresh 15000-step rerun selected; previous 10000-step native failure retained below." + "previous_max_steps": 15000, + "note": "Fresh modified-instruction result selected. Earlier native-instruction runs and their unchanged native verdicts are retained below." }, "attempt_history": [ { @@ -9023,29 +9023,389 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/02/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/02/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/simple-10000/02/episode.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, - "selected_for_formal_metrics": false + "selected_for_formal_metrics": false, + "visual_review": { + "status": "instruction_false_negative", + "confidence": "high", + "review": "Box visibly released on table2 near the basket; native failure requires basket contact rather than ordinary placement on table2. Historical instruction/verifier disagreement." + } + }, + { + "id": "task06-02-seed0-formal-attempt02", + "task_key": "task06/02", + "family": "task06", + "slot": "02", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": false, + "native_reward": 0.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 3875, + "success": false, + "termination": "stopped" + }, + "steps": 3875, + "simulation_time_s": null, + "wall_time_s": 1106.044938, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "pick up the cracker box from table1,locomotion to table2,and place on table2.", + "instruction": "pick up the cracker box from table1,locomotion to table2,and place on table2.", + "instruction_policy": "original_native", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9820486387779543, + "cache_reported_input_tokens": 4989928, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 4989928, + "cached_input_tokens": 4900352, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 4989928, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 4900352, + "known_input_tokens": 4989928, + "known_output_tokens": 18385, + "known_reasoning_output_tokens": 6657, + "output_tokens": 18385, + "reasoning_output_tokens": 6657, + "reasoning_reported_output_tokens": 18385, + "reported_responses": { + "cache_reported_input_tokens": 90, + "cache_write_input_tokens": 90, + "cache_write_reported_input_tokens": 90, + "cached_input_tokens": 90, + "input_tokens": 90, + "output_tokens": 90, + "reasoning_output_tokens": 90, + "reasoning_reported_output_tokens": 90 + }, + "response_count": 90, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 89576, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 89, + "model_tool_calls_by_name": { + "exec": 89 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-stereo-media/2", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1280, + "height": 360, + "native_frames": 776, + "published_frames": 776, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 775, + "duration_s": 19.4, + "sha256": "d5de93036b922175abed15e1d59c2060c131133cb379cd05a322a5f6431fdbe1", + "source_sha256": { + "camera_head_left.mp4": "53abfd74f074d3ee748dbd647ba095793ebd713ba16e2504e2b17a3b5d44ae27", + "camera_head_right.mp4": "a17439e097d2e3ef0aafa45191610b16454bbfe2c3214b6e3628c7088d0f7457" + }, + "all_frames_compared": 776, + "minimum_frame_psnr_db": 37.273305160584975, + "maximum_frame_rgb_mae": 2.372187614440918, + "terminal_frame_psnr_db": 41.26179590726265, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "One decoded stereo pair for each native index; raw RGB input at source_fps * speed", + "scope": "Original native stereo recording, all samples at 4x speed; no simulation replay or rerender", + "recording": { + "accepted_samples": 776, + "captured_samples": 776, + "clock": "simulation", + "dropped_samples": 0, + "encoded_frames": 776, + "end_time_s": 77.50000000000172, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 776, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_left", + "pose": null, + "source": "camera_head_left", + "width": 640 + }, + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_right", + "pose": null, + "source": "camera_head_right", + "width": 640 + } + ] + } + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 3,875 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc", + "dirty": true, + "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2", + "dirty": true, + "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task06-02-transfer-between-tables-codex-seed0-attempt02", + "attempt": 2, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 4, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 8, + "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217", + "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957" + }, + "session_original_sha256": "efee4c63cca91fe21230a740f9b45eb87ac9c145977761b56ae6e54f04b051ef", + "protocol_sha256": "87f34e1db6395a96c29d02fda9e5da869a64e77792776cf2f5d133806f37ede8", + "deployment": "failed10000-rerun15000-20261010T020844Z", + "max_steps": 15000, + "budget_revision": "failed10000-rerun15000-20261010T020844Z", + "same_protocol_replacement": false, + "replaces_user_interrupted_attempt": null, + "replacement_reason": "Explicit user-authorized fresh rerun of a completed 10000-step native failure with 15000 steps.", + "previous_native_failure": "task06-02-transfer-between-tables-codex-seed0-attempt01", + "previous_max_steps": 10000 + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" + }, + "resources": [ + { + "name": "tools/robot.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/resources/tools/robot.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/g1_table_transfer.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/02/seed-0-attempt02/resources/memos/g1_table_transfer.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 198, + "observed_images": 36, + "tool_errors": 1 + }, + "selected_for_formal_metrics": false, + "max_steps": 15000, + "condition": "original-native-instruction", + "visual_review": { + "status": "instruction_false_negative", + "confidence": "high", + "review": "Box is visibly released onto table2 outside the basket. The instruction asks for placement on table2, but active reward requires basket contact and no hand contact; the table2 contact helper is not used." + } } - ] + ], + "instruction_revision": { + "native_instruction": "pick up the cracker box from table1,locomotion to table2,and place on table2.", + "instruction": "Pick up the cracker box from table1, carry it to table2, and release it inside the basket on table2.", + "native_instruction_kind": "source", + "diff": [ + { + "op": "replace", + "original": "pick", + "modified": "Pick" + }, + { + "op": "equal", + "original": " up the cracker box from ", + "modified": " up the cracker box from " + }, + { + "op": "replace", + "original": "table1,locomotion", + "modified": "table1, carry it" + }, + { + "op": "equal", + "original": " to ", + "modified": " to " + }, + { + "op": "replace", + "original": "table2,and", + "modified": "table2," + }, + { + "op": "equal", + "original": " ", + "modified": " " + }, + { + "op": "replace", + "original": "place ", + "modified": "and release it inside the basket " + }, + { + "op": "equal", + "original": "on table2.", + "modified": "on table2." + } + ], + "changed": true, + "preserve_whitespace": true, + "label": "Modified instruction", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task06/02-transfer-between-tables/task.yaml" + }, + "instruction_rerun": { + "attempt": 4, + "condition": "modified-instructions", + "max_steps": 15000, + "seed": 0, + "status": "published", + "native_verifier_changed": false, + "prior_selection_retained": true + } }, { "key": "task06/03", @@ -9109,27 +9469,33 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/03/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/03/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/03/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/simple-10000/03/episode.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, - "selected_for_formal_metrics": false + "selected_for_formal_metrics": false, + "visual_review": { + "status": "no_confirmed_mismatch", + "confidence": "limited", + "review": "No verified sustained pickup above the native lift threshold. Native failure is not contradicted by the sampled footage." + } } ] }, @@ -9140,11 +9506,11 @@ "native_id": "simple/bend-pick-and-place", "catalog_id": "simple/bend-pick-and-place", "title": "Bend pick and place", - "catalog_instruction": "bend to grasp the cracker box and drop it in the basket.", + "catalog_instruction": "Bend down, pick up the cracker box, and release it above the center of the basket so it drops inside.", "native_instruction": "bend to grasp the cracker box and drop it in the basket.", - "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.", + "instruction_source": "Approved benchmark instruction revision; native simulator wording and verifier remain frozen.", "status": "completed", - "episode_id": "task06-04-seed0-formal-attempt02", + "episode_id": "task06-04-bend-pick-and-place-codex-seed0-instruction-attempt04", "run_status": "finished", "status_note": "Native result, execution and usage complete.", "planned_protocol": { @@ -9168,7 +9534,7 @@ }, "timeout_s": 28800, "mode": "stepped", - "instruction_policy": "original_native" + "instruction_policy": "modified" }, "native_identity": { "arm_gravity_compensation": true, @@ -9177,11 +9543,11 @@ "sim_mode": "mujoco_isaac" }, "simple_rerun": { - "attempt": 2, + "attempt": 4, "status": "published", "target_max_steps": 15000, - "previous_max_steps": 10000, - "note": "Fresh 15000-step rerun selected; previous 10000-step native failure retained below." + "previous_max_steps": 15000, + "note": "Fresh modified-instruction result selected. Earlier native-instruction runs and their unchanged native verdicts are retained below." }, "attempt_history": [ { @@ -9195,45 +9561,429 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/04/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/04/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/simple-10000/04/episode.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, - "selected_for_formal_metrics": false - } - ] - }, - { - "key": "task06/05", - "family": "task06", - "slot": "05", - "native_id": "simple/bend-handover", - "catalog_id": "simple/bend-handover", - "title": "Bend handover", - "catalog_instruction": "bend the robot and pick up the cracker box, then place it on the container.", - "native_instruction": "bend the robot and pick up the cracker box, then place it on the container.", - "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.", - "status": "completed", - "episode_id": "task06-05-seed0-formal-attempt02", - "run_status": "finished", - "status_note": "Native result, execution and usage complete.", - "planned_protocol": { + "selected_for_formal_metrics": false, + "visual_review": { + "status": "no_confirmed_mismatch", + "confidence": "high", + "review": "Box finishes on the floor, outside the basket. Native failure agrees with the final state." + } + }, + { + "id": "task06-04-seed0-formal-attempt02", + "task_key": "task06/04", + "family": "task06", + "slot": "04", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": true, + "native_reward": 1.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 116, + "success": true, + "termination": "success" + }, + "steps": 116, + "simulation_time_s": null, + "wall_time_s": 111.95334, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "bend to grasp the cracker box and drop it in the basket.", + "instruction": "bend to grasp the cracker box and drop it in the basket.", + "instruction_policy": "original_native", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9163462297790655, + "cache_reported_input_tokens": 329238, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 329238, + "cached_input_tokens": 301696, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 329238, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 301696, + "known_input_tokens": 329238, + "known_output_tokens": 2595, + "known_reasoning_output_tokens": 187, + "output_tokens": 2595, + "reasoning_output_tokens": 187, + "reasoning_reported_output_tokens": 2595, + "reported_responses": { + "cache_reported_input_tokens": 11, + "cache_write_input_tokens": 11, + "cache_write_reported_input_tokens": 11, + "cached_input_tokens": 11, + "input_tokens": 11, + "output_tokens": 11, + "reasoning_output_tokens": 11, + "reasoning_reported_output_tokens": 11 + }, + "response_count": 11, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 27542, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 10, + "model_tool_calls_by_name": { + "exec": 10 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-stereo-media/2", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1280, + "height": 360, + "native_frames": 24, + "published_frames": 24, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 23, + "duration_s": 0.6, + "sha256": "011fb24c546f4314b7c695a0f553544772086a397afa8dd50a9f0a0c12869e2e", + "source_sha256": { + "camera_head_left.mp4": "1cc8b62ab1d645f322702797da39c5b9c638b1a7b64fac0525e75c81001f3b33", + "camera_head_right.mp4": "ddf92efd54b082dcd847602221236c0de5a90d3e0c596148941f38205314c20f" + }, + "all_frames_compared": 24, + "minimum_frame_psnr_db": 36.0947630714779, + "maximum_frame_rgb_mae": 2.956716537475586, + "terminal_frame_psnr_db": 36.0947630714779, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "One decoded stereo pair for each native index; raw RGB input at source_fps * speed", + "scope": "Original native stereo recording, all samples at 4x speed; no simulation replay or rerender", + "recording": { + "accepted_samples": 25, + "captured_samples": 25, + "clock": "simulation", + "dropped_samples": 0, + "encoded_frames": 24, + "end_time_s": 2.3200000000000016, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 25, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_left", + "pose": null, + "source": "camera_head_left", + "width": 640 + }, + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_right", + "pose": null, + "source": "camera_head_right", + "width": 640 + } + ] + } + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 116 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc", + "dirty": true, + "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2", + "dirty": true, + "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task06-04-bend-pick-and-place-codex-seed0-attempt02", + "attempt": 2, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 2, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 8, + "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217", + "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957" + }, + "session_original_sha256": "ec64abe7c35301ed7902c9a352d65d221a43eda56dfdfd37ff537c0f98ce58d0", + "protocol_sha256": "f1f597064f79f90d61d0cd7b77a96ead0cf0c9c83b37d5abfc9541366e21b97c", + "deployment": "failed10000-rerun15000-20261010T020844Z", + "max_steps": 15000, + "budget_revision": "failed10000-rerun15000-20261010T020844Z", + "same_protocol_replacement": false, + "replaces_user_interrupted_attempt": null, + "replacement_reason": "Explicit user-authorized fresh rerun of a completed 10000-step native failure with 15000 steps.", + "previous_native_failure": "task06-04-bend-pick-and-place-codex-seed0-attempt01", + "previous_max_steps": 10000 + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" + }, + "resources": [ + { + "name": "tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/resources/tools/g1_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/g1_bend_pick.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/04/seed-0-attempt02/resources/memos/g1_bend_pick.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 29, + "observed_images": 5, + "tool_errors": 3 + }, + "selected_for_formal_metrics": false, + "max_steps": 15000, + "condition": "original-native-instruction", + "visual_review": { + "status": "instruction_false_positive", + "confidence": "high", + "review": "Only 116 action frames (2.32 simulation seconds), 12 published frames. Native terminal images show the box next to the basket exterior and forearm, without a grasp-and-drop sequence. Contact-only predicate plus two reward updates per step allows one qualifying step to reach success." + } + } + ], + "instruction_revision": { + "native_instruction": "bend to grasp the cracker box and drop it in the basket.", + "instruction": "Bend down, pick up the cracker box, and release it above the center of the basket so it drops inside.", + "native_instruction_kind": "source", + "diff": [ + { + "op": "replace", + "original": "bend", + "modified": "Bend" + }, + { + "op": "equal", + "original": " ", + "modified": " " + }, + { + "op": "replace", + "original": "to", + "modified": "down," + }, + { + "op": "equal", + "original": " ", + "modified": " " + }, + { + "op": "replace", + "original": "grasp", + "modified": "pick up" + }, + { + "op": "equal", + "original": " the cracker ", + "modified": " the cracker " + }, + { + "op": "replace", + "original": "box", + "modified": "box," + }, + { + "op": "equal", + "original": " and ", + "modified": " and " + }, + { + "op": "replace", + "original": "drop", + "modified": "release" + }, + { + "op": "equal", + "original": " it ", + "modified": " it " + }, + { + "op": "replace", + "original": "in", + "modified": "above" + }, + { + "op": "equal", + "original": " the ", + "modified": " the " + }, + { + "op": "replace", + "original": "basket.", + "modified": "center of the basket so it drops inside." + } + ], + "changed": true, + "preserve_whitespace": true, + "label": "Modified instruction", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task06/04-bend-pick-and-place/task.yaml" + }, + "instruction_rerun": { + "attempt": 4, + "condition": "modified-instructions", + "max_steps": 15000, + "seed": 0, + "status": "published", + "native_verifier_changed": false, + "prior_selection_retained": true + } + }, + { + "key": "task06/05", + "family": "task06", + "slot": "05", + "native_id": "simple/bend-handover", + "catalog_id": "simple/bend-handover", + "title": "Bend handover", + "catalog_instruction": "Bend down, pick up the cracker box, then release it above the basket opening so it drops inside.", + "native_instruction": "bend the robot and pick up the cracker box, then place it on the container.", + "instruction_source": "Approved benchmark instruction revision; native simulator wording and verifier remain frozen.", + "status": "completed", + "episode_id": "task06-05-bend-handover-codex-seed0-instruction-attempt04", + "run_status": "finished", + "status_note": "Native result, execution and usage complete.", + "planned_protocol": { "episodes": 1, "seed": 0, "control_frequency_hz": 50, @@ -9254,7 +10004,7 @@ }, "timeout_s": 28800, "mode": "stepped", - "instruction_policy": "original_native" + "instruction_policy": "modified" }, "native_identity": { "arm_gravity_compensation": true, @@ -9263,11 +10013,11 @@ "sim_mode": "mujoco_isaac" }, "simple_rerun": { - "attempt": 2, + "attempt": 4, "status": "published", "target_max_steps": 15000, - "previous_max_steps": 10000, - "note": "Fresh 15000-step rerun selected; previous 10000-step native failure retained below." + "previous_max_steps": 15000, + "note": "Fresh modified-instruction result selected. Earlier native-instruction runs and their unchanged native verdicts are retained below." }, "attempt_history": [ { @@ -9279,31 +10029,395 @@ "reason": null, "status": "finished" }, - "usage_complete": true, - "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/05/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/05/episode.json" + "usage_complete": true, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/simple-10000/05/episode.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" + }, + "selected_for_formal_metrics": false, + "visual_review": { + "status": "no_confirmed_mismatch", + "confidence": "high", + "review": "Box finishes released on the table, outside the basket; full required placement is not shown. No confirmed false negative." + } + }, + { + "id": "task06-05-seed0-formal-attempt02", + "task_key": "task06/05", + "family": "task06", + "slot": "05", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": true, + "native_reward": 1.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 12320, + "success": true, + "termination": "success" + }, + "steps": 12320, + "simulation_time_s": null, + "wall_time_s": 4359.883363, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "bend the robot and pick up the cracker box, then place it on the container.", + "instruction": "bend the robot and pick up the cracker box, then place it on the container.", + "instruction_policy": "original_native", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9926789916948668, + "cache_reported_input_tokens": 32371907, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 32371907, + "cached_input_tokens": 32134912, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 32371907, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 32134912, + "known_input_tokens": 32371907, + "known_output_tokens": 103454, + "known_reasoning_output_tokens": 69820, + "output_tokens": 103454, + "reasoning_output_tokens": 69820, + "reasoning_reported_output_tokens": 103454, + "reported_responses": { + "cache_reported_input_tokens": 297, + "cache_write_input_tokens": 297, + "cache_write_reported_input_tokens": 297, + "cached_input_tokens": 297, + "input_tokens": 297, + "output_tokens": 297, + "reasoning_output_tokens": 297, + "reasoning_reported_output_tokens": 297 + }, + "response_count": 297, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 236995, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 296, + "model_tool_calls_by_name": { + "exec": 296 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-stereo-media/2", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1280, + "height": 360, + "native_frames": 2465, + "published_frames": 2465, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 2464, + "duration_s": 61.625, + "sha256": "6e375814036f0e820e1bc519f368f7b8bcb8acb0fdc425eaa7b26e37b8322bf5", + "source_sha256": { + "camera_head_left.mp4": "99324d095b871bb7856ba04c0897deee67a7d094b5f7c5340be942c90643cc19", + "camera_head_right.mp4": "bd6a79e4d41523fc54dbf0eb78a4f228429bb16ae6db859e364cddddb65aa26c" + }, + "all_frames_compared": 2465, + "minimum_frame_psnr_db": 36.42045017056597, + "maximum_frame_rgb_mae": 2.767667770385742, + "terminal_frame_psnr_db": 36.993661080255514, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "One decoded stereo pair for each native index; raw RGB input at source_fps * speed", + "scope": "Original native stereo recording, all samples at 4x speed; no simulation replay or rerender", + "recording": { + "accepted_samples": 2465, + "captured_samples": 2465, + "clock": "simulation", + "dropped_samples": 0, + "encoded_frames": 2465, + "end_time_s": 246.40000000005224, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 2465, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_left", + "pose": null, + "source": "camera_head_left", + "width": 640 + }, + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_right", + "pose": null, + "source": "camera_head_right", + "width": 640 + } + ] + } + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 12,320 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1asuccess\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc", + "dirty": true, + "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2", + "dirty": true, + "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task06-05-bend-handover-codex-seed0-attempt02", + "attempt": 2, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 6, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 8, + "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217", + "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957" + }, + "session_original_sha256": "dd94125fcdce620f874702dd716c5257502cf1aef0288654232ff4baf07b2a5b", + "protocol_sha256": "01bd486cc627e7126d7a549e63ec36a66d3b828d4b374814f730b17f8680edc9", + "deployment": "failed10000-rerun15000-20261010T020844Z", + "max_steps": 15000, + "budget_revision": "failed10000-rerun15000-20261010T020844Z", + "same_protocol_replacement": false, + "replaces_user_interrupted_attempt": null, + "replacement_reason": "Explicit user-authorized fresh rerun of a completed 10000-step native failure with 15000 steps.", + "previous_native_failure": "task06-05-bend-handover-codex-seed0-attempt01", + "previous_max_steps": 10000 + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" + }, + "resources": [ + { + "name": "tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/resources/tools/g1_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/g1_dex3.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/05/seed-0-attempt02/resources/memos/g1_dex3.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 639, + "observed_images": 97, + "tool_errors": 3 + }, + "selected_for_formal_metrics": false, + "max_steps": 15000, + "condition": "original-native-instruction", + "visual_review": { + "status": "instruction_false_positive", + "confidence": "high", + "review": "Native terminal stereo frames still show the right hand holding the box against/beside the container, without release and completed placement. Success tests only box-container contact; hand release, containment, stable support and handover are absent from the active predicate." + } + } + ], + "instruction_revision": { + "native_instruction": "bend the robot and pick up the cracker box, then place it on the container.", + "instruction": "Bend down, pick up the cracker box, then release it above the basket opening so it drops inside.", + "native_instruction_kind": "source", + "diff": [ + { + "op": "replace", + "original": "bend", + "modified": "Bend" + }, + { + "op": "equal", + "original": " ", + "modified": " " + }, + { + "op": "replace", + "original": "the robot and", + "modified": "down," + }, + { + "op": "equal", + "original": " pick up the cracker box, then ", + "modified": " pick up the cracker box, then " + }, + { + "op": "replace", + "original": "place", + "modified": "release" }, - "selected_for_formal_metrics": false - } - ] + { + "op": "equal", + "original": " it ", + "modified": " it " + }, + { + "op": "replace", + "original": "on", + "modified": "above" + }, + { + "op": "equal", + "original": " the ", + "modified": " the " + }, + { + "op": "replace", + "original": "container.", + "modified": "basket opening so it drops inside." + } + ], + "changed": true, + "preserve_whitespace": true, + "label": "Modified instruction", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task06/05-bend-handover/task.yaml" + }, + "instruction_rerun": { + "attempt": 4, + "condition": "modified-instructions", + "max_steps": 15000, + "seed": 0, + "status": "published", + "native_verifier_changed": false, + "prior_selection_retained": true + } }, { "key": "task06/06", @@ -9367,27 +10481,33 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/06/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/06/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/06/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/simple-10000/06/episode.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, - "selected_for_formal_metrics": false + "selected_for_formal_metrics": false, + "visual_review": { + "status": "visibility_problem", + "confidence": "high", + "review": "Late viewpoint becomes almost black; original native terminal frames are also black. No completed handover and placement is demonstrated." + } } ] }, @@ -9453,27 +10573,33 @@ }, "usage_complete": true, "links": { - "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/agent/session.jsonl", - "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/agent/trajectory.json", - "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/agent/provider-usage.jsonl", - "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/transcript.json", - "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/verifier/episode.json", - "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/verifier/protocol.json", - "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/instructions.json", - "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/verifier/workspace.tar.gz", - "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/verifier/workspace.json", - "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", - "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", - "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/usage.json", - "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/analysis.json", - "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/provenance.json", - "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/video.mp4", - "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/poster.jpg", - "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/media-validation.json", - "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/episodes/task06/07/seed-0/native/evidence/episode/evidence/final-observation.json", - "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/91d2e52faff2e8558b8063e5203e6b42536b7018/history/simple-10000/07/episode.json" + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/07/seed-0/native/evidence/episode/evidence/final-observation.json", + "episode_record": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/history/simple-10000/07/episode.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" }, - "selected_for_formal_metrics": false + "selected_for_formal_metrics": false, + "visual_review": { + "status": "verifier_mismatch_but_run_failed", + "confidence": "high", + "review": "Apple and basket finish on the floor apart. The task predicate has hidden distractor contact requirements, but this run is visibly unsuccessful." + } } ] }, @@ -9528,11 +10654,11 @@ "native_id": "simple/open-oven", "catalog_id": "simple/open-oven", "title": "Open oven", - "catalog_instruction": "move forward to the oven and open it", + "catalog_instruction": "Move to the oven and open either door fully, to roughly a right angle.", "native_instruction": "move forward to the oven and open it", - "instruction_source": "Unchanged native SIMPLE instruction for the effective default task configuration.", + "instruction_source": "Approved benchmark instruction revision; native simulator wording and verifier remain frozen.", "status": "completed", - "episode_id": "task06-09-seed0-formal", + "episode_id": "task06-09-open-oven-codex-seed0-instruction-attempt04", "run_status": "finished", "status_note": "Native result, execution and usage complete.", "planned_protocol": { @@ -9556,13 +10682,349 @@ }, "timeout_s": 28800, "mode": "stepped", - "instruction_policy": "original_native" + "instruction_policy": "modified" }, "native_identity": { "arm_gravity_compensation": true, "dr_level": 0, "env_id": "simple/G1WholebodyOpenOvenTeleop-v0", "sim_mode": "mujoco_isaac" + }, + "instruction_revision": { + "native_instruction": "move forward to the oven and open it", + "instruction": "Move to the oven and open either door fully, to roughly a right angle.", + "native_instruction_kind": "source", + "diff": [ + { + "op": "replace", + "original": "move forward", + "modified": "Move" + }, + { + "op": "equal", + "original": " to the oven and open ", + "modified": " to the oven and open " + }, + { + "op": "replace", + "original": "it", + "modified": "either door fully, to roughly a right angle." + } + ], + "changed": true, + "preserve_whitespace": true, + "label": "Modified instruction", + "source_url": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/catalog/task06/09-open-oven/task.yaml" + }, + "attempt_history": [ + { + "id": "task06-09-seed0-formal", + "task_key": "task06/09", + "family": "task06", + "slot": "09", + "seed": 0, + "episode": 1, + "phase": "formal", + "status": "completed", + "success": false, + "native_reward": 0.0, + "valid": true, + "execution": { + "reason": null, + "status": "finished" + }, + "verdict": { + "evidence_valid": true, + "steps": 3170, + "success": false, + "termination": "stopped" + }, + "steps": 3170, + "simulation_time_s": null, + "wall_time_s": 918.375583, + "model": "gpt-6-astra", + "effort": "high", + "harness": "codex", + "codex_version": "0.160.0", + "native_instruction": "move forward to the oven and open it", + "instruction": "move forward to the oven and open it", + "instruction_policy": "original_native", + "usage": { + "accounting": "reported-responses", + "audit_complete": true, + "cache_hit_rate": 0.9660542143458919, + "cache_reported_input_tokens": 4133002, + "cache_write_input_tokens": 0, + "cache_write_reported_input_tokens": 4133002, + "cached_input_tokens": 3992704, + "completed_turns": 1, + "cost_usd": null, + "failed_turns": 0, + "input_tokens": 4133002, + "known_cache_write_input_tokens": 0, + "known_cached_input_tokens": 3992704, + "known_input_tokens": 4133002, + "known_output_tokens": 16710, + "known_reasoning_output_tokens": 5193, + "output_tokens": 16710, + "reasoning_output_tokens": 5193, + "reasoning_reported_output_tokens": 16710, + "reported_responses": { + "cache_reported_input_tokens": 84, + "cache_write_input_tokens": 84, + "cache_write_reported_input_tokens": 84, + "cached_input_tokens": 84, + "input_tokens": 84, + "output_tokens": 84, + "reasoning_output_tokens": 84, + "reasoning_reported_output_tokens": 84 + }, + "response_count": 84, + "response_ids_complete": true, + "schema": "rlebench/token-usage/1", + "source": "Codex token_usage_record per response", + "uncached_input_tokens": 140298, + "unidentified_usage_records": 0 + }, + "call_activity": { + "model_tool_calls": 83, + "model_tool_calls_by_name": { + "exec": 83 + }, + "nested_python_tool_invocations": null, + "python_device_rpc_attempts": null, + "python_device_rpc_attempts_by_action": null, + "python_device_rpc_errors": null, + "python_instrumented_model_tool_calls": null, + "python_tool_invocations": null, + "python_tool_invocations_by_origin": null, + "schema": "rlebench/call-activity/1", + "source": "Codex native sessions" + }, + "media": { + "schema": "rlebench/native-stereo-media/2", + "passed": true, + "speed": 4, + "source_fps": 10, + "output_fps": 40, + "width": 1280, + "height": 360, + "native_frames": 635, + "published_frames": 635, + "source_sample_coverage": 1.0, + "terminal_frame_retained": true, + "terminal_source_index": 634, + "duration_s": 15.875, + "sha256": "0cab12ef33849200981cc426d00338fa21e96c5883935cd1ef860e036685f5d5", + "source_sha256": { + "camera_head_left.mp4": "06cf5c373fb2d02854812e0cebe90e8d15e72c76fa928245282d3a7950d822fa", + "camera_head_right.mp4": "8dce7464960955f707ab9e9ae109ec8286cf86652a576e582ce6e5f724bf30ec" + }, + "all_frames_compared": 635, + "minimum_frame_psnr_db": 38.42669493473994, + "maximum_frame_rgb_mae": 2.0199508666992188, + "terminal_frame_psnr_db": 39.10580975354201, + "decode_ok": true, + "pts_verified": true, + "frame_mapping": "One decoded stereo pair for each native index; raw RGB input at source_fps * speed", + "scope": "Original native stereo recording, all samples at 4x speed; no simulation replay or rerender", + "recording": { + "accepted_samples": 635, + "captured_samples": 635, + "clock": "simulation", + "dropped_samples": 0, + "encoded_frames": 635, + "end_time_s": 63.40000000000431, + "error": null, + "experimental": true, + "fps": 10, + "received_samples": 635, + "schema": "roboenv/recording/1", + "state": "closed", + "status": "complete", + "views": [ + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_left", + "pose": null, + "source": "camera_head_left", + "width": 640 + }, + { + "fov_y": 45.0, + "height": 360, + "name": "camera_head_right", + "pose": null, + "source": "camera_head_right", + "width": 640 + } + ] + } + }, + "analysis": { + "outcome": "\u539f\u751f\u5224\u5b9a\u672a\u6210\u529f\u3002", + "duration": "\u4f7f\u7528 3,170 / 15,000 \u4e2a\u52a8\u4f5c\u5e27\uff1b\u7ed3\u675f\u539f\u56e0\uff1astopped\u3002", + "execution": "\u6267\u884c\u6b63\u5e38\u5b8c\u6210\uff0c\u7528\u91cf\u8bb0\u5f55\u5b8c\u6574\u3002", + "cause": "\u539f\u751f\u5931\u8d25\u5df2\u7ecf\u786e\u8ba4\uff1b\u5177\u4f53\u5931\u8d25\u673a\u5236\u5c1a\u672a\u9010\u9879\u5b9a\u4f4d\uff0c\u8bf7\u7ed3\u5408\u7ec8\u6001\u8bc1\u636e\u548c\u5b8c\u6574 session \u67e5\u770b\u3002" + }, + "provenance": { + "sources": { + "RLE-Bench-inhouse": { + "build_inputs": [ + "pyproject.toml", + "src", + "tasks", + "README.md", + "Makefile", + "tests", + "docs" + ], + "content_sha256": "c9c6cd1b554380cfffee56b9a7bb47e102296d7bd38690f3ddcef2a4a5ca7bfc", + "dirty": true, + "revision": "2f315b78ab45a2978c2a69b7c33ec8fa4f78f618", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RLE-Bench-inhouse" + }, + "RoboEnv": { + "build_inputs": [ + "pyproject.toml", + "README.md", + "src", + "runtime/pyproject.toml", + "runtime/README.md", + "runtime/src", + "runtime/environments.json", + "runtime/locks", + "catalog", + "upstreams.lock.json", + "third_party/patches", + "docs/validation" + ], + "content_sha256": "0ea971928890164e2c1b0056a6cc3ebd0d1a306a63746193c33cf0b6e15a9aa2", + "dirty": true, + "revision": "b73356c09507eeffaf9ca9e90b668ef0b7f1fa92", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/RoboEnv" + }, + "kinex": { + "build_inputs": [ + "package.json", + "package-lock.json", + ".nvmrc", + "tsconfig.json", + "VERSION", + "src", + "packages/core", + "packages/setup", + "script", + "assets", + "bin" + ], + "content_sha256": "70f5107905efa45cace74e04b630660aaee4c80e1db2e3d540d6c81e0f4d0d38", + "dirty": false, + "revision": "caac19a8a36272972f762e0f74cfe381e8500048", + "source": "/home/evaluator/PhyRSI/benchmark-v0.10.0/kinex" + } + }, + "job": "task06-09-open-oven-codex-seed0-attempt03", + "attempt": 3, + "harness": "stock Codex CLI", + "control_interface": "public RoboEnv SDK and CLI", + "kinex_agent_runtime_used": false, + "gpu_index": 2, + "gpu_model": "NVIDIA L40S", + "campaign_dispatch_concurrency_limit": 8, + "concurrency_note": "One GPU per active SIMPLE episode; dispatch only on idle GPUs.", + "codex_version": "0.160.0", + "model": "gpt-6-astra", + "effort": "high", + "service_tier": "default", + "fresh_session": true, + "source_jobs": [], + "resume_trajectory": false, + "imported_skills": [], + "automatic_harbor_retries": 0, + "request_policy": { + "max_request_retries": 50, + "configuration": "explicit retry50 SSE", + "usage_accounting": "reported-responses" + }, + "classification": "formal", + "measured_images": { + "agent": "sha256:0093e794ce2fe8d5ab270fccfc8aef2ee8faf15d040d5a83f8c9416335b19217", + "task": "sha256:66c86a017966a02653a3a2278a74002e1d16a717e6b21aa5e884cb94d54a8957" + }, + "session_original_sha256": "6fb961f0b29e07b7221771021776cddb69173e038f2ec5beba7e983b4c767efb", + "protocol_sha256": "cabc50e33ebf5f7a2e34a075aff7a1f66d2b2ffc5f84e941d688a929d7f70681", + "deployment": "steps15000-20261010T012736Z", + "max_steps": 15000, + "budget_revision": "steps15000-20261010T012736Z", + "same_protocol_replacement": false, + "replaces_user_interrupted_attempt": "task06-09-open-oven-codex-seed0-attempt01", + "replacement_reason": "User requested 300 seconds at 50 Hz with updated code." + }, + "links": { + "native_session": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/agent/session.jsonl", + "trace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/agent/trajectory.json", + "provider_usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/agent/provider-usage.jsonl", + "transcript": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/transcript.json", + "verdict": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/verifier/episode.json", + "protocol": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/verifier/protocol.json", + "native_goal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/instructions.json", + "workspace": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/verifier/workspace.tar.gz", + "workspace_changes": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/verifier/workspace.json", + "owner_journal": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/evidence/episode/evidence/journal/actions.jsonl", + "recording_manifest": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/evidence/episode/evidence/recording/manifest.json", + "usage": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/usage.json", + "analysis": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/analysis.json", + "provenance": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/provenance.json", + "video": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/video.mp4", + "poster": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/poster.jpg", + "media_check": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/media-validation.json", + "final_observation": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/native/evidence/episode/evidence/final-observation.json", + "independent_review": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/validation/simple/original-run-review.json" + }, + "resources": [ + { + "name": "tools/g1_control.py", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/resources/tools/g1_control.py", + "kind": "Created during this episode; final workspace snapshot." + }, + { + "name": "memos/g1_oven.md", + "file": "https://huggingface.co/datasets/RLE-Bench/codex-benchmark/resolve/048598de801dd9d2bd75fdb8fe4faab62b7e1bda/episodes/task06/09/seed-0/resources/memos/g1_oven.md", + "kind": "Created during this episode; final workspace snapshot." + } + ], + "session_counts": { + "visible_events": 183, + "observed_images": 31, + "tool_errors": 1 + }, + "selected_for_formal_metrics": false, + "max_steps": 15000, + "condition": "original-native-instruction", + "visual_review": { + "status": "threshold_explained", + "confidence": "high", + "review": "Door looks open, but terminal oven joints are 0.63390 rad (36.32 degrees) and 0.04089 rad (2.34 degrees); native requires either >1.4 rad (80.21 degrees). Not a proven native false negative; instruction omits the required opening angle." + } + } + ], + "instruction_rerun": { + "attempt": 4, + "condition": "modified-instructions", + "max_steps": 15000, + "seed": 0, + "status": "published", + "native_verifier_changed": false, + "prior_selection_retained": true + }, + "simple_rerun": { + "attempt": 4, + "status": "published", + "target_max_steps": 15000, + "previous_max_steps": 15000, + "note": "Fresh modified-instruction result selected. Earlier native-instruction runs and their unchanged native verdicts are retained below." } }, {